GNU Linux-libre 4.19.314-gnu1
[releases.git] / drivers / md / bcache / request.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Main bcache entry point - handle a read or a write request and decide what to
4  * do with it; the make_request functions are called by the block layer.
5  *
6  * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
7  * Copyright 2012 Google, Inc.
8  */
9
10 #include "bcache.h"
11 #include "btree.h"
12 #include "debug.h"
13 #include "request.h"
14 #include "writeback.h"
15
16 #include <linux/module.h>
17 #include <linux/hash.h>
18 #include <linux/random.h>
19 #include <linux/backing-dev.h>
20
21 #include <trace/events/bcache.h>
22
23 #define CUTOFF_CACHE_ADD        95
24 #define CUTOFF_CACHE_READA      90
25
26 struct kmem_cache *bch_search_cache;
27
28 static void bch_data_insert_start(struct closure *cl);
29
30 static unsigned int cache_mode(struct cached_dev *dc)
31 {
32         return BDEV_CACHE_MODE(&dc->sb);
33 }
34
35 static bool verify(struct cached_dev *dc)
36 {
37         return dc->verify;
38 }
39
40 static void bio_csum(struct bio *bio, struct bkey *k)
41 {
42         struct bio_vec bv;
43         struct bvec_iter iter;
44         uint64_t csum = 0;
45
46         bio_for_each_segment(bv, bio, iter) {
47                 void *d = kmap(bv.bv_page) + bv.bv_offset;
48
49                 csum = bch_crc64_update(csum, d, bv.bv_len);
50                 kunmap(bv.bv_page);
51         }
52
53         k->ptr[KEY_PTRS(k)] = csum & (~0ULL >> 1);
54 }
55
56 /* Insert data into cache */
57
58 static void bch_data_insert_keys(struct closure *cl)
59 {
60         struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
61         atomic_t *journal_ref = NULL;
62         struct bkey *replace_key = op->replace ? &op->replace_key : NULL;
63         int ret;
64
65         /*
66          * If we're looping, might already be waiting on
67          * another journal write - can't wait on more than one journal write at
68          * a time
69          *
70          * XXX: this looks wrong
71          */
72 #if 0
73         while (atomic_read(&s->cl.remaining) & CLOSURE_WAITING)
74                 closure_sync(&s->cl);
75 #endif
76
77         if (!op->replace)
78                 journal_ref = bch_journal(op->c, &op->insert_keys,
79                                           op->flush_journal ? cl : NULL);
80
81         ret = bch_btree_insert(op->c, &op->insert_keys,
82                                journal_ref, replace_key);
83         if (ret == -ESRCH) {
84                 op->replace_collision = true;
85         } else if (ret) {
86                 op->status              = BLK_STS_RESOURCE;
87                 op->insert_data_done    = true;
88         }
89
90         if (journal_ref)
91                 atomic_dec_bug(journal_ref);
92
93         if (!op->insert_data_done) {
94                 continue_at(cl, bch_data_insert_start, op->wq);
95                 return;
96         }
97
98         bch_keylist_free(&op->insert_keys);
99         closure_return(cl);
100 }
101
102 static int bch_keylist_realloc(struct keylist *l, unsigned int u64s,
103                                struct cache_set *c)
104 {
105         size_t oldsize = bch_keylist_nkeys(l);
106         size_t newsize = oldsize + u64s;
107
108         /*
109          * The journalling code doesn't handle the case where the keys to insert
110          * is bigger than an empty write: If we just return -ENOMEM here,
111          * bch_data_insert_keys() will insert the keys created so far
112          * and finish the rest when the keylist is empty.
113          */
114         if (newsize * sizeof(uint64_t) > block_bytes(c) - sizeof(struct jset))
115                 return -ENOMEM;
116
117         return __bch_keylist_realloc(l, u64s);
118 }
119
120 static void bch_data_invalidate(struct closure *cl)
121 {
122         struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
123         struct bio *bio = op->bio;
124
125         pr_debug("invalidating %i sectors from %llu",
126                  bio_sectors(bio), (uint64_t) bio->bi_iter.bi_sector);
127
128         while (bio_sectors(bio)) {
129                 unsigned int sectors = min(bio_sectors(bio),
130                                        1U << (KEY_SIZE_BITS - 1));
131
132                 if (bch_keylist_realloc(&op->insert_keys, 2, op->c))
133                         goto out;
134
135                 bio->bi_iter.bi_sector  += sectors;
136                 bio->bi_iter.bi_size    -= sectors << 9;
137
138                 bch_keylist_add(&op->insert_keys,
139                                 &KEY(op->inode,
140                                      bio->bi_iter.bi_sector,
141                                      sectors));
142         }
143
144         op->insert_data_done = true;
145         /* get in bch_data_insert() */
146         bio_put(bio);
147 out:
148         continue_at(cl, bch_data_insert_keys, op->wq);
149 }
150
151 static void bch_data_insert_error(struct closure *cl)
152 {
153         struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
154
155         /*
156          * Our data write just errored, which means we've got a bunch of keys to
157          * insert that point to data that wasn't successfully written.
158          *
159          * We don't have to insert those keys but we still have to invalidate
160          * that region of the cache - so, if we just strip off all the pointers
161          * from the keys we'll accomplish just that.
162          */
163
164         struct bkey *src = op->insert_keys.keys, *dst = op->insert_keys.keys;
165
166         while (src != op->insert_keys.top) {
167                 struct bkey *n = bkey_next(src);
168
169                 SET_KEY_PTRS(src, 0);
170                 memmove(dst, src, bkey_bytes(src));
171
172                 dst = bkey_next(dst);
173                 src = n;
174         }
175
176         op->insert_keys.top = dst;
177
178         bch_data_insert_keys(cl);
179 }
180
181 static void bch_data_insert_endio(struct bio *bio)
182 {
183         struct closure *cl = bio->bi_private;
184         struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
185
186         if (bio->bi_status) {
187                 /* TODO: We could try to recover from this. */
188                 if (op->writeback)
189                         op->status = bio->bi_status;
190                 else if (!op->replace)
191                         set_closure_fn(cl, bch_data_insert_error, op->wq);
192                 else
193                         set_closure_fn(cl, NULL, NULL);
194         }
195
196         bch_bbio_endio(op->c, bio, bio->bi_status, "writing data to cache");
197 }
198
199 static void bch_data_insert_start(struct closure *cl)
200 {
201         struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
202         struct bio *bio = op->bio, *n;
203
204         if (op->bypass)
205                 return bch_data_invalidate(cl);
206
207         if (atomic_sub_return(bio_sectors(bio), &op->c->sectors_to_gc) < 0)
208                 wake_up_gc(op->c);
209
210         /*
211          * Journal writes are marked REQ_PREFLUSH; if the original write was a
212          * flush, it'll wait on the journal write.
213          */
214         bio->bi_opf &= ~(REQ_PREFLUSH|REQ_FUA);
215
216         do {
217                 unsigned int i;
218                 struct bkey *k;
219                 struct bio_set *split = &op->c->bio_split;
220
221                 /* 1 for the device pointer and 1 for the chksum */
222                 if (bch_keylist_realloc(&op->insert_keys,
223                                         3 + (op->csum ? 1 : 0),
224                                         op->c)) {
225                         continue_at(cl, bch_data_insert_keys, op->wq);
226                         return;
227                 }
228
229                 k = op->insert_keys.top;
230                 bkey_init(k);
231                 SET_KEY_INODE(k, op->inode);
232                 SET_KEY_OFFSET(k, bio->bi_iter.bi_sector);
233
234                 if (!bch_alloc_sectors(op->c, k, bio_sectors(bio),
235                                        op->write_point, op->write_prio,
236                                        op->writeback))
237                         goto err;
238
239                 n = bio_next_split(bio, KEY_SIZE(k), GFP_NOIO, split);
240
241                 n->bi_end_io    = bch_data_insert_endio;
242                 n->bi_private   = cl;
243
244                 if (op->writeback) {
245                         SET_KEY_DIRTY(k, true);
246
247                         for (i = 0; i < KEY_PTRS(k); i++)
248                                 SET_GC_MARK(PTR_BUCKET(op->c, k, i),
249                                             GC_MARK_DIRTY);
250                 }
251
252                 SET_KEY_CSUM(k, op->csum);
253                 if (KEY_CSUM(k))
254                         bio_csum(n, k);
255
256                 trace_bcache_cache_insert(k);
257                 bch_keylist_push(&op->insert_keys);
258
259                 bio_set_op_attrs(n, REQ_OP_WRITE, 0);
260                 bch_submit_bbio(n, op->c, k, 0);
261         } while (n != bio);
262
263         op->insert_data_done = true;
264         continue_at(cl, bch_data_insert_keys, op->wq);
265         return;
266 err:
267         /* bch_alloc_sectors() blocks if s->writeback = true */
268         BUG_ON(op->writeback);
269
270         /*
271          * But if it's not a writeback write we'd rather just bail out if
272          * there aren't any buckets ready to write to - it might take awhile and
273          * we might be starving btree writes for gc or something.
274          */
275
276         if (!op->replace) {
277                 /*
278                  * Writethrough write: We can't complete the write until we've
279                  * updated the index. But we don't want to delay the write while
280                  * we wait for buckets to be freed up, so just invalidate the
281                  * rest of the write.
282                  */
283                 op->bypass = true;
284                 return bch_data_invalidate(cl);
285         } else {
286                 /*
287                  * From a cache miss, we can just insert the keys for the data
288                  * we have written or bail out if we didn't do anything.
289                  */
290                 op->insert_data_done = true;
291                 bio_put(bio);
292
293                 if (!bch_keylist_empty(&op->insert_keys))
294                         continue_at(cl, bch_data_insert_keys, op->wq);
295                 else
296                         closure_return(cl);
297         }
298 }
299
300 /**
301  * bch_data_insert - stick some data in the cache
302  * @cl: closure pointer.
303  *
304  * This is the starting point for any data to end up in a cache device; it could
305  * be from a normal write, or a writeback write, or a write to a flash only
306  * volume - it's also used by the moving garbage collector to compact data in
307  * mostly empty buckets.
308  *
309  * It first writes the data to the cache, creating a list of keys to be inserted
310  * (if the data had to be fragmented there will be multiple keys); after the
311  * data is written it calls bch_journal, and after the keys have been added to
312  * the next journal write they're inserted into the btree.
313  *
314  * It inserts the data in s->cache_bio; bi_sector is used for the key offset,
315  * and op->inode is used for the key inode.
316  *
317  * If s->bypass is true, instead of inserting the data it invalidates the
318  * region of the cache represented by s->cache_bio and op->inode.
319  */
320 void bch_data_insert(struct closure *cl)
321 {
322         struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
323
324         trace_bcache_write(op->c, op->inode, op->bio,
325                            op->writeback, op->bypass);
326
327         bch_keylist_init(&op->insert_keys);
328         bio_get(op->bio);
329         bch_data_insert_start(cl);
330 }
331
332 /* Congested? */
333
334 unsigned int bch_get_congested(struct cache_set *c)
335 {
336         int i;
337         long rand;
338
339         if (!c->congested_read_threshold_us &&
340             !c->congested_write_threshold_us)
341                 return 0;
342
343         i = (local_clock_us() - c->congested_last_us) / 1024;
344         if (i < 0)
345                 return 0;
346
347         i += atomic_read(&c->congested);
348         if (i >= 0)
349                 return 0;
350
351         i += CONGESTED_MAX;
352
353         if (i > 0)
354                 i = fract_exp_two(i, 6);
355
356         rand = get_random_int();
357         i -= bitmap_weight(&rand, BITS_PER_LONG);
358
359         return i > 0 ? i : 1;
360 }
361
362 static void add_sequential(struct task_struct *t)
363 {
364         ewma_add(t->sequential_io_avg,
365                  t->sequential_io, 8, 0);
366
367         t->sequential_io = 0;
368 }
369
370 static struct hlist_head *iohash(struct cached_dev *dc, uint64_t k)
371 {
372         return &dc->io_hash[hash_64(k, RECENT_IO_BITS)];
373 }
374
375 static bool check_should_bypass(struct cached_dev *dc, struct bio *bio)
376 {
377         struct cache_set *c = dc->disk.c;
378         unsigned int mode = cache_mode(dc);
379         unsigned int sectors, congested = bch_get_congested(c);
380         struct task_struct *task = current;
381         struct io *i;
382
383         if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) ||
384             c->gc_stats.in_use > CUTOFF_CACHE_ADD ||
385             (bio_op(bio) == REQ_OP_DISCARD))
386                 goto skip;
387
388         if (mode == CACHE_MODE_NONE ||
389             (mode == CACHE_MODE_WRITEAROUND &&
390              op_is_write(bio_op(bio))))
391                 goto skip;
392
393         /*
394          * If the bio is for read-ahead or background IO, bypass it or
395          * not depends on the following situations,
396          * - If the IO is for meta data, always cache it and no bypass
397          * - If the IO is not meta data, check dc->cache_reada_policy,
398          *      BCH_CACHE_READA_ALL: cache it and not bypass
399          *      BCH_CACHE_READA_META_ONLY: not cache it and bypass
400          * That is, read-ahead request for metadata always get cached
401          * (eg, for gfs2 or xfs).
402          */
403         if ((bio->bi_opf & (REQ_RAHEAD|REQ_BACKGROUND))) {
404                 if (!(bio->bi_opf & (REQ_META|REQ_PRIO)) &&
405                     (dc->cache_readahead_policy != BCH_CACHE_READA_ALL))
406                         goto skip;
407         }
408
409         if (bio->bi_iter.bi_sector & (c->sb.block_size - 1) ||
410             bio_sectors(bio) & (c->sb.block_size - 1)) {
411                 pr_debug("skipping unaligned io");
412                 goto skip;
413         }
414
415         if (bypass_torture_test(dc)) {
416                 if ((get_random_int() & 3) == 3)
417                         goto skip;
418                 else
419                         goto rescale;
420         }
421
422         if (!congested && !dc->sequential_cutoff)
423                 goto rescale;
424
425         spin_lock(&dc->io_lock);
426
427         hlist_for_each_entry(i, iohash(dc, bio->bi_iter.bi_sector), hash)
428                 if (i->last == bio->bi_iter.bi_sector &&
429                     time_before(jiffies, i->jiffies))
430                         goto found;
431
432         i = list_first_entry(&dc->io_lru, struct io, lru);
433
434         add_sequential(task);
435         i->sequential = 0;
436 found:
437         if (i->sequential + bio->bi_iter.bi_size > i->sequential)
438                 i->sequential   += bio->bi_iter.bi_size;
439
440         i->last                  = bio_end_sector(bio);
441         i->jiffies               = jiffies + msecs_to_jiffies(5000);
442         task->sequential_io      = i->sequential;
443
444         hlist_del(&i->hash);
445         hlist_add_head(&i->hash, iohash(dc, i->last));
446         list_move_tail(&i->lru, &dc->io_lru);
447
448         spin_unlock(&dc->io_lock);
449
450         sectors = max(task->sequential_io,
451                       task->sequential_io_avg) >> 9;
452
453         if (dc->sequential_cutoff &&
454             sectors >= dc->sequential_cutoff >> 9) {
455                 trace_bcache_bypass_sequential(bio);
456                 goto skip;
457         }
458
459         if (congested && sectors >= congested) {
460                 trace_bcache_bypass_congested(bio);
461                 goto skip;
462         }
463
464 rescale:
465         bch_rescale_priorities(c, bio_sectors(bio));
466         return false;
467 skip:
468         bch_mark_sectors_bypassed(c, dc, bio_sectors(bio));
469         return true;
470 }
471
472 /* Cache lookup */
473
474 struct search {
475         /* Stack frame for bio_complete */
476         struct closure          cl;
477
478         struct bbio             bio;
479         struct bio              *orig_bio;
480         struct bio              *cache_miss;
481         struct bcache_device    *d;
482
483         unsigned int            insert_bio_sectors;
484         unsigned int            recoverable:1;
485         unsigned int            write:1;
486         unsigned int            read_dirty_data:1;
487         unsigned int            cache_missed:1;
488
489         unsigned long           start_time;
490
491         struct btree_op         op;
492         struct data_insert_op   iop;
493 };
494
495 static void bch_cache_read_endio(struct bio *bio)
496 {
497         struct bbio *b = container_of(bio, struct bbio, bio);
498         struct closure *cl = bio->bi_private;
499         struct search *s = container_of(cl, struct search, cl);
500
501         /*
502          * If the bucket was reused while our bio was in flight, we might have
503          * read the wrong data. Set s->error but not error so it doesn't get
504          * counted against the cache device, but we'll still reread the data
505          * from the backing device.
506          */
507
508         if (bio->bi_status)
509                 s->iop.status = bio->bi_status;
510         else if (!KEY_DIRTY(&b->key) &&
511                  ptr_stale(s->iop.c, &b->key, 0)) {
512                 atomic_long_inc(&s->iop.c->cache_read_races);
513                 s->iop.status = BLK_STS_IOERR;
514         }
515
516         bch_bbio_endio(s->iop.c, bio, bio->bi_status, "reading from cache");
517 }
518
519 /*
520  * Read from a single key, handling the initial cache miss if the key starts in
521  * the middle of the bio
522  */
523 static int cache_lookup_fn(struct btree_op *op, struct btree *b, struct bkey *k)
524 {
525         struct search *s = container_of(op, struct search, op);
526         struct bio *n, *bio = &s->bio.bio;
527         struct bkey *bio_key;
528         unsigned int ptr;
529
530         if (bkey_cmp(k, &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0)) <= 0)
531                 return MAP_CONTINUE;
532
533         if (KEY_INODE(k) != s->iop.inode ||
534             KEY_START(k) > bio->bi_iter.bi_sector) {
535                 unsigned int bio_sectors = bio_sectors(bio);
536                 unsigned int sectors = KEY_INODE(k) == s->iop.inode
537                         ? min_t(uint64_t, INT_MAX,
538                                 KEY_START(k) - bio->bi_iter.bi_sector)
539                         : INT_MAX;
540                 int ret = s->d->cache_miss(b, s, bio, sectors);
541
542                 if (ret != MAP_CONTINUE)
543                         return ret;
544
545                 /* if this was a complete miss we shouldn't get here */
546                 BUG_ON(bio_sectors <= sectors);
547         }
548
549         if (!KEY_SIZE(k))
550                 return MAP_CONTINUE;
551
552         /* XXX: figure out best pointer - for multiple cache devices */
553         ptr = 0;
554
555         PTR_BUCKET(b->c, k, ptr)->prio = INITIAL_PRIO;
556
557         if (KEY_DIRTY(k))
558                 s->read_dirty_data = true;
559
560         n = bio_next_split(bio, min_t(uint64_t, INT_MAX,
561                                       KEY_OFFSET(k) - bio->bi_iter.bi_sector),
562                            GFP_NOIO, &s->d->bio_split);
563
564         bio_key = &container_of(n, struct bbio, bio)->key;
565         bch_bkey_copy_single_ptr(bio_key, k, ptr);
566
567         bch_cut_front(&KEY(s->iop.inode, n->bi_iter.bi_sector, 0), bio_key);
568         bch_cut_back(&KEY(s->iop.inode, bio_end_sector(n), 0), bio_key);
569
570         n->bi_end_io    = bch_cache_read_endio;
571         n->bi_private   = &s->cl;
572
573         /*
574          * The bucket we're reading from might be reused while our bio
575          * is in flight, and we could then end up reading the wrong
576          * data.
577          *
578          * We guard against this by checking (in cache_read_endio()) if
579          * the pointer is stale again; if so, we treat it as an error
580          * and reread from the backing device (but we don't pass that
581          * error up anywhere).
582          */
583
584         __bch_submit_bbio(n, b->c);
585         return n == bio ? MAP_DONE : MAP_CONTINUE;
586 }
587
588 static void cache_lookup(struct closure *cl)
589 {
590         struct search *s = container_of(cl, struct search, iop.cl);
591         struct bio *bio = &s->bio.bio;
592         struct cached_dev *dc;
593         int ret;
594
595         bch_btree_op_init(&s->op, -1);
596
597         ret = bch_btree_map_keys(&s->op, s->iop.c,
598                                  &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0),
599                                  cache_lookup_fn, MAP_END_KEY);
600         if (ret == -EAGAIN) {
601                 continue_at(cl, cache_lookup, bcache_wq);
602                 return;
603         }
604
605         /*
606          * We might meet err when searching the btree, If that happens, we will
607          * get negative ret, in this scenario we should not recover data from
608          * backing device (when cache device is dirty) because we don't know
609          * whether bkeys the read request covered are all clean.
610          *
611          * And after that happened, s->iop.status is still its initial value
612          * before we submit s->bio.bio
613          */
614         if (ret < 0) {
615                 BUG_ON(ret == -EINTR);
616                 if (s->d && s->d->c &&
617                                 !UUID_FLASH_ONLY(&s->d->c->uuids[s->d->id])) {
618                         dc = container_of(s->d, struct cached_dev, disk);
619                         if (dc && atomic_read(&dc->has_dirty))
620                                 s->recoverable = false;
621                 }
622                 if (!s->iop.status)
623                         s->iop.status = BLK_STS_IOERR;
624         }
625
626         closure_return(cl);
627 }
628
629 /* Common code for the make_request functions */
630
631 static void request_endio(struct bio *bio)
632 {
633         struct closure *cl = bio->bi_private;
634
635         if (bio->bi_status) {
636                 struct search *s = container_of(cl, struct search, cl);
637
638                 s->iop.status = bio->bi_status;
639                 /* Only cache read errors are recoverable */
640                 s->recoverable = false;
641         }
642
643         bio_put(bio);
644         closure_put(cl);
645 }
646
647 static void backing_request_endio(struct bio *bio)
648 {
649         struct closure *cl = bio->bi_private;
650
651         if (bio->bi_status) {
652                 struct search *s = container_of(cl, struct search, cl);
653                 struct cached_dev *dc = container_of(s->d,
654                                                      struct cached_dev, disk);
655                 /*
656                  * If a bio has REQ_PREFLUSH for writeback mode, it is
657                  * speically assembled in cached_dev_write() for a non-zero
658                  * write request which has REQ_PREFLUSH. we don't set
659                  * s->iop.status by this failure, the status will be decided
660                  * by result of bch_data_insert() operation.
661                  */
662                 if (unlikely(s->iop.writeback &&
663                              bio->bi_opf & REQ_PREFLUSH)) {
664                         pr_err("Can't flush %s: returned bi_status %i",
665                                 dc->backing_dev_name, bio->bi_status);
666                 } else {
667                         /* set to orig_bio->bi_status in bio_complete() */
668                         s->iop.status = bio->bi_status;
669                 }
670                 s->recoverable = false;
671                 /* should count I/O error for backing device here */
672                 bch_count_backing_io_errors(dc, bio);
673         }
674
675         bio_put(bio);
676         closure_put(cl);
677 }
678
679 static void bio_complete(struct search *s)
680 {
681         if (s->orig_bio) {
682                 generic_end_io_acct(s->d->disk->queue, bio_op(s->orig_bio),
683                                     &s->d->disk->part0, s->start_time);
684
685                 trace_bcache_request_end(s->d, s->orig_bio);
686                 s->orig_bio->bi_status = s->iop.status;
687                 bio_endio(s->orig_bio);
688                 s->orig_bio = NULL;
689         }
690 }
691
692 static void do_bio_hook(struct search *s,
693                         struct bio *orig_bio,
694                         bio_end_io_t *end_io_fn)
695 {
696         struct bio *bio = &s->bio.bio;
697
698         bio_init(bio, NULL, 0);
699         __bio_clone_fast(bio, orig_bio);
700         /*
701          * bi_end_io can be set separately somewhere else, e.g. the
702          * variants in,
703          * - cache_bio->bi_end_io from cached_dev_cache_miss()
704          * - n->bi_end_io from cache_lookup_fn()
705          */
706         bio->bi_end_io          = end_io_fn;
707         bio->bi_private         = &s->cl;
708
709         bio_cnt_set(bio, 3);
710 }
711
712 static void search_free(struct closure *cl)
713 {
714         struct search *s = container_of(cl, struct search, cl);
715
716         atomic_dec(&s->d->c->search_inflight);
717
718         if (s->iop.bio)
719                 bio_put(s->iop.bio);
720
721         bio_complete(s);
722         closure_debug_destroy(cl);
723         mempool_free(s, &s->d->c->search);
724 }
725
726 static inline struct search *search_alloc(struct bio *bio,
727                                           struct bcache_device *d)
728 {
729         struct search *s;
730
731         s = mempool_alloc(&d->c->search, GFP_NOIO);
732
733         closure_init(&s->cl, NULL);
734         do_bio_hook(s, bio, request_endio);
735         atomic_inc(&d->c->search_inflight);
736
737         s->orig_bio             = bio;
738         s->cache_miss           = NULL;
739         s->cache_missed         = 0;
740         s->d                    = d;
741         s->recoverable          = 1;
742         s->write                = op_is_write(bio_op(bio));
743         s->read_dirty_data      = 0;
744         s->start_time           = jiffies;
745
746         s->iop.c                = d->c;
747         s->iop.bio              = NULL;
748         s->iop.inode            = d->id;
749         s->iop.write_point      = hash_long((unsigned long) current, 16);
750         s->iop.write_prio       = 0;
751         s->iop.status           = 0;
752         s->iop.flags            = 0;
753         s->iop.flush_journal    = op_is_flush(bio->bi_opf);
754         s->iop.wq               = bcache_wq;
755
756         return s;
757 }
758
759 /* Cached devices */
760
761 static void cached_dev_bio_complete(struct closure *cl)
762 {
763         struct search *s = container_of(cl, struct search, cl);
764         struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
765
766         search_free(cl);
767         cached_dev_put(dc);
768 }
769
770 /* Process reads */
771
772 static void cached_dev_cache_miss_done(struct closure *cl)
773 {
774         struct search *s = container_of(cl, struct search, cl);
775
776         if (s->iop.replace_collision)
777                 bch_mark_cache_miss_collision(s->iop.c, s->d);
778
779         if (s->iop.bio)
780                 bio_free_pages(s->iop.bio);
781
782         cached_dev_bio_complete(cl);
783 }
784
785 static void cached_dev_read_error(struct closure *cl)
786 {
787         struct search *s = container_of(cl, struct search, cl);
788         struct bio *bio = &s->bio.bio;
789
790         /*
791          * If read request hit dirty data (s->read_dirty_data is true),
792          * then recovery a failed read request from cached device may
793          * get a stale data back. So read failure recovery is only
794          * permitted when read request hit clean data in cache device,
795          * or when cache read race happened.
796          */
797         if (s->recoverable && !s->read_dirty_data) {
798                 /* Retry from the backing device: */
799                 trace_bcache_read_retry(s->orig_bio);
800
801                 s->iop.status = 0;
802                 do_bio_hook(s, s->orig_bio, backing_request_endio);
803
804                 /* XXX: invalidate cache */
805
806                 /* I/O request sent to backing device */
807                 closure_bio_submit(s->iop.c, bio, cl);
808         }
809
810         continue_at(cl, cached_dev_cache_miss_done, NULL);
811 }
812
813 static void cached_dev_read_done(struct closure *cl)
814 {
815         struct search *s = container_of(cl, struct search, cl);
816         struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
817
818         /*
819          * We had a cache miss; cache_bio now contains data ready to be inserted
820          * into the cache.
821          *
822          * First, we copy the data we just read from cache_bio's bounce buffers
823          * to the buffers the original bio pointed to:
824          */
825
826         if (s->iop.bio) {
827                 bio_reset(s->iop.bio);
828                 s->iop.bio->bi_iter.bi_sector =
829                         s->cache_miss->bi_iter.bi_sector;
830                 bio_copy_dev(s->iop.bio, s->cache_miss);
831                 s->iop.bio->bi_iter.bi_size = s->insert_bio_sectors << 9;
832                 bch_bio_map(s->iop.bio, NULL);
833
834                 bio_copy_data(s->cache_miss, s->iop.bio);
835
836                 bio_put(s->cache_miss);
837                 s->cache_miss = NULL;
838         }
839
840         if (verify(dc) && s->recoverable && !s->read_dirty_data)
841                 bch_data_verify(dc, s->orig_bio);
842
843         bio_complete(s);
844
845         if (s->iop.bio &&
846             !test_bit(CACHE_SET_STOPPING, &s->iop.c->flags)) {
847                 BUG_ON(!s->iop.replace);
848                 closure_call(&s->iop.cl, bch_data_insert, NULL, cl);
849         }
850
851         continue_at(cl, cached_dev_cache_miss_done, NULL);
852 }
853
854 static void cached_dev_read_done_bh(struct closure *cl)
855 {
856         struct search *s = container_of(cl, struct search, cl);
857         struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
858
859         bch_mark_cache_accounting(s->iop.c, s->d,
860                                   !s->cache_missed, s->iop.bypass);
861         trace_bcache_read(s->orig_bio, !s->cache_missed, s->iop.bypass);
862
863         if (s->iop.status)
864                 continue_at_nobarrier(cl, cached_dev_read_error, bcache_wq);
865         else if (s->iop.bio || verify(dc))
866                 continue_at_nobarrier(cl, cached_dev_read_done, bcache_wq);
867         else
868                 continue_at_nobarrier(cl, cached_dev_bio_complete, NULL);
869 }
870
871 static int cached_dev_cache_miss(struct btree *b, struct search *s,
872                                  struct bio *bio, unsigned int sectors)
873 {
874         int ret = MAP_CONTINUE;
875         unsigned int reada = 0;
876         struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
877         struct bio *miss, *cache_bio;
878
879         s->cache_missed = 1;
880
881         if (s->cache_miss || s->iop.bypass) {
882                 miss = bio_next_split(bio, sectors, GFP_NOIO, &s->d->bio_split);
883                 ret = miss == bio ? MAP_DONE : MAP_CONTINUE;
884                 goto out_submit;
885         }
886
887         if (!(bio->bi_opf & REQ_RAHEAD) &&
888             !(bio->bi_opf & (REQ_META|REQ_PRIO)) &&
889             s->iop.c->gc_stats.in_use < CUTOFF_CACHE_READA)
890                 reada = min_t(sector_t, dc->readahead >> 9,
891                               get_capacity(bio->bi_disk) - bio_end_sector(bio));
892
893         s->insert_bio_sectors = min(sectors, bio_sectors(bio) + reada);
894
895         s->iop.replace_key = KEY(s->iop.inode,
896                                  bio->bi_iter.bi_sector + s->insert_bio_sectors,
897                                  s->insert_bio_sectors);
898
899         ret = bch_btree_insert_check_key(b, &s->op, &s->iop.replace_key);
900         if (ret)
901                 return ret;
902
903         s->iop.replace = true;
904
905         miss = bio_next_split(bio, sectors, GFP_NOIO, &s->d->bio_split);
906
907         /* btree_search_recurse()'s btree iterator is no good anymore */
908         ret = miss == bio ? MAP_DONE : -EINTR;
909
910         cache_bio = bio_alloc_bioset(GFP_NOWAIT,
911                         DIV_ROUND_UP(s->insert_bio_sectors, PAGE_SECTORS),
912                         &dc->disk.bio_split);
913         if (!cache_bio)
914                 goto out_submit;
915
916         cache_bio->bi_iter.bi_sector    = miss->bi_iter.bi_sector;
917         bio_copy_dev(cache_bio, miss);
918         cache_bio->bi_iter.bi_size      = s->insert_bio_sectors << 9;
919
920         cache_bio->bi_end_io    = backing_request_endio;
921         cache_bio->bi_private   = &s->cl;
922
923         bch_bio_map(cache_bio, NULL);
924         if (bch_bio_alloc_pages(cache_bio, __GFP_NOWARN|GFP_NOIO))
925                 goto out_put;
926
927         if (reada)
928                 bch_mark_cache_readahead(s->iop.c, s->d);
929
930         s->cache_miss   = miss;
931         s->iop.bio      = cache_bio;
932         bio_get(cache_bio);
933         /* I/O request sent to backing device */
934         closure_bio_submit(s->iop.c, cache_bio, &s->cl);
935
936         return ret;
937 out_put:
938         bio_put(cache_bio);
939 out_submit:
940         miss->bi_end_io         = backing_request_endio;
941         miss->bi_private        = &s->cl;
942         /* I/O request sent to backing device */
943         closure_bio_submit(s->iop.c, miss, &s->cl);
944         return ret;
945 }
946
947 static void cached_dev_read(struct cached_dev *dc, struct search *s)
948 {
949         struct closure *cl = &s->cl;
950
951         closure_call(&s->iop.cl, cache_lookup, NULL, cl);
952         continue_at(cl, cached_dev_read_done_bh, NULL);
953 }
954
955 /* Process writes */
956
957 static void cached_dev_write_complete(struct closure *cl)
958 {
959         struct search *s = container_of(cl, struct search, cl);
960         struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
961
962         up_read_non_owner(&dc->writeback_lock);
963         cached_dev_bio_complete(cl);
964 }
965
966 static void cached_dev_write(struct cached_dev *dc, struct search *s)
967 {
968         struct closure *cl = &s->cl;
969         struct bio *bio = &s->bio.bio;
970         struct bkey start = KEY(dc->disk.id, bio->bi_iter.bi_sector, 0);
971         struct bkey end = KEY(dc->disk.id, bio_end_sector(bio), 0);
972
973         bch_keybuf_check_overlapping(&s->iop.c->moving_gc_keys, &start, &end);
974
975         down_read_non_owner(&dc->writeback_lock);
976         if (bch_keybuf_check_overlapping(&dc->writeback_keys, &start, &end)) {
977                 /*
978                  * We overlap with some dirty data undergoing background
979                  * writeback, force this write to writeback
980                  */
981                 s->iop.bypass = false;
982                 s->iop.writeback = true;
983         }
984
985         /*
986          * Discards aren't _required_ to do anything, so skipping if
987          * check_overlapping returned true is ok
988          *
989          * But check_overlapping drops dirty keys for which io hasn't started,
990          * so we still want to call it.
991          */
992         if (bio_op(bio) == REQ_OP_DISCARD)
993                 s->iop.bypass = true;
994
995         if (should_writeback(dc, s->orig_bio,
996                              cache_mode(dc),
997                              s->iop.bypass)) {
998                 s->iop.bypass = false;
999                 s->iop.writeback = true;
1000         }
1001
1002         if (s->iop.bypass) {
1003                 s->iop.bio = s->orig_bio;
1004                 bio_get(s->iop.bio);
1005
1006                 if (bio_op(bio) == REQ_OP_DISCARD &&
1007                     !blk_queue_discard(bdev_get_queue(dc->bdev)))
1008                         goto insert_data;
1009
1010                 /* I/O request sent to backing device */
1011                 bio->bi_end_io = backing_request_endio;
1012                 closure_bio_submit(s->iop.c, bio, cl);
1013
1014         } else if (s->iop.writeback) {
1015                 bch_writeback_add(dc);
1016                 s->iop.bio = bio;
1017
1018                 if (bio->bi_opf & REQ_PREFLUSH) {
1019                         /*
1020                          * Also need to send a flush to the backing
1021                          * device.
1022                          */
1023                         struct bio *flush;
1024
1025                         flush = bio_alloc_bioset(GFP_NOIO, 0,
1026                                                  &dc->disk.bio_split);
1027                         if (!flush) {
1028                                 s->iop.status = BLK_STS_RESOURCE;
1029                                 goto insert_data;
1030                         }
1031                         bio_copy_dev(flush, bio);
1032                         flush->bi_end_io = backing_request_endio;
1033                         flush->bi_private = cl;
1034                         flush->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
1035                         /* I/O request sent to backing device */
1036                         closure_bio_submit(s->iop.c, flush, cl);
1037                 }
1038         } else {
1039                 s->iop.bio = bio_clone_fast(bio, GFP_NOIO, &dc->disk.bio_split);
1040                 /* I/O request sent to backing device */
1041                 bio->bi_end_io = backing_request_endio;
1042                 closure_bio_submit(s->iop.c, bio, cl);
1043         }
1044
1045 insert_data:
1046         closure_call(&s->iop.cl, bch_data_insert, NULL, cl);
1047         continue_at(cl, cached_dev_write_complete, NULL);
1048 }
1049
1050 static void cached_dev_nodata(struct closure *cl)
1051 {
1052         struct search *s = container_of(cl, struct search, cl);
1053         struct bio *bio = &s->bio.bio;
1054
1055         if (s->iop.flush_journal)
1056                 bch_journal_meta(s->iop.c, cl);
1057
1058         /* If it's a flush, we send the flush to the backing device too */
1059         bio->bi_end_io = backing_request_endio;
1060         closure_bio_submit(s->iop.c, bio, cl);
1061
1062         continue_at(cl, cached_dev_bio_complete, NULL);
1063 }
1064
1065 struct detached_dev_io_private {
1066         struct bcache_device    *d;
1067         unsigned long           start_time;
1068         bio_end_io_t            *bi_end_io;
1069         void                    *bi_private;
1070 };
1071
1072 static void detached_dev_end_io(struct bio *bio)
1073 {
1074         struct detached_dev_io_private *ddip;
1075
1076         ddip = bio->bi_private;
1077         bio->bi_end_io = ddip->bi_end_io;
1078         bio->bi_private = ddip->bi_private;
1079
1080         generic_end_io_acct(ddip->d->disk->queue, bio_op(bio),
1081                             &ddip->d->disk->part0, ddip->start_time);
1082
1083         if (bio->bi_status) {
1084                 struct cached_dev *dc = container_of(ddip->d,
1085                                                      struct cached_dev, disk);
1086                 /* should count I/O error for backing device here */
1087                 bch_count_backing_io_errors(dc, bio);
1088         }
1089
1090         kfree(ddip);
1091         bio->bi_end_io(bio);
1092 }
1093
1094 static void detached_dev_do_request(struct bcache_device *d, struct bio *bio)
1095 {
1096         struct detached_dev_io_private *ddip;
1097         struct cached_dev *dc = container_of(d, struct cached_dev, disk);
1098
1099         /*
1100          * no need to call closure_get(&dc->disk.cl),
1101          * because upper layer had already opened bcache device,
1102          * which would call closure_get(&dc->disk.cl)
1103          */
1104         ddip = kzalloc(sizeof(struct detached_dev_io_private), GFP_NOIO);
1105         if (!ddip) {
1106                 bio->bi_status = BLK_STS_RESOURCE;
1107                 bio->bi_end_io(bio);
1108                 return;
1109         }
1110
1111         ddip->d = d;
1112         ddip->start_time = jiffies;
1113         ddip->bi_end_io = bio->bi_end_io;
1114         ddip->bi_private = bio->bi_private;
1115         bio->bi_end_io = detached_dev_end_io;
1116         bio->bi_private = ddip;
1117
1118         if ((bio_op(bio) == REQ_OP_DISCARD) &&
1119             !blk_queue_discard(bdev_get_queue(dc->bdev)))
1120                 bio->bi_end_io(bio);
1121         else
1122                 generic_make_request(bio);
1123 }
1124
1125 static void quit_max_writeback_rate(struct cache_set *c,
1126                                     struct cached_dev *this_dc)
1127 {
1128         int i;
1129         struct bcache_device *d;
1130         struct cached_dev *dc;
1131
1132         /*
1133          * mutex bch_register_lock may compete with other parallel requesters,
1134          * or attach/detach operations on other backing device. Waiting to
1135          * the mutex lock may increase I/O request latency for seconds or more.
1136          * To avoid such situation, if mutext_trylock() failed, only writeback
1137          * rate of current cached device is set to 1, and __update_write_back()
1138          * will decide writeback rate of other cached devices (remember now
1139          * c->idle_counter is 0 already).
1140          */
1141         if (mutex_trylock(&bch_register_lock)) {
1142                 for (i = 0; i < c->devices_max_used; i++) {
1143                         if (!c->devices[i])
1144                                 continue;
1145
1146                         if (UUID_FLASH_ONLY(&c->uuids[i]))
1147                                 continue;
1148
1149                         d = c->devices[i];
1150                         dc = container_of(d, struct cached_dev, disk);
1151                         /*
1152                          * set writeback rate to default minimum value,
1153                          * then let update_writeback_rate() to decide the
1154                          * upcoming rate.
1155                          */
1156                         atomic_long_set(&dc->writeback_rate.rate, 1);
1157                 }
1158                 mutex_unlock(&bch_register_lock);
1159         } else
1160                 atomic_long_set(&this_dc->writeback_rate.rate, 1);
1161 }
1162
1163 /* Cached devices - read & write stuff */
1164
1165 static blk_qc_t cached_dev_make_request(struct request_queue *q,
1166                                         struct bio *bio)
1167 {
1168         struct search *s;
1169         struct bcache_device *d = bio->bi_disk->private_data;
1170         struct cached_dev *dc = container_of(d, struct cached_dev, disk);
1171         int rw = bio_data_dir(bio);
1172
1173         if (unlikely((d->c && test_bit(CACHE_SET_IO_DISABLE, &d->c->flags)) ||
1174                      dc->io_disable)) {
1175                 bio->bi_status = BLK_STS_IOERR;
1176                 bio_endio(bio);
1177                 return BLK_QC_T_NONE;
1178         }
1179
1180         if (likely(d->c)) {
1181                 if (atomic_read(&d->c->idle_counter))
1182                         atomic_set(&d->c->idle_counter, 0);
1183                 /*
1184                  * If at_max_writeback_rate of cache set is true and new I/O
1185                  * comes, quit max writeback rate of all cached devices
1186                  * attached to this cache set, and set at_max_writeback_rate
1187                  * to false.
1188                  */
1189                 if (unlikely(atomic_read(&d->c->at_max_writeback_rate) == 1)) {
1190                         atomic_set(&d->c->at_max_writeback_rate, 0);
1191                         quit_max_writeback_rate(d->c, dc);
1192                 }
1193         }
1194
1195         generic_start_io_acct(q,
1196                               bio_op(bio),
1197                               bio_sectors(bio),
1198                               &d->disk->part0);
1199
1200         bio_set_dev(bio, dc->bdev);
1201         bio->bi_iter.bi_sector += dc->sb.data_offset;
1202
1203         if (cached_dev_get(dc)) {
1204                 s = search_alloc(bio, d);
1205                 trace_bcache_request_start(s->d, bio);
1206
1207                 if (!bio->bi_iter.bi_size) {
1208                         /*
1209                          * can't call bch_journal_meta from under
1210                          * generic_make_request
1211                          */
1212                         continue_at_nobarrier(&s->cl,
1213                                               cached_dev_nodata,
1214                                               bcache_wq);
1215                 } else {
1216                         s->iop.bypass = check_should_bypass(dc, bio);
1217
1218                         if (rw)
1219                                 cached_dev_write(dc, s);
1220                         else
1221                                 cached_dev_read(dc, s);
1222                 }
1223         } else
1224                 /* I/O request sent to backing device */
1225                 detached_dev_do_request(d, bio);
1226
1227         return BLK_QC_T_NONE;
1228 }
1229
1230 static int cached_dev_ioctl(struct bcache_device *d, fmode_t mode,
1231                             unsigned int cmd, unsigned long arg)
1232 {
1233         struct cached_dev *dc = container_of(d, struct cached_dev, disk);
1234
1235         if (dc->io_disable)
1236                 return -EIO;
1237
1238         return __blkdev_driver_ioctl(dc->bdev, mode, cmd, arg);
1239 }
1240
1241 static int cached_dev_congested(void *data, int bits)
1242 {
1243         struct bcache_device *d = data;
1244         struct cached_dev *dc = container_of(d, struct cached_dev, disk);
1245         struct request_queue *q = bdev_get_queue(dc->bdev);
1246         int ret = 0;
1247
1248         if (bdi_congested(q->backing_dev_info, bits))
1249                 return 1;
1250
1251         if (cached_dev_get(dc)) {
1252                 unsigned int i;
1253                 struct cache *ca;
1254
1255                 for_each_cache(ca, d->c, i) {
1256                         q = bdev_get_queue(ca->bdev);
1257                         ret |= bdi_congested(q->backing_dev_info, bits);
1258                 }
1259
1260                 cached_dev_put(dc);
1261         }
1262
1263         return ret;
1264 }
1265
1266 void bch_cached_dev_request_init(struct cached_dev *dc)
1267 {
1268         struct gendisk *g = dc->disk.disk;
1269
1270         g->queue->make_request_fn               = cached_dev_make_request;
1271         g->queue->backing_dev_info->congested_fn = cached_dev_congested;
1272         dc->disk.cache_miss                     = cached_dev_cache_miss;
1273         dc->disk.ioctl                          = cached_dev_ioctl;
1274 }
1275
1276 /* Flash backed devices */
1277
1278 static int flash_dev_cache_miss(struct btree *b, struct search *s,
1279                                 struct bio *bio, unsigned int sectors)
1280 {
1281         unsigned int bytes = min(sectors, bio_sectors(bio)) << 9;
1282
1283         swap(bio->bi_iter.bi_size, bytes);
1284         zero_fill_bio(bio);
1285         swap(bio->bi_iter.bi_size, bytes);
1286
1287         bio_advance(bio, bytes);
1288
1289         if (!bio->bi_iter.bi_size)
1290                 return MAP_DONE;
1291
1292         return MAP_CONTINUE;
1293 }
1294
1295 static void flash_dev_nodata(struct closure *cl)
1296 {
1297         struct search *s = container_of(cl, struct search, cl);
1298
1299         if (s->iop.flush_journal)
1300                 bch_journal_meta(s->iop.c, cl);
1301
1302         continue_at(cl, search_free, NULL);
1303 }
1304
1305 static blk_qc_t flash_dev_make_request(struct request_queue *q,
1306                                              struct bio *bio)
1307 {
1308         struct search *s;
1309         struct closure *cl;
1310         struct bcache_device *d = bio->bi_disk->private_data;
1311
1312         if (unlikely(d->c && test_bit(CACHE_SET_IO_DISABLE, &d->c->flags))) {
1313                 bio->bi_status = BLK_STS_IOERR;
1314                 bio_endio(bio);
1315                 return BLK_QC_T_NONE;
1316         }
1317
1318         generic_start_io_acct(q, bio_op(bio), bio_sectors(bio), &d->disk->part0);
1319
1320         s = search_alloc(bio, d);
1321         cl = &s->cl;
1322         bio = &s->bio.bio;
1323
1324         trace_bcache_request_start(s->d, bio);
1325
1326         if (!bio->bi_iter.bi_size) {
1327                 /*
1328                  * can't call bch_journal_meta from under
1329                  * generic_make_request
1330                  */
1331                 continue_at_nobarrier(&s->cl,
1332                                       flash_dev_nodata,
1333                                       bcache_wq);
1334                 return BLK_QC_T_NONE;
1335         } else if (bio_data_dir(bio)) {
1336                 bch_keybuf_check_overlapping(&s->iop.c->moving_gc_keys,
1337                                         &KEY(d->id, bio->bi_iter.bi_sector, 0),
1338                                         &KEY(d->id, bio_end_sector(bio), 0));
1339
1340                 s->iop.bypass           = (bio_op(bio) == REQ_OP_DISCARD) != 0;
1341                 s->iop.writeback        = true;
1342                 s->iop.bio              = bio;
1343
1344                 closure_call(&s->iop.cl, bch_data_insert, NULL, cl);
1345         } else {
1346                 closure_call(&s->iop.cl, cache_lookup, NULL, cl);
1347         }
1348
1349         continue_at(cl, search_free, NULL);
1350         return BLK_QC_T_NONE;
1351 }
1352
1353 static int flash_dev_ioctl(struct bcache_device *d, fmode_t mode,
1354                            unsigned int cmd, unsigned long arg)
1355 {
1356         return -ENOTTY;
1357 }
1358
1359 static int flash_dev_congested(void *data, int bits)
1360 {
1361         struct bcache_device *d = data;
1362         struct request_queue *q;
1363         struct cache *ca;
1364         unsigned int i;
1365         int ret = 0;
1366
1367         for_each_cache(ca, d->c, i) {
1368                 q = bdev_get_queue(ca->bdev);
1369                 ret |= bdi_congested(q->backing_dev_info, bits);
1370         }
1371
1372         return ret;
1373 }
1374
1375 void bch_flash_dev_request_init(struct bcache_device *d)
1376 {
1377         struct gendisk *g = d->disk;
1378
1379         g->queue->make_request_fn               = flash_dev_make_request;
1380         g->queue->backing_dev_info->congested_fn = flash_dev_congested;
1381         d->cache_miss                           = flash_dev_cache_miss;
1382         d->ioctl                                = flash_dev_ioctl;
1383 }
1384
1385 void bch_request_exit(void)
1386 {
1387         kmem_cache_destroy(bch_search_cache);
1388 }
1389
1390 int __init bch_request_init(void)
1391 {
1392         bch_search_cache = KMEM_CACHE(search, 0);
1393         if (!bch_search_cache)
1394                 return -ENOMEM;
1395
1396         return 0;
1397 }