segment.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * fs/f2fs/segment.c
   4  *
   5  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
   6  *             http://www.samsung.com/
   7  */
   8 #include <linux/fs.h>
   9 #include <linux/f2fs_fs.h>
  10 #include <linux/bio.h>
  11 #include <linux/blkdev.h>
  12 #include <linux/sched/mm.h>
  13 #include <linux/prefetch.h>
  14 #include <linux/kthread.h>
  15 #include <linux/swap.h>
  16 #include <linux/timer.h>
  17 #include <linux/freezer.h>
  18 #include <linux/sched/signal.h>
  19 #include <linux/random.h>
  20
  21 #include "f2fs.h"
  22 #include "segment.h"
  23 #include "node.h"
  24 #include "gc.h"
  25 #include "iostat.h"
  26 #include <trace/events/f2fs.h>
  27
  28 #define __reverse_ffz(x) __reverse_ffs(~(x))
  29
  30 static struct kmem_cache *discard_entry_slab;
  31 static struct kmem_cache *discard_cmd_slab;
  32 static struct kmem_cache *sit_entry_set_slab;
  33 static struct kmem_cache *revoke_entry_slab;
  34
  35 static unsigned long __reverse_ulong(unsigned char *str)
  36 {
  37         unsigned long tmp = 0;
  38         int shift = 24, idx = 0;
  39
  40 #if BITS_PER_LONG == 64
  41         shift = 56;
  42 #endif
  43         while (shift >= 0) {
  44                 tmp |= (unsigned long)str[idx++] << shift;
  45                 shift -= BITS_PER_BYTE;
  46         }
  47         return tmp;
  48 }
  49
  50 /*
  51  * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
  52  * MSB and LSB are reversed in a byte by f2fs_set_bit.
  53  */
  54 static inline unsigned long __reverse_ffs(unsigned long word)
  55 {
  56         int num = 0;
  57
  58 #if BITS_PER_LONG == 64
  59         if ((word & 0xffffffff00000000UL) == 0)
  60                 num += 32;
  61         else
  62                 word >>= 32;
  63 #endif
  64         if ((word & 0xffff0000) == 0)
  65                 num += 16;
  66         else
  67                 word >>= 16;
  68
  69         if ((word & 0xff00) == 0)
  70                 num += 8;
  71         else
  72                 word >>= 8;
  73
  74         if ((word & 0xf0) == 0)
  75                 num += 4;
  76         else
  77                 word >>= 4;
  78
  79         if ((word & 0xc) == 0)
  80                 num += 2;
  81         else
  82                 word >>= 2;
  83
  84         if ((word & 0x2) == 0)
  85                 num += 1;
  86         return num;
  87 }
  88
  89 /*
  90  * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
  91  * f2fs_set_bit makes MSB and LSB reversed in a byte.
  92  * @size must be integral times of unsigned long.
  93  * Example:
  94  *                             MSB <--> LSB
  95  *   f2fs_set_bit(0, bitmap) => 1000 0000
  96  *   f2fs_set_bit(7, bitmap) => 0000 0001
  97  */
  98 static unsigned long __find_rev_next_bit(const unsigned long *addr,
  99                         unsigned long size, unsigned long offset)
 100 {
 101         const unsigned long *p = addr + BIT_WORD(offset);
 102         unsigned long result = size;
 103         unsigned long tmp;
 104
 105         if (offset >= size)
 106                 return size;
 107
 108         size -= (offset & ~(BITS_PER_LONG - 1));
 109         offset %= BITS_PER_LONG;
 110
 111         while (1) {
 112                 if (*p == 0)
 113                         goto pass;
 114
 115                 tmp = __reverse_ulong((unsigned char *)p);
 116
 117                 tmp &= ~0UL >> offset;
 118                 if (size < BITS_PER_LONG)
 119                         tmp &= (~0UL << (BITS_PER_LONG - size));
 120                 if (tmp)
 121                         goto found;
 122 pass:
 123                 if (size <= BITS_PER_LONG)
 124                         break;
 125                 size -= BITS_PER_LONG;
 126                 offset = 0;
 127                 p++;
 128         }
 129         return result;
 130 found:
 131         return result - size + __reverse_ffs(tmp);
 132 }
 133
 134 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
 135                         unsigned long size, unsigned long offset)
 136 {
 137         const unsigned long *p = addr + BIT_WORD(offset);
 138         unsigned long result = size;
 139         unsigned long tmp;
 140
 141         if (offset >= size)
 142                 return size;
 143
 144         size -= (offset & ~(BITS_PER_LONG - 1));
 145         offset %= BITS_PER_LONG;
 146
 147         while (1) {
 148                 if (*p == ~0UL)
 149                         goto pass;
 150
 151                 tmp = __reverse_ulong((unsigned char *)p);
 152
 153                 if (offset)
 154                         tmp |= ~0UL << (BITS_PER_LONG - offset);
 155                 if (size < BITS_PER_LONG)
 156                         tmp |= ~0UL >> size;
 157                 if (tmp != ~0UL)
 158                         goto found;
 159 pass:
 160                 if (size <= BITS_PER_LONG)
 161                         break;
 162                 size -= BITS_PER_LONG;
 163                 offset = 0;
 164                 p++;
 165         }
 166         return result;
 167 found:
 168         return result - size + __reverse_ffz(tmp);
 169 }
 170
 171 bool f2fs_need_SSR(struct f2fs_sb_info *sbi)
 172 {
 173         int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);
 174         int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);
 175         int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);
 176
 177         if (f2fs_lfs_mode(sbi))
 178                 return false;
 179         if (sbi->gc_mode == GC_URGENT_HIGH)
 180                 return true;
 181         if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
 182                 return true;
 183
 184         return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs +
 185                         SM_I(sbi)->min_ssr_sections + reserved_sections(sbi));
 186 }
 187
 188 void f2fs_abort_atomic_write(struct inode *inode, bool clean)
 189 {
 190         struct f2fs_inode_info *fi = F2FS_I(inode);
 191
 192         if (!f2fs_is_atomic_file(inode))
 193                 return;
 194
 195         if (clean)
 196                 truncate_inode_pages_final(inode->i_mapping);
 197
 198         release_atomic_write_cnt(inode);
 199         clear_inode_flag(inode, FI_ATOMIC_COMMITTED);
 200         clear_inode_flag(inode, FI_ATOMIC_FILE);
 201         stat_dec_atomic_inode(inode);
 202
 203         F2FS_I(inode)->atomic_write_task = NULL;
 204
 205         if (clean) {
 206                 f2fs_i_size_write(inode, fi->original_i_size);
 207                 fi->original_i_size = 0;
 208         }
 209         /* avoid stale dirty inode during eviction */
 210         sync_inode_metadata(inode, 0);
 211 }
 212
 213 static int __replace_atomic_write_block(struct inode *inode, pgoff_t index,
 214                         block_t new_addr, block_t *old_addr, bool recover)
 215 {
 216         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 217         struct dnode_of_data dn;
 218         struct node_info ni;
 219         int err;
 220
 221 retry:
 222         set_new_dnode(&dn, inode, NULL, NULL, 0);
 223         err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE_RA);
 224         if (err) {
 225                 if (err == -ENOMEM) {
 226                         f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT);
 227                         goto retry;
 228                 }
 229                 return err;
 230         }
 231
 232         err = f2fs_get_node_info(sbi, dn.nid, &ni, false);
 233         if (err) {
 234                 f2fs_put_dnode(&dn);
 235                 return err;
 236         }
 237
 238         if (recover) {
 239                 /* dn.data_blkaddr is always valid */
 240                 if (!__is_valid_data_blkaddr(new_addr)) {
 241                         if (new_addr == NULL_ADDR)
 242                                 dec_valid_block_count(sbi, inode, 1);
 243                         f2fs_invalidate_blocks(sbi, dn.data_blkaddr);
 244                         f2fs_update_data_blkaddr(&dn, new_addr);
 245                 } else {
 246                         f2fs_replace_block(sbi, &dn, dn.data_blkaddr,
 247                                 new_addr, ni.version, true, true);
 248                 }
 249         } else {
 250                 blkcnt_t count = 1;
 251
 252                 err = inc_valid_block_count(sbi, inode, &count, true);
 253                 if (err) {
 254                         f2fs_put_dnode(&dn);
 255                         return err;
 256                 }
 257
 258                 *old_addr = dn.data_blkaddr;
 259                 f2fs_truncate_data_blocks_range(&dn, 1);
 260                 dec_valid_block_count(sbi, F2FS_I(inode)->cow_inode, count);
 261
 262                 f2fs_replace_block(sbi, &dn, dn.data_blkaddr, new_addr,
 263                                         ni.version, true, false);
 264         }
 265
 266         f2fs_put_dnode(&dn);
 267
 268         trace_f2fs_replace_atomic_write_block(inode, F2FS_I(inode)->cow_inode,
 269                         index, old_addr ? *old_addr : 0, new_addr, recover);
 270         return 0;
 271 }
 272
 273 static void __complete_revoke_list(struct inode *inode, struct list_head *head,
 274                                         bool revoke)
 275 {
 276         struct revoke_entry *cur, *tmp;
 277
 278         list_for_each_entry_safe(cur, tmp, head, list) {
 279                 if (revoke)
 280                         __replace_atomic_write_block(inode, cur->index,
 281                                                 cur->old_addr, NULL, true);
 282                 list_del(&cur->list);
 283                 kmem_cache_free(revoke_entry_slab, cur);
 284         }
 285 }
 286
 287 static int __f2fs_commit_atomic_write(struct inode *inode)
 288 {
 289         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 290         struct f2fs_inode_info *fi = F2FS_I(inode);
 291         struct inode *cow_inode = fi->cow_inode;
 292         struct revoke_entry *new;
 293         struct list_head revoke_list;
 294         block_t blkaddr;
 295         struct dnode_of_data dn;
 296         pgoff_t len = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
 297         pgoff_t off = 0, blen, index;
 298         int ret = 0, i;
 299
 300         INIT_LIST_HEAD(&revoke_list);
 301
 302         while (len) {
 303                 blen = min_t(pgoff_t, ADDRS_PER_BLOCK(cow_inode), len);
 304
 305                 set_new_dnode(&dn, cow_inode, NULL, NULL, 0);
 306                 ret = f2fs_get_dnode_of_data(&dn, off, LOOKUP_NODE_RA);
 307                 if (ret && ret != -ENOENT) {
 308                         goto out;
 309                 } else if (ret == -ENOENT) {
 310                         ret = 0;
 311                         if (dn.max_level == 0)
 312                                 goto out;
 313                         goto next;
 314                 }
 315
 316                 blen = min((pgoff_t)ADDRS_PER_PAGE(dn.node_page, cow_inode),
 317                                 len);
 318                 index = off;
 319                 for (i = 0; i < blen; i++, dn.ofs_in_node++, index++) {
 320                         blkaddr = f2fs_data_blkaddr(&dn);
 321
 322                         if (!__is_valid_data_blkaddr(blkaddr)) {
 323                                 continue;
 324                         } else if (!f2fs_is_valid_blkaddr(sbi, blkaddr,
 325                                         DATA_GENERIC_ENHANCE)) {
 326                                 f2fs_put_dnode(&dn);
 327                                 ret = -EFSCORRUPTED;
 328                                 f2fs_handle_error(sbi,
 329                                                 ERROR_INVALID_BLKADDR);
 330                                 goto out;
 331                         }
 332
 333                         new = f2fs_kmem_cache_alloc(revoke_entry_slab, GFP_NOFS,
 334                                                         true, NULL);
 335
 336                         ret = __replace_atomic_write_block(inode, index, blkaddr,
 337                                                         &new->old_addr, false);
 338                         if (ret) {
 339                                 f2fs_put_dnode(&dn);
 340                                 kmem_cache_free(revoke_entry_slab, new);
 341                                 goto out;
 342                         }
 343
 344                         f2fs_update_data_blkaddr(&dn, NULL_ADDR);
 345                         new->index = index;
 346                         list_add_tail(&new->list, &revoke_list);
 347                 }
 348                 f2fs_put_dnode(&dn);
 349 next:
 350                 off += blen;
 351                 len -= blen;
 352         }
 353
 354 out:
 355         if (ret) {
 356                 sbi->revoked_atomic_block += fi->atomic_write_cnt;
 357         } else {
 358                 sbi->committed_atomic_block += fi->atomic_write_cnt;
 359                 set_inode_flag(inode, FI_ATOMIC_COMMITTED);
 360         }
 361
 362         __complete_revoke_list(inode, &revoke_list, ret ? true : false);
 363
 364         return ret;
 365 }
 366
 367 int f2fs_commit_atomic_write(struct inode *inode)
 368 {
 369         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 370         struct f2fs_inode_info *fi = F2FS_I(inode);
 371         int err;
 372
 373         err = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX);
 374         if (err)
 375                 return err;
 376
 377         f2fs_down_write(&fi->i_gc_rwsem[WRITE]);
 378         f2fs_lock_op(sbi);
 379
 380         err = __f2fs_commit_atomic_write(inode);
 381
 382         f2fs_unlock_op(sbi);
 383         f2fs_up_write(&fi->i_gc_rwsem[WRITE]);
 384
 385         return err;
 386 }
 387
 388 /*
 389  * This function balances dirty node and dentry pages.
 390  * In addition, it controls garbage collection.
 391  */
 392 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
 393 {
 394         if (time_to_inject(sbi, FAULT_CHECKPOINT)) {
 395                 f2fs_show_injection_info(sbi, FAULT_CHECKPOINT);
 396                 f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_FAULT_INJECT);
 397         }
 398
 399         /* balance_fs_bg is able to be pending */
 400         if (need && excess_cached_nats(sbi))
 401                 f2fs_balance_fs_bg(sbi, false);
 402
 403         if (!f2fs_is_checkpoint_ready(sbi))
 404                 return;
 405
 406         /*
 407          * We should do GC or end up with checkpoint, if there are so many dirty
 408          * dir/node pages without enough free segments.
 409          */
 410         if (has_not_enough_free_secs(sbi, 0, 0)) {
 411                 if (test_opt(sbi, GC_MERGE) && sbi->gc_thread &&
 412                                         sbi->gc_thread->f2fs_gc_task) {
 413                         DEFINE_WAIT(wait);
 414
 415                         prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait,
 416                                                 TASK_UNINTERRUPTIBLE);
 417                         wake_up(&sbi->gc_thread->gc_wait_queue_head);
 418                         io_schedule();
 419                         finish_wait(&sbi->gc_thread->fggc_wq, &wait);
 420                 } else {
 421                         struct f2fs_gc_control gc_control = {
 422                                 .victim_segno = NULL_SEGNO,
 423                                 .init_gc_type = BG_GC,
 424                                 .no_bg_gc = true,
 425                                 .should_migrate_blocks = false,
 426                                 .err_gc_skipped = false,
 427                                 .nr_free_secs = 1 };
 428                         f2fs_down_write(&sbi->gc_lock);
 429                         f2fs_gc(sbi, &gc_control);
 430                 }
 431         }
 432 }
 433
 434 static inline bool excess_dirty_threshold(struct f2fs_sb_info *sbi)
 435 {
 436         int factor = f2fs_rwsem_is_locked(&sbi->cp_rwsem) ? 3 : 2;
 437         unsigned int dents = get_pages(sbi, F2FS_DIRTY_DENTS);
 438         unsigned int qdata = get_pages(sbi, F2FS_DIRTY_QDATA);
 439         unsigned int nodes = get_pages(sbi, F2FS_DIRTY_NODES);
 440         unsigned int meta = get_pages(sbi, F2FS_DIRTY_META);
 441         unsigned int imeta = get_pages(sbi, F2FS_DIRTY_IMETA);
 442         unsigned int threshold = sbi->blocks_per_seg * factor *
 443                                         DEFAULT_DIRTY_THRESHOLD;
 444         unsigned int global_threshold = threshold * 3 / 2;
 445
 446         if (dents >= threshold || qdata >= threshold ||
 447                 nodes >= threshold || meta >= threshold ||
 448                 imeta >= threshold)
 449                 return true;
 450         return dents + qdata + nodes + meta + imeta >  global_threshold;
 451 }
 452
 453 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg)
 454 {
 455         if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
 456                 return;
 457
 458         /* try to shrink extent cache when there is no enough memory */
 459         if (!f2fs_available_free_memory(sbi, READ_EXTENT_CACHE))
 460                 f2fs_shrink_read_extent_tree(sbi,
 461                                 READ_EXTENT_CACHE_SHRINK_NUMBER);
 462
 463         /* check the # of cached NAT entries */
 464         if (!f2fs_available_free_memory(sbi, NAT_ENTRIES))
 465                 f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
 466
 467         if (!f2fs_available_free_memory(sbi, FREE_NIDS))
 468                 f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS);
 469         else
 470                 f2fs_build_free_nids(sbi, false, false);
 471
 472         if (excess_dirty_nats(sbi) || excess_dirty_threshold(sbi) ||
 473                 excess_prefree_segs(sbi) || !f2fs_space_for_roll_forward(sbi))
 474                 goto do_sync;
 475
 476         /* there is background inflight IO or foreground operation recently */
 477         if (is_inflight_io(sbi, REQ_TIME) ||
 478                 (!f2fs_time_over(sbi, REQ_TIME) && f2fs_rwsem_is_locked(&sbi->cp_rwsem)))
 479                 return;
 480
 481         /* exceed periodical checkpoint timeout threshold */
 482         if (f2fs_time_over(sbi, CP_TIME))
 483                 goto do_sync;
 484
 485         /* checkpoint is the only way to shrink partial cached entries */
 486         if (f2fs_available_free_memory(sbi, NAT_ENTRIES) &&
 487                 f2fs_available_free_memory(sbi, INO_ENTRIES))
 488                 return;
 489
 490 do_sync:
 491         if (test_opt(sbi, DATA_FLUSH) && from_bg) {
 492                 struct blk_plug plug;
 493
 494                 mutex_lock(&sbi->flush_lock);
 495
 496                 blk_start_plug(&plug);
 497                 f2fs_sync_dirty_inodes(sbi, FILE_INODE, false);
 498                 blk_finish_plug(&plug);
 499
 500                 mutex_unlock(&sbi->flush_lock);
 501         }
 502         f2fs_sync_fs(sbi->sb, 1);
 503         stat_inc_bg_cp_count(sbi->stat_info);
 504 }
 505
 506 static int __submit_flush_wait(struct f2fs_sb_info *sbi,
 507                                 struct block_device *bdev)
 508 {
 509         int ret = blkdev_issue_flush(bdev);
 510
 511         trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),
 512                                 test_opt(sbi, FLUSH_MERGE), ret);
 513         return ret;
 514 }
 515
 516 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino)
 517 {
 518         int ret = 0;
 519         int i;
 520
 521         if (!f2fs_is_multi_device(sbi))
 522                 return __submit_flush_wait(sbi, sbi->sb->s_bdev);
 523
 524         for (i = 0; i < sbi->s_ndevs; i++) {
 525                 if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO))
 526                         continue;
 527                 ret = __submit_flush_wait(sbi, FDEV(i).bdev);
 528                 if (ret)
 529                         break;
 530         }
 531         return ret;
 532 }
 533
 534 static int issue_flush_thread(void *data)
 535 {
 536         struct f2fs_sb_info *sbi = data;
 537         struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
 538         wait_queue_head_t *q = &fcc->flush_wait_queue;
 539 repeat:
 540         if (kthread_should_stop())
 541                 return 0;
 542
 543         if (!llist_empty(&fcc->issue_list)) {
 544                 struct flush_cmd *cmd, *next;
 545                 int ret;
 546
 547                 fcc->dispatch_list = llist_del_all(&fcc->issue_list);
 548                 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
 549
 550                 cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode);
 551
 552                 ret = submit_flush_wait(sbi, cmd->ino);
 553                 atomic_inc(&fcc->issued_flush);
 554
 555                 llist_for_each_entry_safe(cmd, next,
 556                                           fcc->dispatch_list, llnode) {
 557                         cmd->ret = ret;
 558                         complete(&cmd->wait);
 559                 }
 560                 fcc->dispatch_list = NULL;
 561         }
 562
 563         wait_event_interruptible(*q,
 564                 kthread_should_stop() || !llist_empty(&fcc->issue_list));
 565         goto repeat;
 566 }
 567
 568 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino)
 569 {
 570         struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
 571         struct flush_cmd cmd;
 572         int ret;
 573
 574         if (test_opt(sbi, NOBARRIER))
 575                 return 0;
 576
 577         if (!test_opt(sbi, FLUSH_MERGE)) {
 578                 atomic_inc(&fcc->queued_flush);
 579                 ret = submit_flush_wait(sbi, ino);
 580                 atomic_dec(&fcc->queued_flush);
 581                 atomic_inc(&fcc->issued_flush);
 582                 return ret;
 583         }
 584
 585         if (atomic_inc_return(&fcc->queued_flush) == 1 ||
 586             f2fs_is_multi_device(sbi)) {
 587                 ret = submit_flush_wait(sbi, ino);
 588                 atomic_dec(&fcc->queued_flush);
 589
 590                 atomic_inc(&fcc->issued_flush);
 591                 return ret;
 592         }
 593
 594         cmd.ino = ino;
 595         init_completion(&cmd.wait);
 596
 597         llist_add(&cmd.llnode, &fcc->issue_list);
 598
 599         /*
 600          * update issue_list before we wake up issue_flush thread, this
 601          * smp_mb() pairs with another barrier in ___wait_event(), see
 602          * more details in comments of waitqueue_active().
 603          */
 604         smp_mb();
 605
 606         if (waitqueue_active(&fcc->flush_wait_queue))
 607                 wake_up(&fcc->flush_wait_queue);
 608
 609         if (fcc->f2fs_issue_flush) {
 610                 wait_for_completion(&cmd.wait);
 611                 atomic_dec(&fcc->queued_flush);
 612         } else {
 613                 struct llist_node *list;
 614
 615                 list = llist_del_all(&fcc->issue_list);
 616                 if (!list) {
 617                         wait_for_completion(&cmd.wait);
 618                         atomic_dec(&fcc->queued_flush);
 619                 } else {
 620                         struct flush_cmd *tmp, *next;
 621
 622                         ret = submit_flush_wait(sbi, ino);
 623
 624                         llist_for_each_entry_safe(tmp, next, list, llnode) {
 625                                 if (tmp == &cmd) {
 626                                         cmd.ret = ret;
 627                                         atomic_dec(&fcc->queued_flush);
 628                                         continue;
 629                                 }
 630                                 tmp->ret = ret;
 631                                 complete(&tmp->wait);
 632                         }
 633                 }
 634         }
 635
 636         return cmd.ret;
 637 }
 638
 639 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi)
 640 {
 641         dev_t dev = sbi->sb->s_bdev->bd_dev;
 642         struct flush_cmd_control *fcc;
 643         int err = 0;
 644
 645         if (SM_I(sbi)->fcc_info) {
 646                 fcc = SM_I(sbi)->fcc_info;
 647                 if (fcc->f2fs_issue_flush)
 648                         return err;
 649                 goto init_thread;
 650         }
 651
 652         fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL);
 653         if (!fcc)
 654                 return -ENOMEM;
 655         atomic_set(&fcc->issued_flush, 0);
 656         atomic_set(&fcc->queued_flush, 0);
 657         init_waitqueue_head(&fcc->flush_wait_queue);
 658         init_llist_head(&fcc->issue_list);
 659         SM_I(sbi)->fcc_info = fcc;
 660         if (!test_opt(sbi, FLUSH_MERGE))
 661                 return err;
 662
 663 init_thread:
 664         fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
 665                                 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
 666         if (IS_ERR(fcc->f2fs_issue_flush)) {
 667                 err = PTR_ERR(fcc->f2fs_issue_flush);
 668                 kfree(fcc);
 669                 SM_I(sbi)->fcc_info = NULL;
 670                 return err;
 671         }
 672
 673         return err;
 674 }
 675
 676 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)
 677 {
 678         struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
 679
 680         if (fcc && fcc->f2fs_issue_flush) {
 681                 struct task_struct *flush_thread = fcc->f2fs_issue_flush;
 682
 683                 fcc->f2fs_issue_flush = NULL;
 684                 kthread_stop(flush_thread);
 685         }
 686         if (free) {
 687                 kfree(fcc);
 688                 SM_I(sbi)->fcc_info = NULL;
 689         }
 690 }
 691
 692 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi)
 693 {
 694         int ret = 0, i;
 695
 696         if (!f2fs_is_multi_device(sbi))
 697                 return 0;
 698
 699         if (test_opt(sbi, NOBARRIER))
 700                 return 0;
 701
 702         for (i = 1; i < sbi->s_ndevs; i++) {
 703                 int count = DEFAULT_RETRY_IO_COUNT;
 704
 705                 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))
 706                         continue;
 707
 708                 do {
 709                         ret = __submit_flush_wait(sbi, FDEV(i).bdev);
 710                         if (ret)
 711                                 f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT);
 712                 } while (ret && --count);
 713
 714                 if (ret) {
 715                         f2fs_stop_checkpoint(sbi, false,
 716                                         STOP_CP_REASON_FLUSH_FAIL);
 717                         break;
 718                 }
 719
 720                 spin_lock(&sbi->dev_lock);
 721                 f2fs_clear_bit(i, (char *)&sbi->dirty_device);
 722                 spin_unlock(&sbi->dev_lock);
 723         }
 724
 725         return ret;
 726 }
 727
 728 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
 729                 enum dirty_type dirty_type)
 730 {
 731         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 732
 733         /* need not be added */
 734         if (IS_CURSEG(sbi, segno))
 735                 return;
 736
 737         if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
 738                 dirty_i->nr_dirty[dirty_type]++;
 739
 740         if (dirty_type == DIRTY) {
 741                 struct seg_entry *sentry = get_seg_entry(sbi, segno);
 742                 enum dirty_type t = sentry->type;
 743
 744                 if (unlikely(t >= DIRTY)) {
 745                         f2fs_bug_on(sbi, 1);
 746                         return;
 747                 }
 748                 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
 749                         dirty_i->nr_dirty[t]++;
 750
 751                 if (__is_large_section(sbi)) {
 752                         unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
 753                         block_t valid_blocks =
 754                                 get_valid_blocks(sbi, segno, true);
 755
 756                         f2fs_bug_on(sbi, unlikely(!valid_blocks ||
 757                                         valid_blocks == CAP_BLKS_PER_SEC(sbi)));
 758
 759                         if (!IS_CURSEC(sbi, secno))
 760                                 set_bit(secno, dirty_i->dirty_secmap);
 761                 }
 762         }
 763 }
 764
 765 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
 766                 enum dirty_type dirty_type)
 767 {
 768         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 769         block_t valid_blocks;
 770
 771         if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
 772                 dirty_i->nr_dirty[dirty_type]--;
 773
 774         if (dirty_type == DIRTY) {
 775                 struct seg_entry *sentry = get_seg_entry(sbi, segno);
 776                 enum dirty_type t = sentry->type;
 777
 778                 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
 779                         dirty_i->nr_dirty[t]--;
 780
 781                 valid_blocks = get_valid_blocks(sbi, segno, true);
 782                 if (valid_blocks == 0) {
 783                         clear_bit(GET_SEC_FROM_SEG(sbi, segno),
 784                                                 dirty_i->victim_secmap);
 785 #ifdef CONFIG_F2FS_CHECK_FS
 786                         clear_bit(segno, SIT_I(sbi)->invalid_segmap);
 787 #endif
 788                 }
 789                 if (__is_large_section(sbi)) {
 790                         unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
 791
 792                         if (!valid_blocks ||
 793                                         valid_blocks == CAP_BLKS_PER_SEC(sbi)) {
 794                                 clear_bit(secno, dirty_i->dirty_secmap);
 795                                 return;
 796                         }
 797
 798                         if (!IS_CURSEC(sbi, secno))
 799                                 set_bit(secno, dirty_i->dirty_secmap);
 800                 }
 801         }
 802 }
 803
 804 /*
 805  * Should not occur error such as -ENOMEM.
 806  * Adding dirty entry into seglist is not critical operation.
 807  * If a given segment is one of current working segments, it won't be added.
 808  */
 809 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
 810 {
 811         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 812         unsigned short valid_blocks, ckpt_valid_blocks;
 813         unsigned int usable_blocks;
 814
 815         if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
 816                 return;
 817
 818         usable_blocks = f2fs_usable_blks_in_seg(sbi, segno);
 819         mutex_lock(&dirty_i->seglist_lock);
 820
 821         valid_blocks = get_valid_blocks(sbi, segno, false);
 822         ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false);
 823
 824         if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) ||
 825                 ckpt_valid_blocks == usable_blocks)) {
 826                 __locate_dirty_segment(sbi, segno, PRE);
 827                 __remove_dirty_segment(sbi, segno, DIRTY);
 828         } else if (valid_blocks < usable_blocks) {
 829                 __locate_dirty_segment(sbi, segno, DIRTY);
 830         } else {
 831                 /* Recovery routine with SSR needs this */
 832                 __remove_dirty_segment(sbi, segno, DIRTY);
 833         }
 834
 835         mutex_unlock(&dirty_i->seglist_lock);
 836 }
 837
 838 /* This moves currently empty dirty blocks to prefree. Must hold seglist_lock */
 839 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi)
 840 {
 841         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 842         unsigned int segno;
 843
 844         mutex_lock(&dirty_i->seglist_lock);
 845         for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
 846                 if (get_valid_blocks(sbi, segno, false))
 847                         continue;
 848                 if (IS_CURSEG(sbi, segno))
 849                         continue;
 850                 __locate_dirty_segment(sbi, segno, PRE);
 851                 __remove_dirty_segment(sbi, segno, DIRTY);
 852         }
 853         mutex_unlock(&dirty_i->seglist_lock);
 854 }
 855
 856 block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi)
 857 {
 858         int ovp_hole_segs =
 859                 (overprovision_segments(sbi) - reserved_segments(sbi));
 860         block_t ovp_holes = ovp_hole_segs << sbi->log_blocks_per_seg;
 861         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 862         block_t holes[2] = {0, 0};      /* DATA and NODE */
 863         block_t unusable;
 864         struct seg_entry *se;
 865         unsigned int segno;
 866
 867         mutex_lock(&dirty_i->seglist_lock);
 868         for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
 869                 se = get_seg_entry(sbi, segno);
 870                 if (IS_NODESEG(se->type))
 871                         holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) -
 872                                                         se->valid_blocks;
 873                 else
 874                         holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) -
 875                                                         se->valid_blocks;
 876         }
 877         mutex_unlock(&dirty_i->seglist_lock);
 878
 879         unusable = holes[DATA] > holes[NODE] ? holes[DATA] : holes[NODE];
 880         if (unusable > ovp_holes)
 881                 return unusable - ovp_holes;
 882         return 0;
 883 }
 884
 885 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable)
 886 {
 887         int ovp_hole_segs =
 888                 (overprovision_segments(sbi) - reserved_segments(sbi));
 889         if (unusable > F2FS_OPTION(sbi).unusable_cap)
 890                 return -EAGAIN;
 891         if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) &&
 892                 dirty_segments(sbi) > ovp_hole_segs)
 893                 return -EAGAIN;
 894         return 0;
 895 }
 896
 897 /* This is only used by SBI_CP_DISABLED */
 898 static unsigned int get_free_segment(struct f2fs_sb_info *sbi)
 899 {
 900         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 901         unsigned int segno = 0;
 902
 903         mutex_lock(&dirty_i->seglist_lock);
 904         for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
 905                 if (get_valid_blocks(sbi, segno, false))
 906                         continue;
 907                 if (get_ckpt_valid_blocks(sbi, segno, false))
 908                         continue;
 909                 mutex_unlock(&dirty_i->seglist_lock);
 910                 return segno;
 911         }
 912         mutex_unlock(&dirty_i->seglist_lock);
 913         return NULL_SEGNO;
 914 }
 915
 916 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,
 917                 struct block_device *bdev, block_t lstart,
 918                 block_t start, block_t len)
 919 {
 920         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
 921         struct list_head *pend_list;
 922         struct discard_cmd *dc;
 923
 924         f2fs_bug_on(sbi, !len);
 925
 926         pend_list = &dcc->pend_list[plist_idx(len)];
 927
 928         dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS, true, NULL);
 929         INIT_LIST_HEAD(&dc->list);
 930         dc->bdev = bdev;
 931         dc->lstart = lstart;
 932         dc->start = start;
 933         dc->len = len;
 934         dc->ref = 0;
 935         dc->state = D_PREP;
 936         dc->queued = 0;
 937         dc->error = 0;
 938         init_completion(&dc->wait);
 939         list_add_tail(&dc->list, pend_list);
 940         spin_lock_init(&dc->lock);
 941         dc->bio_ref = 0;
 942         atomic_inc(&dcc->discard_cmd_cnt);
 943         dcc->undiscard_blks += len;
 944
 945         return dc;
 946 }
 947
 948 static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi,
 949                                 struct block_device *bdev, block_t lstart,
 950                                 block_t start, block_t len,
 951                                 struct rb_node *parent, struct rb_node **p,
 952                                 bool leftmost)
 953 {
 954         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
 955         struct discard_cmd *dc;
 956
 957         dc = __create_discard_cmd(sbi, bdev, lstart, start, len);
 958
 959         rb_link_node(&dc->rb_node, parent, p);
 960         rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost);
 961
 962         return dc;
 963 }
 964
 965 static void __detach_discard_cmd(struct discard_cmd_control *dcc,
 966                                                         struct discard_cmd *dc)
 967 {
 968         if (dc->state == D_DONE)
 969                 atomic_sub(dc->queued, &dcc->queued_discard);
 970
 971         list_del(&dc->list);
 972         rb_erase_cached(&dc->rb_node, &dcc->root);
 973         dcc->undiscard_blks -= dc->len;
 974
 975         kmem_cache_free(discard_cmd_slab, dc);
 976
 977         atomic_dec(&dcc->discard_cmd_cnt);
 978 }
 979
 980 static void __remove_discard_cmd(struct f2fs_sb_info *sbi,
 981                                                         struct discard_cmd *dc)
 982 {
 983         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
 984         unsigned long flags;
 985
 986         trace_f2fs_remove_discard(dc->bdev, dc->start, dc->len);
 987
 988         spin_lock_irqsave(&dc->lock, flags);
 989         if (dc->bio_ref) {
 990                 spin_unlock_irqrestore(&dc->lock, flags);
 991                 return;
 992         }
 993         spin_unlock_irqrestore(&dc->lock, flags);
 994
 995         f2fs_bug_on(sbi, dc->ref);
 996
 997         if (dc->error == -EOPNOTSUPP)
 998                 dc->error = 0;
 999
1000         if (dc->error)
1001                 printk_ratelimited(
1002                         "%sF2FS-fs (%s): Issue discard(%u, %u, %u) failed, ret: %d",
1003                         KERN_INFO, sbi->sb->s_id,
1004                         dc->lstart, dc->start, dc->len, dc->error);
1005         __detach_discard_cmd(dcc, dc);
1006 }
1007
1008 static void f2fs_submit_discard_endio(struct bio *bio)
1009 {
1010         struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private;
1011         unsigned long flags;
1012
1013         spin_lock_irqsave(&dc->lock, flags);
1014         if (!dc->error)
1015                 dc->error = blk_status_to_errno(bio->bi_status);
1016         dc->bio_ref--;
1017         if (!dc->bio_ref && dc->state == D_SUBMIT) {
1018                 dc->state = D_DONE;
1019                 complete_all(&dc->wait);
1020         }
1021         spin_unlock_irqrestore(&dc->lock, flags);
1022         bio_put(bio);
1023 }
1024
1025 static void __check_sit_bitmap(struct f2fs_sb_info *sbi,
1026                                 block_t start, block_t end)
1027 {
1028 #ifdef CONFIG_F2FS_CHECK_FS
1029         struct seg_entry *sentry;
1030         unsigned int segno;
1031         block_t blk = start;
1032         unsigned long offset, size, max_blocks = sbi->blocks_per_seg;
1033         unsigned long *map;
1034
1035         while (blk < end) {
1036                 segno = GET_SEGNO(sbi, blk);
1037                 sentry = get_seg_entry(sbi, segno);
1038                 offset = GET_BLKOFF_FROM_SEG0(sbi, blk);
1039
1040                 if (end < START_BLOCK(sbi, segno + 1))
1041                         size = GET_BLKOFF_FROM_SEG0(sbi, end);
1042                 else
1043                         size = max_blocks;
1044                 map = (unsigned long *)(sentry->cur_valid_map);
1045                 offset = __find_rev_next_bit(map, size, offset);
1046                 f2fs_bug_on(sbi, offset != size);
1047                 blk = START_BLOCK(sbi, segno + 1);
1048         }
1049 #endif
1050 }
1051
1052 static void __init_discard_policy(struct f2fs_sb_info *sbi,
1053                                 struct discard_policy *dpolicy,
1054                                 int discard_type, unsigned int granularity)
1055 {
1056         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1057
1058         /* common policy */
1059         dpolicy->type = discard_type;
1060         dpolicy->sync = true;
1061         dpolicy->ordered = false;
1062         dpolicy->granularity = granularity;
1063
1064         dpolicy->max_requests = dcc->max_discard_request;
1065         dpolicy->io_aware_gran = MAX_PLIST_NUM;
1066         dpolicy->timeout = false;
1067
1068         if (discard_type == DPOLICY_BG) {
1069                 dpolicy->min_interval = dcc->min_discard_issue_time;
1070                 dpolicy->mid_interval = dcc->mid_discard_issue_time;
1071                 dpolicy->max_interval = dcc->max_discard_issue_time;
1072                 dpolicy->io_aware = true;
1073                 dpolicy->sync = false;
1074                 dpolicy->ordered = true;
1075                 if (utilization(sbi) > DEF_DISCARD_URGENT_UTIL) {
1076                         dpolicy->granularity = 1;
1077                         if (atomic_read(&dcc->discard_cmd_cnt))
1078                                 dpolicy->max_interval =
1079                                         dcc->min_discard_issue_time;
1080                 }
1081         } else if (discard_type == DPOLICY_FORCE) {
1082                 dpolicy->min_interval = dcc->min_discard_issue_time;
1083                 dpolicy->mid_interval = dcc->mid_discard_issue_time;
1084                 dpolicy->max_interval = dcc->max_discard_issue_time;
1085                 dpolicy->io_aware = false;
1086         } else if (discard_type == DPOLICY_FSTRIM) {
1087                 dpolicy->io_aware = false;
1088         } else if (discard_type == DPOLICY_UMOUNT) {
1089                 dpolicy->io_aware = false;
1090                 /* we need to issue all to keep CP_TRIMMED_FLAG */
1091                 dpolicy->granularity = 1;
1092                 dpolicy->timeout = true;
1093         }
1094 }
1095
1096 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1097                                 struct block_device *bdev, block_t lstart,
1098                                 block_t start, block_t len);
1099 /* this function is copied from blkdev_issue_discard from block/blk-lib.c */
1100 static int __submit_discard_cmd(struct f2fs_sb_info *sbi,
1101                                                 struct discard_policy *dpolicy,
1102                                                 struct discard_cmd *dc,
1103                                                 unsigned int *issued)
1104 {
1105         struct block_device *bdev = dc->bdev;
1106         unsigned int max_discard_blocks =
1107                         SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev));
1108         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1109         struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1110                                         &(dcc->fstrim_list) : &(dcc->wait_list);
1111         blk_opf_t flag = dpolicy->sync ? REQ_SYNC : 0;
1112         block_t lstart, start, len, total_len;
1113         int err = 0;
1114
1115         if (dc->state != D_PREP)
1116                 return 0;
1117
1118         if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))
1119                 return 0;
1120
1121         trace_f2fs_issue_discard(bdev, dc->start, dc->len);
1122
1123         lstart = dc->lstart;
1124         start = dc->start;
1125         len = dc->len;
1126         total_len = len;
1127
1128         dc->len = 0;
1129
1130         while (total_len && *issued < dpolicy->max_requests && !err) {
1131                 struct bio *bio = NULL;
1132                 unsigned long flags;
1133                 bool last = true;
1134
1135                 if (len > max_discard_blocks) {
1136                         len = max_discard_blocks;
1137                         last = false;
1138                 }
1139
1140                 (*issued)++;
1141                 if (*issued == dpolicy->max_requests)
1142                         last = true;
1143
1144                 dc->len += len;
1145
1146                 if (time_to_inject(sbi, FAULT_DISCARD)) {
1147                         f2fs_show_injection_info(sbi, FAULT_DISCARD);
1148                         err = -EIO;
1149                         goto submit;
1150                 }
1151                 err = __blkdev_issue_discard(bdev,
1152                                         SECTOR_FROM_BLOCK(start),
1153                                         SECTOR_FROM_BLOCK(len),
1154                                         GFP_NOFS, &bio);
1155 submit:
1156                 if (err) {
1157                         spin_lock_irqsave(&dc->lock, flags);
1158                         if (dc->state == D_PARTIAL)
1159                                 dc->state = D_SUBMIT;
1160                         spin_unlock_irqrestore(&dc->lock, flags);
1161
1162                         break;
1163                 }
1164
1165                 f2fs_bug_on(sbi, !bio);
1166
1167                 /*
1168                  * should keep before submission to avoid D_DONE
1169                  * right away
1170                  */
1171                 spin_lock_irqsave(&dc->lock, flags);
1172                 if (last)
1173                         dc->state = D_SUBMIT;
1174                 else
1175                         dc->state = D_PARTIAL;
1176                 dc->bio_ref++;
1177                 spin_unlock_irqrestore(&dc->lock, flags);
1178
1179                 atomic_inc(&dcc->queued_discard);
1180                 dc->queued++;
1181                 list_move_tail(&dc->list, wait_list);
1182
1183                 /* sanity check on discard range */
1184                 __check_sit_bitmap(sbi, lstart, lstart + len);
1185
1186                 bio->bi_private = dc;
1187                 bio->bi_end_io = f2fs_submit_discard_endio;
1188                 bio->bi_opf |= flag;
1189                 submit_bio(bio);
1190
1191                 atomic_inc(&dcc->issued_discard);
1192
1193                 f2fs_update_iostat(sbi, NULL, FS_DISCARD, len * F2FS_BLKSIZE);
1194
1195                 lstart += len;
1196                 start += len;
1197                 total_len -= len;
1198                 len = total_len;
1199         }
1200
1201         if (!err && len) {
1202                 dcc->undiscard_blks -= len;
1203                 __update_discard_tree_range(sbi, bdev, lstart, start, len);
1204         }
1205         return err;
1206 }
1207
1208 static void __insert_discard_tree(struct f2fs_sb_info *sbi,
1209                                 struct block_device *bdev, block_t lstart,
1210                                 block_t start, block_t len,
1211                                 struct rb_node **insert_p,
1212                                 struct rb_node *insert_parent)
1213 {
1214         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1215         struct rb_node **p;
1216         struct rb_node *parent = NULL;
1217         bool leftmost = true;
1218
1219         if (insert_p && insert_parent) {
1220                 parent = insert_parent;
1221                 p = insert_p;
1222                 goto do_insert;
1223         }
1224
1225         p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent,
1226                                                         lstart, &leftmost);
1227 do_insert:
1228         __attach_discard_cmd(sbi, bdev, lstart, start, len, parent,
1229                                                                 p, leftmost);
1230 }
1231
1232 static void __relocate_discard_cmd(struct discard_cmd_control *dcc,
1233                                                 struct discard_cmd *dc)
1234 {
1235         list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->len)]);
1236 }
1237
1238 static void __punch_discard_cmd(struct f2fs_sb_info *sbi,
1239                                 struct discard_cmd *dc, block_t blkaddr)
1240 {
1241         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1242         struct discard_info di = dc->di;
1243         bool modified = false;
1244
1245         if (dc->state == D_DONE || dc->len == 1) {
1246                 __remove_discard_cmd(sbi, dc);
1247                 return;
1248         }
1249
1250         dcc->undiscard_blks -= di.len;
1251
1252         if (blkaddr > di.lstart) {
1253                 dc->len = blkaddr - dc->lstart;
1254                 dcc->undiscard_blks += dc->len;
1255                 __relocate_discard_cmd(dcc, dc);
1256                 modified = true;
1257         }
1258
1259         if (blkaddr < di.lstart + di.len - 1) {
1260                 if (modified) {
1261                         __insert_discard_tree(sbi, dc->bdev, blkaddr + 1,
1262                                         di.start + blkaddr + 1 - di.lstart,
1263                                         di.lstart + di.len - 1 - blkaddr,
1264                                         NULL, NULL);
1265                 } else {
1266                         dc->lstart++;
1267                         dc->len--;
1268                         dc->start++;
1269                         dcc->undiscard_blks += dc->len;
1270                         __relocate_discard_cmd(dcc, dc);
1271                 }
1272         }
1273 }
1274
1275 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1276                                 struct block_device *bdev, block_t lstart,
1277                                 block_t start, block_t len)
1278 {
1279         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1280         struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1281         struct discard_cmd *dc;
1282         struct discard_info di = {0};
1283         struct rb_node **insert_p = NULL, *insert_parent = NULL;
1284         unsigned int max_discard_blocks =
1285                         SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev));
1286         block_t end = lstart + len;
1287
1288         dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
1289                                         NULL, lstart,
1290                                         (struct rb_entry **)&prev_dc,
1291                                         (struct rb_entry **)&next_dc,
1292                                         &insert_p, &insert_parent, true, NULL);
1293         if (dc)
1294                 prev_dc = dc;
1295
1296         if (!prev_dc) {
1297                 di.lstart = lstart;
1298                 di.len = next_dc ? next_dc->lstart - lstart : len;
1299                 di.len = min(di.len, len);
1300                 di.start = start;
1301         }
1302
1303         while (1) {
1304                 struct rb_node *node;
1305                 bool merged = false;
1306                 struct discard_cmd *tdc = NULL;
1307
1308                 if (prev_dc) {
1309                         di.lstart = prev_dc->lstart + prev_dc->len;
1310                         if (di.lstart < lstart)
1311                                 di.lstart = lstart;
1312                         if (di.lstart >= end)
1313                                 break;
1314
1315                         if (!next_dc || next_dc->lstart > end)
1316                                 di.len = end - di.lstart;
1317                         else
1318                                 di.len = next_dc->lstart - di.lstart;
1319                         di.start = start + di.lstart - lstart;
1320                 }
1321
1322                 if (!di.len)
1323                         goto next;
1324
1325                 if (prev_dc && prev_dc->state == D_PREP &&
1326                         prev_dc->bdev == bdev &&
1327                         __is_discard_back_mergeable(&di, &prev_dc->di,
1328                                                         max_discard_blocks)) {
1329                         prev_dc->di.len += di.len;
1330                         dcc->undiscard_blks += di.len;
1331                         __relocate_discard_cmd(dcc, prev_dc);
1332                         di = prev_dc->di;
1333                         tdc = prev_dc;
1334                         merged = true;
1335                 }
1336
1337                 if (next_dc && next_dc->state == D_PREP &&
1338                         next_dc->bdev == bdev &&
1339                         __is_discard_front_mergeable(&di, &next_dc->di,
1340                                                         max_discard_blocks)) {
1341                         next_dc->di.lstart = di.lstart;
1342                         next_dc->di.len += di.len;
1343                         next_dc->di.start = di.start;
1344                         dcc->undiscard_blks += di.len;
1345                         __relocate_discard_cmd(dcc, next_dc);
1346                         if (tdc)
1347                                 __remove_discard_cmd(sbi, tdc);
1348                         merged = true;
1349                 }
1350
1351                 if (!merged) {
1352                         __insert_discard_tree(sbi, bdev, di.lstart, di.start,
1353                                                         di.len, NULL, NULL);
1354                 }
1355  next:
1356                 prev_dc = next_dc;
1357                 if (!prev_dc)
1358                         break;
1359
1360                 node = rb_next(&prev_dc->rb_node);
1361                 next_dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1362         }
1363 }
1364
1365 static int __queue_discard_cmd(struct f2fs_sb_info *sbi,
1366                 struct block_device *bdev, block_t blkstart, block_t blklen)
1367 {
1368         block_t lblkstart = blkstart;
1369
1370         if (!f2fs_bdev_support_discard(bdev))
1371                 return 0;
1372
1373         trace_f2fs_queue_discard(bdev, blkstart, blklen);
1374
1375         if (f2fs_is_multi_device(sbi)) {
1376                 int devi = f2fs_target_device_index(sbi, blkstart);
1377
1378                 blkstart -= FDEV(devi).start_blk;
1379         }
1380         mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);
1381         __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen);
1382         mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);
1383         return 0;
1384 }
1385
1386 static unsigned int __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi,
1387                                         struct discard_policy *dpolicy)
1388 {
1389         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1390         struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1391         struct rb_node **insert_p = NULL, *insert_parent = NULL;
1392         struct discard_cmd *dc;
1393         struct blk_plug plug;
1394         unsigned int pos = dcc->next_pos;
1395         unsigned int issued = 0;
1396         bool io_interrupted = false;
1397
1398         mutex_lock(&dcc->cmd_lock);
1399         dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
1400                                         NULL, pos,
1401                                         (struct rb_entry **)&prev_dc,
1402                                         (struct rb_entry **)&next_dc,
1403                                         &insert_p, &insert_parent, true, NULL);
1404         if (!dc)
1405                 dc = next_dc;
1406
1407         blk_start_plug(&plug);
1408
1409         while (dc) {
1410                 struct rb_node *node;
1411                 int err = 0;
1412
1413                 if (dc->state != D_PREP)
1414                         goto next;
1415
1416                 if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) {
1417                         io_interrupted = true;
1418                         break;
1419                 }
1420
1421                 dcc->next_pos = dc->lstart + dc->len;
1422                 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
1423
1424                 if (issued >= dpolicy->max_requests)
1425                         break;
1426 next:
1427                 node = rb_next(&dc->rb_node);
1428                 if (err)
1429                         __remove_discard_cmd(sbi, dc);
1430                 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1431         }
1432
1433         blk_finish_plug(&plug);
1434
1435         if (!dc)
1436                 dcc->next_pos = 0;
1437
1438         mutex_unlock(&dcc->cmd_lock);
1439
1440         if (!issued && io_interrupted)
1441                 issued = -1;
1442
1443         return issued;
1444 }
1445 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1446                                         struct discard_policy *dpolicy);
1447
1448 static int __issue_discard_cmd(struct f2fs_sb_info *sbi,
1449                                         struct discard_policy *dpolicy)
1450 {
1451         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1452         struct list_head *pend_list;
1453         struct discard_cmd *dc, *tmp;
1454         struct blk_plug plug;
1455         int i, issued;
1456         bool io_interrupted = false;
1457
1458         if (dpolicy->timeout)
1459                 f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT);
1460
1461 retry:
1462         issued = 0;
1463         for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1464                 if (dpolicy->timeout &&
1465                                 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1466                         break;
1467
1468                 if (i + 1 < dpolicy->granularity)
1469                         break;
1470
1471                 if (i + 1 < DEFAULT_DISCARD_GRANULARITY && dpolicy->ordered)
1472                         return __issue_discard_cmd_orderly(sbi, dpolicy);
1473
1474                 pend_list = &dcc->pend_list[i];
1475
1476                 mutex_lock(&dcc->cmd_lock);
1477                 if (list_empty(pend_list))
1478                         goto next;
1479                 if (unlikely(dcc->rbtree_check))
1480                         f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
1481                                                         &dcc->root));
1482                 blk_start_plug(&plug);
1483                 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1484                         f2fs_bug_on(sbi, dc->state != D_PREP);
1485
1486                         if (dpolicy->timeout &&
1487                                 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1488                                 break;
1489
1490                         if (dpolicy->io_aware && i < dpolicy->io_aware_gran &&
1491                                                 !is_idle(sbi, DISCARD_TIME)) {
1492                                 io_interrupted = true;
1493                                 break;
1494                         }
1495
1496                         __submit_discard_cmd(sbi, dpolicy, dc, &issued);
1497
1498                         if (issued >= dpolicy->max_requests)
1499                                 break;
1500                 }
1501                 blk_finish_plug(&plug);
1502 next:
1503                 mutex_unlock(&dcc->cmd_lock);
1504
1505                 if (issued >= dpolicy->max_requests || io_interrupted)
1506                         break;
1507         }
1508
1509         if (dpolicy->type == DPOLICY_UMOUNT && issued) {
1510                 __wait_all_discard_cmd(sbi, dpolicy);
1511                 goto retry;
1512         }
1513
1514         if (!issued && io_interrupted)
1515                 issued = -1;
1516
1517         return issued;
1518 }
1519
1520 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi)
1521 {
1522         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1523         struct list_head *pend_list;
1524         struct discard_cmd *dc, *tmp;
1525         int i;
1526         bool dropped = false;
1527
1528         mutex_lock(&dcc->cmd_lock);
1529         for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1530                 pend_list = &dcc->pend_list[i];
1531                 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1532                         f2fs_bug_on(sbi, dc->state != D_PREP);
1533                         __remove_discard_cmd(sbi, dc);
1534                         dropped = true;
1535                 }
1536         }
1537         mutex_unlock(&dcc->cmd_lock);
1538
1539         return dropped;
1540 }
1541
1542 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi)
1543 {
1544         __drop_discard_cmd(sbi);
1545 }
1546
1547 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi,
1548                                                         struct discard_cmd *dc)
1549 {
1550         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1551         unsigned int len = 0;
1552
1553         wait_for_completion_io(&dc->wait);
1554         mutex_lock(&dcc->cmd_lock);
1555         f2fs_bug_on(sbi, dc->state != D_DONE);
1556         dc->ref--;
1557         if (!dc->ref) {
1558                 if (!dc->error)
1559                         len = dc->len;
1560                 __remove_discard_cmd(sbi, dc);
1561         }
1562         mutex_unlock(&dcc->cmd_lock);
1563
1564         return len;
1565 }
1566
1567 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi,
1568                                                 struct discard_policy *dpolicy,
1569                                                 block_t start, block_t end)
1570 {
1571         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1572         struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1573                                         &(dcc->fstrim_list) : &(dcc->wait_list);
1574         struct discard_cmd *dc = NULL, *iter, *tmp;
1575         unsigned int trimmed = 0;
1576
1577 next:
1578         dc = NULL;
1579
1580         mutex_lock(&dcc->cmd_lock);
1581         list_for_each_entry_safe(iter, tmp, wait_list, list) {
1582                 if (iter->lstart + iter->len <= start || end <= iter->lstart)
1583                         continue;
1584                 if (iter->len < dpolicy->granularity)
1585                         continue;
1586                 if (iter->state == D_DONE && !iter->ref) {
1587                         wait_for_completion_io(&iter->wait);
1588                         if (!iter->error)
1589                                 trimmed += iter->len;
1590                         __remove_discard_cmd(sbi, iter);
1591                 } else {
1592                         iter->ref++;
1593                         dc = iter;
1594                         break;
1595                 }
1596         }
1597         mutex_unlock(&dcc->cmd_lock);
1598
1599         if (dc) {
1600                 trimmed += __wait_one_discard_bio(sbi, dc);
1601                 goto next;
1602         }
1603
1604         return trimmed;
1605 }
1606
1607 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1608                                                 struct discard_policy *dpolicy)
1609 {
1610         struct discard_policy dp;
1611         unsigned int discard_blks;
1612
1613         if (dpolicy)
1614                 return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX);
1615
1616         /* wait all */
1617         __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1);
1618         discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1619         __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1);
1620         discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1621
1622         return discard_blks;
1623 }
1624
1625 /* This should be covered by global mutex, &sit_i->sentry_lock */
1626 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)
1627 {
1628         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1629         struct discard_cmd *dc;
1630         bool need_wait = false;
1631
1632         mutex_lock(&dcc->cmd_lock);
1633         dc = (struct discard_cmd *)f2fs_lookup_rb_tree(&dcc->root,
1634                                                         NULL, blkaddr);
1635         if (dc) {
1636                 if (dc->state == D_PREP) {
1637                         __punch_discard_cmd(sbi, dc, blkaddr);
1638                 } else {
1639                         dc->ref++;
1640                         need_wait = true;
1641                 }
1642         }
1643         mutex_unlock(&dcc->cmd_lock);
1644
1645         if (need_wait)
1646                 __wait_one_discard_bio(sbi, dc);
1647 }
1648
1649 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi)
1650 {
1651         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1652
1653         if (dcc && dcc->f2fs_issue_discard) {
1654                 struct task_struct *discard_thread = dcc->f2fs_issue_discard;
1655
1656                 dcc->f2fs_issue_discard = NULL;
1657                 kthread_stop(discard_thread);
1658         }
1659 }
1660
1661 /* This comes from f2fs_put_super */
1662 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi)
1663 {
1664         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1665         struct discard_policy dpolicy;
1666         bool dropped;
1667
1668         __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT,
1669                                         dcc->discard_granularity);
1670         __issue_discard_cmd(sbi, &dpolicy);
1671         dropped = __drop_discard_cmd(sbi);
1672
1673         /* just to make sure there is no pending discard commands */
1674         __wait_all_discard_cmd(sbi, NULL);
1675
1676         f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt));
1677         return dropped;
1678 }
1679
1680 static int issue_discard_thread(void *data)
1681 {
1682         struct f2fs_sb_info *sbi = data;
1683         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1684         wait_queue_head_t *q = &dcc->discard_wait_queue;
1685         struct discard_policy dpolicy;
1686         unsigned int wait_ms = dcc->min_discard_issue_time;
1687         int issued;
1688
1689         set_freezable();
1690
1691         do {
1692                 if (sbi->gc_mode == GC_URGENT_HIGH ||
1693                         !f2fs_available_free_memory(sbi, DISCARD_CACHE))
1694                         __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1);
1695                 else
1696                         __init_discard_policy(sbi, &dpolicy, DPOLICY_BG,
1697                                                 dcc->discard_granularity);
1698
1699                 if (!atomic_read(&dcc->discard_cmd_cnt))
1700                        wait_ms = dpolicy.max_interval;
1701
1702                 wait_event_interruptible_timeout(*q,
1703                                 kthread_should_stop() || freezing(current) ||
1704                                 dcc->discard_wake,
1705                                 msecs_to_jiffies(wait_ms));
1706
1707                 if (dcc->discard_wake)
1708                         dcc->discard_wake = 0;
1709
1710                 /* clean up pending candidates before going to sleep */
1711                 if (atomic_read(&dcc->queued_discard))
1712                         __wait_all_discard_cmd(sbi, NULL);
1713
1714                 if (try_to_freeze())
1715                         continue;
1716                 if (f2fs_readonly(sbi->sb))
1717                         continue;
1718                 if (kthread_should_stop())
1719                         return 0;
1720                 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
1721                         wait_ms = dpolicy.max_interval;
1722                         continue;
1723                 }
1724                 if (!atomic_read(&dcc->discard_cmd_cnt))
1725                         continue;
1726
1727                 sb_start_intwrite(sbi->sb);
1728
1729                 issued = __issue_discard_cmd(sbi, &dpolicy);
1730                 if (issued > 0) {
1731                         __wait_all_discard_cmd(sbi, &dpolicy);
1732                         wait_ms = dpolicy.min_interval;
1733                 } else if (issued == -1) {
1734                         wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME);
1735                         if (!wait_ms)
1736                                 wait_ms = dpolicy.mid_interval;
1737                 } else {
1738                         wait_ms = dpolicy.max_interval;
1739                 }
1740
1741                 sb_end_intwrite(sbi->sb);
1742
1743         } while (!kthread_should_stop());
1744         return 0;
1745 }
1746
1747 #ifdef CONFIG_BLK_DEV_ZONED
1748 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
1749                 struct block_device *bdev, block_t blkstart, block_t blklen)
1750 {
1751         sector_t sector, nr_sects;
1752         block_t lblkstart = blkstart;
1753         int devi = 0;
1754
1755         if (f2fs_is_multi_device(sbi)) {
1756                 devi = f2fs_target_device_index(sbi, blkstart);
1757                 if (blkstart < FDEV(devi).start_blk ||
1758                     blkstart > FDEV(devi).end_blk) {
1759                         f2fs_err(sbi, "Invalid block %x", blkstart);
1760                         return -EIO;
1761                 }
1762                 blkstart -= FDEV(devi).start_blk;
1763         }
1764
1765         /* For sequential zones, reset the zone write pointer */
1766         if (f2fs_blkz_is_seq(sbi, devi, blkstart)) {
1767                 sector = SECTOR_FROM_BLOCK(blkstart);
1768                 nr_sects = SECTOR_FROM_BLOCK(blklen);
1769
1770                 if (sector & (bdev_zone_sectors(bdev) - 1) ||
1771                                 nr_sects != bdev_zone_sectors(bdev)) {
1772                         f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)",
1773                                  devi, sbi->s_ndevs ? FDEV(devi).path : "",
1774                                  blkstart, blklen);
1775                         return -EIO;
1776                 }
1777                 trace_f2fs_issue_reset_zone(bdev, blkstart);
1778                 return blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET,
1779                                         sector, nr_sects, GFP_NOFS);
1780         }
1781
1782         /* For conventional zones, use regular discard if supported */
1783         return __queue_discard_cmd(sbi, bdev, lblkstart, blklen);
1784 }
1785 #endif
1786
1787 static int __issue_discard_async(struct f2fs_sb_info *sbi,
1788                 struct block_device *bdev, block_t blkstart, block_t blklen)
1789 {
1790 #ifdef CONFIG_BLK_DEV_ZONED
1791         if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev))
1792                 return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen);
1793 #endif
1794         return __queue_discard_cmd(sbi, bdev, blkstart, blklen);
1795 }
1796
1797 static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
1798                                 block_t blkstart, block_t blklen)
1799 {
1800         sector_t start = blkstart, len = 0;
1801         struct block_device *bdev;
1802         struct seg_entry *se;
1803         unsigned int offset;
1804         block_t i;
1805         int err = 0;
1806
1807         bdev = f2fs_target_device(sbi, blkstart, NULL);
1808
1809         for (i = blkstart; i < blkstart + blklen; i++, len++) {
1810                 if (i != start) {
1811                         struct block_device *bdev2 =
1812                                 f2fs_target_device(sbi, i, NULL);
1813
1814                         if (bdev2 != bdev) {
1815                                 err = __issue_discard_async(sbi, bdev,
1816                                                 start, len);
1817                                 if (err)
1818                                         return err;
1819                                 bdev = bdev2;
1820                                 start = i;
1821                                 len = 0;
1822                         }
1823                 }
1824
1825                 se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
1826                 offset = GET_BLKOFF_FROM_SEG0(sbi, i);
1827
1828                 if (f2fs_block_unit_discard(sbi) &&
1829                                 !f2fs_test_and_set_bit(offset, se->discard_map))
1830                         sbi->discard_blks--;
1831         }
1832
1833         if (len)
1834                 err = __issue_discard_async(sbi, bdev, start, len);
1835         return err;
1836 }
1837
1838 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc,
1839                                                         bool check_only)
1840 {
1841         int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
1842         int max_blocks = sbi->blocks_per_seg;
1843         struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
1844         unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
1845         unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
1846         unsigned long *discard_map = (unsigned long *)se->discard_map;
1847         unsigned long *dmap = SIT_I(sbi)->tmp_map;
1848         unsigned int start = 0, end = -1;
1849         bool force = (cpc->reason & CP_DISCARD);
1850         struct discard_entry *de = NULL;
1851         struct list_head *head = &SM_I(sbi)->dcc_info->entry_list;
1852         int i;
1853
1854         if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi) ||
1855                         !f2fs_block_unit_discard(sbi))
1856                 return false;
1857
1858         if (!force) {
1859                 if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks ||
1860                         SM_I(sbi)->dcc_info->nr_discards >=
1861                                 SM_I(sbi)->dcc_info->max_discards)
1862                         return false;
1863         }
1864
1865         /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
1866         for (i = 0; i < entries; i++)
1867                 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
1868                                 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
1869
1870         while (force || SM_I(sbi)->dcc_info->nr_discards <=
1871                                 SM_I(sbi)->dcc_info->max_discards) {
1872                 start = __find_rev_next_bit(dmap, max_blocks, end + 1);
1873                 if (start >= max_blocks)
1874                         break;
1875
1876                 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
1877                 if (force && start && end != max_blocks
1878                                         && (end - start) < cpc->trim_minlen)
1879                         continue;
1880
1881                 if (check_only)
1882                         return true;
1883
1884                 if (!de) {
1885                         de = f2fs_kmem_cache_alloc(discard_entry_slab,
1886                                                 GFP_F2FS_ZERO, true, NULL);
1887                         de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start);
1888                         list_add_tail(&de->list, head);
1889                 }
1890
1891                 for (i = start; i < end; i++)
1892                         __set_bit_le(i, (void *)de->discard_map);
1893
1894                 SM_I(sbi)->dcc_info->nr_discards += end - start;
1895         }
1896         return false;
1897 }
1898
1899 static void release_discard_addr(struct discard_entry *entry)
1900 {
1901         list_del(&entry->list);
1902         kmem_cache_free(discard_entry_slab, entry);
1903 }
1904
1905 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi)
1906 {
1907         struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list);
1908         struct discard_entry *entry, *this;
1909
1910         /* drop caches */
1911         list_for_each_entry_safe(entry, this, head, list)
1912                 release_discard_addr(entry);
1913 }
1914
1915 /*
1916  * Should call f2fs_clear_prefree_segments after checkpoint is done.
1917  */
1918 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
1919 {
1920         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1921         unsigned int segno;
1922
1923         mutex_lock(&dirty_i->seglist_lock);
1924         for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
1925                 __set_test_and_free(sbi, segno, false);
1926         mutex_unlock(&dirty_i->seglist_lock);
1927 }
1928
1929 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
1930                                                 struct cp_control *cpc)
1931 {
1932         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1933         struct list_head *head = &dcc->entry_list;
1934         struct discard_entry *entry, *this;
1935         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1936         unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
1937         unsigned int start = 0, end = -1;
1938         unsigned int secno, start_segno;
1939         bool force = (cpc->reason & CP_DISCARD);
1940         bool section_alignment = F2FS_OPTION(sbi).discard_unit ==
1941                                                 DISCARD_UNIT_SECTION;
1942
1943         if (f2fs_lfs_mode(sbi) && __is_large_section(sbi))
1944                 section_alignment = true;
1945
1946         mutex_lock(&dirty_i->seglist_lock);
1947
1948         while (1) {
1949                 int i;
1950
1951                 if (section_alignment && end != -1)
1952                         end--;
1953                 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
1954                 if (start >= MAIN_SEGS(sbi))
1955                         break;
1956                 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
1957                                                                 start + 1);
1958
1959                 if (section_alignment) {
1960                         start = rounddown(start, sbi->segs_per_sec);
1961                         end = roundup(end, sbi->segs_per_sec);
1962                 }
1963
1964                 for (i = start; i < end; i++) {
1965                         if (test_and_clear_bit(i, prefree_map))
1966                                 dirty_i->nr_dirty[PRE]--;
1967                 }
1968
1969                 if (!f2fs_realtime_discard_enable(sbi))
1970                         continue;
1971
1972                 if (force && start >= cpc->trim_start &&
1973                                         (end - 1) <= cpc->trim_end)
1974                                 continue;
1975
1976                 if (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi)) {
1977                         f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
1978                                 (end - start) << sbi->log_blocks_per_seg);
1979                         continue;
1980                 }
1981 next:
1982                 secno = GET_SEC_FROM_SEG(sbi, start);
1983                 start_segno = GET_SEG_FROM_SEC(sbi, secno);
1984                 if (!IS_CURSEC(sbi, secno) &&
1985                         !get_valid_blocks(sbi, start, true))
1986                         f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno),
1987                                 sbi->segs_per_sec << sbi->log_blocks_per_seg);
1988
1989                 start = start_segno + sbi->segs_per_sec;
1990                 if (start < end)
1991                         goto next;
1992                 else
1993                         end = start - 1;
1994         }
1995         mutex_unlock(&dirty_i->seglist_lock);
1996
1997         if (!f2fs_block_unit_discard(sbi))
1998                 goto wakeup;
1999
2000         /* send small discards */
2001         list_for_each_entry_safe(entry, this, head, list) {
2002                 unsigned int cur_pos = 0, next_pos, len, total_len = 0;
2003                 bool is_valid = test_bit_le(0, entry->discard_map);
2004
2005 find_next:
2006                 if (is_valid) {
2007                         next_pos = find_next_zero_bit_le(entry->discard_map,
2008                                         sbi->blocks_per_seg, cur_pos);
2009                         len = next_pos - cur_pos;
2010
2011                         if (f2fs_sb_has_blkzoned(sbi) ||
2012                             (force && len < cpc->trim_minlen))
2013                                 goto skip;
2014
2015                         f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos,
2016                                                                         len);
2017                         total_len += len;
2018                 } else {
2019                         next_pos = find_next_bit_le(entry->discard_map,
2020                                         sbi->blocks_per_seg, cur_pos);
2021                 }
2022 skip:
2023                 cur_pos = next_pos;
2024                 is_valid = !is_valid;
2025
2026                 if (cur_pos < sbi->blocks_per_seg)
2027                         goto find_next;
2028
2029                 release_discard_addr(entry);
2030                 dcc->nr_discards -= total_len;
2031         }
2032
2033 wakeup:
2034         wake_up_discard_thread(sbi, false);
2035 }
2036
2037 int f2fs_start_discard_thread(struct f2fs_sb_info *sbi)
2038 {
2039         dev_t dev = sbi->sb->s_bdev->bd_dev;
2040         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2041         int err = 0;
2042
2043         if (!f2fs_realtime_discard_enable(sbi))
2044                 return 0;
2045
2046         dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,
2047                                 "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev));
2048         if (IS_ERR(dcc->f2fs_issue_discard)) {
2049                 err = PTR_ERR(dcc->f2fs_issue_discard);
2050                 dcc->f2fs_issue_discard = NULL;
2051         }
2052
2053         return err;
2054 }
2055
2056 static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
2057 {
2058         struct discard_cmd_control *dcc;
2059         int err = 0, i;
2060
2061         if (SM_I(sbi)->dcc_info) {
2062                 dcc = SM_I(sbi)->dcc_info;
2063                 goto init_thread;
2064         }
2065
2066         dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);
2067         if (!dcc)
2068                 return -ENOMEM;
2069
2070         dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY;
2071         if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT)
2072                 dcc->discard_granularity = sbi->blocks_per_seg;
2073         else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION)
2074                 dcc->discard_granularity = BLKS_PER_SEC(sbi);
2075
2076         INIT_LIST_HEAD(&dcc->entry_list);
2077         for (i = 0; i < MAX_PLIST_NUM; i++)
2078                 INIT_LIST_HEAD(&dcc->pend_list[i]);
2079         INIT_LIST_HEAD(&dcc->wait_list);
2080         INIT_LIST_HEAD(&dcc->fstrim_list);
2081         mutex_init(&dcc->cmd_lock);
2082         atomic_set(&dcc->issued_discard, 0);
2083         atomic_set(&dcc->queued_discard, 0);
2084         atomic_set(&dcc->discard_cmd_cnt, 0);
2085         dcc->nr_discards = 0;
2086         dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg;
2087         dcc->max_discard_request = DEF_MAX_DISCARD_REQUEST;
2088         dcc->min_discard_issue_time = DEF_MIN_DISCARD_ISSUE_TIME;
2089         dcc->mid_discard_issue_time = DEF_MID_DISCARD_ISSUE_TIME;
2090         dcc->max_discard_issue_time = DEF_MAX_DISCARD_ISSUE_TIME;
2091         dcc->undiscard_blks = 0;
2092         dcc->next_pos = 0;
2093         dcc->root = RB_ROOT_CACHED;
2094         dcc->rbtree_check = false;
2095
2096         init_waitqueue_head(&dcc->discard_wait_queue);
2097         SM_I(sbi)->dcc_info = dcc;
2098 init_thread:
2099         err = f2fs_start_discard_thread(sbi);
2100         if (err) {
2101                 kfree(dcc);
2102                 SM_I(sbi)->dcc_info = NULL;
2103         }
2104
2105         return err;
2106 }
2107
2108 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)
2109 {
2110         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2111
2112         if (!dcc)
2113                 return;
2114
2115         f2fs_stop_discard_thread(sbi);
2116
2117         /*
2118          * Recovery can cache discard commands, so in error path of
2119          * fill_super(), it needs to give a chance to handle them.
2120          */
2121         if (unlikely(atomic_read(&dcc->discard_cmd_cnt)))
2122                 f2fs_issue_discard_timeout(sbi);
2123
2124         kfree(dcc);
2125         SM_I(sbi)->dcc_info = NULL;
2126 }
2127
2128 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
2129 {
2130         struct sit_info *sit_i = SIT_I(sbi);
2131
2132         if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
2133                 sit_i->dirty_sentries++;
2134                 return false;
2135         }
2136
2137         return true;
2138 }
2139
2140 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
2141                                         unsigned int segno, int modified)
2142 {
2143         struct seg_entry *se = get_seg_entry(sbi, segno);
2144
2145         se->type = type;
2146         if (modified)
2147                 __mark_sit_entry_dirty(sbi, segno);
2148 }
2149
2150 static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi,
2151                                                                 block_t blkaddr)
2152 {
2153         unsigned int segno = GET_SEGNO(sbi, blkaddr);
2154
2155         if (segno == NULL_SEGNO)
2156                 return 0;
2157         return get_seg_entry(sbi, segno)->mtime;
2158 }
2159
2160 static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr,
2161                                                 unsigned long long old_mtime)
2162 {
2163         struct seg_entry *se;
2164         unsigned int segno = GET_SEGNO(sbi, blkaddr);
2165         unsigned long long ctime = get_mtime(sbi, false);
2166         unsigned long long mtime = old_mtime ? old_mtime : ctime;
2167
2168         if (segno == NULL_SEGNO)
2169                 return;
2170
2171         se = get_seg_entry(sbi, segno);
2172
2173         if (!se->mtime)
2174                 se->mtime = mtime;
2175         else
2176                 se->mtime = div_u64(se->mtime * se->valid_blocks + mtime,
2177                                                 se->valid_blocks + 1);
2178
2179         if (ctime > SIT_I(sbi)->max_mtime)
2180                 SIT_I(sbi)->max_mtime = ctime;
2181 }
2182
2183 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
2184 {
2185         struct seg_entry *se;
2186         unsigned int segno, offset;
2187         long int new_vblocks;
2188         bool exist;
2189 #ifdef CONFIG_F2FS_CHECK_FS
2190         bool mir_exist;
2191 #endif
2192
2193         segno = GET_SEGNO(sbi, blkaddr);
2194
2195         se = get_seg_entry(sbi, segno);
2196         new_vblocks = se->valid_blocks + del;
2197         offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2198
2199         f2fs_bug_on(sbi, (new_vblocks < 0 ||
2200                         (new_vblocks > f2fs_usable_blks_in_seg(sbi, segno))));
2201
2202         se->valid_blocks = new_vblocks;
2203
2204         /* Update valid block bitmap */
2205         if (del > 0) {
2206                 exist = f2fs_test_and_set_bit(offset, se->cur_valid_map);
2207 #ifdef CONFIG_F2FS_CHECK_FS
2208                 mir_exist = f2fs_test_and_set_bit(offset,
2209                                                 se->cur_valid_map_mir);
2210                 if (unlikely(exist != mir_exist)) {
2211                         f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d",
2212                                  blkaddr, exist);
2213                         f2fs_bug_on(sbi, 1);
2214                 }
2215 #endif
2216                 if (unlikely(exist)) {
2217                         f2fs_err(sbi, "Bitmap was wrongly set, blk:%u",
2218                                  blkaddr);
2219                         f2fs_bug_on(sbi, 1);
2220                         se->valid_blocks--;
2221                         del = 0;
2222                 }
2223
2224                 if (f2fs_block_unit_discard(sbi) &&
2225                                 !f2fs_test_and_set_bit(offset, se->discard_map))
2226                         sbi->discard_blks--;
2227
2228                 /*
2229                  * SSR should never reuse block which is checkpointed
2230                  * or newly invalidated.
2231                  */
2232                 if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {
2233                         if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map))
2234                                 se->ckpt_valid_blocks++;
2235                 }
2236         } else {
2237                 exist = f2fs_test_and_clear_bit(offset, se->cur_valid_map);
2238 #ifdef CONFIG_F2FS_CHECK_FS
2239                 mir_exist = f2fs_test_and_clear_bit(offset,
2240                                                 se->cur_valid_map_mir);
2241                 if (unlikely(exist != mir_exist)) {
2242                         f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d",
2243                                  blkaddr, exist);
2244                         f2fs_bug_on(sbi, 1);
2245                 }
2246 #endif
2247                 if (unlikely(!exist)) {
2248                         f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u",
2249                                  blkaddr);
2250                         f2fs_bug_on(sbi, 1);
2251                         se->valid_blocks++;
2252                         del = 0;
2253                 } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2254                         /*
2255                          * If checkpoints are off, we must not reuse data that
2256                          * was used in the previous checkpoint. If it was used
2257                          * before, we must track that to know how much space we
2258                          * really have.
2259                          */
2260                         if (f2fs_test_bit(offset, se->ckpt_valid_map)) {
2261                                 spin_lock(&sbi->stat_lock);
2262                                 sbi->unusable_block_count++;
2263                                 spin_unlock(&sbi->stat_lock);
2264                         }
2265                 }
2266
2267                 if (f2fs_block_unit_discard(sbi) &&
2268                         f2fs_test_and_clear_bit(offset, se->discard_map))
2269                         sbi->discard_blks++;
2270         }
2271         if (!f2fs_test_bit(offset, se->ckpt_valid_map))
2272                 se->ckpt_valid_blocks += del;
2273
2274         __mark_sit_entry_dirty(sbi, segno);
2275
2276         /* update total number of valid blocks to be written in ckpt area */
2277         SIT_I(sbi)->written_valid_blocks += del;
2278
2279         if (__is_large_section(sbi))
2280                 get_sec_entry(sbi, segno)->valid_blocks += del;
2281 }
2282
2283 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
2284 {
2285         unsigned int segno = GET_SEGNO(sbi, addr);
2286         struct sit_info *sit_i = SIT_I(sbi);
2287
2288         f2fs_bug_on(sbi, addr == NULL_ADDR);
2289         if (addr == NEW_ADDR || addr == COMPRESS_ADDR)
2290                 return;
2291
2292         invalidate_mapping_pages(META_MAPPING(sbi), addr, addr);
2293         f2fs_invalidate_compress_page(sbi, addr);
2294
2295         /* add it into sit main buffer */
2296         down_write(&sit_i->sentry_lock);
2297
2298         update_segment_mtime(sbi, addr, 0);
2299         update_sit_entry(sbi, addr, -1);
2300
2301         /* add it into dirty seglist */
2302         locate_dirty_segment(sbi, segno);
2303
2304         up_write(&sit_i->sentry_lock);
2305 }
2306
2307 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
2308 {
2309         struct sit_info *sit_i = SIT_I(sbi);
2310         unsigned int segno, offset;
2311         struct seg_entry *se;
2312         bool is_cp = false;
2313
2314         if (!__is_valid_data_blkaddr(blkaddr))
2315                 return true;
2316
2317         down_read(&sit_i->sentry_lock);
2318
2319         segno = GET_SEGNO(sbi, blkaddr);
2320         se = get_seg_entry(sbi, segno);
2321         offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2322
2323         if (f2fs_test_bit(offset, se->ckpt_valid_map))
2324                 is_cp = true;
2325
2326         up_read(&sit_i->sentry_lock);
2327
2328         return is_cp;
2329 }
2330
2331 /*
2332  * This function should be resided under the curseg_mutex lock
2333  */
2334 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
2335                                         struct f2fs_summary *sum)
2336 {
2337         struct curseg_info *curseg = CURSEG_I(sbi, type);
2338         void *addr = curseg->sum_blk;
2339
2340         addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
2341         memcpy(addr, sum, sizeof(struct f2fs_summary));
2342 }
2343
2344 /*
2345  * Calculate the number of current summary pages for writing
2346  */
2347 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
2348 {
2349         int valid_sum_count = 0;
2350         int i, sum_in_page;
2351
2352         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
2353                 if (sbi->ckpt->alloc_type[i] == SSR)
2354                         valid_sum_count += sbi->blocks_per_seg;
2355                 else {
2356                         if (for_ra)
2357                                 valid_sum_count += le16_to_cpu(
2358                                         F2FS_CKPT(sbi)->cur_data_blkoff[i]);
2359                         else
2360                                 valid_sum_count += curseg_blkoff(sbi, i);
2361                 }
2362         }
2363
2364         sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE -
2365                         SUM_FOOTER_SIZE) / SUMMARY_SIZE;
2366         if (valid_sum_count <= sum_in_page)
2367                 return 1;
2368         else if ((valid_sum_count - sum_in_page) <=
2369                 (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
2370                 return 2;
2371         return 3;
2372 }
2373
2374 /*
2375  * Caller should put this summary page
2376  */
2377 struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
2378 {
2379         if (unlikely(f2fs_cp_error(sbi)))
2380                 return ERR_PTR(-EIO);
2381         return f2fs_get_meta_page_retry(sbi, GET_SUM_BLOCK(sbi, segno));
2382 }
2383
2384 void f2fs_update_meta_page(struct f2fs_sb_info *sbi,
2385                                         void *src, block_t blk_addr)
2386 {
2387         struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2388
2389         memcpy(page_address(page), src, PAGE_SIZE);
2390         set_page_dirty(page);
2391         f2fs_put_page(page, 1);
2392 }
2393
2394 static void write_sum_page(struct f2fs_sb_info *sbi,
2395                         struct f2fs_summary_block *sum_blk, block_t blk_addr)
2396 {
2397         f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr);
2398 }
2399
2400 static void write_current_sum_page(struct f2fs_sb_info *sbi,
2401                                                 int type, block_t blk_addr)
2402 {
2403         struct curseg_info *curseg = CURSEG_I(sbi, type);
2404         struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2405         struct f2fs_summary_block *src = curseg->sum_blk;
2406         struct f2fs_summary_block *dst;
2407
2408         dst = (struct f2fs_summary_block *)page_address(page);
2409         memset(dst, 0, PAGE_SIZE);
2410
2411         mutex_lock(&curseg->curseg_mutex);
2412
2413         down_read(&curseg->journal_rwsem);
2414         memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE);
2415         up_read(&curseg->journal_rwsem);
2416
2417         memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE);
2418         memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE);
2419
2420         mutex_unlock(&curseg->curseg_mutex);
2421
2422         set_page_dirty(page);
2423         f2fs_put_page(page, 1);
2424 }
2425
2426 static int is_next_segment_free(struct f2fs_sb_info *sbi,
2427                                 struct curseg_info *curseg, int type)
2428 {
2429         unsigned int segno = curseg->segno + 1;
2430         struct free_segmap_info *free_i = FREE_I(sbi);
2431
2432         if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)
2433                 return !test_bit(segno, free_i->free_segmap);
2434         return 0;
2435 }
2436
2437 /*
2438  * Find a new segment from the free segments bitmap to right order
2439  * This function should be returned with success, otherwise BUG
2440  */
2441 static void get_new_segment(struct f2fs_sb_info *sbi,
2442                         unsigned int *newseg, bool new_sec, int dir)
2443 {
2444         struct free_segmap_info *free_i = FREE_I(sbi);
2445         unsigned int segno, secno, zoneno;
2446         unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
2447         unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg);
2448         unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg);
2449         unsigned int left_start = hint;
2450         bool init = true;
2451         int go_left = 0;
2452         int i;
2453
2454         spin_lock(&free_i->segmap_lock);
2455
2456         if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
2457                 segno = find_next_zero_bit(free_i->free_segmap,
2458                         GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1);
2459                 if (segno < GET_SEG_FROM_SEC(sbi, hint + 1))
2460                         goto got_it;
2461         }
2462 find_other_zone:
2463         secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
2464         if (secno >= MAIN_SECS(sbi)) {
2465                 if (dir == ALLOC_RIGHT) {
2466                         secno = find_first_zero_bit(free_i->free_secmap,
2467                                                         MAIN_SECS(sbi));
2468                         f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));
2469                 } else {
2470                         go_left = 1;
2471                         left_start = hint - 1;
2472                 }
2473         }
2474         if (go_left == 0)
2475                 goto skip_left;
2476
2477         while (test_bit(left_start, free_i->free_secmap)) {
2478                 if (left_start > 0) {
2479                         left_start--;
2480                         continue;
2481                 }
2482                 left_start = find_first_zero_bit(free_i->free_secmap,
2483                                                         MAIN_SECS(sbi));
2484                 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));
2485                 break;
2486         }
2487         secno = left_start;
2488 skip_left:
2489         segno = GET_SEG_FROM_SEC(sbi, secno);
2490         zoneno = GET_ZONE_FROM_SEC(sbi, secno);
2491
2492         /* give up on finding another zone */
2493         if (!init)
2494                 goto got_it;
2495         if (sbi->secs_per_zone == 1)
2496                 goto got_it;
2497         if (zoneno == old_zoneno)
2498                 goto got_it;
2499         if (dir == ALLOC_LEFT) {
2500                 if (!go_left && zoneno + 1 >= total_zones)
2501                         goto got_it;
2502                 if (go_left && zoneno == 0)
2503                         goto got_it;
2504         }
2505         for (i = 0; i < NR_CURSEG_TYPE; i++)
2506                 if (CURSEG_I(sbi, i)->zone == zoneno)
2507                         break;
2508
2509         if (i < NR_CURSEG_TYPE) {
2510                 /* zone is in user, try another */
2511                 if (go_left)
2512                         hint = zoneno * sbi->secs_per_zone - 1;
2513                 else if (zoneno + 1 >= total_zones)
2514                         hint = 0;
2515                 else
2516                         hint = (zoneno + 1) * sbi->secs_per_zone;
2517                 init = false;
2518                 goto find_other_zone;
2519         }
2520 got_it:
2521         /* set it as dirty segment in free segmap */
2522         f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
2523         __set_inuse(sbi, segno);
2524         *newseg = segno;
2525         spin_unlock(&free_i->segmap_lock);
2526 }
2527
2528 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
2529 {
2530         struct curseg_info *curseg = CURSEG_I(sbi, type);
2531         struct summary_footer *sum_footer;
2532         unsigned short seg_type = curseg->seg_type;
2533
2534         curseg->inited = true;
2535         curseg->segno = curseg->next_segno;
2536         curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno);
2537         curseg->next_blkoff = 0;
2538         curseg->next_segno = NULL_SEGNO;
2539
2540         sum_footer = &(curseg->sum_blk->footer);
2541         memset(sum_footer, 0, sizeof(struct summary_footer));
2542
2543         sanity_check_seg_type(sbi, seg_type);
2544
2545         if (IS_DATASEG(seg_type))
2546                 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
2547         if (IS_NODESEG(seg_type))
2548                 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
2549         __set_sit_entry_type(sbi, seg_type, curseg->segno, modified);
2550 }
2551
2552 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)
2553 {
2554         struct curseg_info *curseg = CURSEG_I(sbi, type);
2555         unsigned short seg_type = curseg->seg_type;
2556
2557         sanity_check_seg_type(sbi, seg_type);
2558         if (f2fs_need_rand_seg(sbi))
2559                 return prandom_u32_max(MAIN_SECS(sbi) * sbi->segs_per_sec);
2560
2561         /* if segs_per_sec is large than 1, we need to keep original policy. */
2562         if (__is_large_section(sbi))
2563                 return curseg->segno;
2564
2565         /* inmem log may not locate on any segment after mount */
2566         if (!curseg->inited)
2567                 return 0;
2568
2569         if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2570                 return 0;
2571
2572         if (test_opt(sbi, NOHEAP) &&
2573                 (seg_type == CURSEG_HOT_DATA || IS_NODESEG(seg_type)))
2574                 return 0;
2575
2576         if (SIT_I(sbi)->last_victim[ALLOC_NEXT])
2577                 return SIT_I(sbi)->last_victim[ALLOC_NEXT];
2578
2579         /* find segments from 0 to reuse freed segments */
2580         if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE)
2581                 return 0;
2582
2583         return curseg->segno;
2584 }
2585
2586 /*
2587  * Allocate a current working segment.
2588  * This function always allocates a free segment in LFS manner.
2589  */
2590 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
2591 {
2592         struct curseg_info *curseg = CURSEG_I(sbi, type);
2593         unsigned short seg_type = curseg->seg_type;
2594         unsigned int segno = curseg->segno;
2595         int dir = ALLOC_LEFT;
2596
2597         if (curseg->inited)
2598                 write_sum_page(sbi, curseg->sum_blk,
2599                                 GET_SUM_BLOCK(sbi, segno));
2600         if (seg_type == CURSEG_WARM_DATA || seg_type == CURSEG_COLD_DATA)
2601                 dir = ALLOC_RIGHT;
2602
2603         if (test_opt(sbi, NOHEAP))
2604                 dir = ALLOC_RIGHT;
2605
2606         segno = __get_next_segno(sbi, type);
2607         get_new_segment(sbi, &segno, new_sec, dir);
2608         curseg->next_segno = segno;
2609         reset_curseg(sbi, type, 1);
2610         curseg->alloc_type = LFS;
2611         if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK)
2612                 curseg->fragment_remained_chunk =
2613                                 prandom_u32_max(sbi->max_fragment_chunk) + 1;
2614 }
2615
2616 static int __next_free_blkoff(struct f2fs_sb_info *sbi,
2617                                         int segno, block_t start)
2618 {
2619         struct seg_entry *se = get_seg_entry(sbi, segno);
2620         int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
2621         unsigned long *target_map = SIT_I(sbi)->tmp_map;
2622         unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
2623         unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
2624         int i;
2625
2626         for (i = 0; i < entries; i++)
2627                 target_map[i] = ckpt_map[i] | cur_map[i];
2628
2629         return __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
2630 }
2631
2632 /*
2633  * If a segment is written by LFS manner, next block offset is just obtained
2634  * by increasing the current block offset. However, if a segment is written by
2635  * SSR manner, next block offset obtained by calling __next_free_blkoff
2636  */
2637 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
2638                                 struct curseg_info *seg)
2639 {
2640         if (seg->alloc_type == SSR) {
2641                 seg->next_blkoff =
2642                         __next_free_blkoff(sbi, seg->segno,
2643                                                 seg->next_blkoff + 1);
2644         } else {
2645                 seg->next_blkoff++;
2646                 if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) {
2647                         /* To allocate block chunks in different sizes, use random number */
2648                         if (--seg->fragment_remained_chunk <= 0) {
2649                                 seg->fragment_remained_chunk =
2650                                    prandom_u32_max(sbi->max_fragment_chunk) + 1;
2651                                 seg->next_blkoff +=
2652                                    prandom_u32_max(sbi->max_fragment_hole) + 1;
2653                         }
2654                 }
2655         }
2656 }
2657
2658 bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno)
2659 {
2660         return __next_free_blkoff(sbi, segno, 0) < sbi->blocks_per_seg;
2661 }
2662
2663 /*
2664  * This function always allocates a used segment(from dirty seglist) by SSR
2665  * manner, so it should recover the existing segment information of valid blocks
2666  */
2667 static void change_curseg(struct f2fs_sb_info *sbi, int type, bool flush)
2668 {
2669         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2670         struct curseg_info *curseg = CURSEG_I(sbi, type);
2671         unsigned int new_segno = curseg->next_segno;
2672         struct f2fs_summary_block *sum_node;
2673         struct page *sum_page;
2674
2675         if (flush)
2676                 write_sum_page(sbi, curseg->sum_blk,
2677                                         GET_SUM_BLOCK(sbi, curseg->segno));
2678
2679         __set_test_and_inuse(sbi, new_segno);
2680
2681         mutex_lock(&dirty_i->seglist_lock);
2682         __remove_dirty_segment(sbi, new_segno, PRE);
2683         __remove_dirty_segment(sbi, new_segno, DIRTY);
2684         mutex_unlock(&dirty_i->seglist_lock);
2685
2686         reset_curseg(sbi, type, 1);
2687         curseg->alloc_type = SSR;
2688         curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0);
2689
2690         sum_page = f2fs_get_sum_page(sbi, new_segno);
2691         if (IS_ERR(sum_page)) {
2692                 /* GC won't be able to use stale summary pages by cp_error */
2693                 memset(curseg->sum_blk, 0, SUM_ENTRY_SIZE);
2694                 return;
2695         }
2696         sum_node = (struct f2fs_summary_block *)page_address(sum_page);
2697         memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
2698         f2fs_put_page(sum_page, 1);
2699 }
2700
2701 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2702                                 int alloc_mode, unsigned long long age);
2703
2704 static void get_atssr_segment(struct f2fs_sb_info *sbi, int type,
2705                                         int target_type, int alloc_mode,
2706                                         unsigned long long age)
2707 {
2708         struct curseg_info *curseg = CURSEG_I(sbi, type);
2709
2710         curseg->seg_type = target_type;
2711
2712         if (get_ssr_segment(sbi, type, alloc_mode, age)) {
2713                 struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno);
2714
2715                 curseg->seg_type = se->type;
2716                 change_curseg(sbi, type, true);
2717         } else {
2718                 /* allocate cold segment by default */
2719                 curseg->seg_type = CURSEG_COLD_DATA;
2720                 new_curseg(sbi, type, true);
2721         }
2722         stat_inc_seg_type(sbi, curseg);
2723 }
2724
2725 static void __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi)
2726 {
2727         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC);
2728
2729         if (!sbi->am.atgc_enabled)
2730                 return;
2731
2732         f2fs_down_read(&SM_I(sbi)->curseg_lock);
2733
2734         mutex_lock(&curseg->curseg_mutex);
2735         down_write(&SIT_I(sbi)->sentry_lock);
2736
2737         get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, CURSEG_COLD_DATA, SSR, 0);
2738
2739         up_write(&SIT_I(sbi)->sentry_lock);
2740         mutex_unlock(&curseg->curseg_mutex);
2741
2742         f2fs_up_read(&SM_I(sbi)->curseg_lock);
2743
2744 }
2745 void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi)
2746 {
2747         __f2fs_init_atgc_curseg(sbi);
2748 }
2749
2750 static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type)
2751 {
2752         struct curseg_info *curseg = CURSEG_I(sbi, type);
2753
2754         mutex_lock(&curseg->curseg_mutex);
2755         if (!curseg->inited)
2756                 goto out;
2757
2758         if (get_valid_blocks(sbi, curseg->segno, false)) {
2759                 write_sum_page(sbi, curseg->sum_blk,
2760                                 GET_SUM_BLOCK(sbi, curseg->segno));
2761         } else {
2762                 mutex_lock(&DIRTY_I(sbi)->seglist_lock);
2763                 __set_test_and_free(sbi, curseg->segno, true);
2764                 mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
2765         }
2766 out:
2767         mutex_unlock(&curseg->curseg_mutex);
2768 }
2769
2770 void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi)
2771 {
2772         __f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
2773
2774         if (sbi->am.atgc_enabled)
2775                 __f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
2776 }
2777
2778 static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type)
2779 {
2780         struct curseg_info *curseg = CURSEG_I(sbi, type);
2781
2782         mutex_lock(&curseg->curseg_mutex);
2783         if (!curseg->inited)
2784                 goto out;
2785         if (get_valid_blocks(sbi, curseg->segno, false))
2786                 goto out;
2787
2788         mutex_lock(&DIRTY_I(sbi)->seglist_lock);
2789         __set_test_and_inuse(sbi, curseg->segno);
2790         mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
2791 out:
2792         mutex_unlock(&curseg->curseg_mutex);
2793 }
2794
2795 void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi)
2796 {
2797         __f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
2798
2799         if (sbi->am.atgc_enabled)
2800                 __f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
2801 }
2802
2803 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2804                                 int alloc_mode, unsigned long long age)
2805 {
2806         struct curseg_info *curseg = CURSEG_I(sbi, type);
2807         const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
2808         unsigned segno = NULL_SEGNO;
2809         unsigned short seg_type = curseg->seg_type;
2810         int i, cnt;
2811         bool reversed = false;
2812
2813         sanity_check_seg_type(sbi, seg_type);
2814
2815         /* f2fs_need_SSR() already forces to do this */
2816         if (!v_ops->get_victim(sbi, &segno, BG_GC, seg_type, alloc_mode, age)) {
2817                 curseg->next_segno = segno;
2818                 return 1;
2819         }
2820
2821         /* For node segments, let's do SSR more intensively */
2822         if (IS_NODESEG(seg_type)) {
2823                 if (seg_type >= CURSEG_WARM_NODE) {
2824                         reversed = true;
2825                         i = CURSEG_COLD_NODE;
2826                 } else {
2827                         i = CURSEG_HOT_NODE;
2828                 }
2829                 cnt = NR_CURSEG_NODE_TYPE;
2830         } else {
2831                 if (seg_type >= CURSEG_WARM_DATA) {
2832                         reversed = true;
2833                         i = CURSEG_COLD_DATA;
2834                 } else {
2835                         i = CURSEG_HOT_DATA;
2836                 }
2837                 cnt = NR_CURSEG_DATA_TYPE;
2838         }
2839
2840         for (; cnt-- > 0; reversed ? i-- : i++) {
2841                 if (i == seg_type)
2842                         continue;
2843                 if (!v_ops->get_victim(sbi, &segno, BG_GC, i, alloc_mode, age)) {
2844                         curseg->next_segno = segno;
2845                         return 1;
2846                 }
2847         }
2848
2849         /* find valid_blocks=0 in dirty list */
2850         if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2851                 segno = get_free_segment(sbi);
2852                 if (segno != NULL_SEGNO) {
2853                         curseg->next_segno = segno;
2854                         return 1;
2855                 }
2856         }
2857         return 0;
2858 }
2859
2860 /*
2861  * flush out current segment and replace it with new segment
2862  * This function should be returned with success, otherwise BUG
2863  */
2864 static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
2865                                                 int type, bool force)
2866 {
2867         struct curseg_info *curseg = CURSEG_I(sbi, type);
2868
2869         if (force)
2870                 new_curseg(sbi, type, true);
2871         else if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) &&
2872                                         curseg->seg_type == CURSEG_WARM_NODE)
2873                 new_curseg(sbi, type, false);
2874         else if (curseg->alloc_type == LFS &&
2875                         is_next_segment_free(sbi, curseg, type) &&
2876                         likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2877                 new_curseg(sbi, type, false);
2878         else if (f2fs_need_SSR(sbi) &&
2879                         get_ssr_segment(sbi, type, SSR, 0))
2880                 change_curseg(sbi, type, true);
2881         else
2882                 new_curseg(sbi, type, false);
2883
2884         stat_inc_seg_type(sbi, curseg);
2885 }
2886
2887 void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
2888                                         unsigned int start, unsigned int end)
2889 {
2890         struct curseg_info *curseg = CURSEG_I(sbi, type);
2891         unsigned int segno;
2892
2893         f2fs_down_read(&SM_I(sbi)->curseg_lock);
2894         mutex_lock(&curseg->curseg_mutex);
2895         down_write(&SIT_I(sbi)->sentry_lock);
2896
2897         segno = CURSEG_I(sbi, type)->segno;
2898         if (segno < start || segno > end)
2899                 goto unlock;
2900
2901         if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0))
2902                 change_curseg(sbi, type, true);
2903         else
2904                 new_curseg(sbi, type, true);
2905
2906         stat_inc_seg_type(sbi, curseg);
2907
2908         locate_dirty_segment(sbi, segno);
2909 unlock:
2910         up_write(&SIT_I(sbi)->sentry_lock);
2911
2912         if (segno != curseg->segno)
2913                 f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u",
2914                             type, segno, curseg->segno);
2915
2916         mutex_unlock(&curseg->curseg_mutex);
2917         f2fs_up_read(&SM_I(sbi)->curseg_lock);
2918 }
2919
2920 static void __allocate_new_segment(struct f2fs_sb_info *sbi, int type,
2921                                                 bool new_sec, bool force)
2922 {
2923         struct curseg_info *curseg = CURSEG_I(sbi, type);
2924         unsigned int old_segno;
2925
2926         if (!curseg->inited)
2927                 goto alloc;
2928
2929         if (force || curseg->next_blkoff ||
2930                 get_valid_blocks(sbi, curseg->segno, new_sec))
2931                 goto alloc;
2932
2933         if (!get_ckpt_valid_blocks(sbi, curseg->segno, new_sec))
2934                 return;
2935 alloc:
2936         old_segno = curseg->segno;
2937         SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true);
2938         locate_dirty_segment(sbi, old_segno);
2939 }
2940
2941 static void __allocate_new_section(struct f2fs_sb_info *sbi,
2942                                                 int type, bool force)
2943 {
2944         __allocate_new_segment(sbi, type, true, force);
2945 }
2946
2947 void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force)
2948 {
2949         f2fs_down_read(&SM_I(sbi)->curseg_lock);
2950         down_write(&SIT_I(sbi)->sentry_lock);
2951         __allocate_new_section(sbi, type, force);
2952         up_write(&SIT_I(sbi)->sentry_lock);
2953         f2fs_up_read(&SM_I(sbi)->curseg_lock);
2954 }
2955
2956 void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)
2957 {
2958         int i;
2959
2960         f2fs_down_read(&SM_I(sbi)->curseg_lock);
2961         down_write(&SIT_I(sbi)->sentry_lock);
2962         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
2963                 __allocate_new_segment(sbi, i, false, false);
2964         up_write(&SIT_I(sbi)->sentry_lock);
2965         f2fs_up_read(&SM_I(sbi)->curseg_lock);
2966 }
2967
2968 static const struct segment_allocation default_salloc_ops = {
2969         .allocate_segment = allocate_segment_by_default,
2970 };
2971
2972 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
2973                                                 struct cp_control *cpc)
2974 {
2975         __u64 trim_start = cpc->trim_start;
2976         bool has_candidate = false;
2977
2978         down_write(&SIT_I(sbi)->sentry_lock);
2979         for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) {
2980                 if (add_discard_addrs(sbi, cpc, true)) {
2981                         has_candidate = true;
2982                         break;
2983                 }
2984         }
2985         up_write(&SIT_I(sbi)->sentry_lock);
2986
2987         cpc->trim_start = trim_start;
2988         return has_candidate;
2989 }
2990
2991 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,
2992                                         struct discard_policy *dpolicy,
2993                                         unsigned int start, unsigned int end)
2994 {
2995         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2996         struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
2997         struct rb_node **insert_p = NULL, *insert_parent = NULL;
2998         struct discard_cmd *dc;
2999         struct blk_plug plug;
3000         int issued;
3001         unsigned int trimmed = 0;
3002
3003 next:
3004         issued = 0;
3005
3006         mutex_lock(&dcc->cmd_lock);
3007         if (unlikely(dcc->rbtree_check))
3008                 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
3009                                                         &dcc->root));
3010
3011         dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
3012                                         NULL, start,
3013                                         (struct rb_entry **)&prev_dc,
3014                                         (struct rb_entry **)&next_dc,
3015                                         &insert_p, &insert_parent, true, NULL);
3016         if (!dc)
3017                 dc = next_dc;
3018
3019         blk_start_plug(&plug);
3020
3021         while (dc && dc->lstart <= end) {
3022                 struct rb_node *node;
3023                 int err = 0;
3024
3025                 if (dc->len < dpolicy->granularity)
3026                         goto skip;
3027
3028                 if (dc->state != D_PREP) {
3029                         list_move_tail(&dc->list, &dcc->fstrim_list);
3030                         goto skip;
3031                 }
3032
3033                 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
3034
3035                 if (issued >= dpolicy->max_requests) {
3036                         start = dc->lstart + dc->len;
3037
3038                         if (err)
3039                                 __remove_discard_cmd(sbi, dc);
3040
3041                         blk_finish_plug(&plug);
3042                         mutex_unlock(&dcc->cmd_lock);
3043                         trimmed += __wait_all_discard_cmd(sbi, NULL);
3044                         f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT);
3045                         goto next;
3046                 }
3047 skip:
3048                 node = rb_next(&dc->rb_node);
3049                 if (err)
3050                         __remove_discard_cmd(sbi, dc);
3051                 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
3052
3053                 if (fatal_signal_pending(current))
3054                         break;
3055         }
3056
3057         blk_finish_plug(&plug);
3058         mutex_unlock(&dcc->cmd_lock);
3059
3060         return trimmed;
3061 }
3062
3063 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
3064 {
3065         __u64 start = F2FS_BYTES_TO_BLK(range->start);
3066         __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
3067         unsigned int start_segno, end_segno;
3068         block_t start_block, end_block;
3069         struct cp_control cpc;
3070         struct discard_policy dpolicy;
3071         unsigned long long trimmed = 0;
3072         int err = 0;
3073         bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);
3074
3075         if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
3076                 return -EINVAL;
3077
3078         if (end < MAIN_BLKADDR(sbi))
3079                 goto out;
3080
3081         if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
3082                 f2fs_warn(sbi, "Found FS corruption, run fsck to fix.");
3083                 return -EFSCORRUPTED;
3084         }
3085
3086         /* start/end segment number in main_area */
3087         start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
3088         end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
3089                                                 GET_SEGNO(sbi, end);
3090         if (need_align) {
3091                 start_segno = rounddown(start_segno, sbi->segs_per_sec);
3092                 end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1;
3093         }
3094
3095         cpc.reason = CP_DISCARD;
3096         cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
3097         cpc.trim_start = start_segno;
3098         cpc.trim_end = end_segno;
3099
3100         if (sbi->discard_blks == 0)
3101                 goto out;
3102
3103         f2fs_down_write(&sbi->gc_lock);
3104         err = f2fs_write_checkpoint(sbi, &cpc);
3105         f2fs_up_write(&sbi->gc_lock);
3106         if (err)
3107                 goto out;
3108
3109         /*
3110          * We filed discard candidates, but actually we don't need to wait for
3111          * all of them, since they'll be issued in idle time along with runtime
3112          * discard option. User configuration looks like using runtime discard
3113          * or periodic fstrim instead of it.
3114          */
3115         if (f2fs_realtime_discard_enable(sbi))
3116                 goto out;
3117
3118         start_block = START_BLOCK(sbi, start_segno);
3119         end_block = START_BLOCK(sbi, end_segno + 1);
3120
3121         __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen);
3122         trimmed = __issue_discard_cmd_range(sbi, &dpolicy,
3123                                         start_block, end_block);
3124
3125         trimmed += __wait_discard_cmd_range(sbi, &dpolicy,
3126                                         start_block, end_block);
3127 out:
3128         if (!err)
3129                 range->len = F2FS_BLK_TO_BYTES(trimmed);
3130         return err;
3131 }
3132
3133 static bool __has_curseg_space(struct f2fs_sb_info *sbi,
3134                                         struct curseg_info *curseg)
3135 {
3136         return curseg->next_blkoff < f2fs_usable_blks_in_seg(sbi,
3137                                                         curseg->segno);
3138 }
3139
3140 int f2fs_rw_hint_to_seg_type(enum rw_hint hint)
3141 {
3142         switch (hint) {
3143         case WRITE_LIFE_SHORT:
3144                 return CURSEG_HOT_DATA;
3145         case WRITE_LIFE_EXTREME:
3146                 return CURSEG_COLD_DATA;
3147         default:
3148                 return CURSEG_WARM_DATA;
3149         }
3150 }
3151
3152 static int __get_segment_type_2(struct f2fs_io_info *fio)
3153 {
3154         if (fio->type == DATA)
3155                 return CURSEG_HOT_DATA;
3156         else
3157                 return CURSEG_HOT_NODE;
3158 }
3159
3160 static int __get_segment_type_4(struct f2fs_io_info *fio)
3161 {
3162         if (fio->type == DATA) {
3163                 struct inode *inode = fio->page->mapping->host;
3164
3165                 if (S_ISDIR(inode->i_mode))
3166                         return CURSEG_HOT_DATA;
3167                 else
3168                         return CURSEG_COLD_DATA;
3169         } else {
3170                 if (IS_DNODE(fio->page) && is_cold_node(fio->page))
3171                         return CURSEG_WARM_NODE;
3172                 else
3173                         return CURSEG_COLD_NODE;
3174         }
3175 }
3176
3177 static int __get_segment_type_6(struct f2fs_io_info *fio)
3178 {
3179         if (fio->type == DATA) {
3180                 struct inode *inode = fio->page->mapping->host;
3181
3182                 if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
3183                         return CURSEG_COLD_DATA_PINNED;
3184
3185                 if (page_private_gcing(fio->page)) {
3186                         if (fio->sbi->am.atgc_enabled &&
3187                                 (fio->io_type == FS_DATA_IO) &&
3188                                 (fio->sbi->gc_mode != GC_URGENT_HIGH))
3189                                 return CURSEG_ALL_DATA_ATGC;
3190                         else
3191                                 return CURSEG_COLD_DATA;
3192                 }
3193                 if (file_is_cold(inode) || f2fs_need_compress_data(inode))
3194                         return CURSEG_COLD_DATA;
3195                 if (file_is_hot(inode) ||
3196                                 is_inode_flag_set(inode, FI_HOT_DATA) ||
3197                                 f2fs_is_cow_file(inode))
3198                         return CURSEG_HOT_DATA;
3199                 return f2fs_rw_hint_to_seg_type(inode->i_write_hint);
3200         } else {
3201                 if (IS_DNODE(fio->page))
3202                         return is_cold_node(fio->page) ? CURSEG_WARM_NODE :
3203                                                 CURSEG_HOT_NODE;
3204                 return CURSEG_COLD_NODE;
3205         }
3206 }
3207
3208 static int __get_segment_type(struct f2fs_io_info *fio)
3209 {
3210         int type = 0;
3211
3212         switch (F2FS_OPTION(fio->sbi).active_logs) {
3213         case 2:
3214                 type = __get_segment_type_2(fio);
3215                 break;
3216         case 4:
3217                 type = __get_segment_type_4(fio);
3218                 break;
3219         case 6:
3220                 type = __get_segment_type_6(fio);
3221                 break;
3222         default:
3223                 f2fs_bug_on(fio->sbi, true);
3224         }
3225
3226         if (IS_HOT(type))
3227                 fio->temp = HOT;
3228         else if (IS_WARM(type))
3229                 fio->temp = WARM;
3230         else
3231                 fio->temp = COLD;
3232         return type;
3233 }
3234
3235 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
3236                 block_t old_blkaddr, block_t *new_blkaddr,
3237                 struct f2fs_summary *sum, int type,
3238                 struct f2fs_io_info *fio)
3239 {
3240         struct sit_info *sit_i = SIT_I(sbi);
3241         struct curseg_info *curseg = CURSEG_I(sbi, type);
3242         unsigned long long old_mtime;
3243         bool from_gc = (type == CURSEG_ALL_DATA_ATGC);
3244         struct seg_entry *se = NULL;
3245
3246         f2fs_down_read(&SM_I(sbi)->curseg_lock);
3247
3248         mutex_lock(&curseg->curseg_mutex);
3249         down_write(&sit_i->sentry_lock);
3250
3251         if (from_gc) {
3252                 f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO);
3253                 se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr));
3254                 sanity_check_seg_type(sbi, se->type);
3255                 f2fs_bug_on(sbi, IS_NODESEG(se->type));
3256         }
3257         *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
3258
3259         f2fs_bug_on(sbi, curseg->next_blkoff >= sbi->blocks_per_seg);
3260
3261         f2fs_wait_discard_bio(sbi, *new_blkaddr);
3262
3263         /*
3264          * __add_sum_entry should be resided under the curseg_mutex
3265          * because, this function updates a summary entry in the
3266          * current summary block.
3267          */
3268         __add_sum_entry(sbi, type, sum);
3269
3270         __refresh_next_blkoff(sbi, curseg);
3271
3272         stat_inc_block_count(sbi, curseg);
3273
3274         if (from_gc) {
3275                 old_mtime = get_segment_mtime(sbi, old_blkaddr);
3276         } else {
3277                 update_segment_mtime(sbi, old_blkaddr, 0);
3278                 old_mtime = 0;
3279         }
3280         update_segment_mtime(sbi, *new_blkaddr, old_mtime);
3281
3282         /*
3283          * SIT information should be updated before segment allocation,
3284          * since SSR needs latest valid block information.
3285          */
3286         update_sit_entry(sbi, *new_blkaddr, 1);
3287         if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
3288                 update_sit_entry(sbi, old_blkaddr, -1);
3289
3290         if (!__has_curseg_space(sbi, curseg)) {
3291                 if (from_gc)
3292                         get_atssr_segment(sbi, type, se->type,
3293                                                 AT_SSR, se->mtime);
3294                 else
3295                         sit_i->s_ops->allocate_segment(sbi, type, false);
3296         }
3297         /*
3298          * segment dirty status should be updated after segment allocation,
3299          * so we just need to update status only one time after previous
3300          * segment being closed.
3301          */
3302         locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3303         locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr));
3304
3305         up_write(&sit_i->sentry_lock);
3306
3307         if (page && IS_NODESEG(type)) {
3308                 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
3309
3310                 f2fs_inode_chksum_set(sbi, page);
3311         }
3312
3313         if (fio) {
3314                 struct f2fs_bio_info *io;
3315
3316                 if (F2FS_IO_ALIGNED(sbi))
3317                         fio->retry = 0;
3318
3319                 INIT_LIST_HEAD(&fio->list);
3320                 fio->in_list = 1;
3321                 io = sbi->write_io[fio->type] + fio->temp;
3322                 spin_lock(&io->io_lock);
3323                 list_add_tail(&fio->list, &io->io_list);
3324                 spin_unlock(&io->io_lock);
3325         }
3326
3327         mutex_unlock(&curseg->curseg_mutex);
3328
3329         f2fs_up_read(&SM_I(sbi)->curseg_lock);
3330 }
3331
3332 void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino,
3333                                         block_t blkaddr, unsigned int blkcnt)
3334 {
3335         if (!f2fs_is_multi_device(sbi))
3336                 return;
3337
3338         while (1) {
3339                 unsigned int devidx = f2fs_target_device_index(sbi, blkaddr);
3340                 unsigned int blks = FDEV(devidx).end_blk - blkaddr + 1;
3341
3342                 /* update device state for fsync */
3343                 f2fs_set_dirty_device(sbi, ino, devidx, FLUSH_INO);
3344
3345                 /* update device state for checkpoint */
3346                 if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) {
3347                         spin_lock(&sbi->dev_lock);
3348                         f2fs_set_bit(devidx, (char *)&sbi->dirty_device);
3349                         spin_unlock(&sbi->dev_lock);
3350                 }
3351
3352                 if (blkcnt <= blks)
3353                         break;
3354                 blkcnt -= blks;
3355                 blkaddr += blks;
3356         }
3357 }
3358
3359 static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
3360 {
3361         int type = __get_segment_type(fio);
3362         bool keep_order = (f2fs_lfs_mode(fio->sbi) && type == CURSEG_COLD_DATA);
3363
3364         if (keep_order)
3365                 f2fs_down_read(&fio->sbi->io_order_lock);
3366 reallocate:
3367         f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,
3368                         &fio->new_blkaddr, sum, type, fio);
3369         if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) {
3370                 invalidate_mapping_pages(META_MAPPING(fio->sbi),
3371                                         fio->old_blkaddr, fio->old_blkaddr);
3372                 f2fs_invalidate_compress_page(fio->sbi, fio->old_blkaddr);
3373         }
3374
3375         /* writeout dirty page into bdev */
3376         f2fs_submit_page_write(fio);
3377         if (fio->retry) {
3378                 fio->old_blkaddr = fio->new_blkaddr;
3379                 goto reallocate;
3380         }
3381
3382         f2fs_update_device_state(fio->sbi, fio->ino, fio->new_blkaddr, 1);
3383
3384         if (keep_order)
3385                 f2fs_up_read(&fio->sbi->io_order_lock);
3386 }
3387
3388 void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,
3389                                         enum iostat_type io_type)
3390 {
3391         struct f2fs_io_info fio = {
3392                 .sbi = sbi,
3393                 .type = META,
3394                 .temp = HOT,
3395                 .op = REQ_OP_WRITE,
3396                 .op_flags = REQ_SYNC | REQ_META | REQ_PRIO,
3397                 .old_blkaddr = page->index,
3398                 .new_blkaddr = page->index,
3399                 .page = page,
3400                 .encrypted_page = NULL,
3401                 .in_list = 0,
3402         };
3403
3404         if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
3405                 fio.op_flags &= ~REQ_META;
3406
3407         set_page_writeback(page);
3408         ClearPageError(page);
3409         f2fs_submit_page_write(&fio);
3410
3411         stat_inc_meta_count(sbi, page->index);
3412         f2fs_update_iostat(sbi, NULL, io_type, F2FS_BLKSIZE);
3413 }
3414
3415 void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio)
3416 {
3417         struct f2fs_summary sum;
3418
3419         set_summary(&sum, nid, 0, 0);
3420         do_write_page(&sum, fio);
3421
3422         f2fs_update_iostat(fio->sbi, NULL, fio->io_type, F2FS_BLKSIZE);
3423 }
3424
3425 void f2fs_outplace_write_data(struct dnode_of_data *dn,
3426                                         struct f2fs_io_info *fio)
3427 {
3428         struct f2fs_sb_info *sbi = fio->sbi;
3429         struct f2fs_summary sum;
3430
3431         f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
3432         set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version);
3433         do_write_page(&sum, fio);
3434         f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
3435
3436         f2fs_update_iostat(sbi, dn->inode, fio->io_type, F2FS_BLKSIZE);
3437 }
3438
3439 int f2fs_inplace_write_data(struct f2fs_io_info *fio)
3440 {
3441         int err;
3442         struct f2fs_sb_info *sbi = fio->sbi;
3443         unsigned int segno;
3444
3445         fio->new_blkaddr = fio->old_blkaddr;
3446         /* i/o temperature is needed for passing down write hints */
3447         __get_segment_type(fio);
3448
3449         segno = GET_SEGNO(sbi, fio->new_blkaddr);
3450
3451         if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) {
3452                 set_sbi_flag(sbi, SBI_NEED_FSCK);
3453                 f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",
3454                           __func__, segno);
3455                 err = -EFSCORRUPTED;
3456                 f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE);
3457                 goto drop_bio;
3458         }
3459
3460         if (f2fs_cp_error(sbi)) {
3461                 err = -EIO;
3462                 goto drop_bio;
3463         }
3464
3465         if (fio->post_read)
3466                 invalidate_mapping_pages(META_MAPPING(sbi),
3467                                 fio->new_blkaddr, fio->new_blkaddr);
3468
3469         stat_inc_inplace_blocks(fio->sbi);
3470
3471         if (fio->bio && !(SM_I(sbi)->ipu_policy & (1 << F2FS_IPU_NOCACHE)))
3472                 err = f2fs_merge_page_bio(fio);
3473         else
3474                 err = f2fs_submit_page_bio(fio);
3475         if (!err) {
3476                 f2fs_update_device_state(fio->sbi, fio->ino,
3477                                                 fio->new_blkaddr, 1);
3478                 f2fs_update_iostat(fio->sbi, fio->page->mapping->host,
3479                                                 fio->io_type, F2FS_BLKSIZE);
3480         }
3481
3482         return err;
3483 drop_bio:
3484         if (fio->bio && *(fio->bio)) {
3485                 struct bio *bio = *(fio->bio);
3486
3487                 bio->bi_status = BLK_STS_IOERR;
3488                 bio_endio(bio);
3489                 *(fio->bio) = NULL;
3490         }
3491         return err;
3492 }
3493
3494 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi,
3495                                                 unsigned int segno)
3496 {
3497         int i;
3498
3499         for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) {
3500                 if (CURSEG_I(sbi, i)->segno == segno)
3501                         break;
3502         }
3503         return i;
3504 }
3505
3506 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
3507                                 block_t old_blkaddr, block_t new_blkaddr,
3508                                 bool recover_curseg, bool recover_newaddr,
3509                                 bool from_gc)
3510 {
3511         struct sit_info *sit_i = SIT_I(sbi);
3512         struct curseg_info *curseg;
3513         unsigned int segno, old_cursegno;
3514         struct seg_entry *se;
3515         int type;
3516         unsigned short old_blkoff;
3517         unsigned char old_alloc_type;
3518
3519         segno = GET_SEGNO(sbi, new_blkaddr);
3520         se = get_seg_entry(sbi, segno);
3521         type = se->type;
3522
3523         f2fs_down_write(&SM_I(sbi)->curseg_lock);
3524
3525         if (!recover_curseg) {
3526                 /* for recovery flow */
3527                 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
3528                         if (old_blkaddr == NULL_ADDR)
3529                                 type = CURSEG_COLD_DATA;
3530                         else
3531                                 type = CURSEG_WARM_DATA;
3532                 }
3533         } else {
3534                 if (IS_CURSEG(sbi, segno)) {
3535                         /* se->type is volatile as SSR allocation */
3536                         type = __f2fs_get_curseg(sbi, segno);
3537                         f2fs_bug_on(sbi, type == NO_CHECK_TYPE);
3538                 } else {
3539                         type = CURSEG_WARM_DATA;
3540                 }
3541         }
3542
3543         f2fs_bug_on(sbi, !IS_DATASEG(type));
3544         curseg = CURSEG_I(sbi, type);
3545
3546         mutex_lock(&curseg->curseg_mutex);
3547         down_write(&sit_i->sentry_lock);
3548
3549         old_cursegno = curseg->segno;
3550         old_blkoff = curseg->next_blkoff;
3551         old_alloc_type = curseg->alloc_type;
3552
3553         /* change the current segment */
3554         if (segno != curseg->segno) {
3555                 curseg->next_segno = segno;
3556                 change_curseg(sbi, type, true);
3557         }
3558
3559         curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
3560         __add_sum_entry(sbi, type, sum);
3561
3562         if (!recover_curseg || recover_newaddr) {
3563                 if (!from_gc)
3564                         update_segment_mtime(sbi, new_blkaddr, 0);
3565                 update_sit_entry(sbi, new_blkaddr, 1);
3566         }
3567         if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {
3568                 invalidate_mapping_pages(META_MAPPING(sbi),
3569                                         old_blkaddr, old_blkaddr);
3570                 f2fs_invalidate_compress_page(sbi, old_blkaddr);
3571                 if (!from_gc)
3572                         update_segment_mtime(sbi, old_blkaddr, 0);
3573                 update_sit_entry(sbi, old_blkaddr, -1);
3574         }
3575
3576         locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3577         locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
3578
3579         locate_dirty_segment(sbi, old_cursegno);
3580
3581         if (recover_curseg) {
3582                 if (old_cursegno != curseg->segno) {
3583                         curseg->next_segno = old_cursegno;
3584                         change_curseg(sbi, type, true);
3585                 }
3586                 curseg->next_blkoff = old_blkoff;
3587                 curseg->alloc_type = old_alloc_type;
3588         }
3589
3590         up_write(&sit_i->sentry_lock);
3591         mutex_unlock(&curseg->curseg_mutex);
3592         f2fs_up_write(&SM_I(sbi)->curseg_lock);
3593 }
3594
3595 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
3596                                 block_t old_addr, block_t new_addr,
3597                                 unsigned char version, bool recover_curseg,
3598                                 bool recover_newaddr)
3599 {
3600         struct f2fs_summary sum;
3601
3602         set_summary(&sum, dn->nid, dn->ofs_in_node, version);
3603
3604         f2fs_do_replace_block(sbi, &sum, old_addr, new_addr,
3605                                         recover_curseg, recover_newaddr, false);
3606
3607         f2fs_update_data_blkaddr(dn, new_addr);
3608 }
3609
3610 void f2fs_wait_on_page_writeback(struct page *page,
3611                                 enum page_type type, bool ordered, bool locked)
3612 {
3613         if (PageWriteback(page)) {
3614                 struct f2fs_sb_info *sbi = F2FS_P_SB(page);
3615
3616                 /* submit cached LFS IO */
3617                 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type);
3618                 /* sbumit cached IPU IO */
3619                 f2fs_submit_merged_ipu_write(sbi, NULL, page);
3620                 if (ordered) {
3621                         wait_on_page_writeback(page);
3622                         f2fs_bug_on(sbi, locked && PageWriteback(page));
3623                 } else {
3624                         wait_for_stable_page(page);
3625                 }
3626         }
3627 }
3628
3629 void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr)
3630 {
3631         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3632         struct page *cpage;
3633
3634         if (!f2fs_post_read_required(inode))
3635                 return;
3636
3637         if (!__is_valid_data_blkaddr(blkaddr))
3638                 return;
3639
3640         cpage = find_lock_page(META_MAPPING(sbi), blkaddr);
3641         if (cpage) {
3642                 f2fs_wait_on_page_writeback(cpage, DATA, true, true);
3643                 f2fs_put_page(cpage, 1);
3644         }
3645 }
3646
3647 void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
3648                                                                 block_t len)
3649 {
3650         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3651         block_t i;
3652
3653         if (!f2fs_post_read_required(inode))
3654                 return;
3655
3656         for (i = 0; i < len; i++)
3657                 f2fs_wait_on_block_writeback(inode, blkaddr + i);
3658
3659         invalidate_mapping_pages(META_MAPPING(sbi), blkaddr, blkaddr + len - 1);
3660 }
3661
3662 static int read_compacted_summaries(struct f2fs_sb_info *sbi)
3663 {
3664         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3665         struct curseg_info *seg_i;
3666         unsigned char *kaddr;
3667         struct page *page;
3668         block_t start;
3669         int i, j, offset;
3670
3671         start = start_sum_block(sbi);
3672
3673         page = f2fs_get_meta_page(sbi, start++);
3674         if (IS_ERR(page))
3675                 return PTR_ERR(page);
3676         kaddr = (unsigned char *)page_address(page);
3677
3678         /* Step 1: restore nat cache */
3679         seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
3680         memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE);
3681
3682         /* Step 2: restore sit cache */
3683         seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
3684         memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE);
3685         offset = 2 * SUM_JOURNAL_SIZE;
3686
3687         /* Step 3: restore summary entries */
3688         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
3689                 unsigned short blk_off;
3690                 unsigned int segno;
3691
3692                 seg_i = CURSEG_I(sbi, i);
3693                 segno = le32_to_cpu(ckpt->cur_data_segno[i]);
3694                 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
3695                 seg_i->next_segno = segno;
3696                 reset_curseg(sbi, i, 0);
3697                 seg_i->alloc_type = ckpt->alloc_type[i];
3698                 seg_i->next_blkoff = blk_off;
3699
3700                 if (seg_i->alloc_type == SSR)
3701                         blk_off = sbi->blocks_per_seg;
3702
3703                 for (j = 0; j < blk_off; j++) {
3704                         struct f2fs_summary *s;
3705
3706                         s = (struct f2fs_summary *)(kaddr + offset);
3707                         seg_i->sum_blk->entries[j] = *s;
3708                         offset += SUMMARY_SIZE;
3709                         if (offset + SUMMARY_SIZE <= PAGE_SIZE -
3710                                                 SUM_FOOTER_SIZE)
3711                                 continue;
3712
3713                         f2fs_put_page(page, 1);
3714                         page = NULL;
3715
3716                         page = f2fs_get_meta_page(sbi, start++);
3717                         if (IS_ERR(page))
3718                                 return PTR_ERR(page);
3719                         kaddr = (unsigned char *)page_address(page);
3720                         offset = 0;
3721                 }
3722         }
3723         f2fs_put_page(page, 1);
3724         return 0;
3725 }
3726
3727 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
3728 {
3729         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3730         struct f2fs_summary_block *sum;
3731         struct curseg_info *curseg;
3732         struct page *new;
3733         unsigned short blk_off;
3734         unsigned int segno = 0;
3735         block_t blk_addr = 0;
3736         int err = 0;
3737
3738         /* get segment number and block addr */
3739         if (IS_DATASEG(type)) {
3740                 segno = le32_to_cpu(ckpt->cur_data_segno[type]);
3741                 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
3742                                                         CURSEG_HOT_DATA]);
3743                 if (__exist_node_summaries(sbi))
3744                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type);
3745                 else
3746                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
3747         } else {
3748                 segno = le32_to_cpu(ckpt->cur_node_segno[type -
3749                                                         CURSEG_HOT_NODE]);
3750                 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
3751                                                         CURSEG_HOT_NODE]);
3752                 if (__exist_node_summaries(sbi))
3753                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
3754                                                         type - CURSEG_HOT_NODE);
3755                 else
3756                         blk_addr = GET_SUM_BLOCK(sbi, segno);
3757         }
3758
3759         new = f2fs_get_meta_page(sbi, blk_addr);
3760         if (IS_ERR(new))
3761                 return PTR_ERR(new);
3762         sum = (struct f2fs_summary_block *)page_address(new);
3763
3764         if (IS_NODESEG(type)) {
3765                 if (__exist_node_summaries(sbi)) {
3766                         struct f2fs_summary *ns = &sum->entries[0];
3767                         int i;
3768
3769                         for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
3770                                 ns->version = 0;
3771                                 ns->ofs_in_node = 0;
3772                         }
3773                 } else {
3774                         err = f2fs_restore_node_summary(sbi, segno, sum);
3775                         if (err)
3776                                 goto out;
3777                 }
3778         }
3779
3780         /* set uncompleted segment to curseg */
3781         curseg = CURSEG_I(sbi, type);
3782         mutex_lock(&curseg->curseg_mutex);
3783
3784         /* update journal info */
3785         down_write(&curseg->journal_rwsem);
3786         memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE);
3787         up_write(&curseg->journal_rwsem);
3788
3789         memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE);
3790         memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE);
3791         curseg->next_segno = segno;
3792         reset_curseg(sbi, type, 0);
3793         curseg->alloc_type = ckpt->alloc_type[type];
3794         curseg->next_blkoff = blk_off;
3795         mutex_unlock(&curseg->curseg_mutex);
3796 out:
3797         f2fs_put_page(new, 1);
3798         return err;
3799 }
3800
3801 static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
3802 {
3803         struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal;
3804         struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal;
3805         int type = CURSEG_HOT_DATA;
3806         int err;
3807
3808         if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) {
3809                 int npages = f2fs_npages_for_summary_flush(sbi, true);
3810
3811                 if (npages >= 2)
3812                         f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages,
3813                                                         META_CP, true);
3814
3815                 /* restore for compacted data summary */
3816                 err = read_compacted_summaries(sbi);
3817                 if (err)
3818                         return err;
3819                 type = CURSEG_HOT_NODE;
3820         }
3821
3822         if (__exist_node_summaries(sbi))
3823                 f2fs_ra_meta_pages(sbi,
3824                                 sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type),
3825                                 NR_CURSEG_PERSIST_TYPE - type, META_CP, true);
3826
3827         for (; type <= CURSEG_COLD_NODE; type++) {
3828                 err = read_normal_summaries(sbi, type);
3829                 if (err)
3830                         return err;
3831         }
3832
3833         /* sanity check for summary blocks */
3834         if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES ||
3835                         sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) {
3836                 f2fs_err(sbi, "invalid journal entries nats %u sits %u",
3837                          nats_in_cursum(nat_j), sits_in_cursum(sit_j));
3838                 return -EINVAL;
3839         }
3840
3841         return 0;
3842 }
3843
3844 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
3845 {
3846         struct page *page;
3847         unsigned char *kaddr;
3848         struct f2fs_summary *summary;
3849         struct curseg_info *seg_i;
3850         int written_size = 0;
3851         int i, j;
3852
3853         page = f2fs_grab_meta_page(sbi, blkaddr++);
3854         kaddr = (unsigned char *)page_address(page);
3855         memset(kaddr, 0, PAGE_SIZE);
3856
3857         /* Step 1: write nat cache */
3858         seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
3859         memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE);
3860         written_size += SUM_JOURNAL_SIZE;
3861
3862         /* Step 2: write sit cache */
3863         seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
3864         memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE);
3865         written_size += SUM_JOURNAL_SIZE;
3866
3867         /* Step 3: write summary entries */
3868         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
3869                 unsigned short blkoff;
3870
3871                 seg_i = CURSEG_I(sbi, i);
3872                 if (sbi->ckpt->alloc_type[i] == SSR)
3873                         blkoff = sbi->blocks_per_seg;
3874                 else
3875                         blkoff = curseg_blkoff(sbi, i);
3876
3877                 for (j = 0; j < blkoff; j++) {
3878                         if (!page) {
3879                                 page = f2fs_grab_meta_page(sbi, blkaddr++);
3880                                 kaddr = (unsigned char *)page_address(page);
3881                                 memset(kaddr, 0, PAGE_SIZE);
3882                                 written_size = 0;
3883                         }
3884                         summary = (struct f2fs_summary *)(kaddr + written_size);
3885                         *summary = seg_i->sum_blk->entries[j];
3886                         written_size += SUMMARY_SIZE;
3887
3888                         if (written_size + SUMMARY_SIZE <= PAGE_SIZE -
3889                                                         SUM_FOOTER_SIZE)
3890                                 continue;
3891
3892                         set_page_dirty(page);
3893                         f2fs_put_page(page, 1);
3894                         page = NULL;
3895                 }
3896         }
3897         if (page) {
3898                 set_page_dirty(page);
3899                 f2fs_put_page(page, 1);
3900         }
3901 }
3902
3903 static void write_normal_summaries(struct f2fs_sb_info *sbi,
3904                                         block_t blkaddr, int type)
3905 {
3906         int i, end;
3907
3908         if (IS_DATASEG(type))
3909                 end = type + NR_CURSEG_DATA_TYPE;
3910         else
3911                 end = type + NR_CURSEG_NODE_TYPE;
3912
3913         for (i = type; i < end; i++)
3914                 write_current_sum_page(sbi, i, blkaddr + (i - type));
3915 }
3916
3917 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
3918 {
3919         if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG))
3920                 write_compacted_summaries(sbi, start_blk);
3921         else
3922                 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
3923 }
3924
3925 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
3926 {
3927         write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
3928 }
3929
3930 int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
3931                                         unsigned int val, int alloc)
3932 {
3933         int i;
3934
3935         if (type == NAT_JOURNAL) {
3936                 for (i = 0; i < nats_in_cursum(journal); i++) {
3937                         if (le32_to_cpu(nid_in_journal(journal, i)) == val)
3938                                 return i;
3939                 }
3940                 if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL))
3941                         return update_nats_in_cursum(journal, 1);
3942         } else if (type == SIT_JOURNAL) {
3943                 for (i = 0; i < sits_in_cursum(journal); i++)
3944                         if (le32_to_cpu(segno_in_journal(journal, i)) == val)
3945                                 return i;
3946                 if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL))
3947                         return update_sits_in_cursum(journal, 1);
3948         }
3949         return -1;
3950 }
3951
3952 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
3953                                         unsigned int segno)
3954 {
3955         return f2fs_get_meta_page(sbi, current_sit_addr(sbi, segno));
3956 }
3957
3958 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
3959                                         unsigned int start)
3960 {
3961         struct sit_info *sit_i = SIT_I(sbi);
3962         struct page *page;
3963         pgoff_t src_off, dst_off;
3964
3965         src_off = current_sit_addr(sbi, start);
3966         dst_off = next_sit_addr(sbi, src_off);
3967
3968         page = f2fs_grab_meta_page(sbi, dst_off);
3969         seg_info_to_sit_page(sbi, page, start);
3970
3971         set_page_dirty(page);
3972         set_to_next_sit(sit_i, start);
3973
3974         return page;
3975 }
3976
3977 static struct sit_entry_set *grab_sit_entry_set(void)
3978 {
3979         struct sit_entry_set *ses =
3980                         f2fs_kmem_cache_alloc(sit_entry_set_slab,
3981                                                 GFP_NOFS, true, NULL);
3982
3983         ses->entry_cnt = 0;
3984         INIT_LIST_HEAD(&ses->set_list);
3985         return ses;
3986 }
3987
3988 static void release_sit_entry_set(struct sit_entry_set *ses)
3989 {
3990         list_del(&ses->set_list);
3991         kmem_cache_free(sit_entry_set_slab, ses);
3992 }
3993
3994 static void adjust_sit_entry_set(struct sit_entry_set *ses,
3995                                                 struct list_head *head)
3996 {
3997         struct sit_entry_set *next = ses;
3998
3999         if (list_is_last(&ses->set_list, head))
4000                 return;
4001
4002         list_for_each_entry_continue(next, head, set_list)
4003                 if (ses->entry_cnt <= next->entry_cnt) {
4004                         list_move_tail(&ses->set_list, &next->set_list);
4005                         return;
4006                 }
4007
4008         list_move_tail(&ses->set_list, head);
4009 }
4010
4011 static void add_sit_entry(unsigned int segno, struct list_head *head)
4012 {
4013         struct sit_entry_set *ses;
4014         unsigned int start_segno = START_SEGNO(segno);
4015
4016         list_for_each_entry(ses, head, set_list) {
4017                 if (ses->start_segno == start_segno) {
4018                         ses->entry_cnt++;
4019                         adjust_sit_entry_set(ses, head);
4020                         return;
4021                 }
4022         }
4023
4024         ses = grab_sit_entry_set();
4025
4026         ses->start_segno = start_segno;
4027         ses->entry_cnt++;
4028         list_add(&ses->set_list, head);
4029 }
4030
4031 static void add_sits_in_set(struct f2fs_sb_info *sbi)
4032 {
4033         struct f2fs_sm_info *sm_info = SM_I(sbi);
4034         struct list_head *set_list = &sm_info->sit_entry_set;
4035         unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
4036         unsigned int segno;
4037
4038         for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
4039                 add_sit_entry(segno, set_list);
4040 }
4041
4042 static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
4043 {
4044         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4045         struct f2fs_journal *journal = curseg->journal;
4046         int i;
4047
4048         down_write(&curseg->journal_rwsem);
4049         for (i = 0; i < sits_in_cursum(journal); i++) {
4050                 unsigned int segno;
4051                 bool dirtied;
4052
4053                 segno = le32_to_cpu(segno_in_journal(journal, i));
4054                 dirtied = __mark_sit_entry_dirty(sbi, segno);
4055
4056                 if (!dirtied)
4057                         add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
4058         }
4059         update_sits_in_cursum(journal, -i);
4060         up_write(&curseg->journal_rwsem);
4061 }
4062
4063 /*
4064  * CP calls this function, which flushes SIT entries including sit_journal,
4065  * and moves prefree segs to free segs.
4066  */
4067 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
4068 {
4069         struct sit_info *sit_i = SIT_I(sbi);
4070         unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
4071         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4072         struct f2fs_journal *journal = curseg->journal;
4073         struct sit_entry_set *ses, *tmp;
4074         struct list_head *head = &SM_I(sbi)->sit_entry_set;
4075         bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS);
4076         struct seg_entry *se;
4077
4078         down_write(&sit_i->sentry_lock);
4079
4080         if (!sit_i->dirty_sentries)
4081                 goto out;
4082
4083         /*
4084          * add and account sit entries of dirty bitmap in sit entry
4085          * set temporarily
4086          */
4087         add_sits_in_set(sbi);
4088
4089         /*
4090          * if there are no enough space in journal to store dirty sit
4091          * entries, remove all entries from journal and add and account
4092          * them in sit entry set.
4093          */
4094         if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL) ||
4095                                                                 !to_journal)
4096                 remove_sits_in_journal(sbi);
4097
4098         /*
4099          * there are two steps to flush sit entries:
4100          * #1, flush sit entries to journal in current cold data summary block.
4101          * #2, flush sit entries to sit page.
4102          */
4103         list_for_each_entry_safe(ses, tmp, head, set_list) {
4104                 struct page *page = NULL;
4105                 struct f2fs_sit_block *raw_sit = NULL;
4106                 unsigned int start_segno = ses->start_segno;
4107                 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
4108                                                 (unsigned long)MAIN_SEGS(sbi));
4109                 unsigned int segno = start_segno;
4110
4111                 if (to_journal &&
4112                         !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL))
4113                         to_journal = false;
4114
4115                 if (to_journal) {
4116                         down_write(&curseg->journal_rwsem);
4117                 } else {
4118                         page = get_next_sit_page(sbi, start_segno);
4119                         raw_sit = page_address(page);
4120                 }
4121
4122                 /* flush dirty sit entries in region of current sit set */
4123                 for_each_set_bit_from(segno, bitmap, end) {
4124                         int offset, sit_offset;
4125
4126                         se = get_seg_entry(sbi, segno);
4127 #ifdef CONFIG_F2FS_CHECK_FS
4128                         if (memcmp(se->cur_valid_map, se->cur_valid_map_mir,
4129                                                 SIT_VBLOCK_MAP_SIZE))
4130                                 f2fs_bug_on(sbi, 1);
4131 #endif
4132
4133                         /* add discard candidates */
4134                         if (!(cpc->reason & CP_DISCARD)) {
4135                                 cpc->trim_start = segno;
4136                                 add_discard_addrs(sbi, cpc, false);
4137                         }
4138
4139                         if (to_journal) {
4140                                 offset = f2fs_lookup_journal_in_cursum(journal,
4141                                                         SIT_JOURNAL, segno, 1);
4142                                 f2fs_bug_on(sbi, offset < 0);
4143                                 segno_in_journal(journal, offset) =
4144                                                         cpu_to_le32(segno);
4145                                 seg_info_to_raw_sit(se,
4146                                         &sit_in_journal(journal, offset));
4147                                 check_block_count(sbi, segno,
4148                                         &sit_in_journal(journal, offset));
4149                         } else {
4150                                 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
4151                                 seg_info_to_raw_sit(se,
4152                                                 &raw_sit->entries[sit_offset]);
4153                                 check_block_count(sbi, segno,
4154                                                 &raw_sit->entries[sit_offset]);
4155                         }
4156
4157                         __clear_bit(segno, bitmap);
4158                         sit_i->dirty_sentries--;
4159                         ses->entry_cnt--;
4160                 }
4161
4162                 if (to_journal)
4163                         up_write(&curseg->journal_rwsem);
4164                 else
4165                         f2fs_put_page(page, 1);
4166
4167                 f2fs_bug_on(sbi, ses->entry_cnt);
4168                 release_sit_entry_set(ses);
4169         }
4170
4171         f2fs_bug_on(sbi, !list_empty(head));
4172         f2fs_bug_on(sbi, sit_i->dirty_sentries);
4173 out:
4174         if (cpc->reason & CP_DISCARD) {
4175                 __u64 trim_start = cpc->trim_start;
4176
4177                 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
4178                         add_discard_addrs(sbi, cpc, false);
4179
4180                 cpc->trim_start = trim_start;
4181         }
4182         up_write(&sit_i->sentry_lock);
4183
4184         set_prefree_as_free_segments(sbi);
4185 }
4186
4187 static int build_sit_info(struct f2fs_sb_info *sbi)
4188 {
4189         struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
4190         struct sit_info *sit_i;
4191         unsigned int sit_segs, start;
4192         char *src_bitmap, *bitmap;
4193         unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size;
4194         unsigned int discard_map = f2fs_block_unit_discard(sbi) ? 1 : 0;
4195
4196         /* allocate memory for SIT information */
4197         sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL);
4198         if (!sit_i)
4199                 return -ENOMEM;
4200
4201         SM_I(sbi)->sit_info = sit_i;
4202
4203         sit_i->sentries =
4204                 f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry),
4205                                               MAIN_SEGS(sbi)),
4206                               GFP_KERNEL);
4207         if (!sit_i->sentries)
4208                 return -ENOMEM;
4209
4210         main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4211         sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size,
4212                                                                 GFP_KERNEL);
4213         if (!sit_i->dirty_sentries_bitmap)
4214                 return -ENOMEM;
4215
4216 #ifdef CONFIG_F2FS_CHECK_FS
4217         bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (3 + discard_map);
4218 #else
4219         bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (2 + discard_map);
4220 #endif
4221         sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4222         if (!sit_i->bitmap)
4223                 return -ENOMEM;
4224
4225         bitmap = sit_i->bitmap;
4226
4227         for (start = 0; start < MAIN_SEGS(sbi); start++) {
4228                 sit_i->sentries[start].cur_valid_map = bitmap;
4229                 bitmap += SIT_VBLOCK_MAP_SIZE;
4230
4231                 sit_i->sentries[start].ckpt_valid_map = bitmap;
4232                 bitmap += SIT_VBLOCK_MAP_SIZE;
4233
4234 #ifdef CONFIG_F2FS_CHECK_FS
4235                 sit_i->sentries[start].cur_valid_map_mir = bitmap;
4236                 bitmap += SIT_VBLOCK_MAP_SIZE;
4237 #endif
4238
4239                 if (discard_map) {
4240                         sit_i->sentries[start].discard_map = bitmap;
4241                         bitmap += SIT_VBLOCK_MAP_SIZE;
4242                 }
4243         }
4244
4245         sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
4246         if (!sit_i->tmp_map)
4247                 return -ENOMEM;
4248
4249         if (__is_large_section(sbi)) {
4250                 sit_i->sec_entries =
4251                         f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry),
4252                                                       MAIN_SECS(sbi)),
4253                                       GFP_KERNEL);
4254                 if (!sit_i->sec_entries)
4255                         return -ENOMEM;
4256         }
4257
4258         /* get information related with SIT */
4259         sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
4260
4261         /* setup SIT bitmap from ckeckpoint pack */
4262         sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
4263         src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
4264
4265         sit_i->sit_bitmap = kmemdup(src_bitmap, sit_bitmap_size, GFP_KERNEL);
4266         if (!sit_i->sit_bitmap)
4267                 return -ENOMEM;
4268
4269 #ifdef CONFIG_F2FS_CHECK_FS
4270         sit_i->sit_bitmap_mir = kmemdup(src_bitmap,
4271                                         sit_bitmap_size, GFP_KERNEL);
4272         if (!sit_i->sit_bitmap_mir)
4273                 return -ENOMEM;
4274
4275         sit_i->invalid_segmap = f2fs_kvzalloc(sbi,
4276                                         main_bitmap_size, GFP_KERNEL);
4277         if (!sit_i->invalid_segmap)
4278                 return -ENOMEM;
4279 #endif
4280
4281         /* init SIT information */
4282         sit_i->s_ops = &default_salloc_ops;
4283
4284         sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
4285         sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
4286         sit_i->written_valid_blocks = 0;
4287         sit_i->bitmap_size = sit_bitmap_size;
4288         sit_i->dirty_sentries = 0;
4289         sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
4290         sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
4291         sit_i->mounted_time = ktime_get_boottime_seconds();
4292         init_rwsem(&sit_i->sentry_lock);
4293         return 0;
4294 }
4295
4296 static int build_free_segmap(struct f2fs_sb_info *sbi)
4297 {
4298         struct free_segmap_info *free_i;
4299         unsigned int bitmap_size, sec_bitmap_size;
4300
4301         /* allocate memory for free segmap information */
4302         free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL);
4303         if (!free_i)
4304                 return -ENOMEM;
4305
4306         SM_I(sbi)->free_info = free_i;
4307
4308         bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4309         free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL);
4310         if (!free_i->free_segmap)
4311                 return -ENOMEM;
4312
4313         sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4314         free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL);
4315         if (!free_i->free_secmap)
4316                 return -ENOMEM;
4317
4318         /* set all segments as dirty temporarily */
4319         memset(free_i->free_segmap, 0xff, bitmap_size);
4320         memset(free_i->free_secmap, 0xff, sec_bitmap_size);
4321
4322         /* init free segmap information */
4323         free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
4324         free_i->free_segments = 0;
4325         free_i->free_sections = 0;
4326         spin_lock_init(&free_i->segmap_lock);
4327         return 0;
4328 }
4329
4330 static int build_curseg(struct f2fs_sb_info *sbi)
4331 {
4332         struct curseg_info *array;
4333         int i;
4334
4335         array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE,
4336                                         sizeof(*array)), GFP_KERNEL);
4337         if (!array)
4338                 return -ENOMEM;
4339
4340         SM_I(sbi)->curseg_array = array;
4341
4342         for (i = 0; i < NO_CHECK_TYPE; i++) {
4343                 mutex_init(&array[i].curseg_mutex);
4344                 array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL);
4345                 if (!array[i].sum_blk)
4346                         return -ENOMEM;
4347                 init_rwsem(&array[i].journal_rwsem);
4348                 array[i].journal = f2fs_kzalloc(sbi,
4349                                 sizeof(struct f2fs_journal), GFP_KERNEL);
4350                 if (!array[i].journal)
4351                         return -ENOMEM;
4352                 if (i < NR_PERSISTENT_LOG)
4353                         array[i].seg_type = CURSEG_HOT_DATA + i;
4354                 else if (i == CURSEG_COLD_DATA_PINNED)
4355                         array[i].seg_type = CURSEG_COLD_DATA;
4356                 else if (i == CURSEG_ALL_DATA_ATGC)
4357                         array[i].seg_type = CURSEG_COLD_DATA;
4358                 array[i].segno = NULL_SEGNO;
4359                 array[i].next_blkoff = 0;
4360                 array[i].inited = false;
4361         }
4362         return restore_curseg_summaries(sbi);
4363 }
4364
4365 static int build_sit_entries(struct f2fs_sb_info *sbi)
4366 {
4367         struct sit_info *sit_i = SIT_I(sbi);
4368         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4369         struct f2fs_journal *journal = curseg->journal;
4370         struct seg_entry *se;
4371         struct f2fs_sit_entry sit;
4372         int sit_blk_cnt = SIT_BLK_CNT(sbi);
4373         unsigned int i, start, end;
4374         unsigned int readed, start_blk = 0;
4375         int err = 0;
4376         block_t sit_valid_blocks[2] = {0, 0};
4377
4378         do {
4379                 readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_VECS,
4380                                                         META_SIT, true);
4381
4382                 start = start_blk * sit_i->sents_per_block;
4383                 end = (start_blk + readed) * sit_i->sents_per_block;
4384
4385                 for (; start < end && start < MAIN_SEGS(sbi); start++) {
4386                         struct f2fs_sit_block *sit_blk;
4387                         struct page *page;
4388
4389                         se = &sit_i->sentries[start];
4390                         page = get_current_sit_page(sbi, start);
4391                         if (IS_ERR(page))
4392                                 return PTR_ERR(page);
4393                         sit_blk = (struct f2fs_sit_block *)page_address(page);
4394                         sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
4395                         f2fs_put_page(page, 1);
4396
4397                         err = check_block_count(sbi, start, &sit);
4398                         if (err)
4399                                 return err;
4400                         seg_info_from_raw_sit(se, &sit);
4401
4402                         if (se->type >= NR_PERSISTENT_LOG) {
4403                                 f2fs_err(sbi, "Invalid segment type: %u, segno: %u",
4404                                                         se->type, start);
4405                                 f2fs_handle_error(sbi,
4406                                                 ERROR_INCONSISTENT_SUM_TYPE);
4407                                 return -EFSCORRUPTED;
4408                         }
4409
4410                         sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks;
4411
4412                         if (f2fs_block_unit_discard(sbi)) {
4413                                 /* build discard map only one time */
4414                                 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4415                                         memset(se->discard_map, 0xff,
4416                                                 SIT_VBLOCK_MAP_SIZE);
4417                                 } else {
4418                                         memcpy(se->discard_map,
4419                                                 se->cur_valid_map,
4420                                                 SIT_VBLOCK_MAP_SIZE);
4421                                         sbi->discard_blks +=
4422                                                 sbi->blocks_per_seg -
4423                                                 se->valid_blocks;
4424                                 }
4425                         }
4426
4427                         if (__is_large_section(sbi))
4428                                 get_sec_entry(sbi, start)->valid_blocks +=
4429                                                         se->valid_blocks;
4430                 }
4431                 start_blk += readed;
4432         } while (start_blk < sit_blk_cnt);
4433
4434         down_read(&curseg->journal_rwsem);
4435         for (i = 0; i < sits_in_cursum(journal); i++) {
4436                 unsigned int old_valid_blocks;
4437
4438                 start = le32_to_cpu(segno_in_journal(journal, i));
4439                 if (start >= MAIN_SEGS(sbi)) {
4440                         f2fs_err(sbi, "Wrong journal entry on segno %u",
4441                                  start);
4442                         err = -EFSCORRUPTED;
4443                         f2fs_handle_error(sbi, ERROR_CORRUPTED_JOURNAL);
4444                         break;
4445                 }
4446
4447                 se = &sit_i->sentries[start];
4448                 sit = sit_in_journal(journal, i);
4449
4450                 old_valid_blocks = se->valid_blocks;
4451
4452                 sit_valid_blocks[SE_PAGETYPE(se)] -= old_valid_blocks;
4453
4454                 err = check_block_count(sbi, start, &sit);
4455                 if (err)
4456                         break;
4457                 seg_info_from_raw_sit(se, &sit);
4458
4459                 if (se->type >= NR_PERSISTENT_LOG) {
4460                         f2fs_err(sbi, "Invalid segment type: %u, segno: %u",
4461                                                         se->type, start);
4462                         err = -EFSCORRUPTED;
4463                         f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE);
4464                         break;
4465                 }
4466
4467                 sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks;
4468
4469                 if (f2fs_block_unit_discard(sbi)) {
4470                         if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4471                                 memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE);
4472                         } else {
4473                                 memcpy(se->discard_map, se->cur_valid_map,
4474                                                         SIT_VBLOCK_MAP_SIZE);
4475                                 sbi->discard_blks += old_valid_blocks;
4476                                 sbi->discard_blks -= se->valid_blocks;
4477                         }
4478                 }
4479
4480                 if (__is_large_section(sbi)) {
4481                         get_sec_entry(sbi, start)->valid_blocks +=
4482                                                         se->valid_blocks;
4483                         get_sec_entry(sbi, start)->valid_blocks -=
4484                                                         old_valid_blocks;
4485                 }
4486         }
4487         up_read(&curseg->journal_rwsem);
4488
4489         if (err)
4490                 return err;
4491
4492         if (sit_valid_blocks[NODE] != valid_node_count(sbi)) {
4493                 f2fs_err(sbi, "SIT is corrupted node# %u vs %u",
4494                          sit_valid_blocks[NODE], valid_node_count(sbi));
4495                 f2fs_handle_error(sbi, ERROR_INCONSISTENT_NODE_COUNT);
4496                 return -EFSCORRUPTED;
4497         }
4498
4499         if (sit_valid_blocks[DATA] + sit_valid_blocks[NODE] >
4500                                 valid_user_blocks(sbi)) {
4501                 f2fs_err(sbi, "SIT is corrupted data# %u %u vs %u",
4502                          sit_valid_blocks[DATA], sit_valid_blocks[NODE],
4503                          valid_user_blocks(sbi));
4504                 f2fs_handle_error(sbi, ERROR_INCONSISTENT_BLOCK_COUNT);
4505                 return -EFSCORRUPTED;
4506         }
4507
4508         return 0;
4509 }
4510
4511 static void init_free_segmap(struct f2fs_sb_info *sbi)
4512 {
4513         unsigned int start;
4514         int type;
4515         struct seg_entry *sentry;
4516
4517         for (start = 0; start < MAIN_SEGS(sbi); start++) {
4518                 if (f2fs_usable_blks_in_seg(sbi, start) == 0)
4519                         continue;
4520                 sentry = get_seg_entry(sbi, start);
4521                 if (!sentry->valid_blocks)
4522                         __set_free(sbi, start);
4523                 else
4524                         SIT_I(sbi)->written_valid_blocks +=
4525                                                 sentry->valid_blocks;
4526         }
4527
4528         /* set use the current segments */
4529         for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
4530                 struct curseg_info *curseg_t = CURSEG_I(sbi, type);
4531
4532                 __set_test_and_inuse(sbi, curseg_t->segno);
4533         }
4534 }
4535
4536 static void init_dirty_segmap(struct f2fs_sb_info *sbi)
4537 {
4538         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4539         struct free_segmap_info *free_i = FREE_I(sbi);
4540         unsigned int segno = 0, offset = 0, secno;
4541         block_t valid_blocks, usable_blks_in_seg;
4542
4543         while (1) {
4544                 /* find dirty segment based on free segmap */
4545                 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
4546                 if (segno >= MAIN_SEGS(sbi))
4547                         break;
4548                 offset = segno + 1;
4549                 valid_blocks = get_valid_blocks(sbi, segno, false);
4550                 usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);
4551                 if (valid_blocks == usable_blks_in_seg || !valid_blocks)
4552                         continue;
4553                 if (valid_blocks > usable_blks_in_seg) {
4554                         f2fs_bug_on(sbi, 1);
4555                         continue;
4556                 }
4557                 mutex_lock(&dirty_i->seglist_lock);
4558                 __locate_dirty_segment(sbi, segno, DIRTY);
4559                 mutex_unlock(&dirty_i->seglist_lock);
4560         }
4561
4562         if (!__is_large_section(sbi))
4563                 return;
4564
4565         mutex_lock(&dirty_i->seglist_lock);
4566         for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
4567                 valid_blocks = get_valid_blocks(sbi, segno, true);
4568                 secno = GET_SEC_FROM_SEG(sbi, segno);
4569
4570                 if (!valid_blocks || valid_blocks == CAP_BLKS_PER_SEC(sbi))
4571                         continue;
4572                 if (IS_CURSEC(sbi, secno))
4573                         continue;
4574                 set_bit(secno, dirty_i->dirty_secmap);
4575         }
4576         mutex_unlock(&dirty_i->seglist_lock);
4577 }
4578
4579 static int init_victim_secmap(struct f2fs_sb_info *sbi)
4580 {
4581         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4582         unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4583
4584         dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4585         if (!dirty_i->victim_secmap)
4586                 return -ENOMEM;
4587
4588         dirty_i->pinned_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4589         if (!dirty_i->pinned_secmap)
4590                 return -ENOMEM;
4591
4592         dirty_i->pinned_secmap_cnt = 0;
4593         dirty_i->enable_pin_section = true;
4594         return 0;
4595 }
4596
4597 static int build_dirty_segmap(struct f2fs_sb_info *sbi)
4598 {
4599         struct dirty_seglist_info *dirty_i;
4600         unsigned int bitmap_size, i;
4601
4602         /* allocate memory for dirty segments list information */
4603         dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info),
4604                                                                 GFP_KERNEL);
4605         if (!dirty_i)
4606                 return -ENOMEM;
4607
4608         SM_I(sbi)->dirty_info = dirty_i;
4609         mutex_init(&dirty_i->seglist_lock);
4610
4611         bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4612
4613         for (i = 0; i < NR_DIRTY_TYPE; i++) {
4614                 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size,
4615                                                                 GFP_KERNEL);
4616                 if (!dirty_i->dirty_segmap[i])
4617                         return -ENOMEM;
4618         }
4619
4620         if (__is_large_section(sbi)) {
4621                 bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4622                 dirty_i->dirty_secmap = f2fs_kvzalloc(sbi,
4623                                                 bitmap_size, GFP_KERNEL);
4624                 if (!dirty_i->dirty_secmap)
4625                         return -ENOMEM;
4626         }
4627
4628         init_dirty_segmap(sbi);
4629         return init_victim_secmap(sbi);
4630 }
4631
4632 static int sanity_check_curseg(struct f2fs_sb_info *sbi)
4633 {
4634         int i;
4635
4636         /*
4637          * In LFS/SSR curseg, .next_blkoff should point to an unused blkaddr;
4638          * In LFS curseg, all blkaddr after .next_blkoff should be unused.
4639          */
4640         for (i = 0; i < NR_PERSISTENT_LOG; i++) {
4641                 struct curseg_info *curseg = CURSEG_I(sbi, i);
4642                 struct seg_entry *se = get_seg_entry(sbi, curseg->segno);
4643                 unsigned int blkofs = curseg->next_blkoff;
4644
4645                 if (f2fs_sb_has_readonly(sbi) &&
4646                         i != CURSEG_HOT_DATA && i != CURSEG_HOT_NODE)
4647                         continue;
4648
4649                 sanity_check_seg_type(sbi, curseg->seg_type);
4650
4651                 if (curseg->alloc_type != LFS && curseg->alloc_type != SSR) {
4652                         f2fs_err(sbi,
4653                                  "Current segment has invalid alloc_type:%d",
4654                                  curseg->alloc_type);
4655                         f2fs_handle_error(sbi, ERROR_INVALID_CURSEG);
4656                         return -EFSCORRUPTED;
4657                 }
4658
4659                 if (f2fs_test_bit(blkofs, se->cur_valid_map))
4660                         goto out;
4661
4662                 if (curseg->alloc_type == SSR)
4663                         continue;
4664
4665                 for (blkofs += 1; blkofs < sbi->blocks_per_seg; blkofs++) {
4666                         if (!f2fs_test_bit(blkofs, se->cur_valid_map))
4667                                 continue;
4668 out:
4669                         f2fs_err(sbi,
4670                                  "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u",
4671                                  i, curseg->segno, curseg->alloc_type,
4672                                  curseg->next_blkoff, blkofs);
4673                         f2fs_handle_error(sbi, ERROR_INVALID_CURSEG);
4674                         return -EFSCORRUPTED;
4675                 }
4676         }
4677         return 0;
4678 }
4679
4680 #ifdef CONFIG_BLK_DEV_ZONED
4681
4682 static int check_zone_write_pointer(struct f2fs_sb_info *sbi,
4683                                     struct f2fs_dev_info *fdev,
4684                                     struct blk_zone *zone)
4685 {
4686         unsigned int wp_segno, wp_blkoff, zone_secno, zone_segno, segno;
4687         block_t zone_block, wp_block, last_valid_block;
4688         unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
4689         int i, s, b, ret;
4690         struct seg_entry *se;
4691
4692         if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4693                 return 0;
4694
4695         wp_block = fdev->start_blk + (zone->wp >> log_sectors_per_block);
4696         wp_segno = GET_SEGNO(sbi, wp_block);
4697         wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
4698         zone_block = fdev->start_blk + (zone->start >> log_sectors_per_block);
4699         zone_segno = GET_SEGNO(sbi, zone_block);
4700         zone_secno = GET_SEC_FROM_SEG(sbi, zone_segno);
4701
4702         if (zone_segno >= MAIN_SEGS(sbi))
4703                 return 0;
4704
4705         /*
4706          * Skip check of zones cursegs point to, since
4707          * fix_curseg_write_pointer() checks them.
4708          */
4709         for (i = 0; i < NO_CHECK_TYPE; i++)
4710                 if (zone_secno == GET_SEC_FROM_SEG(sbi,
4711                                                    CURSEG_I(sbi, i)->segno))
4712                         return 0;
4713
4714         /*
4715          * Get last valid block of the zone.
4716          */
4717         last_valid_block = zone_block - 1;
4718         for (s = sbi->segs_per_sec - 1; s >= 0; s--) {
4719                 segno = zone_segno + s;
4720                 se = get_seg_entry(sbi, segno);
4721                 for (b = sbi->blocks_per_seg - 1; b >= 0; b--)
4722                         if (f2fs_test_bit(b, se->cur_valid_map)) {
4723                                 last_valid_block = START_BLOCK(sbi, segno) + b;
4724                                 break;
4725                         }
4726                 if (last_valid_block >= zone_block)
4727                         break;
4728         }
4729
4730         /*
4731          * If last valid block is beyond the write pointer, report the
4732          * inconsistency. This inconsistency does not cause write error
4733          * because the zone will not be selected for write operation until
4734          * it get discarded. Just report it.
4735          */
4736         if (last_valid_block >= wp_block) {
4737                 f2fs_notice(sbi, "Valid block beyond write pointer: "
4738                             "valid block[0x%x,0x%x] wp[0x%x,0x%x]",
4739                             GET_SEGNO(sbi, last_valid_block),
4740                             GET_BLKOFF_FROM_SEG0(sbi, last_valid_block),
4741                             wp_segno, wp_blkoff);
4742                 return 0;
4743         }
4744
4745         /*
4746          * If there is no valid block in the zone and if write pointer is
4747          * not at zone start, reset the write pointer.
4748          */
4749         if (last_valid_block + 1 == zone_block && zone->wp != zone->start) {
4750                 f2fs_notice(sbi,
4751                             "Zone without valid block has non-zero write "
4752                             "pointer. Reset the write pointer: wp[0x%x,0x%x]",
4753                             wp_segno, wp_blkoff);
4754                 ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block,
4755                                         zone->len >> log_sectors_per_block);
4756                 if (ret) {
4757                         f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
4758                                  fdev->path, ret);
4759                         return ret;
4760                 }
4761         }
4762
4763         return 0;
4764 }
4765
4766 static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi,
4767                                                   block_t zone_blkaddr)
4768 {
4769         int i;
4770
4771         for (i = 0; i < sbi->s_ndevs; i++) {
4772                 if (!bdev_is_zoned(FDEV(i).bdev))
4773                         continue;
4774                 if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr &&
4775                                 zone_blkaddr <= FDEV(i).end_blk))
4776                         return &FDEV(i);
4777         }
4778
4779         return NULL;
4780 }
4781
4782 static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx,
4783                               void *data)
4784 {
4785         memcpy(data, zone, sizeof(struct blk_zone));
4786         return 0;
4787 }
4788
4789 static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type)
4790 {
4791         struct curseg_info *cs = CURSEG_I(sbi, type);
4792         struct f2fs_dev_info *zbd;
4793         struct blk_zone zone;
4794         unsigned int cs_section, wp_segno, wp_blkoff, wp_sector_off;
4795         block_t cs_zone_block, wp_block;
4796         unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
4797         sector_t zone_sector;
4798         int err;
4799
4800         cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
4801         cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
4802
4803         zbd = get_target_zoned_dev(sbi, cs_zone_block);
4804         if (!zbd)
4805                 return 0;
4806
4807         /* report zone for the sector the curseg points to */
4808         zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
4809                 << log_sectors_per_block;
4810         err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
4811                                   report_one_zone_cb, &zone);
4812         if (err != 1) {
4813                 f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
4814                          zbd->path, err);
4815                 return err;
4816         }
4817
4818         if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4819                 return 0;
4820
4821         wp_block = zbd->start_blk + (zone.wp >> log_sectors_per_block);
4822         wp_segno = GET_SEGNO(sbi, wp_block);
4823         wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
4824         wp_sector_off = zone.wp & GENMASK(log_sectors_per_block - 1, 0);
4825
4826         if (cs->segno == wp_segno && cs->next_blkoff == wp_blkoff &&
4827                 wp_sector_off == 0)
4828                 return 0;
4829
4830         f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: "
4831                     "curseg[0x%x,0x%x] wp[0x%x,0x%x]",
4832                     type, cs->segno, cs->next_blkoff, wp_segno, wp_blkoff);
4833
4834         f2fs_notice(sbi, "Assign new section to curseg[%d]: "
4835                     "curseg[0x%x,0x%x]", type, cs->segno, cs->next_blkoff);
4836
4837         f2fs_allocate_new_section(sbi, type, true);
4838
4839         /* check consistency of the zone curseg pointed to */
4840         if (check_zone_write_pointer(sbi, zbd, &zone))
4841                 return -EIO;
4842
4843         /* check newly assigned zone */
4844         cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
4845         cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
4846
4847         zbd = get_target_zoned_dev(sbi, cs_zone_block);
4848         if (!zbd)
4849                 return 0;
4850
4851         zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
4852                 << log_sectors_per_block;
4853         err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
4854                                   report_one_zone_cb, &zone);
4855         if (err != 1) {
4856                 f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
4857                          zbd->path, err);
4858                 return err;
4859         }
4860
4861         if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4862                 return 0;
4863
4864         if (zone.wp != zone.start) {
4865                 f2fs_notice(sbi,
4866                             "New zone for curseg[%d] is not yet discarded. "
4867                             "Reset the zone: curseg[0x%x,0x%x]",
4868                             type, cs->segno, cs->next_blkoff);
4869                 err = __f2fs_issue_discard_zone(sbi, zbd->bdev,
4870                                 zone_sector >> log_sectors_per_block,
4871                                 zone.len >> log_sectors_per_block);
4872                 if (err) {
4873                         f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
4874                                  zbd->path, err);
4875                         return err;
4876                 }
4877         }
4878
4879         return 0;
4880 }
4881
4882 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
4883 {
4884         int i, ret;
4885
4886         for (i = 0; i < NR_PERSISTENT_LOG; i++) {
4887                 ret = fix_curseg_write_pointer(sbi, i);
4888                 if (ret)
4889                         return ret;
4890         }
4891
4892         return 0;
4893 }
4894
4895 struct check_zone_write_pointer_args {
4896         struct f2fs_sb_info *sbi;
4897         struct f2fs_dev_info *fdev;
4898 };
4899
4900 static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx,
4901                                       void *data)
4902 {
4903         struct check_zone_write_pointer_args *args;
4904
4905         args = (struct check_zone_write_pointer_args *)data;
4906
4907         return check_zone_write_pointer(args->sbi, args->fdev, zone);
4908 }
4909
4910 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
4911 {
4912         int i, ret;
4913         struct check_zone_write_pointer_args args;
4914
4915         for (i = 0; i < sbi->s_ndevs; i++) {
4916                 if (!bdev_is_zoned(FDEV(i).bdev))
4917                         continue;
4918
4919                 args.sbi = sbi;
4920                 args.fdev = &FDEV(i);
4921                 ret = blkdev_report_zones(FDEV(i).bdev, 0, BLK_ALL_ZONES,
4922                                           check_zone_write_pointer_cb, &args);
4923                 if (ret < 0)
4924                         return ret;
4925         }
4926
4927         return 0;
4928 }
4929
4930 /*
4931  * Return the number of usable blocks in a segment. The number of blocks
4932  * returned is always equal to the number of blocks in a segment for
4933  * segments fully contained within a sequential zone capacity or a
4934  * conventional zone. For segments partially contained in a sequential
4935  * zone capacity, the number of usable blocks up to the zone capacity
4936  * is returned. 0 is returned in all other cases.
4937  */
4938 static inline unsigned int f2fs_usable_zone_blks_in_seg(
4939                         struct f2fs_sb_info *sbi, unsigned int segno)
4940 {
4941         block_t seg_start, sec_start_blkaddr, sec_cap_blkaddr;
4942         unsigned int secno;
4943
4944         if (!sbi->unusable_blocks_per_sec)
4945                 return sbi->blocks_per_seg;
4946
4947         secno = GET_SEC_FROM_SEG(sbi, segno);
4948         seg_start = START_BLOCK(sbi, segno);
4949         sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));
4950         sec_cap_blkaddr = sec_start_blkaddr + CAP_BLKS_PER_SEC(sbi);
4951
4952         /*
4953          * If segment starts before zone capacity and spans beyond
4954          * zone capacity, then usable blocks are from seg start to
4955          * zone capacity. If the segment starts after the zone capacity,
4956          * then there are no usable blocks.
4957          */
4958         if (seg_start >= sec_cap_blkaddr)
4959                 return 0;
4960         if (seg_start + sbi->blocks_per_seg > sec_cap_blkaddr)
4961                 return sec_cap_blkaddr - seg_start;
4962
4963         return sbi->blocks_per_seg;
4964 }
4965 #else
4966 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
4967 {
4968         return 0;
4969 }
4970
4971 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
4972 {
4973         return 0;
4974 }
4975
4976 static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi,
4977                                                         unsigned int segno)
4978 {
4979         return 0;
4980 }
4981
4982 #endif
4983 unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
4984                                         unsigned int segno)
4985 {
4986         if (f2fs_sb_has_blkzoned(sbi))
4987                 return f2fs_usable_zone_blks_in_seg(sbi, segno);
4988
4989         return sbi->blocks_per_seg;
4990 }
4991
4992 unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,
4993                                         unsigned int segno)
4994 {
4995         if (f2fs_sb_has_blkzoned(sbi))
4996                 return CAP_SEGS_PER_SEC(sbi);
4997
4998         return sbi->segs_per_sec;
4999 }
5000
5001 /*
5002  * Update min, max modified time for cost-benefit GC algorithm
5003  */
5004 static void init_min_max_mtime(struct f2fs_sb_info *sbi)
5005 {
5006         struct sit_info *sit_i = SIT_I(sbi);
5007         unsigned int segno;
5008
5009         down_write(&sit_i->sentry_lock);
5010
5011         sit_i->min_mtime = ULLONG_MAX;
5012
5013         for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
5014                 unsigned int i;
5015                 unsigned long long mtime = 0;
5016
5017                 for (i = 0; i < sbi->segs_per_sec; i++)
5018                         mtime += get_seg_entry(sbi, segno + i)->mtime;
5019
5020                 mtime = div_u64(mtime, sbi->segs_per_sec);
5021
5022                 if (sit_i->min_mtime > mtime)
5023                         sit_i->min_mtime = mtime;
5024         }
5025         sit_i->max_mtime = get_mtime(sbi, false);
5026         sit_i->dirty_max_mtime = 0;
5027         up_write(&sit_i->sentry_lock);
5028 }
5029
5030 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi)
5031 {
5032         struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
5033         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
5034         struct f2fs_sm_info *sm_info;
5035         int err;
5036
5037         sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL);
5038         if (!sm_info)
5039                 return -ENOMEM;
5040
5041         /* init sm info */
5042         sbi->sm_info = sm_info;
5043         sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
5044         sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
5045         sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
5046         sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
5047         sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
5048         sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
5049         sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
5050         sm_info->rec_prefree_segments = sm_info->main_segments *
5051                                         DEF_RECLAIM_PREFREE_SEGMENTS / 100;
5052         if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS)
5053                 sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS;
5054
5055         if (!f2fs_lfs_mode(sbi))
5056                 sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC;
5057         sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
5058         sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
5059         sm_info->min_seq_blocks = sbi->blocks_per_seg;
5060         sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS;
5061         sm_info->min_ssr_sections = reserved_sections(sbi);
5062
5063         INIT_LIST_HEAD(&sm_info->sit_entry_set);
5064
5065         init_f2fs_rwsem(&sm_info->curseg_lock);
5066
5067         if (!f2fs_readonly(sbi->sb)) {
5068                 err = f2fs_create_flush_cmd_control(sbi);
5069                 if (err)
5070                         return err;
5071         }
5072
5073         err = create_discard_cmd_control(sbi);
5074         if (err)
5075                 return err;
5076
5077         err = build_sit_info(sbi);
5078         if (err)
5079                 return err;
5080         err = build_free_segmap(sbi);
5081         if (err)
5082                 return err;
5083         err = build_curseg(sbi);
5084         if (err)
5085                 return err;
5086
5087         /* reinit free segmap based on SIT */
5088         err = build_sit_entries(sbi);
5089         if (err)
5090                 return err;
5091
5092         init_free_segmap(sbi);
5093         err = build_dirty_segmap(sbi);
5094         if (err)
5095                 return err;
5096
5097         err = sanity_check_curseg(sbi);
5098         if (err)
5099                 return err;
5100
5101         init_min_max_mtime(sbi);
5102         return 0;
5103 }
5104
5105 static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
5106                 enum dirty_type dirty_type)
5107 {
5108         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5109
5110         mutex_lock(&dirty_i->seglist_lock);
5111         kvfree(dirty_i->dirty_segmap[dirty_type]);
5112         dirty_i->nr_dirty[dirty_type] = 0;
5113         mutex_unlock(&dirty_i->seglist_lock);
5114 }
5115
5116 static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
5117 {
5118         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5119
5120         kvfree(dirty_i->pinned_secmap);
5121         kvfree(dirty_i->victim_secmap);
5122 }
5123
5124 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
5125 {
5126         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5127         int i;
5128
5129         if (!dirty_i)
5130                 return;
5131
5132         /* discard pre-free/dirty segments list */
5133         for (i = 0; i < NR_DIRTY_TYPE; i++)
5134                 discard_dirty_segmap(sbi, i);
5135
5136         if (__is_large_section(sbi)) {
5137                 mutex_lock(&dirty_i->seglist_lock);
5138                 kvfree(dirty_i->dirty_secmap);
5139                 mutex_unlock(&dirty_i->seglist_lock);
5140         }
5141
5142         destroy_victim_secmap(sbi);
5143         SM_I(sbi)->dirty_info = NULL;
5144         kfree(dirty_i);
5145 }
5146
5147 static void destroy_curseg(struct f2fs_sb_info *sbi)
5148 {
5149         struct curseg_info *array = SM_I(sbi)->curseg_array;
5150         int i;
5151
5152         if (!array)
5153                 return;
5154         SM_I(sbi)->curseg_array = NULL;
5155         for (i = 0; i < NR_CURSEG_TYPE; i++) {
5156                 kfree(array[i].sum_blk);
5157                 kfree(array[i].journal);
5158         }
5159         kfree(array);
5160 }
5161
5162 static void destroy_free_segmap(struct f2fs_sb_info *sbi)
5163 {
5164         struct free_segmap_info *free_i = SM_I(sbi)->free_info;
5165
5166         if (!free_i)
5167                 return;
5168         SM_I(sbi)->free_info = NULL;
5169         kvfree(free_i->free_segmap);
5170         kvfree(free_i->free_secmap);
5171         kfree(free_i);
5172 }
5173
5174 static void destroy_sit_info(struct f2fs_sb_info *sbi)
5175 {
5176         struct sit_info *sit_i = SIT_I(sbi);
5177
5178         if (!sit_i)
5179                 return;
5180
5181         if (sit_i->sentries)
5182                 kvfree(sit_i->bitmap);
5183         kfree(sit_i->tmp_map);
5184
5185         kvfree(sit_i->sentries);
5186         kvfree(sit_i->sec_entries);
5187         kvfree(sit_i->dirty_sentries_bitmap);
5188
5189         SM_I(sbi)->sit_info = NULL;
5190         kvfree(sit_i->sit_bitmap);
5191 #ifdef CONFIG_F2FS_CHECK_FS
5192         kvfree(sit_i->sit_bitmap_mir);
5193         kvfree(sit_i->invalid_segmap);
5194 #endif
5195         kfree(sit_i);
5196 }
5197
5198 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)
5199 {
5200         struct f2fs_sm_info *sm_info = SM_I(sbi);
5201
5202         if (!sm_info)
5203                 return;
5204         f2fs_destroy_flush_cmd_control(sbi, true);
5205         destroy_discard_cmd_control(sbi);
5206         destroy_dirty_segmap(sbi);
5207         destroy_curseg(sbi);
5208         destroy_free_segmap(sbi);
5209         destroy_sit_info(sbi);
5210         sbi->sm_info = NULL;
5211         kfree(sm_info);
5212 }
5213
5214 int __init f2fs_create_segment_manager_caches(void)
5215 {
5216         discard_entry_slab = f2fs_kmem_cache_create("f2fs_discard_entry",
5217                         sizeof(struct discard_entry));
5218         if (!discard_entry_slab)
5219                 goto fail;
5220
5221         discard_cmd_slab = f2fs_kmem_cache_create("f2fs_discard_cmd",
5222                         sizeof(struct discard_cmd));
5223         if (!discard_cmd_slab)
5224                 goto destroy_discard_entry;
5225
5226         sit_entry_set_slab = f2fs_kmem_cache_create("f2fs_sit_entry_set",
5227                         sizeof(struct sit_entry_set));
5228         if (!sit_entry_set_slab)
5229                 goto destroy_discard_cmd;
5230
5231         revoke_entry_slab = f2fs_kmem_cache_create("f2fs_revoke_entry",
5232                         sizeof(struct revoke_entry));
5233         if (!revoke_entry_slab)
5234                 goto destroy_sit_entry_set;
5235         return 0;
5236
5237 destroy_sit_entry_set:
5238         kmem_cache_destroy(sit_entry_set_slab);
5239 destroy_discard_cmd:
5240         kmem_cache_destroy(discard_cmd_slab);
5241 destroy_discard_entry:
5242         kmem_cache_destroy(discard_entry_slab);
5243 fail:
5244         return -ENOMEM;
5245 }
5246
5247 void f2fs_destroy_segment_manager_caches(void)
5248 {
5249         kmem_cache_destroy(sit_entry_set_slab);
5250         kmem_cache_destroy(discard_cmd_slab);
5251         kmem_cache_destroy(discard_entry_slab);
5252         kmem_cache_destroy(revoke_entry_slab);
5253 }