fs/f2fs/checkpoint.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * fs/f2fs/checkpoint.c
   4  *
   5  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
   6  *             http://www.samsung.com/
   7  */
   8 #include <linux/fs.h>
   9 #include <linux/bio.h>
  10 #include <linux/mpage.h>
  11 #include <linux/writeback.h>
  12 #include <linux/blkdev.h>
  13 #include <linux/f2fs_fs.h>
  14 #include <linux/pagevec.h>
  15 #include <linux/swap.h>
  16 #include <linux/kthread.h>
  17
  18 #include "f2fs.h"
  19 #include "node.h"
  20 #include "segment.h"
  21 #include "iostat.h"
  22 #include <trace/events/f2fs.h>
  23
  24 #define DEFAULT_CHECKPOINT_IOPRIO (IOPRIO_PRIO_VALUE(IOPRIO_CLASS_BE, 3))
  25
  26 static struct kmem_cache *ino_entry_slab;
  27 struct kmem_cache *f2fs_inode_entry_slab;
  28
  29 void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io)
  30 {
  31         f2fs_build_fault_attr(sbi, 0, 0);
  32         set_ckpt_flags(sbi, CP_ERROR_FLAG);
  33         if (!end_io)
  34                 f2fs_flush_merged_writes(sbi);
  35 }
  36
  37 /*
  38  * We guarantee no failure on the returned page.
  39  */
  40 struct page *f2fs_grab_meta_page(struct f2fs_sb_info *sbi, pgoff_t index)
  41 {
  42         struct address_space *mapping = META_MAPPING(sbi);
  43         struct page *page;
  44 repeat:
  45         page = f2fs_grab_cache_page(mapping, index, false);
  46         if (!page) {
  47                 cond_resched();
  48                 goto repeat;
  49         }
  50         f2fs_wait_on_page_writeback(page, META, true, true);
  51         if (!PageUptodate(page))
  52                 SetPageUptodate(page);
  53         return page;
  54 }
  55
  56 static struct page *__get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index,
  57                                                         bool is_meta)
  58 {
  59         struct address_space *mapping = META_MAPPING(sbi);
  60         struct page *page;
  61         struct f2fs_io_info fio = {
  62                 .sbi = sbi,
  63                 .type = META,
  64                 .op = REQ_OP_READ,
  65                 .op_flags = REQ_META | REQ_PRIO,
  66                 .old_blkaddr = index,
  67                 .new_blkaddr = index,
  68                 .encrypted_page = NULL,
  69                 .is_por = !is_meta,
  70         };
  71         int err;
  72
  73         if (unlikely(!is_meta))
  74                 fio.op_flags &= ~REQ_META;
  75 repeat:
  76         page = f2fs_grab_cache_page(mapping, index, false);
  77         if (!page) {
  78                 cond_resched();
  79                 goto repeat;
  80         }
  81         if (PageUptodate(page))
  82                 goto out;
  83
  84         fio.page = page;
  85
  86         err = f2fs_submit_page_bio(&fio);
  87         if (err) {
  88                 f2fs_put_page(page, 1);
  89                 return ERR_PTR(err);
  90         }
  91
  92         f2fs_update_iostat(sbi, FS_META_READ_IO, F2FS_BLKSIZE);
  93
  94         lock_page(page);
  95         if (unlikely(page->mapping != mapping)) {
  96                 f2fs_put_page(page, 1);
  97                 goto repeat;
  98         }
  99
 100         if (unlikely(!PageUptodate(page))) {
 101                 f2fs_put_page(page, 1);
 102                 return ERR_PTR(-EIO);
 103         }
 104 out:
 105         return page;
 106 }
 107
 108 struct page *f2fs_get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index)
 109 {
 110         return __get_meta_page(sbi, index, true);
 111 }
 112
 113 struct page *f2fs_get_meta_page_retry(struct f2fs_sb_info *sbi, pgoff_t index)
 114 {
 115         struct page *page;
 116         int count = 0;
 117
 118 retry:
 119         page = __get_meta_page(sbi, index, true);
 120         if (IS_ERR(page)) {
 121                 if (PTR_ERR(page) == -EIO &&
 122                                 ++count <= DEFAULT_RETRY_IO_COUNT)
 123                         goto retry;
 124                 f2fs_stop_checkpoint(sbi, false);
 125         }
 126         return page;
 127 }
 128
 129 /* for POR only */
 130 struct page *f2fs_get_tmp_page(struct f2fs_sb_info *sbi, pgoff_t index)
 131 {
 132         return __get_meta_page(sbi, index, false);
 133 }
 134
 135 static bool __is_bitmap_valid(struct f2fs_sb_info *sbi, block_t blkaddr,
 136                                                         int type)
 137 {
 138         struct seg_entry *se;
 139         unsigned int segno, offset;
 140         bool exist;
 141
 142         if (type == DATA_GENERIC)
 143                 return true;
 144
 145         segno = GET_SEGNO(sbi, blkaddr);
 146         offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
 147         se = get_seg_entry(sbi, segno);
 148
 149         exist = f2fs_test_bit(offset, se->cur_valid_map);
 150         if (exist && type == DATA_GENERIC_ENHANCE_UPDATE) {
 151                 f2fs_err(sbi, "Inconsistent error blkaddr:%u, sit bitmap:%d",
 152                          blkaddr, exist);
 153                 set_sbi_flag(sbi, SBI_NEED_FSCK);
 154                 return exist;
 155         }
 156
 157         if (!exist && type == DATA_GENERIC_ENHANCE) {
 158                 f2fs_err(sbi, "Inconsistent error blkaddr:%u, sit bitmap:%d",
 159                          blkaddr, exist);
 160                 set_sbi_flag(sbi, SBI_NEED_FSCK);
 161                 dump_stack();
 162         }
 163         return exist;
 164 }
 165
 166 bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi,
 167                                         block_t blkaddr, int type)
 168 {
 169         switch (type) {
 170         case META_NAT:
 171                 break;
 172         case META_SIT:
 173                 if (unlikely(blkaddr >= SIT_BLK_CNT(sbi)))
 174                         return false;
 175                 break;
 176         case META_SSA:
 177                 if (unlikely(blkaddr >= MAIN_BLKADDR(sbi) ||
 178                         blkaddr < SM_I(sbi)->ssa_blkaddr))
 179                         return false;
 180                 break;
 181         case META_CP:
 182                 if (unlikely(blkaddr >= SIT_I(sbi)->sit_base_addr ||
 183                         blkaddr < __start_cp_addr(sbi)))
 184                         return false;
 185                 break;
 186         case META_POR:
 187                 if (unlikely(blkaddr >= MAX_BLKADDR(sbi) ||
 188                         blkaddr < MAIN_BLKADDR(sbi)))
 189                         return false;
 190                 break;
 191         case DATA_GENERIC:
 192         case DATA_GENERIC_ENHANCE:
 193         case DATA_GENERIC_ENHANCE_READ:
 194         case DATA_GENERIC_ENHANCE_UPDATE:
 195                 if (unlikely(blkaddr >= MAX_BLKADDR(sbi) ||
 196                                 blkaddr < MAIN_BLKADDR(sbi))) {
 197                         f2fs_warn(sbi, "access invalid blkaddr:%u",
 198                                   blkaddr);
 199                         set_sbi_flag(sbi, SBI_NEED_FSCK);
 200                         dump_stack();
 201                         return false;
 202                 } else {
 203                         return __is_bitmap_valid(sbi, blkaddr, type);
 204                 }
 205                 break;
 206         case META_GENERIC:
 207                 if (unlikely(blkaddr < SEG0_BLKADDR(sbi) ||
 208                         blkaddr >= MAIN_BLKADDR(sbi)))
 209                         return false;
 210                 break;
 211         default:
 212                 BUG();
 213         }
 214
 215         return true;
 216 }
 217
 218 /*
 219  * Readahead CP/NAT/SIT/SSA/POR pages
 220  */
 221 int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages,
 222                                                         int type, bool sync)
 223 {
 224         struct page *page;
 225         block_t blkno = start;
 226         struct f2fs_io_info fio = {
 227                 .sbi = sbi,
 228                 .type = META,
 229                 .op = REQ_OP_READ,
 230                 .op_flags = sync ? (REQ_META | REQ_PRIO) : REQ_RAHEAD,
 231                 .encrypted_page = NULL,
 232                 .in_list = false,
 233                 .is_por = (type == META_POR),
 234         };
 235         struct blk_plug plug;
 236         int err;
 237
 238         if (unlikely(type == META_POR))
 239                 fio.op_flags &= ~REQ_META;
 240
 241         blk_start_plug(&plug);
 242         for (; nrpages-- > 0; blkno++) {
 243
 244                 if (!f2fs_is_valid_blkaddr(sbi, blkno, type))
 245                         goto out;
 246
 247                 switch (type) {
 248                 case META_NAT:
 249                         if (unlikely(blkno >=
 250                                         NAT_BLOCK_OFFSET(NM_I(sbi)->max_nid)))
 251                                 blkno = 0;
 252                         /* get nat block addr */
 253                         fio.new_blkaddr = current_nat_addr(sbi,
 254                                         blkno * NAT_ENTRY_PER_BLOCK);
 255                         break;
 256                 case META_SIT:
 257                         if (unlikely(blkno >= TOTAL_SEGS(sbi)))
 258                                 goto out;
 259                         /* get sit block addr */
 260                         fio.new_blkaddr = current_sit_addr(sbi,
 261                                         blkno * SIT_ENTRY_PER_BLOCK);
 262                         break;
 263                 case META_SSA:
 264                 case META_CP:
 265                 case META_POR:
 266                         fio.new_blkaddr = blkno;
 267                         break;
 268                 default:
 269                         BUG();
 270                 }
 271
 272                 page = f2fs_grab_cache_page(META_MAPPING(sbi),
 273                                                 fio.new_blkaddr, false);
 274                 if (!page)
 275                         continue;
 276                 if (PageUptodate(page)) {
 277                         f2fs_put_page(page, 1);
 278                         continue;
 279                 }
 280
 281                 fio.page = page;
 282                 err = f2fs_submit_page_bio(&fio);
 283                 f2fs_put_page(page, err ? 1 : 0);
 284
 285                 if (!err)
 286                         f2fs_update_iostat(sbi, FS_META_READ_IO, F2FS_BLKSIZE);
 287         }
 288 out:
 289         blk_finish_plug(&plug);
 290         return blkno - start;
 291 }
 292
 293 void f2fs_ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index)
 294 {
 295         struct page *page;
 296         bool readahead = false;
 297
 298         page = find_get_page(META_MAPPING(sbi), index);
 299         if (!page || !PageUptodate(page))
 300                 readahead = true;
 301         f2fs_put_page(page, 0);
 302
 303         if (readahead)
 304                 f2fs_ra_meta_pages(sbi, index, BIO_MAX_VECS, META_POR, true);
 305 }
 306
 307 static int __f2fs_write_meta_page(struct page *page,
 308                                 struct writeback_control *wbc,
 309                                 enum iostat_type io_type)
 310 {
 311         struct f2fs_sb_info *sbi = F2FS_P_SB(page);
 312
 313         trace_f2fs_writepage(page, META);
 314
 315         if (unlikely(f2fs_cp_error(sbi))) {
 316                 if (is_sbi_flag_set(sbi, SBI_IS_CLOSE)) {
 317                         ClearPageUptodate(page);
 318                         dec_page_count(sbi, F2FS_DIRTY_META);
 319                         unlock_page(page);
 320                         return 0;
 321                 }
 322                 goto redirty_out;
 323         }
 324         if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
 325                 goto redirty_out;
 326         if (wbc->for_reclaim && page->index < GET_SUM_BLOCK(sbi, 0))
 327                 goto redirty_out;
 328
 329         f2fs_do_write_meta_page(sbi, page, io_type);
 330         dec_page_count(sbi, F2FS_DIRTY_META);
 331
 332         if (wbc->for_reclaim)
 333                 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, META);
 334
 335         unlock_page(page);
 336
 337         if (unlikely(f2fs_cp_error(sbi)))
 338                 f2fs_submit_merged_write(sbi, META);
 339
 340         return 0;
 341
 342 redirty_out:
 343         redirty_page_for_writepage(wbc, page);
 344         return AOP_WRITEPAGE_ACTIVATE;
 345 }
 346
 347 static int f2fs_write_meta_page(struct page *page,
 348                                 struct writeback_control *wbc)
 349 {
 350         return __f2fs_write_meta_page(page, wbc, FS_META_IO);
 351 }
 352
 353 static int f2fs_write_meta_pages(struct address_space *mapping,
 354                                 struct writeback_control *wbc)
 355 {
 356         struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);
 357         long diff, written;
 358
 359         if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
 360                 goto skip_write;
 361
 362         /* collect a number of dirty meta pages and write together */
 363         if (wbc->sync_mode != WB_SYNC_ALL &&
 364                         get_pages(sbi, F2FS_DIRTY_META) <
 365                                         nr_pages_to_skip(sbi, META))
 366                 goto skip_write;
 367
 368         /* if locked failed, cp will flush dirty pages instead */
 369         if (!down_write_trylock(&sbi->cp_global_sem))
 370                 goto skip_write;
 371
 372         trace_f2fs_writepages(mapping->host, wbc, META);
 373         diff = nr_pages_to_write(sbi, META, wbc);
 374         written = f2fs_sync_meta_pages(sbi, META, wbc->nr_to_write, FS_META_IO);
 375         up_write(&sbi->cp_global_sem);
 376         wbc->nr_to_write = max((long)0, wbc->nr_to_write - written - diff);
 377         return 0;
 378
 379 skip_write:
 380         wbc->pages_skipped += get_pages(sbi, F2FS_DIRTY_META);
 381         trace_f2fs_writepages(mapping->host, wbc, META);
 382         return 0;
 383 }
 384
 385 long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, enum page_type type,
 386                                 long nr_to_write, enum iostat_type io_type)
 387 {
 388         struct address_space *mapping = META_MAPPING(sbi);
 389         pgoff_t index = 0, prev = ULONG_MAX;
 390         struct pagevec pvec;
 391         long nwritten = 0;
 392         int nr_pages;
 393         struct writeback_control wbc = {
 394                 .for_reclaim = 0,
 395         };
 396         struct blk_plug plug;
 397
 398         pagevec_init(&pvec);
 399
 400         blk_start_plug(&plug);
 401
 402         while ((nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
 403                                 PAGECACHE_TAG_DIRTY))) {
 404                 int i;
 405
 406                 for (i = 0; i < nr_pages; i++) {
 407                         struct page *page = pvec.pages[i];
 408
 409                         if (prev == ULONG_MAX)
 410                                 prev = page->index - 1;
 411                         if (nr_to_write != LONG_MAX && page->index != prev + 1) {
 412                                 pagevec_release(&pvec);
 413                                 goto stop;
 414                         }
 415
 416                         lock_page(page);
 417
 418                         if (unlikely(page->mapping != mapping)) {
 419 continue_unlock:
 420                                 unlock_page(page);
 421                                 continue;
 422                         }
 423                         if (!PageDirty(page)) {
 424                                 /* someone wrote it for us */
 425                                 goto continue_unlock;
 426                         }
 427
 428                         f2fs_wait_on_page_writeback(page, META, true, true);
 429
 430                         if (!clear_page_dirty_for_io(page))
 431                                 goto continue_unlock;
 432
 433                         if (__f2fs_write_meta_page(page, &wbc, io_type)) {
 434                                 unlock_page(page);
 435                                 break;
 436                         }
 437                         nwritten++;
 438                         prev = page->index;
 439                         if (unlikely(nwritten >= nr_to_write))
 440                                 break;
 441                 }
 442                 pagevec_release(&pvec);
 443                 cond_resched();
 444         }
 445 stop:
 446         if (nwritten)
 447                 f2fs_submit_merged_write(sbi, type);
 448
 449         blk_finish_plug(&plug);
 450
 451         return nwritten;
 452 }
 453
 454 static int f2fs_set_meta_page_dirty(struct page *page)
 455 {
 456         trace_f2fs_set_page_dirty(page, META);
 457
 458         if (!PageUptodate(page))
 459                 SetPageUptodate(page);
 460         if (!PageDirty(page)) {
 461                 __set_page_dirty_nobuffers(page);
 462                 inc_page_count(F2FS_P_SB(page), F2FS_DIRTY_META);
 463                 set_page_private_reference(page);
 464                 return 1;
 465         }
 466         return 0;
 467 }
 468
 469 const struct address_space_operations f2fs_meta_aops = {
 470         .writepage      = f2fs_write_meta_page,
 471         .writepages     = f2fs_write_meta_pages,
 472         .set_page_dirty = f2fs_set_meta_page_dirty,
 473         .invalidatepage = f2fs_invalidate_page,
 474         .releasepage    = f2fs_release_page,
 475 #ifdef CONFIG_MIGRATION
 476         .migratepage    = f2fs_migrate_page,
 477 #endif
 478 };
 479
 480 static void __add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino,
 481                                                 unsigned int devidx, int type)
 482 {
 483         struct inode_management *im = &sbi->im[type];
 484         struct ino_entry *e = NULL, *new = NULL;
 485
 486         if (type == FLUSH_INO) {
 487                 rcu_read_lock();
 488                 e = radix_tree_lookup(&im->ino_root, ino);
 489                 rcu_read_unlock();
 490         }
 491
 492 retry:
 493         if (!e)
 494                 new = f2fs_kmem_cache_alloc(ino_entry_slab,
 495                                                 GFP_NOFS, true, NULL);
 496
 497         radix_tree_preload(GFP_NOFS | __GFP_NOFAIL);
 498
 499         spin_lock(&im->ino_lock);
 500         e = radix_tree_lookup(&im->ino_root, ino);
 501         if (!e) {
 502                 if (!new) {
 503                         spin_unlock(&im->ino_lock);
 504                         goto retry;
 505                 }
 506                 e = new;
 507                 if (unlikely(radix_tree_insert(&im->ino_root, ino, e)))
 508                         f2fs_bug_on(sbi, 1);
 509
 510                 memset(e, 0, sizeof(struct ino_entry));
 511                 e->ino = ino;
 512
 513                 list_add_tail(&e->list, &im->ino_list);
 514                 if (type != ORPHAN_INO)
 515                         im->ino_num++;
 516         }
 517
 518         if (type == FLUSH_INO)
 519                 f2fs_set_bit(devidx, (char *)&e->dirty_device);
 520
 521         spin_unlock(&im->ino_lock);
 522         radix_tree_preload_end();
 523
 524         if (new && e != new)
 525                 kmem_cache_free(ino_entry_slab, new);
 526 }
 527
 528 static void __remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type)
 529 {
 530         struct inode_management *im = &sbi->im[type];
 531         struct ino_entry *e;
 532
 533         spin_lock(&im->ino_lock);
 534         e = radix_tree_lookup(&im->ino_root, ino);
 535         if (e) {
 536                 list_del(&e->list);
 537                 radix_tree_delete(&im->ino_root, ino);
 538                 im->ino_num--;
 539                 spin_unlock(&im->ino_lock);
 540                 kmem_cache_free(ino_entry_slab, e);
 541                 return;
 542         }
 543         spin_unlock(&im->ino_lock);
 544 }
 545
 546 void f2fs_add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type)
 547 {
 548         /* add new dirty ino entry into list */
 549         __add_ino_entry(sbi, ino, 0, type);
 550 }
 551
 552 void f2fs_remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type)
 553 {
 554         /* remove dirty ino entry from list */
 555         __remove_ino_entry(sbi, ino, type);
 556 }
 557
 558 /* mode should be APPEND_INO, UPDATE_INO or TRANS_DIR_INO */
 559 bool f2fs_exist_written_data(struct f2fs_sb_info *sbi, nid_t ino, int mode)
 560 {
 561         struct inode_management *im = &sbi->im[mode];
 562         struct ino_entry *e;
 563
 564         spin_lock(&im->ino_lock);
 565         e = radix_tree_lookup(&im->ino_root, ino);
 566         spin_unlock(&im->ino_lock);
 567         return e ? true : false;
 568 }
 569
 570 void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all)
 571 {
 572         struct ino_entry *e, *tmp;
 573         int i;
 574
 575         for (i = all ? ORPHAN_INO : APPEND_INO; i < MAX_INO_ENTRY; i++) {
 576                 struct inode_management *im = &sbi->im[i];
 577
 578                 spin_lock(&im->ino_lock);
 579                 list_for_each_entry_safe(e, tmp, &im->ino_list, list) {
 580                         list_del(&e->list);
 581                         radix_tree_delete(&im->ino_root, e->ino);
 582                         kmem_cache_free(ino_entry_slab, e);
 583                         im->ino_num--;
 584                 }
 585                 spin_unlock(&im->ino_lock);
 586         }
 587 }
 588
 589 void f2fs_set_dirty_device(struct f2fs_sb_info *sbi, nid_t ino,
 590                                         unsigned int devidx, int type)
 591 {
 592         __add_ino_entry(sbi, ino, devidx, type);
 593 }
 594
 595 bool f2fs_is_dirty_device(struct f2fs_sb_info *sbi, nid_t ino,
 596                                         unsigned int devidx, int type)
 597 {
 598         struct inode_management *im = &sbi->im[type];
 599         struct ino_entry *e;
 600         bool is_dirty = false;
 601
 602         spin_lock(&im->ino_lock);
 603         e = radix_tree_lookup(&im->ino_root, ino);
 604         if (e && f2fs_test_bit(devidx, (char *)&e->dirty_device))
 605                 is_dirty = true;
 606         spin_unlock(&im->ino_lock);
 607         return is_dirty;
 608 }
 609
 610 int f2fs_acquire_orphan_inode(struct f2fs_sb_info *sbi)
 611 {
 612         struct inode_management *im = &sbi->im[ORPHAN_INO];
 613         int err = 0;
 614
 615         spin_lock(&im->ino_lock);
 616
 617         if (time_to_inject(sbi, FAULT_ORPHAN)) {
 618                 spin_unlock(&im->ino_lock);
 619                 f2fs_show_injection_info(sbi, FAULT_ORPHAN);
 620                 return -ENOSPC;
 621         }
 622
 623         if (unlikely(im->ino_num >= sbi->max_orphans))
 624                 err = -ENOSPC;
 625         else
 626                 im->ino_num++;
 627         spin_unlock(&im->ino_lock);
 628
 629         return err;
 630 }
 631
 632 void f2fs_release_orphan_inode(struct f2fs_sb_info *sbi)
 633 {
 634         struct inode_management *im = &sbi->im[ORPHAN_INO];
 635
 636         spin_lock(&im->ino_lock);
 637         f2fs_bug_on(sbi, im->ino_num == 0);
 638         im->ino_num--;
 639         spin_unlock(&im->ino_lock);
 640 }
 641
 642 void f2fs_add_orphan_inode(struct inode *inode)
 643 {
 644         /* add new orphan ino entry into list */
 645         __add_ino_entry(F2FS_I_SB(inode), inode->i_ino, 0, ORPHAN_INO);
 646         f2fs_update_inode_page(inode);
 647 }
 648
 649 void f2fs_remove_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino)
 650 {
 651         /* remove orphan entry from orphan list */
 652         __remove_ino_entry(sbi, ino, ORPHAN_INO);
 653 }
 654
 655 static int recover_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino)
 656 {
 657         struct inode *inode;
 658         struct node_info ni;
 659         int err;
 660
 661         inode = f2fs_iget_retry(sbi->sb, ino);
 662         if (IS_ERR(inode)) {
 663                 /*
 664                  * there should be a bug that we can't find the entry
 665                  * to orphan inode.
 666                  */
 667                 f2fs_bug_on(sbi, PTR_ERR(inode) == -ENOENT);
 668                 return PTR_ERR(inode);
 669         }
 670
 671         err = f2fs_dquot_initialize(inode);
 672         if (err) {
 673                 iput(inode);
 674                 goto err_out;
 675         }
 676
 677         clear_nlink(inode);
 678
 679         /* truncate all the data during iput */
 680         iput(inode);
 681
 682         err = f2fs_get_node_info(sbi, ino, &ni, false);
 683         if (err)
 684                 goto err_out;
 685
 686         /* ENOMEM was fully retried in f2fs_evict_inode. */
 687         if (ni.blk_addr != NULL_ADDR) {
 688                 err = -EIO;
 689                 goto err_out;
 690         }
 691         return 0;
 692
 693 err_out:
 694         set_sbi_flag(sbi, SBI_NEED_FSCK);
 695         f2fs_warn(sbi, "%s: orphan failed (ino=%x), run fsck to fix.",
 696                   __func__, ino);
 697         return err;
 698 }
 699
 700 int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi)
 701 {
 702         block_t start_blk, orphan_blocks, i, j;
 703         unsigned int s_flags = sbi->sb->s_flags;
 704         int err = 0;
 705 #ifdef CONFIG_QUOTA
 706         int quota_enabled;
 707 #endif
 708
 709         if (!is_set_ckpt_flags(sbi, CP_ORPHAN_PRESENT_FLAG))
 710                 return 0;
 711
 712         if (bdev_read_only(sbi->sb->s_bdev)) {
 713                 f2fs_info(sbi, "write access unavailable, skipping orphan cleanup");
 714                 return 0;
 715         }
 716
 717         if (s_flags & SB_RDONLY) {
 718                 f2fs_info(sbi, "orphan cleanup on readonly fs");
 719                 sbi->sb->s_flags &= ~SB_RDONLY;
 720         }
 721
 722 #ifdef CONFIG_QUOTA
 723         /* Needed for iput() to work correctly and not trash data */
 724         sbi->sb->s_flags |= SB_ACTIVE;
 725
 726         /*
 727          * Turn on quotas which were not enabled for read-only mounts if
 728          * filesystem has quota feature, so that they are updated correctly.
 729          */
 730         quota_enabled = f2fs_enable_quota_files(sbi, s_flags & SB_RDONLY);
 731 #endif
 732
 733         start_blk = __start_cp_addr(sbi) + 1 + __cp_payload(sbi);
 734         orphan_blocks = __start_sum_addr(sbi) - 1 - __cp_payload(sbi);
 735
 736         f2fs_ra_meta_pages(sbi, start_blk, orphan_blocks, META_CP, true);
 737
 738         for (i = 0; i < orphan_blocks; i++) {
 739                 struct page *page;
 740                 struct f2fs_orphan_block *orphan_blk;
 741
 742                 page = f2fs_get_meta_page(sbi, start_blk + i);
 743                 if (IS_ERR(page)) {
 744                         err = PTR_ERR(page);
 745                         goto out;
 746                 }
 747
 748                 orphan_blk = (struct f2fs_orphan_block *)page_address(page);
 749                 for (j = 0; j < le32_to_cpu(orphan_blk->entry_count); j++) {
 750                         nid_t ino = le32_to_cpu(orphan_blk->ino[j]);
 751
 752                         err = recover_orphan_inode(sbi, ino);
 753                         if (err) {
 754                                 f2fs_put_page(page, 1);
 755                                 goto out;
 756                         }
 757                 }
 758                 f2fs_put_page(page, 1);
 759         }
 760         /* clear Orphan Flag */
 761         clear_ckpt_flags(sbi, CP_ORPHAN_PRESENT_FLAG);
 762 out:
 763         set_sbi_flag(sbi, SBI_IS_RECOVERED);
 764
 765 #ifdef CONFIG_QUOTA
 766         /* Turn quotas off */
 767         if (quota_enabled)
 768                 f2fs_quota_off_umount(sbi->sb);
 769 #endif
 770         sbi->sb->s_flags = s_flags; /* Restore SB_RDONLY status */
 771
 772         return err;
 773 }
 774
 775 static void write_orphan_inodes(struct f2fs_sb_info *sbi, block_t start_blk)
 776 {
 777         struct list_head *head;
 778         struct f2fs_orphan_block *orphan_blk = NULL;
 779         unsigned int nentries = 0;
 780         unsigned short index = 1;
 781         unsigned short orphan_blocks;
 782         struct page *page = NULL;
 783         struct ino_entry *orphan = NULL;
 784         struct inode_management *im = &sbi->im[ORPHAN_INO];
 785
 786         orphan_blocks = GET_ORPHAN_BLOCKS(im->ino_num);
 787
 788         /*
 789          * we don't need to do spin_lock(&im->ino_lock) here, since all the
 790          * orphan inode operations are covered under f2fs_lock_op().
 791          * And, spin_lock should be avoided due to page operations below.
 792          */
 793         head = &im->ino_list;
 794
 795         /* loop for each orphan inode entry and write them in Jornal block */
 796         list_for_each_entry(orphan, head, list) {
 797                 if (!page) {
 798                         page = f2fs_grab_meta_page(sbi, start_blk++);
 799                         orphan_blk =
 800                                 (struct f2fs_orphan_block *)page_address(page);
 801                         memset(orphan_blk, 0, sizeof(*orphan_blk));
 802                 }
 803
 804                 orphan_blk->ino[nentries++] = cpu_to_le32(orphan->ino);
 805
 806                 if (nentries == F2FS_ORPHANS_PER_BLOCK) {
 807                         /*
 808                          * an orphan block is full of 1020 entries,
 809                          * then we need to flush current orphan blocks
 810                          * and bring another one in memory
 811                          */
 812                         orphan_blk->blk_addr = cpu_to_le16(index);
 813                         orphan_blk->blk_count = cpu_to_le16(orphan_blocks);
 814                         orphan_blk->entry_count = cpu_to_le32(nentries);
 815                         set_page_dirty(page);
 816                         f2fs_put_page(page, 1);
 817                         index++;
 818                         nentries = 0;
 819                         page = NULL;
 820                 }
 821         }
 822
 823         if (page) {
 824                 orphan_blk->blk_addr = cpu_to_le16(index);
 825                 orphan_blk->blk_count = cpu_to_le16(orphan_blocks);
 826                 orphan_blk->entry_count = cpu_to_le32(nentries);
 827                 set_page_dirty(page);
 828                 f2fs_put_page(page, 1);
 829         }
 830 }
 831
 832 static __u32 f2fs_checkpoint_chksum(struct f2fs_sb_info *sbi,
 833                                                 struct f2fs_checkpoint *ckpt)
 834 {
 835         unsigned int chksum_ofs = le32_to_cpu(ckpt->checksum_offset);
 836         __u32 chksum;
 837
 838         chksum = f2fs_crc32(sbi, ckpt, chksum_ofs);
 839         if (chksum_ofs < CP_CHKSUM_OFFSET) {
 840                 chksum_ofs += sizeof(chksum);
 841                 chksum = f2fs_chksum(sbi, chksum, (__u8 *)ckpt + chksum_ofs,
 842                                                 F2FS_BLKSIZE - chksum_ofs);
 843         }
 844         return chksum;
 845 }
 846
 847 static int get_checkpoint_version(struct f2fs_sb_info *sbi, block_t cp_addr,
 848                 struct f2fs_checkpoint **cp_block, struct page **cp_page,
 849                 unsigned long long *version)
 850 {
 851         size_t crc_offset = 0;
 852         __u32 crc;
 853
 854         *cp_page = f2fs_get_meta_page(sbi, cp_addr);
 855         if (IS_ERR(*cp_page))
 856                 return PTR_ERR(*cp_page);
 857
 858         *cp_block = (struct f2fs_checkpoint *)page_address(*cp_page);
 859
 860         crc_offset = le32_to_cpu((*cp_block)->checksum_offset);
 861         if (crc_offset < CP_MIN_CHKSUM_OFFSET ||
 862                         crc_offset > CP_CHKSUM_OFFSET) {
 863                 f2fs_put_page(*cp_page, 1);
 864                 f2fs_warn(sbi, "invalid crc_offset: %zu", crc_offset);
 865                 return -EINVAL;
 866         }
 867
 868         crc = f2fs_checkpoint_chksum(sbi, *cp_block);
 869         if (crc != cur_cp_crc(*cp_block)) {
 870                 f2fs_put_page(*cp_page, 1);
 871                 f2fs_warn(sbi, "invalid crc value");
 872                 return -EINVAL;
 873         }
 874
 875         *version = cur_cp_version(*cp_block);
 876         return 0;
 877 }
 878
 879 static struct page *validate_checkpoint(struct f2fs_sb_info *sbi,
 880                                 block_t cp_addr, unsigned long long *version)
 881 {
 882         struct page *cp_page_1 = NULL, *cp_page_2 = NULL;
 883         struct f2fs_checkpoint *cp_block = NULL;
 884         unsigned long long cur_version = 0, pre_version = 0;
 885         unsigned int cp_blocks;
 886         int err;
 887
 888         err = get_checkpoint_version(sbi, cp_addr, &cp_block,
 889                                         &cp_page_1, version);
 890         if (err)
 891                 return NULL;
 892
 893         cp_blocks = le32_to_cpu(cp_block->cp_pack_total_block_count);
 894
 895         if (cp_blocks > sbi->blocks_per_seg || cp_blocks <= F2FS_CP_PACKS) {
 896                 f2fs_warn(sbi, "invalid cp_pack_total_block_count:%u",
 897                           le32_to_cpu(cp_block->cp_pack_total_block_count));
 898                 goto invalid_cp;
 899         }
 900         pre_version = *version;
 901
 902         cp_addr += cp_blocks - 1;
 903         err = get_checkpoint_version(sbi, cp_addr, &cp_block,
 904                                         &cp_page_2, version);
 905         if (err)
 906                 goto invalid_cp;
 907         cur_version = *version;
 908
 909         if (cur_version == pre_version) {
 910                 *version = cur_version;
 911                 f2fs_put_page(cp_page_2, 1);
 912                 return cp_page_1;
 913         }
 914         f2fs_put_page(cp_page_2, 1);
 915 invalid_cp:
 916         f2fs_put_page(cp_page_1, 1);
 917         return NULL;
 918 }
 919
 920 int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi)
 921 {
 922         struct f2fs_checkpoint *cp_block;
 923         struct f2fs_super_block *fsb = sbi->raw_super;
 924         struct page *cp1, *cp2, *cur_page;
 925         unsigned long blk_size = sbi->blocksize;
 926         unsigned long long cp1_version = 0, cp2_version = 0;
 927         unsigned long long cp_start_blk_no;
 928         unsigned int cp_blks = 1 + __cp_payload(sbi);
 929         block_t cp_blk_no;
 930         int i;
 931         int err;
 932
 933         sbi->ckpt = f2fs_kvzalloc(sbi, array_size(blk_size, cp_blks),
 934                                   GFP_KERNEL);
 935         if (!sbi->ckpt)
 936                 return -ENOMEM;
 937         /*
 938          * Finding out valid cp block involves read both
 939          * sets( cp pack 1 and cp pack 2)
 940          */
 941         cp_start_blk_no = le32_to_cpu(fsb->cp_blkaddr);
 942         cp1 = validate_checkpoint(sbi, cp_start_blk_no, &cp1_version);
 943
 944         /* The second checkpoint pack should start at the next segment */
 945         cp_start_blk_no += ((unsigned long long)1) <<
 946                                 le32_to_cpu(fsb->log_blocks_per_seg);
 947         cp2 = validate_checkpoint(sbi, cp_start_blk_no, &cp2_version);
 948
 949         if (cp1 && cp2) {
 950                 if (ver_after(cp2_version, cp1_version))
 951                         cur_page = cp2;
 952                 else
 953                         cur_page = cp1;
 954         } else if (cp1) {
 955                 cur_page = cp1;
 956         } else if (cp2) {
 957                 cur_page = cp2;
 958         } else {
 959                 err = -EFSCORRUPTED;
 960                 goto fail_no_cp;
 961         }
 962
 963         cp_block = (struct f2fs_checkpoint *)page_address(cur_page);
 964         memcpy(sbi->ckpt, cp_block, blk_size);
 965
 966         if (cur_page == cp1)
 967                 sbi->cur_cp_pack = 1;
 968         else
 969                 sbi->cur_cp_pack = 2;
 970
 971         /* Sanity checking of checkpoint */
 972         if (f2fs_sanity_check_ckpt(sbi)) {
 973                 err = -EFSCORRUPTED;
 974                 goto free_fail_no_cp;
 975         }
 976
 977         if (cp_blks <= 1)
 978                 goto done;
 979
 980         cp_blk_no = le32_to_cpu(fsb->cp_blkaddr);
 981         if (cur_page == cp2)
 982                 cp_blk_no += 1 << le32_to_cpu(fsb->log_blocks_per_seg);
 983
 984         for (i = 1; i < cp_blks; i++) {
 985                 void *sit_bitmap_ptr;
 986                 unsigned char *ckpt = (unsigned char *)sbi->ckpt;
 987
 988                 cur_page = f2fs_get_meta_page(sbi, cp_blk_no + i);
 989                 if (IS_ERR(cur_page)) {
 990                         err = PTR_ERR(cur_page);
 991                         goto free_fail_no_cp;
 992                 }
 993                 sit_bitmap_ptr = page_address(cur_page);
 994                 memcpy(ckpt + i * blk_size, sit_bitmap_ptr, blk_size);
 995                 f2fs_put_page(cur_page, 1);
 996         }
 997 done:
 998         f2fs_put_page(cp1, 1);
 999         f2fs_put_page(cp2, 1);
1000         return 0;
1001
1002 free_fail_no_cp:
1003         f2fs_put_page(cp1, 1);
1004         f2fs_put_page(cp2, 1);
1005 fail_no_cp:
1006         kvfree(sbi->ckpt);
1007         return err;
1008 }
1009
1010 static void __add_dirty_inode(struct inode *inode, enum inode_type type)
1011 {
1012         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1013         int flag = (type == DIR_INODE) ? FI_DIRTY_DIR : FI_DIRTY_FILE;
1014
1015         if (is_inode_flag_set(inode, flag))
1016                 return;
1017
1018         set_inode_flag(inode, flag);
1019         if (!f2fs_is_volatile_file(inode))
1020                 list_add_tail(&F2FS_I(inode)->dirty_list,
1021                                                 &sbi->inode_list[type]);
1022         stat_inc_dirty_inode(sbi, type);
1023 }
1024
1025 static void __remove_dirty_inode(struct inode *inode, enum inode_type type)
1026 {
1027         int flag = (type == DIR_INODE) ? FI_DIRTY_DIR : FI_DIRTY_FILE;
1028
1029         if (get_dirty_pages(inode) || !is_inode_flag_set(inode, flag))
1030                 return;
1031
1032         list_del_init(&F2FS_I(inode)->dirty_list);
1033         clear_inode_flag(inode, flag);
1034         stat_dec_dirty_inode(F2FS_I_SB(inode), type);
1035 }
1036
1037 void f2fs_update_dirty_page(struct inode *inode, struct page *page)
1038 {
1039         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1040         enum inode_type type = S_ISDIR(inode->i_mode) ? DIR_INODE : FILE_INODE;
1041
1042         if (!S_ISDIR(inode->i_mode) && !S_ISREG(inode->i_mode) &&
1043                         !S_ISLNK(inode->i_mode))
1044                 return;
1045
1046         spin_lock(&sbi->inode_lock[type]);
1047         if (type != FILE_INODE || test_opt(sbi, DATA_FLUSH))
1048                 __add_dirty_inode(inode, type);
1049         inode_inc_dirty_pages(inode);
1050         spin_unlock(&sbi->inode_lock[type]);
1051
1052         set_page_private_reference(page);
1053 }
1054
1055 void f2fs_remove_dirty_inode(struct inode *inode)
1056 {
1057         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1058         enum inode_type type = S_ISDIR(inode->i_mode) ? DIR_INODE : FILE_INODE;
1059
1060         if (!S_ISDIR(inode->i_mode) && !S_ISREG(inode->i_mode) &&
1061                         !S_ISLNK(inode->i_mode))
1062                 return;
1063
1064         if (type == FILE_INODE && !test_opt(sbi, DATA_FLUSH))
1065                 return;
1066
1067         spin_lock(&sbi->inode_lock[type]);
1068         __remove_dirty_inode(inode, type);
1069         spin_unlock(&sbi->inode_lock[type]);
1070 }
1071
1072 int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type,
1073                                                 bool from_cp)
1074 {
1075         struct list_head *head;
1076         struct inode *inode;
1077         struct f2fs_inode_info *fi;
1078         bool is_dir = (type == DIR_INODE);
1079         unsigned long ino = 0;
1080
1081         trace_f2fs_sync_dirty_inodes_enter(sbi->sb, is_dir,
1082                                 get_pages(sbi, is_dir ?
1083                                 F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA));
1084 retry:
1085         if (unlikely(f2fs_cp_error(sbi))) {
1086                 trace_f2fs_sync_dirty_inodes_exit(sbi->sb, is_dir,
1087                                 get_pages(sbi, is_dir ?
1088                                 F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA));
1089                 return -EIO;
1090         }
1091
1092         spin_lock(&sbi->inode_lock[type]);
1093
1094         head = &sbi->inode_list[type];
1095         if (list_empty(head)) {
1096                 spin_unlock(&sbi->inode_lock[type]);
1097                 trace_f2fs_sync_dirty_inodes_exit(sbi->sb, is_dir,
1098                                 get_pages(sbi, is_dir ?
1099                                 F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA));
1100                 return 0;
1101         }
1102         fi = list_first_entry(head, struct f2fs_inode_info, dirty_list);
1103         inode = igrab(&fi->vfs_inode);
1104         spin_unlock(&sbi->inode_lock[type]);
1105         if (inode) {
1106                 unsigned long cur_ino = inode->i_ino;
1107
1108                 if (from_cp)
1109                         F2FS_I(inode)->cp_task = current;
1110                 F2FS_I(inode)->wb_task = current;
1111
1112                 filemap_fdatawrite(inode->i_mapping);
1113
1114                 F2FS_I(inode)->wb_task = NULL;
1115                 if (from_cp)
1116                         F2FS_I(inode)->cp_task = NULL;
1117
1118                 iput(inode);
1119                 /* We need to give cpu to another writers. */
1120                 if (ino == cur_ino)
1121                         cond_resched();
1122                 else
1123                         ino = cur_ino;
1124         } else {
1125                 /*
1126                  * We should submit bio, since it exists several
1127                  * wribacking dentry pages in the freeing inode.
1128                  */
1129                 f2fs_submit_merged_write(sbi, DATA);
1130                 cond_resched();
1131         }
1132         goto retry;
1133 }
1134
1135 int f2fs_sync_inode_meta(struct f2fs_sb_info *sbi)
1136 {
1137         struct list_head *head = &sbi->inode_list[DIRTY_META];
1138         struct inode *inode;
1139         struct f2fs_inode_info *fi;
1140         s64 total = get_pages(sbi, F2FS_DIRTY_IMETA);
1141
1142         while (total--) {
1143                 if (unlikely(f2fs_cp_error(sbi)))
1144                         return -EIO;
1145
1146                 spin_lock(&sbi->inode_lock[DIRTY_META]);
1147                 if (list_empty(head)) {
1148                         spin_unlock(&sbi->inode_lock[DIRTY_META]);
1149                         return 0;
1150                 }
1151                 fi = list_first_entry(head, struct f2fs_inode_info,
1152                                                         gdirty_list);
1153                 inode = igrab(&fi->vfs_inode);
1154                 spin_unlock(&sbi->inode_lock[DIRTY_META]);
1155                 if (inode) {
1156                         sync_inode_metadata(inode, 0);
1157
1158                         /* it's on eviction */
1159                         if (is_inode_flag_set(inode, FI_DIRTY_INODE))
1160                                 f2fs_update_inode_page(inode);
1161                         iput(inode);
1162                 }
1163         }
1164         return 0;
1165 }
1166
1167 static void __prepare_cp_block(struct f2fs_sb_info *sbi)
1168 {
1169         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1170         struct f2fs_nm_info *nm_i = NM_I(sbi);
1171         nid_t last_nid = nm_i->next_scan_nid;
1172
1173         next_free_nid(sbi, &last_nid);
1174         ckpt->valid_block_count = cpu_to_le64(valid_user_blocks(sbi));
1175         ckpt->valid_node_count = cpu_to_le32(valid_node_count(sbi));
1176         ckpt->valid_inode_count = cpu_to_le32(valid_inode_count(sbi));
1177         ckpt->next_free_nid = cpu_to_le32(last_nid);
1178 }
1179
1180 static bool __need_flush_quota(struct f2fs_sb_info *sbi)
1181 {
1182         bool ret = false;
1183
1184         if (!is_journalled_quota(sbi))
1185                 return false;
1186
1187         if (!down_write_trylock(&sbi->quota_sem))
1188                 return true;
1189         if (is_sbi_flag_set(sbi, SBI_QUOTA_SKIP_FLUSH)) {
1190                 ret = false;
1191         } else if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_REPAIR)) {
1192                 ret = false;
1193         } else if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_FLUSH)) {
1194                 clear_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH);
1195                 ret = true;
1196         } else if (get_pages(sbi, F2FS_DIRTY_QDATA)) {
1197                 ret = true;
1198         }
1199         up_write(&sbi->quota_sem);
1200         return ret;
1201 }
1202
1203 /*
1204  * Freeze all the FS-operations for checkpoint.
1205  */
1206 static int block_operations(struct f2fs_sb_info *sbi)
1207 {
1208         struct writeback_control wbc = {
1209                 .sync_mode = WB_SYNC_ALL,
1210                 .nr_to_write = LONG_MAX,
1211                 .for_reclaim = 0,
1212         };
1213         int err = 0, cnt = 0;
1214
1215         /*
1216          * Let's flush inline_data in dirty node pages.
1217          */
1218         f2fs_flush_inline_data(sbi);
1219
1220 retry_flush_quotas:
1221         f2fs_lock_all(sbi);
1222         if (__need_flush_quota(sbi)) {
1223                 int locked;
1224
1225                 if (++cnt > DEFAULT_RETRY_QUOTA_FLUSH_COUNT) {
1226                         set_sbi_flag(sbi, SBI_QUOTA_SKIP_FLUSH);
1227                         set_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH);
1228                         goto retry_flush_dents;
1229                 }
1230                 f2fs_unlock_all(sbi);
1231
1232                 /* only failed during mount/umount/freeze/quotactl */
1233                 locked = down_read_trylock(&sbi->sb->s_umount);
1234                 f2fs_quota_sync(sbi->sb, -1);
1235                 if (locked)
1236                         up_read(&sbi->sb->s_umount);
1237                 cond_resched();
1238                 goto retry_flush_quotas;
1239         }
1240
1241 retry_flush_dents:
1242         /* write all the dirty dentry pages */
1243         if (get_pages(sbi, F2FS_DIRTY_DENTS)) {
1244                 f2fs_unlock_all(sbi);
1245                 err = f2fs_sync_dirty_inodes(sbi, DIR_INODE, true);
1246                 if (err)
1247                         return err;
1248                 cond_resched();
1249                 goto retry_flush_quotas;
1250         }
1251
1252         /*
1253          * POR: we should ensure that there are no dirty node pages
1254          * until finishing nat/sit flush. inode->i_blocks can be updated.
1255          */
1256         down_write(&sbi->node_change);
1257
1258         if (get_pages(sbi, F2FS_DIRTY_IMETA)) {
1259                 up_write(&sbi->node_change);
1260                 f2fs_unlock_all(sbi);
1261                 err = f2fs_sync_inode_meta(sbi);
1262                 if (err)
1263                         return err;
1264                 cond_resched();
1265                 goto retry_flush_quotas;
1266         }
1267
1268 retry_flush_nodes:
1269         down_write(&sbi->node_write);
1270
1271         if (get_pages(sbi, F2FS_DIRTY_NODES)) {
1272                 up_write(&sbi->node_write);
1273                 atomic_inc(&sbi->wb_sync_req[NODE]);
1274                 err = f2fs_sync_node_pages(sbi, &wbc, false, FS_CP_NODE_IO);
1275                 atomic_dec(&sbi->wb_sync_req[NODE]);
1276                 if (err) {
1277                         up_write(&sbi->node_change);
1278                         f2fs_unlock_all(sbi);
1279                         return err;
1280                 }
1281                 cond_resched();
1282                 goto retry_flush_nodes;
1283         }
1284
1285         /*
1286          * sbi->node_change is used only for AIO write_begin path which produces
1287          * dirty node blocks and some checkpoint values by block allocation.
1288          */
1289         __prepare_cp_block(sbi);
1290         up_write(&sbi->node_change);
1291         return err;
1292 }
1293
1294 static void unblock_operations(struct f2fs_sb_info *sbi)
1295 {
1296         up_write(&sbi->node_write);
1297         f2fs_unlock_all(sbi);
1298 }
1299
1300 void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type)
1301 {
1302         DEFINE_WAIT(wait);
1303
1304         for (;;) {
1305                 if (!get_pages(sbi, type))
1306                         break;
1307
1308                 if (unlikely(f2fs_cp_error(sbi) &&
1309                         !is_sbi_flag_set(sbi, SBI_IS_CLOSE)))
1310                         break;
1311
1312                 if (type == F2FS_DIRTY_META)
1313                         f2fs_sync_meta_pages(sbi, META, LONG_MAX,
1314                                                         FS_CP_META_IO);
1315                 else if (type == F2FS_WB_CP_DATA)
1316                         f2fs_submit_merged_write(sbi, DATA);
1317
1318                 prepare_to_wait(&sbi->cp_wait, &wait, TASK_UNINTERRUPTIBLE);
1319                 io_schedule_timeout(DEFAULT_IO_TIMEOUT);
1320         }
1321         finish_wait(&sbi->cp_wait, &wait);
1322 }
1323
1324 static void update_ckpt_flags(struct f2fs_sb_info *sbi, struct cp_control *cpc)
1325 {
1326         unsigned long orphan_num = sbi->im[ORPHAN_INO].ino_num;
1327         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1328         unsigned long flags;
1329
1330         if (cpc->reason & CP_UMOUNT) {
1331                 if (le32_to_cpu(ckpt->cp_pack_total_block_count) +
1332                         NM_I(sbi)->nat_bits_blocks > sbi->blocks_per_seg) {
1333                         clear_ckpt_flags(sbi, CP_NAT_BITS_FLAG);
1334                         f2fs_notice(sbi, "Disable nat_bits due to no space");
1335                 } else if (!is_set_ckpt_flags(sbi, CP_NAT_BITS_FLAG) &&
1336                                                 f2fs_nat_bitmap_enabled(sbi)) {
1337                         f2fs_enable_nat_bits(sbi);
1338                         set_ckpt_flags(sbi, CP_NAT_BITS_FLAG);
1339                         f2fs_notice(sbi, "Rebuild and enable nat_bits");
1340                 }
1341         }
1342
1343         spin_lock_irqsave(&sbi->cp_lock, flags);
1344
1345         if (cpc->reason & CP_TRIMMED)
1346                 __set_ckpt_flags(ckpt, CP_TRIMMED_FLAG);
1347         else
1348                 __clear_ckpt_flags(ckpt, CP_TRIMMED_FLAG);
1349
1350         if (cpc->reason & CP_UMOUNT)
1351                 __set_ckpt_flags(ckpt, CP_UMOUNT_FLAG);
1352         else
1353                 __clear_ckpt_flags(ckpt, CP_UMOUNT_FLAG);
1354
1355         if (cpc->reason & CP_FASTBOOT)
1356                 __set_ckpt_flags(ckpt, CP_FASTBOOT_FLAG);
1357         else
1358                 __clear_ckpt_flags(ckpt, CP_FASTBOOT_FLAG);
1359
1360         if (orphan_num)
1361                 __set_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG);
1362         else
1363                 __clear_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG);
1364
1365         if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))
1366                 __set_ckpt_flags(ckpt, CP_FSCK_FLAG);
1367
1368         if (is_sbi_flag_set(sbi, SBI_IS_RESIZEFS))
1369                 __set_ckpt_flags(ckpt, CP_RESIZEFS_FLAG);
1370         else
1371                 __clear_ckpt_flags(ckpt, CP_RESIZEFS_FLAG);
1372
1373         if (is_sbi_flag_set(sbi, SBI_CP_DISABLED))
1374                 __set_ckpt_flags(ckpt, CP_DISABLED_FLAG);
1375         else
1376                 __clear_ckpt_flags(ckpt, CP_DISABLED_FLAG);
1377
1378         if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK))
1379                 __set_ckpt_flags(ckpt, CP_DISABLED_QUICK_FLAG);
1380         else
1381                 __clear_ckpt_flags(ckpt, CP_DISABLED_QUICK_FLAG);
1382
1383         if (is_sbi_flag_set(sbi, SBI_QUOTA_SKIP_FLUSH))
1384                 __set_ckpt_flags(ckpt, CP_QUOTA_NEED_FSCK_FLAG);
1385         else
1386                 __clear_ckpt_flags(ckpt, CP_QUOTA_NEED_FSCK_FLAG);
1387
1388         if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_REPAIR))
1389                 __set_ckpt_flags(ckpt, CP_QUOTA_NEED_FSCK_FLAG);
1390
1391         /* set this flag to activate crc|cp_ver for recovery */
1392         __set_ckpt_flags(ckpt, CP_CRC_RECOVERY_FLAG);
1393         __clear_ckpt_flags(ckpt, CP_NOCRC_RECOVERY_FLAG);
1394
1395         spin_unlock_irqrestore(&sbi->cp_lock, flags);
1396 }
1397
1398 static void commit_checkpoint(struct f2fs_sb_info *sbi,
1399         void *src, block_t blk_addr)
1400 {
1401         struct writeback_control wbc = {
1402                 .for_reclaim = 0,
1403         };
1404
1405         /*
1406          * pagevec_lookup_tag and lock_page again will take
1407          * some extra time. Therefore, f2fs_update_meta_pages and
1408          * f2fs_sync_meta_pages are combined in this function.
1409          */
1410         struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
1411         int err;
1412
1413         f2fs_wait_on_page_writeback(page, META, true, true);
1414
1415         memcpy(page_address(page), src, PAGE_SIZE);
1416
1417         set_page_dirty(page);
1418         if (unlikely(!clear_page_dirty_for_io(page)))
1419                 f2fs_bug_on(sbi, 1);
1420
1421         /* writeout cp pack 2 page */
1422         err = __f2fs_write_meta_page(page, &wbc, FS_CP_META_IO);
1423         if (unlikely(err && f2fs_cp_error(sbi))) {
1424                 f2fs_put_page(page, 1);
1425                 return;
1426         }
1427
1428         f2fs_bug_on(sbi, err);
1429         f2fs_put_page(page, 0);
1430
1431         /* submit checkpoint (with barrier if NOBARRIER is not set) */
1432         f2fs_submit_merged_write(sbi, META_FLUSH);
1433 }
1434
1435 static inline u64 get_sectors_written(struct block_device *bdev)
1436 {
1437         return (u64)part_stat_read(bdev, sectors[STAT_WRITE]);
1438 }
1439
1440 u64 f2fs_get_sectors_written(struct f2fs_sb_info *sbi)
1441 {
1442         if (f2fs_is_multi_device(sbi)) {
1443                 u64 sectors = 0;
1444                 int i;
1445
1446                 for (i = 0; i < sbi->s_ndevs; i++)
1447                         sectors += get_sectors_written(FDEV(i).bdev);
1448
1449                 return sectors;
1450         }
1451
1452         return get_sectors_written(sbi->sb->s_bdev);
1453 }
1454
1455 static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
1456 {
1457         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1458         struct f2fs_nm_info *nm_i = NM_I(sbi);
1459         unsigned long orphan_num = sbi->im[ORPHAN_INO].ino_num, flags;
1460         block_t start_blk;
1461         unsigned int data_sum_blocks, orphan_blocks;
1462         __u32 crc32 = 0;
1463         int i;
1464         int cp_payload_blks = __cp_payload(sbi);
1465         struct curseg_info *seg_i = CURSEG_I(sbi, CURSEG_HOT_NODE);
1466         u64 kbytes_written;
1467         int err;
1468
1469         /* Flush all the NAT/SIT pages */
1470         f2fs_sync_meta_pages(sbi, META, LONG_MAX, FS_CP_META_IO);
1471
1472         /* start to update checkpoint, cp ver is already updated previously */
1473         ckpt->elapsed_time = cpu_to_le64(get_mtime(sbi, true));
1474         ckpt->free_segment_count = cpu_to_le32(free_segments(sbi));
1475         for (i = 0; i < NR_CURSEG_NODE_TYPE; i++) {
1476                 ckpt->cur_node_segno[i] =
1477                         cpu_to_le32(curseg_segno(sbi, i + CURSEG_HOT_NODE));
1478                 ckpt->cur_node_blkoff[i] =
1479                         cpu_to_le16(curseg_blkoff(sbi, i + CURSEG_HOT_NODE));
1480                 ckpt->alloc_type[i + CURSEG_HOT_NODE] =
1481                                 curseg_alloc_type(sbi, i + CURSEG_HOT_NODE);
1482         }
1483         for (i = 0; i < NR_CURSEG_DATA_TYPE; i++) {
1484                 ckpt->cur_data_segno[i] =
1485                         cpu_to_le32(curseg_segno(sbi, i + CURSEG_HOT_DATA));
1486                 ckpt->cur_data_blkoff[i] =
1487                         cpu_to_le16(curseg_blkoff(sbi, i + CURSEG_HOT_DATA));
1488                 ckpt->alloc_type[i + CURSEG_HOT_DATA] =
1489                                 curseg_alloc_type(sbi, i + CURSEG_HOT_DATA);
1490         }
1491
1492         /* 2 cp + n data seg summary + orphan inode blocks */
1493         data_sum_blocks = f2fs_npages_for_summary_flush(sbi, false);
1494         spin_lock_irqsave(&sbi->cp_lock, flags);
1495         if (data_sum_blocks < NR_CURSEG_DATA_TYPE)
1496                 __set_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG);
1497         else
1498                 __clear_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG);
1499         spin_unlock_irqrestore(&sbi->cp_lock, flags);
1500
1501         orphan_blocks = GET_ORPHAN_BLOCKS(orphan_num);
1502         ckpt->cp_pack_start_sum = cpu_to_le32(1 + cp_payload_blks +
1503                         orphan_blocks);
1504
1505         if (__remain_node_summaries(cpc->reason))
1506                 ckpt->cp_pack_total_block_count = cpu_to_le32(F2FS_CP_PACKS +
1507                                 cp_payload_blks + data_sum_blocks +
1508                                 orphan_blocks + NR_CURSEG_NODE_TYPE);
1509         else
1510                 ckpt->cp_pack_total_block_count = cpu_to_le32(F2FS_CP_PACKS +
1511                                 cp_payload_blks + data_sum_blocks +
1512                                 orphan_blocks);
1513
1514         /* update ckpt flag for checkpoint */
1515         update_ckpt_flags(sbi, cpc);
1516
1517         /* update SIT/NAT bitmap */
1518         get_sit_bitmap(sbi, __bitmap_ptr(sbi, SIT_BITMAP));
1519         get_nat_bitmap(sbi, __bitmap_ptr(sbi, NAT_BITMAP));
1520
1521         crc32 = f2fs_checkpoint_chksum(sbi, ckpt);
1522         *((__le32 *)((unsigned char *)ckpt +
1523                                 le32_to_cpu(ckpt->checksum_offset)))
1524                                 = cpu_to_le32(crc32);
1525
1526         start_blk = __start_cp_next_addr(sbi);
1527
1528         /* write nat bits */
1529         if ((cpc->reason & CP_UMOUNT) &&
1530                         is_set_ckpt_flags(sbi, CP_NAT_BITS_FLAG)) {
1531                 __u64 cp_ver = cur_cp_version(ckpt);
1532                 block_t blk;
1533
1534                 cp_ver |= ((__u64)crc32 << 32);
1535                 *(__le64 *)nm_i->nat_bits = cpu_to_le64(cp_ver);
1536
1537                 blk = start_blk + sbi->blocks_per_seg - nm_i->nat_bits_blocks;
1538                 for (i = 0; i < nm_i->nat_bits_blocks; i++)
1539                         f2fs_update_meta_page(sbi, nm_i->nat_bits +
1540                                         (i << F2FS_BLKSIZE_BITS), blk + i);
1541         }
1542
1543         /* write out checkpoint buffer at block 0 */
1544         f2fs_update_meta_page(sbi, ckpt, start_blk++);
1545
1546         for (i = 1; i < 1 + cp_payload_blks; i++)
1547                 f2fs_update_meta_page(sbi, (char *)ckpt + i * F2FS_BLKSIZE,
1548                                                         start_blk++);
1549
1550         if (orphan_num) {
1551                 write_orphan_inodes(sbi, start_blk);
1552                 start_blk += orphan_blocks;
1553         }
1554
1555         f2fs_write_data_summaries(sbi, start_blk);
1556         start_blk += data_sum_blocks;
1557
1558         /* Record write statistics in the hot node summary */
1559         kbytes_written = sbi->kbytes_written;
1560         kbytes_written += (f2fs_get_sectors_written(sbi) -
1561                                 sbi->sectors_written_start) >> 1;
1562         seg_i->journal->info.kbytes_written = cpu_to_le64(kbytes_written);
1563
1564         if (__remain_node_summaries(cpc->reason)) {
1565                 f2fs_write_node_summaries(sbi, start_blk);
1566                 start_blk += NR_CURSEG_NODE_TYPE;
1567         }
1568
1569         /* update user_block_counts */
1570         sbi->last_valid_block_count = sbi->total_valid_block_count;
1571         percpu_counter_set(&sbi->alloc_valid_block_count, 0);
1572
1573         /* Here, we have one bio having CP pack except cp pack 2 page */
1574         f2fs_sync_meta_pages(sbi, META, LONG_MAX, FS_CP_META_IO);
1575         /* Wait for all dirty meta pages to be submitted for IO */
1576         f2fs_wait_on_all_pages(sbi, F2FS_DIRTY_META);
1577
1578         /* wait for previous submitted meta pages writeback */
1579         f2fs_wait_on_all_pages(sbi, F2FS_WB_CP_DATA);
1580
1581         /* flush all device cache */
1582         err = f2fs_flush_device_cache(sbi);
1583         if (err)
1584                 return err;
1585
1586         /* barrier and flush checkpoint cp pack 2 page if it can */
1587         commit_checkpoint(sbi, ckpt, start_blk);
1588         f2fs_wait_on_all_pages(sbi, F2FS_WB_CP_DATA);
1589
1590         /*
1591          * invalidate intermediate page cache borrowed from meta inode which are
1592          * used for migration of encrypted, verity or compressed inode's blocks.
1593          */
1594         if (f2fs_sb_has_encrypt(sbi) || f2fs_sb_has_verity(sbi) ||
1595                 f2fs_sb_has_compression(sbi))
1596                 invalidate_mapping_pages(META_MAPPING(sbi),
1597                                 MAIN_BLKADDR(sbi), MAX_BLKADDR(sbi) - 1);
1598
1599         f2fs_release_ino_entry(sbi, false);
1600
1601         f2fs_reset_fsync_node_info(sbi);
1602
1603         clear_sbi_flag(sbi, SBI_IS_DIRTY);
1604         clear_sbi_flag(sbi, SBI_NEED_CP);
1605         clear_sbi_flag(sbi, SBI_QUOTA_SKIP_FLUSH);
1606
1607         spin_lock(&sbi->stat_lock);
1608         sbi->unusable_block_count = 0;
1609         spin_unlock(&sbi->stat_lock);
1610
1611         __set_cp_next_pack(sbi);
1612
1613         /*
1614          * redirty superblock if metadata like node page or inode cache is
1615          * updated during writing checkpoint.
1616          */
1617         if (get_pages(sbi, F2FS_DIRTY_NODES) ||
1618                         get_pages(sbi, F2FS_DIRTY_IMETA))
1619                 set_sbi_flag(sbi, SBI_IS_DIRTY);
1620
1621         f2fs_bug_on(sbi, get_pages(sbi, F2FS_DIRTY_DENTS));
1622
1623         return unlikely(f2fs_cp_error(sbi)) ? -EIO : 0;
1624 }
1625
1626 int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
1627 {
1628         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1629         unsigned long long ckpt_ver;
1630         int err = 0;
1631
1632         if (f2fs_readonly(sbi->sb) || f2fs_hw_is_readonly(sbi))
1633                 return -EROFS;
1634
1635         if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
1636                 if (cpc->reason != CP_PAUSE)
1637                         return 0;
1638                 f2fs_warn(sbi, "Start checkpoint disabled!");
1639         }
1640         if (cpc->reason != CP_RESIZE)
1641                 down_write(&sbi->cp_global_sem);
1642
1643         if (!is_sbi_flag_set(sbi, SBI_IS_DIRTY) &&
1644                 ((cpc->reason & CP_FASTBOOT) || (cpc->reason & CP_SYNC) ||
1645                 ((cpc->reason & CP_DISCARD) && !sbi->discard_blks)))
1646                 goto out;
1647         if (unlikely(f2fs_cp_error(sbi))) {
1648                 err = -EIO;
1649                 goto out;
1650         }
1651
1652         trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, "start block_ops");
1653
1654         err = block_operations(sbi);
1655         if (err)
1656                 goto out;
1657
1658         trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, "finish block_ops");
1659
1660         f2fs_flush_merged_writes(sbi);
1661
1662         /* this is the case of multiple fstrims without any changes */
1663         if (cpc->reason & CP_DISCARD) {
1664                 if (!f2fs_exist_trim_candidates(sbi, cpc)) {
1665                         unblock_operations(sbi);
1666                         goto out;
1667                 }
1668
1669                 if (NM_I(sbi)->nat_cnt[DIRTY_NAT] == 0 &&
1670                                 SIT_I(sbi)->dirty_sentries == 0 &&
1671                                 prefree_segments(sbi) == 0) {
1672                         f2fs_flush_sit_entries(sbi, cpc);
1673                         f2fs_clear_prefree_segments(sbi, cpc);
1674                         unblock_operations(sbi);
1675                         goto out;
1676                 }
1677         }
1678
1679         /*
1680          * update checkpoint pack index
1681          * Increase the version number so that
1682          * SIT entries and seg summaries are written at correct place
1683          */
1684         ckpt_ver = cur_cp_version(ckpt);
1685         ckpt->checkpoint_ver = cpu_to_le64(++ckpt_ver);
1686
1687         /* write cached NAT/SIT entries to NAT/SIT area */
1688         err = f2fs_flush_nat_entries(sbi, cpc);
1689         if (err) {
1690                 f2fs_err(sbi, "f2fs_flush_nat_entries failed err:%d, stop checkpoint", err);
1691                 f2fs_bug_on(sbi, !f2fs_cp_error(sbi));
1692                 goto stop;
1693         }
1694
1695         f2fs_flush_sit_entries(sbi, cpc);
1696
1697         /* save inmem log status */
1698         f2fs_save_inmem_curseg(sbi);
1699
1700         err = do_checkpoint(sbi, cpc);
1701         if (err) {
1702                 f2fs_err(sbi, "do_checkpoint failed err:%d, stop checkpoint", err);
1703                 f2fs_bug_on(sbi, !f2fs_cp_error(sbi));
1704                 f2fs_release_discard_addrs(sbi);
1705         } else {
1706                 f2fs_clear_prefree_segments(sbi, cpc);
1707         }
1708
1709         f2fs_restore_inmem_curseg(sbi);
1710 stop:
1711         unblock_operations(sbi);
1712         stat_inc_cp_count(sbi->stat_info);
1713
1714         if (cpc->reason & CP_RECOVERY)
1715                 f2fs_notice(sbi, "checkpoint: version = %llx", ckpt_ver);
1716
1717         /* update CP_TIME to trigger checkpoint periodically */
1718         f2fs_update_time(sbi, CP_TIME);
1719         trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, "finish checkpoint");
1720 out:
1721         if (cpc->reason != CP_RESIZE)
1722                 up_write(&sbi->cp_global_sem);
1723         return err;
1724 }
1725
1726 void f2fs_init_ino_entry_info(struct f2fs_sb_info *sbi)
1727 {
1728         int i;
1729
1730         for (i = 0; i < MAX_INO_ENTRY; i++) {
1731                 struct inode_management *im = &sbi->im[i];
1732
1733                 INIT_RADIX_TREE(&im->ino_root, GFP_ATOMIC);
1734                 spin_lock_init(&im->ino_lock);
1735                 INIT_LIST_HEAD(&im->ino_list);
1736                 im->ino_num = 0;
1737         }
1738
1739         sbi->max_orphans = (sbi->blocks_per_seg - F2FS_CP_PACKS -
1740                         NR_CURSEG_PERSIST_TYPE - __cp_payload(sbi)) *
1741                                 F2FS_ORPHANS_PER_BLOCK;
1742 }
1743
1744 int __init f2fs_create_checkpoint_caches(void)
1745 {
1746         ino_entry_slab = f2fs_kmem_cache_create("f2fs_ino_entry",
1747                         sizeof(struct ino_entry));
1748         if (!ino_entry_slab)
1749                 return -ENOMEM;
1750         f2fs_inode_entry_slab = f2fs_kmem_cache_create("f2fs_inode_entry",
1751                         sizeof(struct inode_entry));
1752         if (!f2fs_inode_entry_slab) {
1753                 kmem_cache_destroy(ino_entry_slab);
1754                 return -ENOMEM;
1755         }
1756         return 0;
1757 }
1758
1759 void f2fs_destroy_checkpoint_caches(void)
1760 {
1761         kmem_cache_destroy(ino_entry_slab);
1762         kmem_cache_destroy(f2fs_inode_entry_slab);
1763 }
1764
1765 static int __write_checkpoint_sync(struct f2fs_sb_info *sbi)
1766 {
1767         struct cp_control cpc = { .reason = CP_SYNC, };
1768         int err;
1769
1770         down_write(&sbi->gc_lock);
1771         err = f2fs_write_checkpoint(sbi, &cpc);
1772         up_write(&sbi->gc_lock);
1773
1774         return err;
1775 }
1776
1777 static void __checkpoint_and_complete_reqs(struct f2fs_sb_info *sbi)
1778 {
1779         struct ckpt_req_control *cprc = &sbi->cprc_info;
1780         struct ckpt_req *req, *next;
1781         struct llist_node *dispatch_list;
1782         u64 sum_diff = 0, diff, count = 0;
1783         int ret;
1784
1785         dispatch_list = llist_del_all(&cprc->issue_list);
1786         if (!dispatch_list)
1787                 return;
1788         dispatch_list = llist_reverse_order(dispatch_list);
1789
1790         ret = __write_checkpoint_sync(sbi);
1791         atomic_inc(&cprc->issued_ckpt);
1792
1793         llist_for_each_entry_safe(req, next, dispatch_list, llnode) {
1794                 diff = (u64)ktime_ms_delta(ktime_get(), req->queue_time);
1795                 req->ret = ret;
1796                 complete(&req->wait);
1797
1798                 sum_diff += diff;
1799                 count++;
1800         }
1801         atomic_sub(count, &cprc->queued_ckpt);
1802         atomic_add(count, &cprc->total_ckpt);
1803
1804         spin_lock(&cprc->stat_lock);
1805         cprc->cur_time = (unsigned int)div64_u64(sum_diff, count);
1806         if (cprc->peak_time < cprc->cur_time)
1807                 cprc->peak_time = cprc->cur_time;
1808         spin_unlock(&cprc->stat_lock);
1809 }
1810
1811 static int issue_checkpoint_thread(void *data)
1812 {
1813         struct f2fs_sb_info *sbi = data;
1814         struct ckpt_req_control *cprc = &sbi->cprc_info;
1815         wait_queue_head_t *q = &cprc->ckpt_wait_queue;
1816 repeat:
1817         if (kthread_should_stop())
1818                 return 0;
1819
1820         if (!llist_empty(&cprc->issue_list))
1821                 __checkpoint_and_complete_reqs(sbi);
1822
1823         wait_event_interruptible(*q,
1824                 kthread_should_stop() || !llist_empty(&cprc->issue_list));
1825         goto repeat;
1826 }
1827
1828 static void flush_remained_ckpt_reqs(struct f2fs_sb_info *sbi,
1829                 struct ckpt_req *wait_req)
1830 {
1831         struct ckpt_req_control *cprc = &sbi->cprc_info;
1832
1833         if (!llist_empty(&cprc->issue_list)) {
1834                 __checkpoint_and_complete_reqs(sbi);
1835         } else {
1836                 /* already dispatched by issue_checkpoint_thread */
1837                 if (wait_req)
1838                         wait_for_completion(&wait_req->wait);
1839         }
1840 }
1841
1842 static void init_ckpt_req(struct ckpt_req *req)
1843 {
1844         memset(req, 0, sizeof(struct ckpt_req));
1845
1846         init_completion(&req->wait);
1847         req->queue_time = ktime_get();
1848 }
1849
1850 int f2fs_issue_checkpoint(struct f2fs_sb_info *sbi)
1851 {
1852         struct ckpt_req_control *cprc = &sbi->cprc_info;
1853         struct ckpt_req req;
1854         struct cp_control cpc;
1855
1856         cpc.reason = __get_cp_reason(sbi);
1857         if (!test_opt(sbi, MERGE_CHECKPOINT) || cpc.reason != CP_SYNC) {
1858                 int ret;
1859
1860                 down_write(&sbi->gc_lock);
1861                 ret = f2fs_write_checkpoint(sbi, &cpc);
1862                 up_write(&sbi->gc_lock);
1863
1864                 return ret;
1865         }
1866
1867         if (!cprc->f2fs_issue_ckpt)
1868                 return __write_checkpoint_sync(sbi);
1869
1870         init_ckpt_req(&req);
1871
1872         llist_add(&req.llnode, &cprc->issue_list);
1873         atomic_inc(&cprc->queued_ckpt);
1874
1875         /*
1876          * update issue_list before we wake up issue_checkpoint thread,
1877          * this smp_mb() pairs with another barrier in ___wait_event(),
1878          * see more details in comments of waitqueue_active().
1879          */
1880         smp_mb();
1881
1882         if (waitqueue_active(&cprc->ckpt_wait_queue))
1883                 wake_up(&cprc->ckpt_wait_queue);
1884
1885         if (cprc->f2fs_issue_ckpt)
1886                 wait_for_completion(&req.wait);
1887         else
1888                 flush_remained_ckpt_reqs(sbi, &req);
1889
1890         return req.ret;
1891 }
1892
1893 int f2fs_start_ckpt_thread(struct f2fs_sb_info *sbi)
1894 {
1895         dev_t dev = sbi->sb->s_bdev->bd_dev;
1896         struct ckpt_req_control *cprc = &sbi->cprc_info;
1897
1898         if (cprc->f2fs_issue_ckpt)
1899                 return 0;
1900
1901         cprc->f2fs_issue_ckpt = kthread_run(issue_checkpoint_thread, sbi,
1902                         "f2fs_ckpt-%u:%u", MAJOR(dev), MINOR(dev));
1903         if (IS_ERR(cprc->f2fs_issue_ckpt)) {
1904                 cprc->f2fs_issue_ckpt = NULL;
1905                 return -ENOMEM;
1906         }
1907
1908         set_task_ioprio(cprc->f2fs_issue_ckpt, cprc->ckpt_thread_ioprio);
1909
1910         return 0;
1911 }
1912
1913 void f2fs_stop_ckpt_thread(struct f2fs_sb_info *sbi)
1914 {
1915         struct ckpt_req_control *cprc = &sbi->cprc_info;
1916         struct task_struct *ckpt_task;
1917
1918         if (!cprc->f2fs_issue_ckpt)
1919                 return;
1920
1921         ckpt_task = cprc->f2fs_issue_ckpt;
1922         cprc->f2fs_issue_ckpt = NULL;
1923         kthread_stop(ckpt_task);
1924
1925         f2fs_flush_ckpt_thread(sbi);
1926 }
1927
1928 void f2fs_flush_ckpt_thread(struct f2fs_sb_info *sbi)
1929 {
1930         struct ckpt_req_control *cprc = &sbi->cprc_info;
1931
1932         flush_remained_ckpt_reqs(sbi, NULL);
1933
1934         /* Let's wait for the previous dispatched checkpoint. */
1935         while (atomic_read(&cprc->queued_ckpt))
1936                 io_schedule_timeout(DEFAULT_IO_TIMEOUT);
1937 }
1938
1939 void f2fs_init_ckpt_req_control(struct f2fs_sb_info *sbi)
1940 {
1941         struct ckpt_req_control *cprc = &sbi->cprc_info;
1942
1943         atomic_set(&cprc->issued_ckpt, 0);
1944         atomic_set(&cprc->total_ckpt, 0);
1945         atomic_set(&cprc->queued_ckpt, 0);
1946         cprc->ckpt_thread_ioprio = DEFAULT_CHECKPOINT_IOPRIO;
1947         init_waitqueue_head(&cprc->ckpt_wait_queue);
1948         init_llist_head(&cprc->issue_list);
1949         spin_lock_init(&cprc->stat_lock);
1950 }