fs/ext4/fast_commit.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * fs/ext4/fast_commit.c
   5  *
   6  * Written by Harshad Shirwadkar <harshadshirwadkar@gmail.com>
   7  *
   8  * Ext4 fast commits routines.
   9  */
  10 #include "ext4.h"
  11 #include "ext4_jbd2.h"
  12 #include "ext4_extents.h"
  13 #include "mballoc.h"
  14
  15 /*
  16  * Ext4 Fast Commits
  17  * -----------------
  18  *
  19  * Ext4 fast commits implement fine grained journalling for Ext4.
  20  *
  21  * Fast commits are organized as a log of tag-length-value (TLV) structs. (See
  22  * struct ext4_fc_tl). Each TLV contains some delta that is replayed TLV by
  23  * TLV during the recovery phase. For the scenarios for which we currently
  24  * don't have replay code, fast commit falls back to full commits.
  25  * Fast commits record delta in one of the following three categories.
  26  *
  27  * (A) Directory entry updates:
  28  *
  29  * - EXT4_FC_TAG_UNLINK         - records directory entry unlink
  30  * - EXT4_FC_TAG_LINK           - records directory entry link
  31  * - EXT4_FC_TAG_CREAT          - records inode and directory entry creation
  32  *
  33  * (B) File specific data range updates:
  34  *
  35  * - EXT4_FC_TAG_ADD_RANGE      - records addition of new blocks to an inode
  36  * - EXT4_FC_TAG_DEL_RANGE      - records deletion of blocks from an inode
  37  *
  38  * (C) Inode metadata (mtime / ctime etc):
  39  *
  40  * - EXT4_FC_TAG_INODE          - record the inode that should be replayed
  41  *                                during recovery. Note that iblocks field is
  42  *                                not replayed and instead derived during
  43  *                                replay.
  44  * Commit Operation
  45  * ----------------
  46  * With fast commits, we maintain all the directory entry operations in the
  47  * order in which they are issued in an in-memory queue. This queue is flushed
  48  * to disk during the commit operation. We also maintain a list of inodes
  49  * that need to be committed during a fast commit in another in memory queue of
  50  * inodes. During the commit operation, we commit in the following order:
  51  *
  52  * [1] Lock inodes for any further data updates by setting COMMITTING state
  53  * [2] Submit data buffers of all the inodes
  54  * [3] Wait for [2] to complete
  55  * [4] Commit all the directory entry updates in the fast commit space
  56  * [5] Commit all the changed inode structures
  57  * [6] Write tail tag (this tag ensures the atomicity, please read the following
  58  *     section for more details).
  59  * [7] Wait for [4], [5] and [6] to complete.
  60  *
  61  * All the inode updates must call ext4_fc_start_update() before starting an
  62  * update. If such an ongoing update is present, fast commit waits for it to
  63  * complete. The completion of such an update is marked by
  64  * ext4_fc_stop_update().
  65  *
  66  * Fast Commit Ineligibility
  67  * -------------------------
  68  * Not all operations are supported by fast commits today (e.g extended
  69  * attributes). Fast commit ineligibility is marked by calling one of the
  70  * two following functions:
  71  *
  72  * - ext4_fc_mark_ineligible(): This makes next fast commit operation to fall
  73  *   back to full commit. This is useful in case of transient errors.
  74  *
  75  * - ext4_fc_start_ineligible() and ext4_fc_stop_ineligible() - This makes all
  76  *   the fast commits happening between ext4_fc_start_ineligible() and
  77  *   ext4_fc_stop_ineligible() and one fast commit after the call to
  78  *   ext4_fc_stop_ineligible() to fall back to full commits. It is important to
  79  *   make one more fast commit to fall back to full commit after stop call so
  80  *   that it guaranteed that the fast commit ineligible operation contained
  81  *   within ext4_fc_start_ineligible() and ext4_fc_stop_ineligible() is
  82  *   followed by at least 1 full commit.
  83  *
  84  * Atomicity of commits
  85  * --------------------
  86  * In order to guarantee atomicity during the commit operation, fast commit
  87  * uses "EXT4_FC_TAG_TAIL" tag that marks a fast commit as complete. Tail
  88  * tag contains CRC of the contents and TID of the transaction after which
  89  * this fast commit should be applied. Recovery code replays fast commit
  90  * logs only if there's at least 1 valid tail present. For every fast commit
  91  * operation, there is 1 tail. This means, we may end up with multiple tails
  92  * in the fast commit space. Here's an example:
  93  *
  94  * - Create a new file A and remove existing file B
  95  * - fsync()
  96  * - Append contents to file A
  97  * - Truncate file A
  98  * - fsync()
  99  *
 100  * The fast commit space at the end of above operations would look like this:
 101  *      [HEAD] [CREAT A] [UNLINK B] [TAIL] [ADD_RANGE A] [DEL_RANGE A] [TAIL]
 102  *             |<---  Fast Commit 1   --->|<---      Fast Commit 2     ---->|
 103  *
 104  * Replay code should thus check for all the valid tails in the FC area.
 105  *
 106  * TODOs
 107  * -----
 108  * 1) Make fast commit atomic updates more fine grained. Today, a fast commit
 109  *    eligible update must be protected within ext4_fc_start_update() and
 110  *    ext4_fc_stop_update(). These routines are called at much higher
 111  *    routines. This can be made more fine grained by combining with
 112  *    ext4_journal_start().
 113  *
 114  * 2) Same above for ext4_fc_start_ineligible() and ext4_fc_stop_ineligible()
 115  *
 116  * 3) Handle more ineligible cases.
 117  */
 118
 119 #include <trace/events/ext4.h>
 120 static struct kmem_cache *ext4_fc_dentry_cachep;
 121
 122 static void ext4_end_buffer_io_sync(struct buffer_head *bh, int uptodate)
 123 {
 124         BUFFER_TRACE(bh, "");
 125         if (uptodate) {
 126                 ext4_debug("%s: Block %lld up-to-date",
 127                            __func__, bh->b_blocknr);
 128                 set_buffer_uptodate(bh);
 129         } else {
 130                 ext4_debug("%s: Block %lld not up-to-date",
 131                            __func__, bh->b_blocknr);
 132                 clear_buffer_uptodate(bh);
 133         }
 134
 135         unlock_buffer(bh);
 136 }
 137
 138 static inline void ext4_fc_reset_inode(struct inode *inode)
 139 {
 140         struct ext4_inode_info *ei = EXT4_I(inode);
 141
 142         ei->i_fc_lblk_start = 0;
 143         ei->i_fc_lblk_len = 0;
 144 }
 145
 146 void ext4_fc_init_inode(struct inode *inode)
 147 {
 148         struct ext4_inode_info *ei = EXT4_I(inode);
 149
 150         ext4_fc_reset_inode(inode);
 151         ext4_clear_inode_state(inode, EXT4_STATE_FC_COMMITTING);
 152         INIT_LIST_HEAD(&ei->i_fc_list);
 153         init_waitqueue_head(&ei->i_fc_wait);
 154         atomic_set(&ei->i_fc_updates, 0);
 155 }
 156
 157 /* This function must be called with sbi->s_fc_lock held. */
 158 static void ext4_fc_wait_committing_inode(struct inode *inode)
 159 __releases(&EXT4_SB(inode->i_sb)->s_fc_lock)
 160 {
 161         wait_queue_head_t *wq;
 162         struct ext4_inode_info *ei = EXT4_I(inode);
 163
 164 #if (BITS_PER_LONG < 64)
 165         DEFINE_WAIT_BIT(wait, &ei->i_state_flags,
 166                         EXT4_STATE_FC_COMMITTING);
 167         wq = bit_waitqueue(&ei->i_state_flags,
 168                                 EXT4_STATE_FC_COMMITTING);
 169 #else
 170         DEFINE_WAIT_BIT(wait, &ei->i_flags,
 171                         EXT4_STATE_FC_COMMITTING);
 172         wq = bit_waitqueue(&ei->i_flags,
 173                                 EXT4_STATE_FC_COMMITTING);
 174 #endif
 175         lockdep_assert_held(&EXT4_SB(inode->i_sb)->s_fc_lock);
 176         prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
 177         spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
 178         schedule();
 179         finish_wait(wq, &wait.wq_entry);
 180 }
 181
 182 /*
 183  * Inform Ext4's fast about start of an inode update
 184  *
 185  * This function is called by the high level call VFS callbacks before
 186  * performing any inode update. This function blocks if there's an ongoing
 187  * fast commit on the inode in question.
 188  */
 189 void ext4_fc_start_update(struct inode *inode)
 190 {
 191         struct ext4_inode_info *ei = EXT4_I(inode);
 192
 193         if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) ||
 194             (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY))
 195                 return;
 196
 197 restart:
 198         spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock);
 199         if (list_empty(&ei->i_fc_list))
 200                 goto out;
 201
 202         if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) {
 203                 ext4_fc_wait_committing_inode(inode);
 204                 goto restart;
 205         }
 206 out:
 207         atomic_inc(&ei->i_fc_updates);
 208         spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
 209 }
 210
 211 /*
 212  * Stop inode update and wake up waiting fast commits if any.
 213  */
 214 void ext4_fc_stop_update(struct inode *inode)
 215 {
 216         struct ext4_inode_info *ei = EXT4_I(inode);
 217
 218         if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) ||
 219             (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY))
 220                 return;
 221
 222         if (atomic_dec_and_test(&ei->i_fc_updates))
 223                 wake_up_all(&ei->i_fc_wait);
 224 }
 225
 226 /*
 227  * Remove inode from fast commit list. If the inode is being committed
 228  * we wait until inode commit is done.
 229  */
 230 void ext4_fc_del(struct inode *inode)
 231 {
 232         struct ext4_inode_info *ei = EXT4_I(inode);
 233
 234         if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) ||
 235             (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY))
 236                 return;
 237
 238 restart:
 239         spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock);
 240         if (list_empty(&ei->i_fc_list)) {
 241                 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
 242                 return;
 243         }
 244
 245         if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) {
 246                 ext4_fc_wait_committing_inode(inode);
 247                 goto restart;
 248         }
 249         list_del_init(&ei->i_fc_list);
 250         spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
 251 }
 252
 253 /*
 254  * Mark file system as fast commit ineligible. This means that next commit
 255  * operation would result in a full jbd2 commit.
 256  */
 257 void ext4_fc_mark_ineligible(struct super_block *sb, int reason)
 258 {
 259         struct ext4_sb_info *sbi = EXT4_SB(sb);
 260
 261         if (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
 262             (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))
 263                 return;
 264
 265         ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
 266         WARN_ON(reason >= EXT4_FC_REASON_MAX);
 267         sbi->s_fc_stats.fc_ineligible_reason_count[reason]++;
 268 }
 269
 270 /*
 271  * Start a fast commit ineligible update. Any commits that happen while
 272  * such an operation is in progress fall back to full commits.
 273  */
 274 void ext4_fc_start_ineligible(struct super_block *sb, int reason)
 275 {
 276         struct ext4_sb_info *sbi = EXT4_SB(sb);
 277
 278         if (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
 279             (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))
 280                 return;
 281
 282         WARN_ON(reason >= EXT4_FC_REASON_MAX);
 283         sbi->s_fc_stats.fc_ineligible_reason_count[reason]++;
 284         atomic_inc(&sbi->s_fc_ineligible_updates);
 285 }
 286
 287 /*
 288  * Stop a fast commit ineligible update. We set EXT4_MF_FC_INELIGIBLE flag here
 289  * to ensure that after stopping the ineligible update, at least one full
 290  * commit takes place.
 291  */
 292 void ext4_fc_stop_ineligible(struct super_block *sb)
 293 {
 294         if (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
 295             (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))
 296                 return;
 297
 298         ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
 299         atomic_dec(&EXT4_SB(sb)->s_fc_ineligible_updates);
 300 }
 301
 302 static inline int ext4_fc_is_ineligible(struct super_block *sb)
 303 {
 304         return (ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE) ||
 305                 atomic_read(&EXT4_SB(sb)->s_fc_ineligible_updates));
 306 }
 307
 308 /*
 309  * Generic fast commit tracking function. If this is the first time this we are
 310  * called after a full commit, we initialize fast commit fields and then call
 311  * __fc_track_fn() with update = 0. If we have already been called after a full
 312  * commit, we pass update = 1. Based on that, the track function can determine
 313  * if it needs to track a field for the first time or if it needs to just
 314  * update the previously tracked value.
 315  *
 316  * If enqueue is set, this function enqueues the inode in fast commit list.
 317  */
 318 static int ext4_fc_track_template(
 319         handle_t *handle, struct inode *inode,
 320         int (*__fc_track_fn)(struct inode *, void *, bool),
 321         void *args, int enqueue)
 322 {
 323         bool update = false;
 324         struct ext4_inode_info *ei = EXT4_I(inode);
 325         struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
 326         tid_t tid = 0;
 327         int ret;
 328
 329         if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) ||
 330             (sbi->s_mount_state & EXT4_FC_REPLAY))
 331                 return -EOPNOTSUPP;
 332
 333         if (ext4_fc_is_ineligible(inode->i_sb))
 334                 return -EINVAL;
 335
 336         tid = handle->h_transaction->t_tid;
 337         mutex_lock(&ei->i_fc_lock);
 338         if (tid == ei->i_sync_tid) {
 339                 update = true;
 340         } else {
 341                 ext4_fc_reset_inode(inode);
 342                 ei->i_sync_tid = tid;
 343         }
 344         ret = __fc_track_fn(inode, args, update);
 345         mutex_unlock(&ei->i_fc_lock);
 346
 347         if (!enqueue)
 348                 return ret;
 349
 350         spin_lock(&sbi->s_fc_lock);
 351         if (list_empty(&EXT4_I(inode)->i_fc_list))
 352                 list_add_tail(&EXT4_I(inode)->i_fc_list,
 353                                 (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_COMMITTING)) ?
 354                                 &sbi->s_fc_q[FC_Q_STAGING] :
 355                                 &sbi->s_fc_q[FC_Q_MAIN]);
 356         spin_unlock(&sbi->s_fc_lock);
 357
 358         return ret;
 359 }
 360
 361 struct __track_dentry_update_args {
 362         struct dentry *dentry;
 363         int op;
 364 };
 365
 366 /* __track_fn for directory entry updates. Called with ei->i_fc_lock. */
 367 static int __track_dentry_update(struct inode *inode, void *arg, bool update)
 368 {
 369         struct ext4_fc_dentry_update *node;
 370         struct ext4_inode_info *ei = EXT4_I(inode);
 371         struct __track_dentry_update_args *dentry_update =
 372                 (struct __track_dentry_update_args *)arg;
 373         struct dentry *dentry = dentry_update->dentry;
 374         struct inode *dir = dentry->d_parent->d_inode;
 375         struct super_block *sb = inode->i_sb;
 376         struct ext4_sb_info *sbi = EXT4_SB(sb);
 377
 378         mutex_unlock(&ei->i_fc_lock);
 379
 380         if (IS_ENCRYPTED(dir)) {
 381                 ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_ENCRYPTED_FILENAME);
 382                 mutex_lock(&ei->i_fc_lock);
 383                 return -EOPNOTSUPP;
 384         }
 385
 386         node = kmem_cache_alloc(ext4_fc_dentry_cachep, GFP_NOFS);
 387         if (!node) {
 388                 ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_NOMEM);
 389                 mutex_lock(&ei->i_fc_lock);
 390                 return -ENOMEM;
 391         }
 392
 393         node->fcd_op = dentry_update->op;
 394         node->fcd_parent = dir->i_ino;
 395         node->fcd_ino = inode->i_ino;
 396         if (dentry->d_name.len > DNAME_INLINE_LEN) {
 397                 node->fcd_name.name = kmalloc(dentry->d_name.len, GFP_NOFS);
 398                 if (!node->fcd_name.name) {
 399                         kmem_cache_free(ext4_fc_dentry_cachep, node);
 400                         ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_NOMEM);
 401                         mutex_lock(&ei->i_fc_lock);
 402                         return -ENOMEM;
 403                 }
 404                 memcpy((u8 *)node->fcd_name.name, dentry->d_name.name,
 405                         dentry->d_name.len);
 406         } else {
 407                 memcpy(node->fcd_iname, dentry->d_name.name,
 408                         dentry->d_name.len);
 409                 node->fcd_name.name = node->fcd_iname;
 410         }
 411         node->fcd_name.len = dentry->d_name.len;
 412
 413         spin_lock(&sbi->s_fc_lock);
 414         if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_COMMITTING))
 415                 list_add_tail(&node->fcd_list,
 416                                 &sbi->s_fc_dentry_q[FC_Q_STAGING]);
 417         else
 418                 list_add_tail(&node->fcd_list, &sbi->s_fc_dentry_q[FC_Q_MAIN]);
 419         spin_unlock(&sbi->s_fc_lock);
 420         mutex_lock(&ei->i_fc_lock);
 421
 422         return 0;
 423 }
 424
 425 void __ext4_fc_track_unlink(handle_t *handle,
 426                 struct inode *inode, struct dentry *dentry)
 427 {
 428         struct __track_dentry_update_args args;
 429         int ret;
 430
 431         args.dentry = dentry;
 432         args.op = EXT4_FC_TAG_UNLINK;
 433
 434         ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
 435                                         (void *)&args, 0);
 436         trace_ext4_fc_track_unlink(inode, dentry, ret);
 437 }
 438
 439 void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry)
 440 {
 441         __ext4_fc_track_unlink(handle, d_inode(dentry), dentry);
 442 }
 443
 444 void __ext4_fc_track_link(handle_t *handle,
 445         struct inode *inode, struct dentry *dentry)
 446 {
 447         struct __track_dentry_update_args args;
 448         int ret;
 449
 450         args.dentry = dentry;
 451         args.op = EXT4_FC_TAG_LINK;
 452
 453         ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
 454                                         (void *)&args, 0);
 455         trace_ext4_fc_track_link(inode, dentry, ret);
 456 }
 457
 458 void ext4_fc_track_link(handle_t *handle, struct dentry *dentry)
 459 {
 460         __ext4_fc_track_link(handle, d_inode(dentry), dentry);
 461 }
 462
 463 void __ext4_fc_track_create(handle_t *handle, struct inode *inode,
 464                           struct dentry *dentry)
 465 {
 466         struct __track_dentry_update_args args;
 467         int ret;
 468
 469         args.dentry = dentry;
 470         args.op = EXT4_FC_TAG_CREAT;
 471
 472         ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
 473                                         (void *)&args, 0);
 474         trace_ext4_fc_track_create(inode, dentry, ret);
 475 }
 476
 477 void ext4_fc_track_create(handle_t *handle, struct dentry *dentry)
 478 {
 479         __ext4_fc_track_create(handle, d_inode(dentry), dentry);
 480 }
 481
 482 /* __track_fn for inode tracking */
 483 static int __track_inode(struct inode *inode, void *arg, bool update)
 484 {
 485         if (update)
 486                 return -EEXIST;
 487
 488         EXT4_I(inode)->i_fc_lblk_len = 0;
 489
 490         return 0;
 491 }
 492
 493 void ext4_fc_track_inode(handle_t *handle, struct inode *inode)
 494 {
 495         int ret;
 496
 497         if (S_ISDIR(inode->i_mode))
 498                 return;
 499
 500         if (ext4_should_journal_data(inode)) {
 501                 ext4_fc_mark_ineligible(inode->i_sb,
 502                                         EXT4_FC_REASON_INODE_JOURNAL_DATA);
 503                 return;
 504         }
 505
 506         ret = ext4_fc_track_template(handle, inode, __track_inode, NULL, 1);
 507         trace_ext4_fc_track_inode(inode, ret);
 508 }
 509
 510 struct __track_range_args {
 511         ext4_lblk_t start, end;
 512 };
 513
 514 /* __track_fn for tracking data updates */
 515 static int __track_range(struct inode *inode, void *arg, bool update)
 516 {
 517         struct ext4_inode_info *ei = EXT4_I(inode);
 518         ext4_lblk_t oldstart;
 519         struct __track_range_args *__arg =
 520                 (struct __track_range_args *)arg;
 521
 522         if (inode->i_ino < EXT4_FIRST_INO(inode->i_sb)) {
 523                 ext4_debug("Special inode %ld being modified\n", inode->i_ino);
 524                 return -ECANCELED;
 525         }
 526
 527         oldstart = ei->i_fc_lblk_start;
 528
 529         if (update && ei->i_fc_lblk_len > 0) {
 530                 ei->i_fc_lblk_start = min(ei->i_fc_lblk_start, __arg->start);
 531                 ei->i_fc_lblk_len =
 532                         max(oldstart + ei->i_fc_lblk_len - 1, __arg->end) -
 533                                 ei->i_fc_lblk_start + 1;
 534         } else {
 535                 ei->i_fc_lblk_start = __arg->start;
 536                 ei->i_fc_lblk_len = __arg->end - __arg->start + 1;
 537         }
 538
 539         return 0;
 540 }
 541
 542 void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t start,
 543                          ext4_lblk_t end)
 544 {
 545         struct __track_range_args args;
 546         int ret;
 547
 548         if (S_ISDIR(inode->i_mode))
 549                 return;
 550
 551         args.start = start;
 552         args.end = end;
 553
 554         ret = ext4_fc_track_template(handle, inode,  __track_range, &args, 1);
 555
 556         trace_ext4_fc_track_range(inode, start, end, ret);
 557 }
 558
 559 static void ext4_fc_submit_bh(struct super_block *sb)
 560 {
 561         int write_flags = REQ_SYNC;
 562         struct buffer_head *bh = EXT4_SB(sb)->s_fc_bh;
 563
 564         /* TODO: REQ_FUA | REQ_PREFLUSH is unnecessarily expensive. */
 565         if (test_opt(sb, BARRIER))
 566                 write_flags |= REQ_FUA | REQ_PREFLUSH;
 567         lock_buffer(bh);
 568         set_buffer_dirty(bh);
 569         set_buffer_uptodate(bh);
 570         bh->b_end_io = ext4_end_buffer_io_sync;
 571         submit_bh(REQ_OP_WRITE, write_flags, bh);
 572         EXT4_SB(sb)->s_fc_bh = NULL;
 573 }
 574
 575 /* Ext4 commit path routines */
 576
 577 /* memzero and update CRC */
 578 static void *ext4_fc_memzero(struct super_block *sb, void *dst, int len,
 579                                 u32 *crc)
 580 {
 581         void *ret;
 582
 583         ret = memset(dst, 0, len);
 584         if (crc)
 585                 *crc = ext4_chksum(EXT4_SB(sb), *crc, dst, len);
 586         return ret;
 587 }
 588
 589 /*
 590  * Allocate len bytes on a fast commit buffer.
 591  *
 592  * During the commit time this function is used to manage fast commit
 593  * block space. We don't split a fast commit log onto different
 594  * blocks. So this function makes sure that if there's not enough space
 595  * on the current block, the remaining space in the current block is
 596  * marked as unused by adding EXT4_FC_TAG_PAD tag. In that case,
 597  * new block is from jbd2 and CRC is updated to reflect the padding
 598  * we added.
 599  */
 600 static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc)
 601 {
 602         struct ext4_fc_tl *tl;
 603         struct ext4_sb_info *sbi = EXT4_SB(sb);
 604         struct buffer_head *bh;
 605         int bsize = sbi->s_journal->j_blocksize;
 606         int ret, off = sbi->s_fc_bytes % bsize;
 607         int pad_len;
 608
 609         /*
 610          * After allocating len, we should have space at least for a 0 byte
 611          * padding.
 612          */
 613         if (len + sizeof(struct ext4_fc_tl) > bsize)
 614                 return NULL;
 615
 616         if (bsize - off - 1 > len + sizeof(struct ext4_fc_tl)) {
 617                 /*
 618                  * Only allocate from current buffer if we have enough space for
 619                  * this request AND we have space to add a zero byte padding.
 620                  */
 621                 if (!sbi->s_fc_bh) {
 622                         ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh);
 623                         if (ret)
 624                                 return NULL;
 625                         sbi->s_fc_bh = bh;
 626                 }
 627                 sbi->s_fc_bytes += len;
 628                 return sbi->s_fc_bh->b_data + off;
 629         }
 630         /* Need to add PAD tag */
 631         tl = (struct ext4_fc_tl *)(sbi->s_fc_bh->b_data + off);
 632         tl->fc_tag = cpu_to_le16(EXT4_FC_TAG_PAD);
 633         pad_len = bsize - off - 1 - sizeof(struct ext4_fc_tl);
 634         tl->fc_len = cpu_to_le16(pad_len);
 635         if (crc)
 636                 *crc = ext4_chksum(sbi, *crc, tl, sizeof(*tl));
 637         if (pad_len > 0)
 638                 ext4_fc_memzero(sb, tl + 1, pad_len, crc);
 639         /* Don't leak uninitialized memory in the unused last byte. */
 640         *((u8 *)(tl + 1) + pad_len) = 0;
 641
 642         ext4_fc_submit_bh(sb);
 643
 644         ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh);
 645         if (ret)
 646                 return NULL;
 647         sbi->s_fc_bh = bh;
 648         sbi->s_fc_bytes = (sbi->s_fc_bytes / bsize + 1) * bsize + len;
 649         return sbi->s_fc_bh->b_data;
 650 }
 651
 652 /* memcpy to fc reserved space and update CRC */
 653 static void *ext4_fc_memcpy(struct super_block *sb, void *dst, const void *src,
 654                                 int len, u32 *crc)
 655 {
 656         if (crc)
 657                 *crc = ext4_chksum(EXT4_SB(sb), *crc, src, len);
 658         return memcpy(dst, src, len);
 659 }
 660
 661 /*
 662  * Complete a fast commit by writing tail tag.
 663  *
 664  * Writing tail tag marks the end of a fast commit. In order to guarantee
 665  * atomicity, after writing tail tag, even if there's space remaining
 666  * in the block, next commit shouldn't use it. That's why tail tag
 667  * has the length as that of the remaining space on the block.
 668  */
 669 static int ext4_fc_write_tail(struct super_block *sb, u32 crc)
 670 {
 671         struct ext4_sb_info *sbi = EXT4_SB(sb);
 672         struct ext4_fc_tl tl;
 673         struct ext4_fc_tail tail;
 674         int off, bsize = sbi->s_journal->j_blocksize;
 675         u8 *dst;
 676
 677         /*
 678          * ext4_fc_reserve_space takes care of allocating an extra block if
 679          * there's no enough space on this block for accommodating this tail.
 680          */
 681         dst = ext4_fc_reserve_space(sb, sizeof(tl) + sizeof(tail), &crc);
 682         if (!dst)
 683                 return -ENOSPC;
 684
 685         off = sbi->s_fc_bytes % bsize;
 686
 687         tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_TAIL);
 688         tl.fc_len = cpu_to_le16(bsize - off - 1 + sizeof(struct ext4_fc_tail));
 689         sbi->s_fc_bytes = round_up(sbi->s_fc_bytes, bsize);
 690
 691         ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), &crc);
 692         dst += sizeof(tl);
 693         tail.fc_tid = cpu_to_le32(sbi->s_journal->j_running_transaction->t_tid);
 694         ext4_fc_memcpy(sb, dst, &tail.fc_tid, sizeof(tail.fc_tid), &crc);
 695         dst += sizeof(tail.fc_tid);
 696         tail.fc_crc = cpu_to_le32(crc);
 697         ext4_fc_memcpy(sb, dst, &tail.fc_crc, sizeof(tail.fc_crc), NULL);
 698         dst += sizeof(tail.fc_crc);
 699         memset(dst, 0, bsize - off); /* Don't leak uninitialized memory. */
 700
 701         ext4_fc_submit_bh(sb);
 702
 703         return 0;
 704 }
 705
 706 /*
 707  * Adds tag, length, value and updates CRC. Returns true if tlv was added.
 708  * Returns false if there's not enough space.
 709  */
 710 static bool ext4_fc_add_tlv(struct super_block *sb, u16 tag, u16 len, u8 *val,
 711                            u32 *crc)
 712 {
 713         struct ext4_fc_tl tl;
 714         u8 *dst;
 715
 716         dst = ext4_fc_reserve_space(sb, sizeof(tl) + len, crc);
 717         if (!dst)
 718                 return false;
 719
 720         tl.fc_tag = cpu_to_le16(tag);
 721         tl.fc_len = cpu_to_le16(len);
 722
 723         ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), crc);
 724         ext4_fc_memcpy(sb, dst + sizeof(tl), val, len, crc);
 725
 726         return true;
 727 }
 728
 729 /* Same as above, but adds dentry tlv. */
 730 static  bool ext4_fc_add_dentry_tlv(struct super_block *sb, u16 tag,
 731                                         int parent_ino, int ino, int dlen,
 732                                         const unsigned char *dname,
 733                                         u32 *crc)
 734 {
 735         struct ext4_fc_dentry_info fcd;
 736         struct ext4_fc_tl tl;
 737         u8 *dst = ext4_fc_reserve_space(sb, sizeof(tl) + sizeof(fcd) + dlen,
 738                                         crc);
 739
 740         if (!dst)
 741                 return false;
 742
 743         fcd.fc_parent_ino = cpu_to_le32(parent_ino);
 744         fcd.fc_ino = cpu_to_le32(ino);
 745         tl.fc_tag = cpu_to_le16(tag);
 746         tl.fc_len = cpu_to_le16(sizeof(fcd) + dlen);
 747         ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), crc);
 748         dst += sizeof(tl);
 749         ext4_fc_memcpy(sb, dst, &fcd, sizeof(fcd), crc);
 750         dst += sizeof(fcd);
 751         ext4_fc_memcpy(sb, dst, dname, dlen, crc);
 752         dst += dlen;
 753
 754         return true;
 755 }
 756
 757 /*
 758  * Writes inode in the fast commit space under TLV with tag @tag.
 759  * Returns 0 on success, error on failure.
 760  */
 761 static int ext4_fc_write_inode(struct inode *inode, u32 *crc)
 762 {
 763         struct ext4_inode_info *ei = EXT4_I(inode);
 764         int inode_len = EXT4_GOOD_OLD_INODE_SIZE;
 765         int ret;
 766         struct ext4_iloc iloc;
 767         struct ext4_fc_inode fc_inode;
 768         struct ext4_fc_tl tl;
 769         u8 *dst;
 770
 771         ret = ext4_get_inode_loc(inode, &iloc);
 772         if (ret)
 773                 return ret;
 774
 775         if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE)
 776                 inode_len += ei->i_extra_isize;
 777
 778         fc_inode.fc_ino = cpu_to_le32(inode->i_ino);
 779         tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_INODE);
 780         tl.fc_len = cpu_to_le16(inode_len + sizeof(fc_inode.fc_ino));
 781
 782         ret = -ECANCELED;
 783         dst = ext4_fc_reserve_space(inode->i_sb,
 784                         sizeof(tl) + inode_len + sizeof(fc_inode.fc_ino), crc);
 785         if (!dst)
 786                 goto err;
 787
 788         if (!ext4_fc_memcpy(inode->i_sb, dst, &tl, sizeof(tl), crc))
 789                 goto err;
 790         dst += sizeof(tl);
 791         if (!ext4_fc_memcpy(inode->i_sb, dst, &fc_inode, sizeof(fc_inode), crc))
 792                 goto err;
 793         dst += sizeof(fc_inode);
 794         if (!ext4_fc_memcpy(inode->i_sb, dst, (u8 *)ext4_raw_inode(&iloc),
 795                                         inode_len, crc))
 796                 goto err;
 797         ret = 0;
 798 err:
 799         brelse(iloc.bh);
 800         return ret;
 801 }
 802
 803 /*
 804  * Writes updated data ranges for the inode in question. Updates CRC.
 805  * Returns 0 on success, error otherwise.
 806  */
 807 static int ext4_fc_write_inode_data(struct inode *inode, u32 *crc)
 808 {
 809         ext4_lblk_t old_blk_size, cur_lblk_off, new_blk_size;
 810         struct ext4_inode_info *ei = EXT4_I(inode);
 811         struct ext4_map_blocks map;
 812         struct ext4_fc_add_range fc_ext;
 813         struct ext4_fc_del_range lrange;
 814         struct ext4_extent *ex;
 815         int ret;
 816
 817         mutex_lock(&ei->i_fc_lock);
 818         if (ei->i_fc_lblk_len == 0) {
 819                 mutex_unlock(&ei->i_fc_lock);
 820                 return 0;
 821         }
 822         old_blk_size = ei->i_fc_lblk_start;
 823         new_blk_size = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1;
 824         ei->i_fc_lblk_len = 0;
 825         mutex_unlock(&ei->i_fc_lock);
 826
 827         cur_lblk_off = old_blk_size;
 828         jbd_debug(1, "%s: will try writing %d to %d for inode %ld\n",
 829                   __func__, cur_lblk_off, new_blk_size, inode->i_ino);
 830
 831         while (cur_lblk_off <= new_blk_size) {
 832                 map.m_lblk = cur_lblk_off;
 833                 map.m_len = new_blk_size - cur_lblk_off + 1;
 834                 ret = ext4_map_blocks(NULL, inode, &map, 0);
 835                 if (ret < 0)
 836                         return -ECANCELED;
 837
 838                 if (map.m_len == 0) {
 839                         cur_lblk_off++;
 840                         continue;
 841                 }
 842
 843                 if (ret == 0) {
 844                         lrange.fc_ino = cpu_to_le32(inode->i_ino);
 845                         lrange.fc_lblk = cpu_to_le32(map.m_lblk);
 846                         lrange.fc_len = cpu_to_le32(map.m_len);
 847                         if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_DEL_RANGE,
 848                                             sizeof(lrange), (u8 *)&lrange, crc))
 849                                 return -ENOSPC;
 850                 } else {
 851                         unsigned int max = (map.m_flags & EXT4_MAP_UNWRITTEN) ?
 852                                 EXT_UNWRITTEN_MAX_LEN : EXT_INIT_MAX_LEN;
 853
 854                         /* Limit the number of blocks in one extent */
 855                         map.m_len = min(max, map.m_len);
 856
 857                         fc_ext.fc_ino = cpu_to_le32(inode->i_ino);
 858                         ex = (struct ext4_extent *)&fc_ext.fc_ex;
 859                         ex->ee_block = cpu_to_le32(map.m_lblk);
 860                         ex->ee_len = cpu_to_le16(map.m_len);
 861                         ext4_ext_store_pblock(ex, map.m_pblk);
 862                         if (map.m_flags & EXT4_MAP_UNWRITTEN)
 863                                 ext4_ext_mark_unwritten(ex);
 864                         else
 865                                 ext4_ext_mark_initialized(ex);
 866                         if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_ADD_RANGE,
 867                                             sizeof(fc_ext), (u8 *)&fc_ext, crc))
 868                                 return -ENOSPC;
 869                 }
 870
 871                 cur_lblk_off += map.m_len;
 872         }
 873
 874         return 0;
 875 }
 876
 877
 878 /* Submit data for all the fast commit inodes */
 879 static int ext4_fc_submit_inode_data_all(journal_t *journal)
 880 {
 881         struct super_block *sb = (struct super_block *)(journal->j_private);
 882         struct ext4_sb_info *sbi = EXT4_SB(sb);
 883         struct ext4_inode_info *ei;
 884         struct list_head *pos;
 885         int ret = 0;
 886
 887         spin_lock(&sbi->s_fc_lock);
 888         ext4_set_mount_flag(sb, EXT4_MF_FC_COMMITTING);
 889         list_for_each(pos, &sbi->s_fc_q[FC_Q_MAIN]) {
 890                 ei = list_entry(pos, struct ext4_inode_info, i_fc_list);
 891                 ext4_set_inode_state(&ei->vfs_inode, EXT4_STATE_FC_COMMITTING);
 892                 while (atomic_read(&ei->i_fc_updates)) {
 893                         DEFINE_WAIT(wait);
 894
 895                         prepare_to_wait(&ei->i_fc_wait, &wait,
 896                                                 TASK_UNINTERRUPTIBLE);
 897                         if (atomic_read(&ei->i_fc_updates)) {
 898                                 spin_unlock(&sbi->s_fc_lock);
 899                                 schedule();
 900                                 spin_lock(&sbi->s_fc_lock);
 901                         }
 902                         finish_wait(&ei->i_fc_wait, &wait);
 903                 }
 904                 spin_unlock(&sbi->s_fc_lock);
 905                 ret = jbd2_submit_inode_data(ei->jinode);
 906                 if (ret)
 907                         return ret;
 908                 spin_lock(&sbi->s_fc_lock);
 909         }
 910         spin_unlock(&sbi->s_fc_lock);
 911
 912         return ret;
 913 }
 914
 915 /* Wait for completion of data for all the fast commit inodes */
 916 static int ext4_fc_wait_inode_data_all(journal_t *journal)
 917 {
 918         struct super_block *sb = (struct super_block *)(journal->j_private);
 919         struct ext4_sb_info *sbi = EXT4_SB(sb);
 920         struct ext4_inode_info *pos, *n;
 921         int ret = 0;
 922
 923         spin_lock(&sbi->s_fc_lock);
 924         list_for_each_entry_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
 925                 if (!ext4_test_inode_state(&pos->vfs_inode,
 926                                            EXT4_STATE_FC_COMMITTING))
 927                         continue;
 928                 spin_unlock(&sbi->s_fc_lock);
 929
 930                 ret = jbd2_wait_inode_data(journal, pos->jinode);
 931                 if (ret)
 932                         return ret;
 933                 spin_lock(&sbi->s_fc_lock);
 934         }
 935         spin_unlock(&sbi->s_fc_lock);
 936
 937         return 0;
 938 }
 939
 940 /* Commit all the directory entry updates */
 941 static int ext4_fc_commit_dentry_updates(journal_t *journal, u32 *crc)
 942 __acquires(&sbi->s_fc_lock)
 943 __releases(&sbi->s_fc_lock)
 944 {
 945         struct super_block *sb = (struct super_block *)(journal->j_private);
 946         struct ext4_sb_info *sbi = EXT4_SB(sb);
 947         struct ext4_fc_dentry_update *fc_dentry;
 948         struct inode *inode;
 949         struct list_head *pos, *n, *fcd_pos, *fcd_n;
 950         struct ext4_inode_info *ei;
 951         int ret;
 952
 953         if (list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN]))
 954                 return 0;
 955         list_for_each_safe(fcd_pos, fcd_n, &sbi->s_fc_dentry_q[FC_Q_MAIN]) {
 956                 fc_dentry = list_entry(fcd_pos, struct ext4_fc_dentry_update,
 957                                         fcd_list);
 958                 if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) {
 959                         spin_unlock(&sbi->s_fc_lock);
 960                         if (!ext4_fc_add_dentry_tlv(
 961                                 sb, fc_dentry->fcd_op,
 962                                 fc_dentry->fcd_parent, fc_dentry->fcd_ino,
 963                                 fc_dentry->fcd_name.len,
 964                                 fc_dentry->fcd_name.name, crc)) {
 965                                 ret = -ENOSPC;
 966                                 goto lock_and_exit;
 967                         }
 968                         spin_lock(&sbi->s_fc_lock);
 969                         continue;
 970                 }
 971
 972                 inode = NULL;
 973                 list_for_each_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN]) {
 974                         ei = list_entry(pos, struct ext4_inode_info, i_fc_list);
 975                         if (ei->vfs_inode.i_ino == fc_dentry->fcd_ino) {
 976                                 inode = &ei->vfs_inode;
 977                                 break;
 978                         }
 979                 }
 980                 /*
 981                  * If we don't find inode in our list, then it was deleted,
 982                  * in which case, we don't need to record it's create tag.
 983                  */
 984                 if (!inode)
 985                         continue;
 986                 spin_unlock(&sbi->s_fc_lock);
 987
 988                 /*
 989                  * We first write the inode and then the create dirent. This
 990                  * allows the recovery code to create an unnamed inode first
 991                  * and then link it to a directory entry. This allows us
 992                  * to use namei.c routines almost as is and simplifies
 993                  * the recovery code.
 994                  */
 995                 ret = ext4_fc_write_inode(inode, crc);
 996                 if (ret)
 997                         goto lock_and_exit;
 998
 999                 ret = ext4_fc_write_inode_data(inode, crc);
1000                 if (ret)
1001                         goto lock_and_exit;
1002
1003                 if (!ext4_fc_add_dentry_tlv(
1004                         sb, fc_dentry->fcd_op,
1005                         fc_dentry->fcd_parent, fc_dentry->fcd_ino,
1006                         fc_dentry->fcd_name.len,
1007                         fc_dentry->fcd_name.name, crc)) {
1008                         ret = -ENOSPC;
1009                         goto lock_and_exit;
1010                 }
1011
1012                 spin_lock(&sbi->s_fc_lock);
1013         }
1014         return 0;
1015 lock_and_exit:
1016         spin_lock(&sbi->s_fc_lock);
1017         return ret;
1018 }
1019
1020 static int ext4_fc_perform_commit(journal_t *journal)
1021 {
1022         struct super_block *sb = (struct super_block *)(journal->j_private);
1023         struct ext4_sb_info *sbi = EXT4_SB(sb);
1024         struct ext4_inode_info *iter;
1025         struct ext4_fc_head head;
1026         struct list_head *pos;
1027         struct inode *inode;
1028         struct blk_plug plug;
1029         int ret = 0;
1030         u32 crc = 0;
1031
1032         ret = ext4_fc_submit_inode_data_all(journal);
1033         if (ret)
1034                 return ret;
1035
1036         ret = ext4_fc_wait_inode_data_all(journal);
1037         if (ret)
1038                 return ret;
1039
1040         /*
1041          * If file system device is different from journal device, issue a cache
1042          * flush before we start writing fast commit blocks.
1043          */
1044         if (journal->j_fs_dev != journal->j_dev)
1045                 blkdev_issue_flush(journal->j_fs_dev, GFP_NOFS);
1046
1047         blk_start_plug(&plug);
1048         if (sbi->s_fc_bytes == 0) {
1049                 /*
1050                  * Add a head tag only if this is the first fast commit
1051                  * in this TID.
1052                  */
1053                 head.fc_features = cpu_to_le32(EXT4_FC_SUPPORTED_FEATURES);
1054                 head.fc_tid = cpu_to_le32(
1055                         sbi->s_journal->j_running_transaction->t_tid);
1056                 if (!ext4_fc_add_tlv(sb, EXT4_FC_TAG_HEAD, sizeof(head),
1057                         (u8 *)&head, &crc)) {
1058                         ret = -ENOSPC;
1059                         goto out;
1060                 }
1061         }
1062
1063         spin_lock(&sbi->s_fc_lock);
1064         ret = ext4_fc_commit_dentry_updates(journal, &crc);
1065         if (ret) {
1066                 spin_unlock(&sbi->s_fc_lock);
1067                 goto out;
1068         }
1069
1070         list_for_each(pos, &sbi->s_fc_q[FC_Q_MAIN]) {
1071                 iter = list_entry(pos, struct ext4_inode_info, i_fc_list);
1072                 inode = &iter->vfs_inode;
1073                 if (!ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING))
1074                         continue;
1075
1076                 spin_unlock(&sbi->s_fc_lock);
1077                 ret = ext4_fc_write_inode_data(inode, &crc);
1078                 if (ret)
1079                         goto out;
1080                 ret = ext4_fc_write_inode(inode, &crc);
1081                 if (ret)
1082                         goto out;
1083                 spin_lock(&sbi->s_fc_lock);
1084         }
1085         spin_unlock(&sbi->s_fc_lock);
1086
1087         ret = ext4_fc_write_tail(sb, crc);
1088
1089 out:
1090         blk_finish_plug(&plug);
1091         return ret;
1092 }
1093
1094 /*
1095  * The main commit entry point. Performs a fast commit for transaction
1096  * commit_tid if needed. If it's not possible to perform a fast commit
1097  * due to various reasons, we fall back to full commit. Returns 0
1098  * on success, error otherwise.
1099  */
1100 int ext4_fc_commit(journal_t *journal, tid_t commit_tid)
1101 {
1102         struct super_block *sb = (struct super_block *)(journal->j_private);
1103         struct ext4_sb_info *sbi = EXT4_SB(sb);
1104         int nblks = 0, ret, bsize = journal->j_blocksize;
1105         int subtid = atomic_read(&sbi->s_fc_subtid);
1106         int reason = EXT4_FC_REASON_OK, fc_bufs_before = 0;
1107         ktime_t start_time, commit_time;
1108
1109         trace_ext4_fc_commit_start(sb);
1110
1111         start_time = ktime_get();
1112
1113         if (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
1114                 (ext4_fc_is_ineligible(sb))) {
1115                 reason = EXT4_FC_REASON_INELIGIBLE;
1116                 goto out;
1117         }
1118
1119 restart_fc:
1120         ret = jbd2_fc_begin_commit(journal, commit_tid);
1121         if (ret == -EALREADY) {
1122                 /* There was an ongoing commit, check if we need to restart */
1123                 if (atomic_read(&sbi->s_fc_subtid) <= subtid &&
1124                         commit_tid > journal->j_commit_sequence)
1125                         goto restart_fc;
1126                 reason = EXT4_FC_REASON_ALREADY_COMMITTED;
1127                 goto out;
1128         } else if (ret) {
1129                 sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++;
1130                 reason = EXT4_FC_REASON_FC_START_FAILED;
1131                 goto out;
1132         }
1133
1134         fc_bufs_before = (sbi->s_fc_bytes + bsize - 1) / bsize;
1135         ret = ext4_fc_perform_commit(journal);
1136         if (ret < 0) {
1137                 sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++;
1138                 reason = EXT4_FC_REASON_FC_FAILED;
1139                 goto out;
1140         }
1141         nblks = (sbi->s_fc_bytes + bsize - 1) / bsize - fc_bufs_before;
1142         ret = jbd2_fc_wait_bufs(journal, nblks);
1143         if (ret < 0) {
1144                 sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++;
1145                 reason = EXT4_FC_REASON_FC_FAILED;
1146                 goto out;
1147         }
1148         atomic_inc(&sbi->s_fc_subtid);
1149         jbd2_fc_end_commit(journal);
1150 out:
1151         /* Has any ineligible update happened since we started? */
1152         if (reason == EXT4_FC_REASON_OK && ext4_fc_is_ineligible(sb)) {
1153                 sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++;
1154                 reason = EXT4_FC_REASON_INELIGIBLE;
1155         }
1156
1157         spin_lock(&sbi->s_fc_lock);
1158         if (reason != EXT4_FC_REASON_OK &&
1159                 reason != EXT4_FC_REASON_ALREADY_COMMITTED) {
1160                 sbi->s_fc_stats.fc_ineligible_commits++;
1161         } else {
1162                 sbi->s_fc_stats.fc_num_commits++;
1163                 sbi->s_fc_stats.fc_numblks += nblks;
1164         }
1165         spin_unlock(&sbi->s_fc_lock);
1166         nblks = (reason == EXT4_FC_REASON_OK) ? nblks : 0;
1167         trace_ext4_fc_commit_stop(sb, nblks, reason);
1168         commit_time = ktime_to_ns(ktime_sub(ktime_get(), start_time));
1169         /*
1170          * weight the commit time higher than the average time so we don't
1171          * react too strongly to vast changes in the commit time
1172          */
1173         if (likely(sbi->s_fc_avg_commit_time))
1174                 sbi->s_fc_avg_commit_time = (commit_time +
1175                                 sbi->s_fc_avg_commit_time * 3) / 4;
1176         else
1177                 sbi->s_fc_avg_commit_time = commit_time;
1178         jbd_debug(1,
1179                 "Fast commit ended with blks = %d, reason = %d, subtid - %d",
1180                 nblks, reason, subtid);
1181         if (reason == EXT4_FC_REASON_FC_FAILED)
1182                 return jbd2_fc_end_commit_fallback(journal);
1183         if (reason == EXT4_FC_REASON_FC_START_FAILED ||
1184                 reason == EXT4_FC_REASON_INELIGIBLE)
1185                 return jbd2_complete_transaction(journal, commit_tid);
1186         return 0;
1187 }
1188
1189 /*
1190  * Fast commit cleanup routine. This is called after every fast commit and
1191  * full commit. full is true if we are called after a full commit.
1192  */
1193 static void ext4_fc_cleanup(journal_t *journal, int full)
1194 {
1195         struct super_block *sb = journal->j_private;
1196         struct ext4_sb_info *sbi = EXT4_SB(sb);
1197         struct ext4_inode_info *iter;
1198         struct ext4_fc_dentry_update *fc_dentry;
1199         struct list_head *pos, *n;
1200
1201         if (full && sbi->s_fc_bh)
1202                 sbi->s_fc_bh = NULL;
1203
1204         jbd2_fc_release_bufs(journal);
1205
1206         spin_lock(&sbi->s_fc_lock);
1207         list_for_each_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN]) {
1208                 iter = list_entry(pos, struct ext4_inode_info, i_fc_list);
1209                 list_del_init(&iter->i_fc_list);
1210                 ext4_clear_inode_state(&iter->vfs_inode,
1211                                        EXT4_STATE_FC_COMMITTING);
1212                 ext4_fc_reset_inode(&iter->vfs_inode);
1213                 /* Make sure EXT4_STATE_FC_COMMITTING bit is clear */
1214                 smp_mb();
1215 #if (BITS_PER_LONG < 64)
1216                 wake_up_bit(&iter->i_state_flags, EXT4_STATE_FC_COMMITTING);
1217 #else
1218                 wake_up_bit(&iter->i_flags, EXT4_STATE_FC_COMMITTING);
1219 #endif
1220         }
1221
1222         while (!list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) {
1223                 fc_dentry = list_first_entry(&sbi->s_fc_dentry_q[FC_Q_MAIN],
1224                                              struct ext4_fc_dentry_update,
1225                                              fcd_list);
1226                 list_del_init(&fc_dentry->fcd_list);
1227                 spin_unlock(&sbi->s_fc_lock);
1228
1229                 if (fc_dentry->fcd_name.name &&
1230                         fc_dentry->fcd_name.len > DNAME_INLINE_LEN)
1231                         kfree(fc_dentry->fcd_name.name);
1232                 kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry);
1233                 spin_lock(&sbi->s_fc_lock);
1234         }
1235
1236         list_splice_init(&sbi->s_fc_dentry_q[FC_Q_STAGING],
1237                                 &sbi->s_fc_dentry_q[FC_Q_MAIN]);
1238         list_splice_init(&sbi->s_fc_q[FC_Q_STAGING],
1239                                 &sbi->s_fc_q[FC_Q_MAIN]);
1240
1241         ext4_clear_mount_flag(sb, EXT4_MF_FC_COMMITTING);
1242         ext4_clear_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
1243
1244         if (full)
1245                 sbi->s_fc_bytes = 0;
1246         spin_unlock(&sbi->s_fc_lock);
1247         trace_ext4_fc_stats(sb);
1248 }
1249
1250 /* Ext4 Replay Path Routines */
1251
1252 /* Helper struct for dentry replay routines */
1253 struct dentry_info_args {
1254         int parent_ino, dname_len, ino, inode_len;
1255         char *dname;
1256 };
1257
1258 static inline void tl_to_darg(struct dentry_info_args *darg,
1259                               struct  ext4_fc_tl *tl, u8 *val)
1260 {
1261         struct ext4_fc_dentry_info fcd;
1262
1263         memcpy(&fcd, val, sizeof(fcd));
1264
1265         darg->parent_ino = le32_to_cpu(fcd.fc_parent_ino);
1266         darg->ino = le32_to_cpu(fcd.fc_ino);
1267         darg->dname = val + offsetof(struct ext4_fc_dentry_info, fc_dname);
1268         darg->dname_len = le16_to_cpu(tl->fc_len) -
1269                 sizeof(struct ext4_fc_dentry_info);
1270 }
1271
1272 /* Unlink replay function */
1273 static int ext4_fc_replay_unlink(struct super_block *sb, struct ext4_fc_tl *tl,
1274                                  u8 *val)
1275 {
1276         struct inode *inode, *old_parent;
1277         struct qstr entry;
1278         struct dentry_info_args darg;
1279         int ret = 0;
1280
1281         tl_to_darg(&darg, tl, val);
1282
1283         trace_ext4_fc_replay(sb, EXT4_FC_TAG_UNLINK, darg.ino,
1284                         darg.parent_ino, darg.dname_len);
1285
1286         entry.name = darg.dname;
1287         entry.len = darg.dname_len;
1288         inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
1289
1290         if (IS_ERR(inode)) {
1291                 jbd_debug(1, "Inode %d not found", darg.ino);
1292                 return 0;
1293         }
1294
1295         old_parent = ext4_iget(sb, darg.parent_ino,
1296                                 EXT4_IGET_NORMAL);
1297         if (IS_ERR(old_parent)) {
1298                 jbd_debug(1, "Dir with inode  %d not found", darg.parent_ino);
1299                 iput(inode);
1300                 return 0;
1301         }
1302
1303         ret = __ext4_unlink(old_parent, &entry, inode, NULL);
1304         /* -ENOENT ok coz it might not exist anymore. */
1305         if (ret == -ENOENT)
1306                 ret = 0;
1307         iput(old_parent);
1308         iput(inode);
1309         return ret;
1310 }
1311
1312 static int ext4_fc_replay_link_internal(struct super_block *sb,
1313                                 struct dentry_info_args *darg,
1314                                 struct inode *inode)
1315 {
1316         struct inode *dir = NULL;
1317         struct dentry *dentry_dir = NULL, *dentry_inode = NULL;
1318         struct qstr qstr_dname = QSTR_INIT(darg->dname, darg->dname_len);
1319         int ret = 0;
1320
1321         dir = ext4_iget(sb, darg->parent_ino, EXT4_IGET_NORMAL);
1322         if (IS_ERR(dir)) {
1323                 jbd_debug(1, "Dir with inode %d not found.", darg->parent_ino);
1324                 dir = NULL;
1325                 goto out;
1326         }
1327
1328         dentry_dir = d_obtain_alias(dir);
1329         if (IS_ERR(dentry_dir)) {
1330                 jbd_debug(1, "Failed to obtain dentry");
1331                 dentry_dir = NULL;
1332                 goto out;
1333         }
1334
1335         dentry_inode = d_alloc(dentry_dir, &qstr_dname);
1336         if (!dentry_inode) {
1337                 jbd_debug(1, "Inode dentry not created.");
1338                 ret = -ENOMEM;
1339                 goto out;
1340         }
1341
1342         ret = __ext4_link(dir, inode, dentry_inode);
1343         /*
1344          * It's possible that link already existed since data blocks
1345          * for the dir in question got persisted before we crashed OR
1346          * we replayed this tag and crashed before the entire replay
1347          * could complete.
1348          */
1349         if (ret && ret != -EEXIST) {
1350                 jbd_debug(1, "Failed to link\n");
1351                 goto out;
1352         }
1353
1354         ret = 0;
1355 out:
1356         if (dentry_dir) {
1357                 d_drop(dentry_dir);
1358                 dput(dentry_dir);
1359         } else if (dir) {
1360                 iput(dir);
1361         }
1362         if (dentry_inode) {
1363                 d_drop(dentry_inode);
1364                 dput(dentry_inode);
1365         }
1366
1367         return ret;
1368 }
1369
1370 /* Link replay function */
1371 static int ext4_fc_replay_link(struct super_block *sb, struct ext4_fc_tl *tl,
1372                                u8 *val)
1373 {
1374         struct inode *inode;
1375         struct dentry_info_args darg;
1376         int ret = 0;
1377
1378         tl_to_darg(&darg, tl, val);
1379         trace_ext4_fc_replay(sb, EXT4_FC_TAG_LINK, darg.ino,
1380                         darg.parent_ino, darg.dname_len);
1381
1382         inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
1383         if (IS_ERR(inode)) {
1384                 jbd_debug(1, "Inode not found.");
1385                 return 0;
1386         }
1387
1388         ret = ext4_fc_replay_link_internal(sb, &darg, inode);
1389         iput(inode);
1390         return ret;
1391 }
1392
1393 /*
1394  * Record all the modified inodes during replay. We use this later to setup
1395  * block bitmaps correctly.
1396  */
1397 static int ext4_fc_record_modified_inode(struct super_block *sb, int ino)
1398 {
1399         struct ext4_fc_replay_state *state;
1400         int i;
1401
1402         state = &EXT4_SB(sb)->s_fc_replay_state;
1403         for (i = 0; i < state->fc_modified_inodes_used; i++)
1404                 if (state->fc_modified_inodes[i] == ino)
1405                         return 0;
1406         if (state->fc_modified_inodes_used == state->fc_modified_inodes_size) {
1407                 int *fc_modified_inodes;
1408
1409                 fc_modified_inodes = krealloc(state->fc_modified_inodes,
1410                                 sizeof(int) * (state->fc_modified_inodes_size +
1411                                 EXT4_FC_REPLAY_REALLOC_INCREMENT),
1412                                 GFP_KERNEL);
1413                 if (!fc_modified_inodes)
1414                         return -ENOMEM;
1415                 state->fc_modified_inodes = fc_modified_inodes;
1416                 state->fc_modified_inodes_size +=
1417                         EXT4_FC_REPLAY_REALLOC_INCREMENT;
1418         }
1419         state->fc_modified_inodes[state->fc_modified_inodes_used++] = ino;
1420         return 0;
1421 }
1422
1423 /*
1424  * Inode replay function
1425  */
1426 static int ext4_fc_replay_inode(struct super_block *sb, struct ext4_fc_tl *tl,
1427                                 u8 *val)
1428 {
1429         struct ext4_fc_inode fc_inode;
1430         struct ext4_inode *raw_inode;
1431         struct ext4_inode *raw_fc_inode;
1432         struct inode *inode = NULL;
1433         struct ext4_iloc iloc;
1434         int inode_len, ino, ret, tag = le16_to_cpu(tl->fc_tag);
1435         struct ext4_extent_header *eh;
1436
1437         memcpy(&fc_inode, val, sizeof(fc_inode));
1438
1439         ino = le32_to_cpu(fc_inode.fc_ino);
1440         trace_ext4_fc_replay(sb, tag, ino, 0, 0);
1441
1442         inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL);
1443         if (!IS_ERR(inode)) {
1444                 ext4_ext_clear_bb(inode);
1445                 iput(inode);
1446         }
1447         inode = NULL;
1448
1449         ret = ext4_fc_record_modified_inode(sb, ino);
1450         if (ret)
1451                 goto out;
1452
1453         raw_fc_inode = (struct ext4_inode *)
1454                 (val + offsetof(struct ext4_fc_inode, fc_raw_inode));
1455         ret = ext4_get_fc_inode_loc(sb, ino, &iloc);
1456         if (ret)
1457                 goto out;
1458
1459         inode_len = le16_to_cpu(tl->fc_len) - sizeof(struct ext4_fc_inode);
1460         raw_inode = ext4_raw_inode(&iloc);
1461
1462         memcpy(raw_inode, raw_fc_inode, offsetof(struct ext4_inode, i_block));
1463         memcpy(&raw_inode->i_generation, &raw_fc_inode->i_generation,
1464                 inode_len - offsetof(struct ext4_inode, i_generation));
1465         if (le32_to_cpu(raw_inode->i_flags) & EXT4_EXTENTS_FL) {
1466                 eh = (struct ext4_extent_header *)(&raw_inode->i_block[0]);
1467                 if (eh->eh_magic != EXT4_EXT_MAGIC) {
1468                         memset(eh, 0, sizeof(*eh));
1469                         eh->eh_magic = EXT4_EXT_MAGIC;
1470                         eh->eh_max = cpu_to_le16(
1471                                 (sizeof(raw_inode->i_block) -
1472                                  sizeof(struct ext4_extent_header))
1473                                  / sizeof(struct ext4_extent));
1474                 }
1475         } else if (le32_to_cpu(raw_inode->i_flags) & EXT4_INLINE_DATA_FL) {
1476                 memcpy(raw_inode->i_block, raw_fc_inode->i_block,
1477                         sizeof(raw_inode->i_block));
1478         }
1479
1480         /* Immediately update the inode on disk. */
1481         ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh);
1482         if (ret)
1483                 goto out;
1484         ret = sync_dirty_buffer(iloc.bh);
1485         if (ret)
1486                 goto out;
1487         ret = ext4_mark_inode_used(sb, ino);
1488         if (ret)
1489                 goto out;
1490
1491         /* Given that we just wrote the inode on disk, this SHOULD succeed. */
1492         inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL);
1493         if (IS_ERR(inode)) {
1494                 jbd_debug(1, "Inode not found.");
1495                 return -EFSCORRUPTED;
1496         }
1497
1498         /*
1499          * Our allocator could have made different decisions than before
1500          * crashing. This should be fixed but until then, we calculate
1501          * the number of blocks the inode.
1502          */
1503         ext4_ext_replay_set_iblocks(inode);
1504
1505         inode->i_generation = le32_to_cpu(ext4_raw_inode(&iloc)->i_generation);
1506         ext4_reset_inode_seed(inode);
1507
1508         ext4_inode_csum_set(inode, ext4_raw_inode(&iloc), EXT4_I(inode));
1509         ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh);
1510         sync_dirty_buffer(iloc.bh);
1511         brelse(iloc.bh);
1512 out:
1513         iput(inode);
1514         if (!ret)
1515                 blkdev_issue_flush(sb->s_bdev, GFP_KERNEL);
1516
1517         return 0;
1518 }
1519
1520 /*
1521  * Dentry create replay function.
1522  *
1523  * EXT4_FC_TAG_CREAT is preceded by EXT4_FC_TAG_INODE_FULL. Which means, the
1524  * inode for which we are trying to create a dentry here, should already have
1525  * been replayed before we start here.
1526  */
1527 static int ext4_fc_replay_create(struct super_block *sb, struct ext4_fc_tl *tl,
1528                                  u8 *val)
1529 {
1530         int ret = 0;
1531         struct inode *inode = NULL;
1532         struct inode *dir = NULL;
1533         struct dentry_info_args darg;
1534
1535         tl_to_darg(&darg, tl, val);
1536
1537         trace_ext4_fc_replay(sb, EXT4_FC_TAG_CREAT, darg.ino,
1538                         darg.parent_ino, darg.dname_len);
1539
1540         /* This takes care of update group descriptor and other metadata */
1541         ret = ext4_mark_inode_used(sb, darg.ino);
1542         if (ret)
1543                 goto out;
1544
1545         inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
1546         if (IS_ERR(inode)) {
1547                 jbd_debug(1, "inode %d not found.", darg.ino);
1548                 inode = NULL;
1549                 ret = -EINVAL;
1550                 goto out;
1551         }
1552
1553         if (S_ISDIR(inode->i_mode)) {
1554                 /*
1555                  * If we are creating a directory, we need to make sure that the
1556                  * dot and dot dot dirents are setup properly.
1557                  */
1558                 dir = ext4_iget(sb, darg.parent_ino, EXT4_IGET_NORMAL);
1559                 if (IS_ERR(dir)) {
1560                         jbd_debug(1, "Dir %d not found.", darg.ino);
1561                         goto out;
1562                 }
1563                 ret = ext4_init_new_dir(NULL, dir, inode);
1564                 iput(dir);
1565                 if (ret) {
1566                         ret = 0;
1567                         goto out;
1568                 }
1569         }
1570         ret = ext4_fc_replay_link_internal(sb, &darg, inode);
1571         if (ret)
1572                 goto out;
1573         set_nlink(inode, 1);
1574         ext4_mark_inode_dirty(NULL, inode);
1575 out:
1576         if (inode)
1577                 iput(inode);
1578         return ret;
1579 }
1580
1581 /*
1582  * Record physical disk regions which are in use as per fast commit area,
1583  * and used by inodes during replay phase. Our simple replay phase
1584  * allocator excludes these regions from allocation.
1585  */
1586 int ext4_fc_record_regions(struct super_block *sb, int ino,
1587                 ext4_lblk_t lblk, ext4_fsblk_t pblk, int len, int replay)
1588 {
1589         struct ext4_fc_replay_state *state;
1590         struct ext4_fc_alloc_region *region;
1591
1592         state = &EXT4_SB(sb)->s_fc_replay_state;
1593         /*
1594          * during replay phase, the fc_regions_valid may not same as
1595          * fc_regions_used, update it when do new additions.
1596          */
1597         if (replay && state->fc_regions_used != state->fc_regions_valid)
1598                 state->fc_regions_used = state->fc_regions_valid;
1599         if (state->fc_regions_used == state->fc_regions_size) {
1600                 struct ext4_fc_alloc_region *fc_regions;
1601
1602                 fc_regions = krealloc(state->fc_regions,
1603                                       sizeof(struct ext4_fc_alloc_region) *
1604                                       (state->fc_regions_size +
1605                                        EXT4_FC_REPLAY_REALLOC_INCREMENT),
1606                                       GFP_KERNEL);
1607                 if (!fc_regions)
1608                         return -ENOMEM;
1609                 state->fc_regions_size +=
1610                         EXT4_FC_REPLAY_REALLOC_INCREMENT;
1611                 state->fc_regions = fc_regions;
1612         }
1613         region = &state->fc_regions[state->fc_regions_used++];
1614         region->ino = ino;
1615         region->lblk = lblk;
1616         region->pblk = pblk;
1617         region->len = len;
1618
1619         if (replay)
1620                 state->fc_regions_valid++;
1621
1622         return 0;
1623 }
1624
1625 /* Replay add range tag */
1626 static int ext4_fc_replay_add_range(struct super_block *sb,
1627                                     struct ext4_fc_tl *tl, u8 *val)
1628 {
1629         struct ext4_fc_add_range fc_add_ex;
1630         struct ext4_extent newex, *ex;
1631         struct inode *inode;
1632         ext4_lblk_t start, cur;
1633         int remaining, len;
1634         ext4_fsblk_t start_pblk;
1635         struct ext4_map_blocks map;
1636         struct ext4_ext_path *path = NULL;
1637         int ret;
1638
1639         memcpy(&fc_add_ex, val, sizeof(fc_add_ex));
1640         ex = (struct ext4_extent *)&fc_add_ex.fc_ex;
1641
1642         trace_ext4_fc_replay(sb, EXT4_FC_TAG_ADD_RANGE,
1643                 le32_to_cpu(fc_add_ex.fc_ino), le32_to_cpu(ex->ee_block),
1644                 ext4_ext_get_actual_len(ex));
1645
1646         inode = ext4_iget(sb, le32_to_cpu(fc_add_ex.fc_ino), EXT4_IGET_NORMAL);
1647         if (IS_ERR(inode)) {
1648                 jbd_debug(1, "Inode not found.");
1649                 return 0;
1650         }
1651
1652         ret = ext4_fc_record_modified_inode(sb, inode->i_ino);
1653         if (ret)
1654                 goto out;
1655
1656         start = le32_to_cpu(ex->ee_block);
1657         start_pblk = ext4_ext_pblock(ex);
1658         len = ext4_ext_get_actual_len(ex);
1659
1660         cur = start;
1661         remaining = len;
1662         jbd_debug(1, "ADD_RANGE, lblk %d, pblk %lld, len %d, unwritten %d, inode %ld\n",
1663                   start, start_pblk, len, ext4_ext_is_unwritten(ex),
1664                   inode->i_ino);
1665
1666         while (remaining > 0) {
1667                 map.m_lblk = cur;
1668                 map.m_len = remaining;
1669                 map.m_pblk = 0;
1670                 ret = ext4_map_blocks(NULL, inode, &map, 0);
1671
1672                 if (ret < 0)
1673                         goto out;
1674
1675                 if (ret == 0) {
1676                         /* Range is not mapped */
1677                         path = ext4_find_extent(inode, cur, NULL, 0);
1678                         if (IS_ERR(path))
1679                                 goto out;
1680                         memset(&newex, 0, sizeof(newex));
1681                         newex.ee_block = cpu_to_le32(cur);
1682                         ext4_ext_store_pblock(
1683                                 &newex, start_pblk + cur - start);
1684                         newex.ee_len = cpu_to_le16(map.m_len);
1685                         if (ext4_ext_is_unwritten(ex))
1686                                 ext4_ext_mark_unwritten(&newex);
1687                         down_write(&EXT4_I(inode)->i_data_sem);
1688                         ret = ext4_ext_insert_extent(
1689                                 NULL, inode, &path, &newex, 0);
1690                         up_write((&EXT4_I(inode)->i_data_sem));
1691                         ext4_ext_drop_refs(path);
1692                         kfree(path);
1693                         if (ret)
1694                                 goto out;
1695                         goto next;
1696                 }
1697
1698                 if (start_pblk + cur - start != map.m_pblk) {
1699                         /*
1700                          * Logical to physical mapping changed. This can happen
1701                          * if this range was removed and then reallocated to
1702                          * map to new physical blocks during a fast commit.
1703                          */
1704                         ret = ext4_ext_replay_update_ex(inode, cur, map.m_len,
1705                                         ext4_ext_is_unwritten(ex),
1706                                         start_pblk + cur - start);
1707                         if (ret)
1708                                 goto out;
1709                         /*
1710                          * Mark the old blocks as free since they aren't used
1711                          * anymore. We maintain an array of all the modified
1712                          * inodes. In case these blocks are still used at either
1713                          * a different logical range in the same inode or in
1714                          * some different inode, we will mark them as allocated
1715                          * at the end of the FC replay using our array of
1716                          * modified inodes.
1717                          */
1718                         ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, 0);
1719                         goto next;
1720                 }
1721
1722                 /* Range is mapped and needs a state change */
1723                 jbd_debug(1, "Converting from %ld to %d %lld",
1724                                 map.m_flags & EXT4_MAP_UNWRITTEN,
1725                         ext4_ext_is_unwritten(ex), map.m_pblk);
1726                 ret = ext4_ext_replay_update_ex(inode, cur, map.m_len,
1727                                         ext4_ext_is_unwritten(ex), map.m_pblk);
1728                 if (ret)
1729                         goto out;
1730                 /*
1731                  * We may have split the extent tree while toggling the state.
1732                  * Try to shrink the extent tree now.
1733                  */
1734                 ext4_ext_replay_shrink_inode(inode, start + len);
1735 next:
1736                 cur += map.m_len;
1737                 remaining -= map.m_len;
1738         }
1739         ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >>
1740                                         sb->s_blocksize_bits);
1741 out:
1742         iput(inode);
1743         return 0;
1744 }
1745
1746 /* Replay DEL_RANGE tag */
1747 static int
1748 ext4_fc_replay_del_range(struct super_block *sb, struct ext4_fc_tl *tl,
1749                          u8 *val)
1750 {
1751         struct inode *inode;
1752         struct ext4_fc_del_range lrange;
1753         struct ext4_map_blocks map;
1754         ext4_lblk_t cur, remaining;
1755         int ret;
1756
1757         memcpy(&lrange, val, sizeof(lrange));
1758         cur = le32_to_cpu(lrange.fc_lblk);
1759         remaining = le32_to_cpu(lrange.fc_len);
1760
1761         trace_ext4_fc_replay(sb, EXT4_FC_TAG_DEL_RANGE,
1762                 le32_to_cpu(lrange.fc_ino), cur, remaining);
1763
1764         inode = ext4_iget(sb, le32_to_cpu(lrange.fc_ino), EXT4_IGET_NORMAL);
1765         if (IS_ERR(inode)) {
1766                 jbd_debug(1, "Inode %d not found", le32_to_cpu(lrange.fc_ino));
1767                 return 0;
1768         }
1769
1770         ret = ext4_fc_record_modified_inode(sb, inode->i_ino);
1771         if (ret)
1772                 goto out;
1773
1774         jbd_debug(1, "DEL_RANGE, inode %ld, lblk %d, len %d\n",
1775                         inode->i_ino, le32_to_cpu(lrange.fc_lblk),
1776                         le32_to_cpu(lrange.fc_len));
1777         while (remaining > 0) {
1778                 map.m_lblk = cur;
1779                 map.m_len = remaining;
1780
1781                 ret = ext4_map_blocks(NULL, inode, &map, 0);
1782                 if (ret < 0)
1783                         goto out;
1784                 if (ret > 0) {
1785                         remaining -= ret;
1786                         cur += ret;
1787                         ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, 0);
1788                 } else {
1789                         remaining -= map.m_len;
1790                         cur += map.m_len;
1791                 }
1792         }
1793
1794         down_write(&EXT4_I(inode)->i_data_sem);
1795         ret = ext4_ext_remove_space(inode, le32_to_cpu(lrange.fc_lblk),
1796                                 le32_to_cpu(lrange.fc_lblk) +
1797                                 le32_to_cpu(lrange.fc_len) - 1);
1798         up_write(&EXT4_I(inode)->i_data_sem);
1799         if (ret)
1800                 goto out;
1801         ext4_ext_replay_shrink_inode(inode,
1802                 i_size_read(inode) >> sb->s_blocksize_bits);
1803         ext4_mark_inode_dirty(NULL, inode);
1804 out:
1805         iput(inode);
1806         return 0;
1807 }
1808
1809 static inline const char *tag2str(u16 tag)
1810 {
1811         switch (tag) {
1812         case EXT4_FC_TAG_LINK:
1813                 return "TAG_ADD_ENTRY";
1814         case EXT4_FC_TAG_UNLINK:
1815                 return "TAG_DEL_ENTRY";
1816         case EXT4_FC_TAG_ADD_RANGE:
1817                 return "TAG_ADD_RANGE";
1818         case EXT4_FC_TAG_CREAT:
1819                 return "TAG_CREAT_DENTRY";
1820         case EXT4_FC_TAG_DEL_RANGE:
1821                 return "TAG_DEL_RANGE";
1822         case EXT4_FC_TAG_INODE:
1823                 return "TAG_INODE";
1824         case EXT4_FC_TAG_PAD:
1825                 return "TAG_PAD";
1826         case EXT4_FC_TAG_TAIL:
1827                 return "TAG_TAIL";
1828         case EXT4_FC_TAG_HEAD:
1829                 return "TAG_HEAD";
1830         default:
1831                 return "TAG_ERROR";
1832         }
1833 }
1834
1835 static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb)
1836 {
1837         struct ext4_fc_replay_state *state;
1838         struct inode *inode;
1839         struct ext4_ext_path *path = NULL;
1840         struct ext4_map_blocks map;
1841         int i, ret, j;
1842         ext4_lblk_t cur, end;
1843
1844         state = &EXT4_SB(sb)->s_fc_replay_state;
1845         for (i = 0; i < state->fc_modified_inodes_used; i++) {
1846                 inode = ext4_iget(sb, state->fc_modified_inodes[i],
1847                         EXT4_IGET_NORMAL);
1848                 if (IS_ERR(inode)) {
1849                         jbd_debug(1, "Inode %d not found.",
1850                                 state->fc_modified_inodes[i]);
1851                         continue;
1852                 }
1853                 cur = 0;
1854                 end = EXT_MAX_BLOCKS;
1855                 while (cur < end) {
1856                         map.m_lblk = cur;
1857                         map.m_len = end - cur;
1858
1859                         ret = ext4_map_blocks(NULL, inode, &map, 0);
1860                         if (ret < 0)
1861                                 break;
1862
1863                         if (ret > 0) {
1864                                 path = ext4_find_extent(inode, map.m_lblk, NULL, 0);
1865                                 if (!IS_ERR(path)) {
1866                                         for (j = 0; j < path->p_depth; j++)
1867                                                 ext4_mb_mark_bb(inode->i_sb,
1868                                                         path[j].p_block, 1, 1);
1869                                         ext4_ext_drop_refs(path);
1870                                         kfree(path);
1871                                 }
1872                                 cur += ret;
1873                                 ext4_mb_mark_bb(inode->i_sb, map.m_pblk,
1874                                                         map.m_len, 1);
1875                         } else {
1876                                 cur = cur + (map.m_len ? map.m_len : 1);
1877                         }
1878                 }
1879                 iput(inode);
1880         }
1881 }
1882
1883 /*
1884  * Check if block is in excluded regions for block allocation. The simple
1885  * allocator that runs during replay phase is calls this function to see
1886  * if it is okay to use a block.
1887  */
1888 bool ext4_fc_replay_check_excluded(struct super_block *sb, ext4_fsblk_t blk)
1889 {
1890         int i;
1891         struct ext4_fc_replay_state *state;
1892
1893         state = &EXT4_SB(sb)->s_fc_replay_state;
1894         for (i = 0; i < state->fc_regions_valid; i++) {
1895                 if (state->fc_regions[i].ino == 0 ||
1896                         state->fc_regions[i].len == 0)
1897                         continue;
1898                 if (blk >= state->fc_regions[i].pblk &&
1899                     blk < state->fc_regions[i].pblk + state->fc_regions[i].len)
1900                         return true;
1901         }
1902         return false;
1903 }
1904
1905 /* Cleanup function called after replay */
1906 void ext4_fc_replay_cleanup(struct super_block *sb)
1907 {
1908         struct ext4_sb_info *sbi = EXT4_SB(sb);
1909
1910         sbi->s_mount_state &= ~EXT4_FC_REPLAY;
1911         kfree(sbi->s_fc_replay_state.fc_regions);
1912         kfree(sbi->s_fc_replay_state.fc_modified_inodes);
1913 }
1914
1915 /*
1916  * Recovery Scan phase handler
1917  *
1918  * This function is called during the scan phase and is responsible
1919  * for doing following things:
1920  * - Make sure the fast commit area has valid tags for replay
1921  * - Count number of tags that need to be replayed by the replay handler
1922  * - Verify CRC
1923  * - Create a list of excluded blocks for allocation during replay phase
1924  *
1925  * This function returns JBD2_FC_REPLAY_CONTINUE to indicate that SCAN is
1926  * incomplete and JBD2 should send more blocks. It returns JBD2_FC_REPLAY_STOP
1927  * to indicate that scan has finished and JBD2 can now start replay phase.
1928  * It returns a negative error to indicate that there was an error. At the end
1929  * of a successful scan phase, sbi->s_fc_replay_state.fc_replay_num_tags is set
1930  * to indicate the number of tags that need to replayed during the replay phase.
1931  */
1932 static int ext4_fc_replay_scan(journal_t *journal,
1933                                 struct buffer_head *bh, int off,
1934                                 tid_t expected_tid)
1935 {
1936         struct super_block *sb = journal->j_private;
1937         struct ext4_sb_info *sbi = EXT4_SB(sb);
1938         struct ext4_fc_replay_state *state;
1939         int ret = JBD2_FC_REPLAY_CONTINUE;
1940         struct ext4_fc_add_range ext;
1941         struct ext4_fc_tl tl;
1942         struct ext4_fc_tail tail;
1943         __u8 *start, *end, *cur, *val;
1944         struct ext4_fc_head head;
1945         struct ext4_extent *ex;
1946
1947         state = &sbi->s_fc_replay_state;
1948
1949         start = (u8 *)bh->b_data;
1950         end = (__u8 *)bh->b_data + journal->j_blocksize - 1;
1951
1952         if (state->fc_replay_expected_off == 0) {
1953                 state->fc_cur_tag = 0;
1954                 state->fc_replay_num_tags = 0;
1955                 state->fc_crc = 0;
1956                 state->fc_regions = NULL;
1957                 state->fc_regions_valid = state->fc_regions_used =
1958                         state->fc_regions_size = 0;
1959                 /* Check if we can stop early */
1960                 if (le16_to_cpu(((struct ext4_fc_tl *)start)->fc_tag)
1961                         != EXT4_FC_TAG_HEAD)
1962                         return 0;
1963         }
1964
1965         if (off != state->fc_replay_expected_off) {
1966                 ret = -EFSCORRUPTED;
1967                 goto out_err;
1968         }
1969
1970         state->fc_replay_expected_off++;
1971         for (cur = start; cur < end; cur = cur + sizeof(tl) + le16_to_cpu(tl.fc_len)) {
1972                 memcpy(&tl, cur, sizeof(tl));
1973                 val = cur + sizeof(tl);
1974                 jbd_debug(3, "Scan phase, tag:%s, blk %lld\n",
1975                           tag2str(le16_to_cpu(tl.fc_tag)), bh->b_blocknr);
1976                 switch (le16_to_cpu(tl.fc_tag)) {
1977                 case EXT4_FC_TAG_ADD_RANGE:
1978                         memcpy(&ext, val, sizeof(ext));
1979                         ex = (struct ext4_extent *)&ext.fc_ex;
1980                         ret = ext4_fc_record_regions(sb,
1981                                 le32_to_cpu(ext.fc_ino),
1982                                 le32_to_cpu(ex->ee_block), ext4_ext_pblock(ex),
1983                                 ext4_ext_get_actual_len(ex), 0);
1984                         if (ret < 0)
1985                                 break;
1986                         ret = JBD2_FC_REPLAY_CONTINUE;
1987                         fallthrough;
1988                 case EXT4_FC_TAG_DEL_RANGE:
1989                 case EXT4_FC_TAG_LINK:
1990                 case EXT4_FC_TAG_UNLINK:
1991                 case EXT4_FC_TAG_CREAT:
1992                 case EXT4_FC_TAG_INODE:
1993                 case EXT4_FC_TAG_PAD:
1994                         state->fc_cur_tag++;
1995                         state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur,
1996                                         sizeof(tl) + le16_to_cpu(tl.fc_len));
1997                         break;
1998                 case EXT4_FC_TAG_TAIL:
1999                         state->fc_cur_tag++;
2000                         memcpy(&tail, val, sizeof(tail));
2001                         state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur,
2002                                                 sizeof(tl) +
2003                                                 offsetof(struct ext4_fc_tail,
2004                                                 fc_crc));
2005                         if (le32_to_cpu(tail.fc_tid) == expected_tid &&
2006                                 le32_to_cpu(tail.fc_crc) == state->fc_crc) {
2007                                 state->fc_replay_num_tags = state->fc_cur_tag;
2008                                 state->fc_regions_valid =
2009                                         state->fc_regions_used;
2010                         } else {
2011                                 ret = state->fc_replay_num_tags ?
2012                                         JBD2_FC_REPLAY_STOP : -EFSBADCRC;
2013                         }
2014                         state->fc_crc = 0;
2015                         break;
2016                 case EXT4_FC_TAG_HEAD:
2017                         memcpy(&head, val, sizeof(head));
2018                         if (le32_to_cpu(head.fc_features) &
2019                                 ~EXT4_FC_SUPPORTED_FEATURES) {
2020                                 ret = -EOPNOTSUPP;
2021                                 break;
2022                         }
2023                         if (le32_to_cpu(head.fc_tid) != expected_tid) {
2024                                 ret = JBD2_FC_REPLAY_STOP;
2025                                 break;
2026                         }
2027                         state->fc_cur_tag++;
2028                         state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur,
2029                                             sizeof(tl) + le16_to_cpu(tl.fc_len));
2030                         break;
2031                 default:
2032                         ret = state->fc_replay_num_tags ?
2033                                 JBD2_FC_REPLAY_STOP : -ECANCELED;
2034                 }
2035                 if (ret < 0 || ret == JBD2_FC_REPLAY_STOP)
2036                         break;
2037         }
2038
2039 out_err:
2040         trace_ext4_fc_replay_scan(sb, ret, off);
2041         return ret;
2042 }
2043
2044 /*
2045  * Main recovery path entry point.
2046  * The meaning of return codes is similar as above.
2047  */
2048 static int ext4_fc_replay(journal_t *journal, struct buffer_head *bh,
2049                                 enum passtype pass, int off, tid_t expected_tid)
2050 {
2051         struct super_block *sb = journal->j_private;
2052         struct ext4_sb_info *sbi = EXT4_SB(sb);
2053         struct ext4_fc_tl tl;
2054         __u8 *start, *end, *cur, *val;
2055         int ret = JBD2_FC_REPLAY_CONTINUE;
2056         struct ext4_fc_replay_state *state = &sbi->s_fc_replay_state;
2057         struct ext4_fc_tail tail;
2058
2059         if (pass == PASS_SCAN) {
2060                 state->fc_current_pass = PASS_SCAN;
2061                 return ext4_fc_replay_scan(journal, bh, off, expected_tid);
2062         }
2063
2064         if (state->fc_current_pass != pass) {
2065                 state->fc_current_pass = pass;
2066                 sbi->s_mount_state |= EXT4_FC_REPLAY;
2067         }
2068         if (!sbi->s_fc_replay_state.fc_replay_num_tags) {
2069                 jbd_debug(1, "Replay stops\n");
2070                 ext4_fc_set_bitmaps_and_counters(sb);
2071                 return 0;
2072         }
2073
2074 #ifdef CONFIG_EXT4_DEBUG
2075         if (sbi->s_fc_debug_max_replay && off >= sbi->s_fc_debug_max_replay) {
2076                 pr_warn("Dropping fc block %d because max_replay set\n", off);
2077                 return JBD2_FC_REPLAY_STOP;
2078         }
2079 #endif
2080
2081         start = (u8 *)bh->b_data;
2082         end = (__u8 *)bh->b_data + journal->j_blocksize - 1;
2083
2084         for (cur = start; cur < end; cur = cur + sizeof(tl) + le16_to_cpu(tl.fc_len)) {
2085                 memcpy(&tl, cur, sizeof(tl));
2086                 val = cur + sizeof(tl);
2087
2088                 if (state->fc_replay_num_tags == 0) {
2089                         ret = JBD2_FC_REPLAY_STOP;
2090                         ext4_fc_set_bitmaps_and_counters(sb);
2091                         break;
2092                 }
2093                 jbd_debug(3, "Replay phase, tag:%s\n",
2094                                 tag2str(le16_to_cpu(tl.fc_tag)));
2095                 state->fc_replay_num_tags--;
2096                 switch (le16_to_cpu(tl.fc_tag)) {
2097                 case EXT4_FC_TAG_LINK:
2098                         ret = ext4_fc_replay_link(sb, &tl, val);
2099                         break;
2100                 case EXT4_FC_TAG_UNLINK:
2101                         ret = ext4_fc_replay_unlink(sb, &tl, val);
2102                         break;
2103                 case EXT4_FC_TAG_ADD_RANGE:
2104                         ret = ext4_fc_replay_add_range(sb, &tl, val);
2105                         break;
2106                 case EXT4_FC_TAG_CREAT:
2107                         ret = ext4_fc_replay_create(sb, &tl, val);
2108                         break;
2109                 case EXT4_FC_TAG_DEL_RANGE:
2110                         ret = ext4_fc_replay_del_range(sb, &tl, val);
2111                         break;
2112                 case EXT4_FC_TAG_INODE:
2113                         ret = ext4_fc_replay_inode(sb, &tl, val);
2114                         break;
2115                 case EXT4_FC_TAG_PAD:
2116                         trace_ext4_fc_replay(sb, EXT4_FC_TAG_PAD, 0,
2117                                              le16_to_cpu(tl.fc_len), 0);
2118                         break;
2119                 case EXT4_FC_TAG_TAIL:
2120                         trace_ext4_fc_replay(sb, EXT4_FC_TAG_TAIL, 0,
2121                                              le16_to_cpu(tl.fc_len), 0);
2122                         memcpy(&tail, val, sizeof(tail));
2123                         WARN_ON(le32_to_cpu(tail.fc_tid) != expected_tid);
2124                         break;
2125                 case EXT4_FC_TAG_HEAD:
2126                         break;
2127                 default:
2128                         trace_ext4_fc_replay(sb, le16_to_cpu(tl.fc_tag), 0,
2129                                              le16_to_cpu(tl.fc_len), 0);
2130                         ret = -ECANCELED;
2131                         break;
2132                 }
2133                 if (ret < 0)
2134                         break;
2135                 ret = JBD2_FC_REPLAY_CONTINUE;
2136         }
2137         return ret;
2138 }
2139
2140 void ext4_fc_init(struct super_block *sb, journal_t *journal)
2141 {
2142         /*
2143          * We set replay callback even if fast commit disabled because we may
2144          * could still have fast commit blocks that need to be replayed even if
2145          * fast commit has now been turned off.
2146          */
2147         journal->j_fc_replay_callback = ext4_fc_replay;
2148         if (!test_opt2(sb, JOURNAL_FAST_COMMIT))
2149                 return;
2150         journal->j_fc_cleanup_callback = ext4_fc_cleanup;
2151 }
2152
2153 static const char * const fc_ineligible_reasons[] = {
2154         [EXT4_FC_REASON_XATTR] = "Extended attributes changed",
2155         [EXT4_FC_REASON_CROSS_RENAME] = "Cross rename",
2156         [EXT4_FC_REASON_JOURNAL_FLAG_CHANGE] = "Journal flag changed",
2157         [EXT4_FC_REASON_NOMEM] = "Insufficient memory",
2158         [EXT4_FC_REASON_SWAP_BOOT] = "Swap boot",
2159         [EXT4_FC_REASON_RESIZE] = "Resize",
2160         [EXT4_FC_REASON_RENAME_DIR] = "Dir renamed",
2161         [EXT4_FC_REASON_FALLOC_RANGE] = "Falloc range op",
2162         [EXT4_FC_REASON_INODE_JOURNAL_DATA] = "Data journalling",
2163         [EXT4_FC_REASON_ENCRYPTED_FILENAME] = "Encrypted filename",
2164 };
2165
2166 int ext4_fc_info_show(struct seq_file *seq, void *v)
2167 {
2168         struct ext4_sb_info *sbi = EXT4_SB((struct super_block *)seq->private);
2169         struct ext4_fc_stats *stats = &sbi->s_fc_stats;
2170         int i;
2171
2172         if (v != SEQ_START_TOKEN)
2173                 return 0;
2174
2175         seq_printf(seq,
2176                 "fc stats:\n%ld commits\n%ld ineligible\n%ld numblks\n%lluus avg_commit_time\n",
2177                    stats->fc_num_commits, stats->fc_ineligible_commits,
2178                    stats->fc_numblks,
2179                    div_u64(sbi->s_fc_avg_commit_time, 1000));
2180         seq_puts(seq, "Ineligible reasons:\n");
2181         for (i = 0; i < EXT4_FC_REASON_MAX; i++)
2182                 seq_printf(seq, "\"%s\":\t%d\n", fc_ineligible_reasons[i],
2183                         stats->fc_ineligible_reason_count[i]);
2184
2185         return 0;
2186 }
2187
2188 int __init ext4_fc_init_dentry_cache(void)
2189 {
2190         ext4_fc_dentry_cachep = KMEM_CACHE(ext4_fc_dentry_update,
2191                                            SLAB_RECLAIM_ACCOUNT);
2192
2193         if (ext4_fc_dentry_cachep == NULL)
2194                 return -ENOMEM;
2195
2196         return 0;
2197 }
2198
2199 void ext4_fc_destroy_dentry_cache(void)
2200 {
2201         kmem_cache_destroy(ext4_fc_dentry_cachep);
2202 }