fs/ext4/fast_commit.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * fs/ext4/fast_commit.c
   5  *
   6  * Written by Harshad Shirwadkar <harshadshirwadkar@gmail.com>
   7  *
   8  * Ext4 fast commits routines.
   9  */
  10 #include "ext4.h"
  11 #include "ext4_jbd2.h"
  12 #include "ext4_extents.h"
  13 #include "mballoc.h"
  14
  15 /*
  16  * Ext4 Fast Commits
  17  * -----------------
  18  *
  19  * Ext4 fast commits implement fine grained journalling for Ext4.
  20  *
  21  * Fast commits are organized as a log of tag-length-value (TLV) structs. (See
  22  * struct ext4_fc_tl). Each TLV contains some delta that is replayed TLV by
  23  * TLV during the recovery phase. For the scenarios for which we currently
  24  * don't have replay code, fast commit falls back to full commits.
  25  * Fast commits record delta in one of the following three categories.
  26  *
  27  * (A) Directory entry updates:
  28  *
  29  * - EXT4_FC_TAG_UNLINK         - records directory entry unlink
  30  * - EXT4_FC_TAG_LINK           - records directory entry link
  31  * - EXT4_FC_TAG_CREAT          - records inode and directory entry creation
  32  *
  33  * (B) File specific data range updates:
  34  *
  35  * - EXT4_FC_TAG_ADD_RANGE      - records addition of new blocks to an inode
  36  * - EXT4_FC_TAG_DEL_RANGE      - records deletion of blocks from an inode
  37  *
  38  * (C) Inode metadata (mtime / ctime etc):
  39  *
  40  * - EXT4_FC_TAG_INODE          - record the inode that should be replayed
  41  *                                during recovery. Note that iblocks field is
  42  *                                not replayed and instead derived during
  43  *                                replay.
  44  * Commit Operation
  45  * ----------------
  46  * With fast commits, we maintain all the directory entry operations in the
  47  * order in which they are issued in an in-memory queue. This queue is flushed
  48  * to disk during the commit operation. We also maintain a list of inodes
  49  * that need to be committed during a fast commit in another in memory queue of
  50  * inodes. During the commit operation, we commit in the following order:
  51  *
  52  * [1] Lock inodes for any further data updates by setting COMMITTING state
  53  * [2] Submit data buffers of all the inodes
  54  * [3] Wait for [2] to complete
  55  * [4] Commit all the directory entry updates in the fast commit space
  56  * [5] Commit all the changed inode structures
  57  * [6] Write tail tag (this tag ensures the atomicity, please read the following
  58  *     section for more details).
  59  * [7] Wait for [4], [5] and [6] to complete.
  60  *
  61  * All the inode updates must call ext4_fc_start_update() before starting an
  62  * update. If such an ongoing update is present, fast commit waits for it to
  63  * complete. The completion of such an update is marked by
  64  * ext4_fc_stop_update().
  65  *
  66  * Fast Commit Ineligibility
  67  * -------------------------
  68  *
  69  * Not all operations are supported by fast commits today (e.g extended
  70  * attributes). Fast commit ineligibility is marked by calling
  71  * ext4_fc_mark_ineligible(): This makes next fast commit operation to fall back
  72  * to full commit.
  73  *
  74  * Atomicity of commits
  75  * --------------------
  76  * In order to guarantee atomicity during the commit operation, fast commit
  77  * uses "EXT4_FC_TAG_TAIL" tag that marks a fast commit as complete. Tail
  78  * tag contains CRC of the contents and TID of the transaction after which
  79  * this fast commit should be applied. Recovery code replays fast commit
  80  * logs only if there's at least 1 valid tail present. For every fast commit
  81  * operation, there is 1 tail. This means, we may end up with multiple tails
  82  * in the fast commit space. Here's an example:
  83  *
  84  * - Create a new file A and remove existing file B
  85  * - fsync()
  86  * - Append contents to file A
  87  * - Truncate file A
  88  * - fsync()
  89  *
  90  * The fast commit space at the end of above operations would look like this:
  91  *      [HEAD] [CREAT A] [UNLINK B] [TAIL] [ADD_RANGE A] [DEL_RANGE A] [TAIL]
  92  *             |<---  Fast Commit 1   --->|<---      Fast Commit 2     ---->|
  93  *
  94  * Replay code should thus check for all the valid tails in the FC area.
  95  *
  96  * Fast Commit Replay Idempotence
  97  * ------------------------------
  98  *
  99  * Fast commits tags are idempotent in nature provided the recovery code follows
 100  * certain rules. The guiding principle that the commit path follows while
 101  * committing is that it stores the result of a particular operation instead of
 102  * storing the procedure.
 103  *
 104  * Let's consider this rename operation: 'mv /a /b'. Let's assume dirent '/a'
 105  * was associated with inode 10. During fast commit, instead of storing this
 106  * operation as a procedure "rename a to b", we store the resulting file system
 107  * state as a "series" of outcomes:
 108  *
 109  * - Link dirent b to inode 10
 110  * - Unlink dirent a
 111  * - Inode <10> with valid refcount
 112  *
 113  * Now when recovery code runs, it needs "enforce" this state on the file
 114  * system. This is what guarantees idempotence of fast commit replay.
 115  *
 116  * Let's take an example of a procedure that is not idempotent and see how fast
 117  * commits make it idempotent. Consider following sequence of operations:
 118  *
 119  *     rm A;    mv B A;    read A
 120  *  (x)     (y)        (z)
 121  *
 122  * (x), (y) and (z) are the points at which we can crash. If we store this
 123  * sequence of operations as is then the replay is not idempotent. Let's say
 124  * while in replay, we crash at (z). During the second replay, file A (which was
 125  * actually created as a result of "mv B A" operation) would get deleted. Thus,
 126  * file named A would be absent when we try to read A. So, this sequence of
 127  * operations is not idempotent. However, as mentioned above, instead of storing
 128  * the procedure fast commits store the outcome of each procedure. Thus the fast
 129  * commit log for above procedure would be as follows:
 130  *
 131  * (Let's assume dirent A was linked to inode 10 and dirent B was linked to
 132  * inode 11 before the replay)
 133  *
 134  *    [Unlink A]   [Link A to inode 11]   [Unlink B]   [Inode 11]
 135  * (w)          (x)                    (y)          (z)
 136  *
 137  * If we crash at (z), we will have file A linked to inode 11. During the second
 138  * replay, we will remove file A (inode 11). But we will create it back and make
 139  * it point to inode 11. We won't find B, so we'll just skip that step. At this
 140  * point, the refcount for inode 11 is not reliable, but that gets fixed by the
 141  * replay of last inode 11 tag. Crashes at points (w), (x) and (y) get handled
 142  * similarly. Thus, by converting a non-idempotent procedure into a series of
 143  * idempotent outcomes, fast commits ensured idempotence during the replay.
 144  *
 145  * TODOs
 146  * -----
 147  *
 148  * 0) Fast commit replay path hardening: Fast commit replay code should use
 149  *    journal handles to make sure all the updates it does during the replay
 150  *    path are atomic. With that if we crash during fast commit replay, after
 151  *    trying to do recovery again, we will find a file system where fast commit
 152  *    area is invalid (because new full commit would be found). In order to deal
 153  *    with that, fast commit replay code should ensure that the "FC_REPLAY"
 154  *    superblock state is persisted before starting the replay, so that after
 155  *    the crash, fast commit recovery code can look at that flag and perform
 156  *    fast commit recovery even if that area is invalidated by later full
 157  *    commits.
 158  *
 159  * 1) Fast commit's commit path locks the entire file system during fast
 160  *    commit. This has significant performance penalty. Instead of that, we
 161  *    should use ext4_fc_start/stop_update functions to start inode level
 162  *    updates from ext4_journal_start/stop. Once we do that we can drop file
 163  *    system locking during commit path.
 164  *
 165  * 2) Handle more ineligible cases.
 166  */
 167
 168 #include <trace/events/ext4.h>
 169 static struct kmem_cache *ext4_fc_dentry_cachep;
 170
 171 static void ext4_end_buffer_io_sync(struct buffer_head *bh, int uptodate)
 172 {
 173         BUFFER_TRACE(bh, "");
 174         if (uptodate) {
 175                 ext4_debug("%s: Block %lld up-to-date",
 176                            __func__, bh->b_blocknr);
 177                 set_buffer_uptodate(bh);
 178         } else {
 179                 ext4_debug("%s: Block %lld not up-to-date",
 180                            __func__, bh->b_blocknr);
 181                 clear_buffer_uptodate(bh);
 182         }
 183
 184         unlock_buffer(bh);
 185 }
 186
 187 static inline void ext4_fc_reset_inode(struct inode *inode)
 188 {
 189         struct ext4_inode_info *ei = EXT4_I(inode);
 190
 191         ei->i_fc_lblk_start = 0;
 192         ei->i_fc_lblk_len = 0;
 193 }
 194
 195 void ext4_fc_init_inode(struct inode *inode)
 196 {
 197         struct ext4_inode_info *ei = EXT4_I(inode);
 198
 199         ext4_fc_reset_inode(inode);
 200         ext4_clear_inode_state(inode, EXT4_STATE_FC_COMMITTING);
 201         INIT_LIST_HEAD(&ei->i_fc_list);
 202         INIT_LIST_HEAD(&ei->i_fc_dilist);
 203         init_waitqueue_head(&ei->i_fc_wait);
 204         atomic_set(&ei->i_fc_updates, 0);
 205 }
 206
 207 /* This function must be called with sbi->s_fc_lock held. */
 208 static void ext4_fc_wait_committing_inode(struct inode *inode)
 209 __releases(&EXT4_SB(inode->i_sb)->s_fc_lock)
 210 {
 211         wait_queue_head_t *wq;
 212         struct ext4_inode_info *ei = EXT4_I(inode);
 213
 214 #if (BITS_PER_LONG < 64)
 215         DEFINE_WAIT_BIT(wait, &ei->i_state_flags,
 216                         EXT4_STATE_FC_COMMITTING);
 217         wq = bit_waitqueue(&ei->i_state_flags,
 218                                 EXT4_STATE_FC_COMMITTING);
 219 #else
 220         DEFINE_WAIT_BIT(wait, &ei->i_flags,
 221                         EXT4_STATE_FC_COMMITTING);
 222         wq = bit_waitqueue(&ei->i_flags,
 223                                 EXT4_STATE_FC_COMMITTING);
 224 #endif
 225         lockdep_assert_held(&EXT4_SB(inode->i_sb)->s_fc_lock);
 226         prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
 227         spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
 228         schedule();
 229         finish_wait(wq, &wait.wq_entry);
 230 }
 231
 232 static bool ext4_fc_disabled(struct super_block *sb)
 233 {
 234         return (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
 235                 (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY));
 236 }
 237
 238 /*
 239  * Inform Ext4's fast about start of an inode update
 240  *
 241  * This function is called by the high level call VFS callbacks before
 242  * performing any inode update. This function blocks if there's an ongoing
 243  * fast commit on the inode in question.
 244  */
 245 void ext4_fc_start_update(struct inode *inode)
 246 {
 247         struct ext4_inode_info *ei = EXT4_I(inode);
 248
 249         if (ext4_fc_disabled(inode->i_sb))
 250                 return;
 251
 252 restart:
 253         spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock);
 254         if (list_empty(&ei->i_fc_list))
 255                 goto out;
 256
 257         if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) {
 258                 ext4_fc_wait_committing_inode(inode);
 259                 goto restart;
 260         }
 261 out:
 262         atomic_inc(&ei->i_fc_updates);
 263         spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
 264 }
 265
 266 /*
 267  * Stop inode update and wake up waiting fast commits if any.
 268  */
 269 void ext4_fc_stop_update(struct inode *inode)
 270 {
 271         struct ext4_inode_info *ei = EXT4_I(inode);
 272
 273         if (ext4_fc_disabled(inode->i_sb))
 274                 return;
 275
 276         if (atomic_dec_and_test(&ei->i_fc_updates))
 277                 wake_up_all(&ei->i_fc_wait);
 278 }
 279
 280 /*
 281  * Remove inode from fast commit list. If the inode is being committed
 282  * we wait until inode commit is done.
 283  */
 284 void ext4_fc_del(struct inode *inode)
 285 {
 286         struct ext4_inode_info *ei = EXT4_I(inode);
 287         struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
 288         struct ext4_fc_dentry_update *fc_dentry;
 289
 290         if (ext4_fc_disabled(inode->i_sb))
 291                 return;
 292
 293 restart:
 294         spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock);
 295         if (list_empty(&ei->i_fc_list) && list_empty(&ei->i_fc_dilist)) {
 296                 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
 297                 return;
 298         }
 299
 300         if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) {
 301                 ext4_fc_wait_committing_inode(inode);
 302                 goto restart;
 303         }
 304
 305         if (!list_empty(&ei->i_fc_list))
 306                 list_del_init(&ei->i_fc_list);
 307
 308         /*
 309          * Since this inode is getting removed, let's also remove all FC
 310          * dentry create references, since it is not needed to log it anyways.
 311          */
 312         if (list_empty(&ei->i_fc_dilist)) {
 313                 spin_unlock(&sbi->s_fc_lock);
 314                 return;
 315         }
 316
 317         fc_dentry = list_first_entry(&ei->i_fc_dilist, struct ext4_fc_dentry_update, fcd_dilist);
 318         WARN_ON(fc_dentry->fcd_op != EXT4_FC_TAG_CREAT);
 319         list_del_init(&fc_dentry->fcd_list);
 320         list_del_init(&fc_dentry->fcd_dilist);
 321
 322         WARN_ON(!list_empty(&ei->i_fc_dilist));
 323         spin_unlock(&sbi->s_fc_lock);
 324
 325         if (fc_dentry->fcd_name.name &&
 326                 fc_dentry->fcd_name.len > DNAME_INLINE_LEN)
 327                 kfree(fc_dentry->fcd_name.name);
 328         kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry);
 329
 330         return;
 331 }
 332
 333 /*
 334  * Mark file system as fast commit ineligible, and record latest
 335  * ineligible transaction tid. This means until the recorded
 336  * transaction, commit operation would result in a full jbd2 commit.
 337  */
 338 void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handle)
 339 {
 340         struct ext4_sb_info *sbi = EXT4_SB(sb);
 341         tid_t tid;
 342
 343         if (ext4_fc_disabled(sb))
 344                 return;
 345
 346         ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
 347         if (handle && !IS_ERR(handle))
 348                 tid = handle->h_transaction->t_tid;
 349         else {
 350                 read_lock(&sbi->s_journal->j_state_lock);
 351                 tid = sbi->s_journal->j_running_transaction ?
 352                                 sbi->s_journal->j_running_transaction->t_tid : 0;
 353                 read_unlock(&sbi->s_journal->j_state_lock);
 354         }
 355         spin_lock(&sbi->s_fc_lock);
 356         if (sbi->s_fc_ineligible_tid < tid)
 357                 sbi->s_fc_ineligible_tid = tid;
 358         spin_unlock(&sbi->s_fc_lock);
 359         WARN_ON(reason >= EXT4_FC_REASON_MAX);
 360         sbi->s_fc_stats.fc_ineligible_reason_count[reason]++;
 361 }
 362
 363 /*
 364  * Generic fast commit tracking function. If this is the first time this we are
 365  * called after a full commit, we initialize fast commit fields and then call
 366  * __fc_track_fn() with update = 0. If we have already been called after a full
 367  * commit, we pass update = 1. Based on that, the track function can determine
 368  * if it needs to track a field for the first time or if it needs to just
 369  * update the previously tracked value.
 370  *
 371  * If enqueue is set, this function enqueues the inode in fast commit list.
 372  */
 373 static int ext4_fc_track_template(
 374         handle_t *handle, struct inode *inode,
 375         int (*__fc_track_fn)(struct inode *, void *, bool),
 376         void *args, int enqueue)
 377 {
 378         bool update = false;
 379         struct ext4_inode_info *ei = EXT4_I(inode);
 380         struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
 381         tid_t tid = 0;
 382         int ret;
 383
 384         tid = handle->h_transaction->t_tid;
 385         mutex_lock(&ei->i_fc_lock);
 386         if (tid == ei->i_sync_tid) {
 387                 update = true;
 388         } else {
 389                 ext4_fc_reset_inode(inode);
 390                 ei->i_sync_tid = tid;
 391         }
 392         ret = __fc_track_fn(inode, args, update);
 393         mutex_unlock(&ei->i_fc_lock);
 394
 395         if (!enqueue)
 396                 return ret;
 397
 398         spin_lock(&sbi->s_fc_lock);
 399         if (list_empty(&EXT4_I(inode)->i_fc_list))
 400                 list_add_tail(&EXT4_I(inode)->i_fc_list,
 401                                 (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING ||
 402                                  sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) ?
 403                                 &sbi->s_fc_q[FC_Q_STAGING] :
 404                                 &sbi->s_fc_q[FC_Q_MAIN]);
 405         spin_unlock(&sbi->s_fc_lock);
 406
 407         return ret;
 408 }
 409
 410 struct __track_dentry_update_args {
 411         struct dentry *dentry;
 412         int op;
 413 };
 414
 415 /* __track_fn for directory entry updates. Called with ei->i_fc_lock. */
 416 static int __track_dentry_update(struct inode *inode, void *arg, bool update)
 417 {
 418         struct ext4_fc_dentry_update *node;
 419         struct ext4_inode_info *ei = EXT4_I(inode);
 420         struct __track_dentry_update_args *dentry_update =
 421                 (struct __track_dentry_update_args *)arg;
 422         struct dentry *dentry = dentry_update->dentry;
 423         struct inode *dir = dentry->d_parent->d_inode;
 424         struct super_block *sb = inode->i_sb;
 425         struct ext4_sb_info *sbi = EXT4_SB(sb);
 426
 427         mutex_unlock(&ei->i_fc_lock);
 428
 429         if (IS_ENCRYPTED(dir)) {
 430                 ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_ENCRYPTED_FILENAME,
 431                                         NULL);
 432                 mutex_lock(&ei->i_fc_lock);
 433                 return -EOPNOTSUPP;
 434         }
 435
 436         node = kmem_cache_alloc(ext4_fc_dentry_cachep, GFP_NOFS);
 437         if (!node) {
 438                 ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_NOMEM, NULL);
 439                 mutex_lock(&ei->i_fc_lock);
 440                 return -ENOMEM;
 441         }
 442
 443         node->fcd_op = dentry_update->op;
 444         node->fcd_parent = dir->i_ino;
 445         node->fcd_ino = inode->i_ino;
 446         if (dentry->d_name.len > DNAME_INLINE_LEN) {
 447                 node->fcd_name.name = kmalloc(dentry->d_name.len, GFP_NOFS);
 448                 if (!node->fcd_name.name) {
 449                         kmem_cache_free(ext4_fc_dentry_cachep, node);
 450                         ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_NOMEM, NULL);
 451                         mutex_lock(&ei->i_fc_lock);
 452                         return -ENOMEM;
 453                 }
 454                 memcpy((u8 *)node->fcd_name.name, dentry->d_name.name,
 455                         dentry->d_name.len);
 456         } else {
 457                 memcpy(node->fcd_iname, dentry->d_name.name,
 458                         dentry->d_name.len);
 459                 node->fcd_name.name = node->fcd_iname;
 460         }
 461         node->fcd_name.len = dentry->d_name.len;
 462         INIT_LIST_HEAD(&node->fcd_dilist);
 463         spin_lock(&sbi->s_fc_lock);
 464         if (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING ||
 465                 sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING)
 466                 list_add_tail(&node->fcd_list,
 467                                 &sbi->s_fc_dentry_q[FC_Q_STAGING]);
 468         else
 469                 list_add_tail(&node->fcd_list, &sbi->s_fc_dentry_q[FC_Q_MAIN]);
 470
 471         /*
 472          * This helps us keep a track of all fc_dentry updates which is part of
 473          * this ext4 inode. So in case the inode is getting unlinked, before
 474          * even we get a chance to fsync, we could remove all fc_dentry
 475          * references while evicting the inode in ext4_fc_del().
 476          * Also with this, we don't need to loop over all the inodes in
 477          * sbi->s_fc_q to get the corresponding inode in
 478          * ext4_fc_commit_dentry_updates().
 479          */
 480         if (dentry_update->op == EXT4_FC_TAG_CREAT) {
 481                 WARN_ON(!list_empty(&ei->i_fc_dilist));
 482                 list_add_tail(&node->fcd_dilist, &ei->i_fc_dilist);
 483         }
 484         spin_unlock(&sbi->s_fc_lock);
 485         mutex_lock(&ei->i_fc_lock);
 486
 487         return 0;
 488 }
 489
 490 void __ext4_fc_track_unlink(handle_t *handle,
 491                 struct inode *inode, struct dentry *dentry)
 492 {
 493         struct __track_dentry_update_args args;
 494         int ret;
 495
 496         args.dentry = dentry;
 497         args.op = EXT4_FC_TAG_UNLINK;
 498
 499         ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
 500                                         (void *)&args, 0);
 501         trace_ext4_fc_track_unlink(handle, inode, dentry, ret);
 502 }
 503
 504 void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry)
 505 {
 506         struct inode *inode = d_inode(dentry);
 507
 508         if (ext4_fc_disabled(inode->i_sb))
 509                 return;
 510
 511         if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE))
 512                 return;
 513
 514         __ext4_fc_track_unlink(handle, inode, dentry);
 515 }
 516
 517 void __ext4_fc_track_link(handle_t *handle,
 518         struct inode *inode, struct dentry *dentry)
 519 {
 520         struct __track_dentry_update_args args;
 521         int ret;
 522
 523         args.dentry = dentry;
 524         args.op = EXT4_FC_TAG_LINK;
 525
 526         ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
 527                                         (void *)&args, 0);
 528         trace_ext4_fc_track_link(handle, inode, dentry, ret);
 529 }
 530
 531 void ext4_fc_track_link(handle_t *handle, struct dentry *dentry)
 532 {
 533         struct inode *inode = d_inode(dentry);
 534
 535         if (ext4_fc_disabled(inode->i_sb))
 536                 return;
 537
 538         if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE))
 539                 return;
 540
 541         __ext4_fc_track_link(handle, inode, dentry);
 542 }
 543
 544 void __ext4_fc_track_create(handle_t *handle, struct inode *inode,
 545                           struct dentry *dentry)
 546 {
 547         struct __track_dentry_update_args args;
 548         int ret;
 549
 550         args.dentry = dentry;
 551         args.op = EXT4_FC_TAG_CREAT;
 552
 553         ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
 554                                         (void *)&args, 0);
 555         trace_ext4_fc_track_create(handle, inode, dentry, ret);
 556 }
 557
 558 void ext4_fc_track_create(handle_t *handle, struct dentry *dentry)
 559 {
 560         struct inode *inode = d_inode(dentry);
 561
 562         if (ext4_fc_disabled(inode->i_sb))
 563                 return;
 564
 565         if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE))
 566                 return;
 567
 568         __ext4_fc_track_create(handle, inode, dentry);
 569 }
 570
 571 /* __track_fn for inode tracking */
 572 static int __track_inode(struct inode *inode, void *arg, bool update)
 573 {
 574         if (update)
 575                 return -EEXIST;
 576
 577         EXT4_I(inode)->i_fc_lblk_len = 0;
 578
 579         return 0;
 580 }
 581
 582 void ext4_fc_track_inode(handle_t *handle, struct inode *inode)
 583 {
 584         int ret;
 585
 586         if (S_ISDIR(inode->i_mode))
 587                 return;
 588
 589         if (ext4_fc_disabled(inode->i_sb))
 590                 return;
 591
 592         if (ext4_should_journal_data(inode)) {
 593                 ext4_fc_mark_ineligible(inode->i_sb,
 594                                         EXT4_FC_REASON_INODE_JOURNAL_DATA, handle);
 595                 return;
 596         }
 597
 598         if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE))
 599                 return;
 600
 601         ret = ext4_fc_track_template(handle, inode, __track_inode, NULL, 1);
 602         trace_ext4_fc_track_inode(handle, inode, ret);
 603 }
 604
 605 struct __track_range_args {
 606         ext4_lblk_t start, end;
 607 };
 608
 609 /* __track_fn for tracking data updates */
 610 static int __track_range(struct inode *inode, void *arg, bool update)
 611 {
 612         struct ext4_inode_info *ei = EXT4_I(inode);
 613         ext4_lblk_t oldstart;
 614         struct __track_range_args *__arg =
 615                 (struct __track_range_args *)arg;
 616
 617         if (inode->i_ino < EXT4_FIRST_INO(inode->i_sb)) {
 618                 ext4_debug("Special inode %ld being modified\n", inode->i_ino);
 619                 return -ECANCELED;
 620         }
 621
 622         oldstart = ei->i_fc_lblk_start;
 623
 624         if (update && ei->i_fc_lblk_len > 0) {
 625                 ei->i_fc_lblk_start = min(ei->i_fc_lblk_start, __arg->start);
 626                 ei->i_fc_lblk_len =
 627                         max(oldstart + ei->i_fc_lblk_len - 1, __arg->end) -
 628                                 ei->i_fc_lblk_start + 1;
 629         } else {
 630                 ei->i_fc_lblk_start = __arg->start;
 631                 ei->i_fc_lblk_len = __arg->end - __arg->start + 1;
 632         }
 633
 634         return 0;
 635 }
 636
 637 void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t start,
 638                          ext4_lblk_t end)
 639 {
 640         struct __track_range_args args;
 641         int ret;
 642
 643         if (S_ISDIR(inode->i_mode))
 644                 return;
 645
 646         if (ext4_fc_disabled(inode->i_sb))
 647                 return;
 648
 649         if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE))
 650                 return;
 651
 652         args.start = start;
 653         args.end = end;
 654
 655         ret = ext4_fc_track_template(handle, inode,  __track_range, &args, 1);
 656
 657         trace_ext4_fc_track_range(handle, inode, start, end, ret);
 658 }
 659
 660 static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail)
 661 {
 662         blk_opf_t write_flags = REQ_SYNC;
 663         struct buffer_head *bh = EXT4_SB(sb)->s_fc_bh;
 664
 665         /* Add REQ_FUA | REQ_PREFLUSH only its tail */
 666         if (test_opt(sb, BARRIER) && is_tail)
 667                 write_flags |= REQ_FUA | REQ_PREFLUSH;
 668         lock_buffer(bh);
 669         set_buffer_dirty(bh);
 670         set_buffer_uptodate(bh);
 671         bh->b_end_io = ext4_end_buffer_io_sync;
 672         submit_bh(REQ_OP_WRITE | write_flags, bh);
 673         EXT4_SB(sb)->s_fc_bh = NULL;
 674 }
 675
 676 /* Ext4 commit path routines */
 677
 678 /* memcpy to fc reserved space and update CRC */
 679 static void *ext4_fc_memcpy(struct super_block *sb, void *dst, const void *src,
 680                                 int len, u32 *crc)
 681 {
 682         if (crc)
 683                 *crc = ext4_chksum(EXT4_SB(sb), *crc, src, len);
 684         return memcpy(dst, src, len);
 685 }
 686
 687 /* memzero and update CRC */
 688 static void *ext4_fc_memzero(struct super_block *sb, void *dst, int len,
 689                                 u32 *crc)
 690 {
 691         void *ret;
 692
 693         ret = memset(dst, 0, len);
 694         if (crc)
 695                 *crc = ext4_chksum(EXT4_SB(sb), *crc, dst, len);
 696         return ret;
 697 }
 698
 699 /*
 700  * Allocate len bytes on a fast commit buffer.
 701  *
 702  * During the commit time this function is used to manage fast commit
 703  * block space. We don't split a fast commit log onto different
 704  * blocks. So this function makes sure that if there's not enough space
 705  * on the current block, the remaining space in the current block is
 706  * marked as unused by adding EXT4_FC_TAG_PAD tag. In that case,
 707  * new block is from jbd2 and CRC is updated to reflect the padding
 708  * we added.
 709  */
 710 static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc)
 711 {
 712         struct ext4_fc_tl tl;
 713         struct ext4_sb_info *sbi = EXT4_SB(sb);
 714         struct buffer_head *bh;
 715         int bsize = sbi->s_journal->j_blocksize;
 716         int ret, off = sbi->s_fc_bytes % bsize;
 717         int remaining;
 718         u8 *dst;
 719
 720         /*
 721          * If 'len' is too long to fit in any block alongside a PAD tlv, then we
 722          * cannot fulfill the request.
 723          */
 724         if (len > bsize - EXT4_FC_TAG_BASE_LEN)
 725                 return NULL;
 726
 727         if (!sbi->s_fc_bh) {
 728                 ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh);
 729                 if (ret)
 730                         return NULL;
 731                 sbi->s_fc_bh = bh;
 732         }
 733         dst = sbi->s_fc_bh->b_data + off;
 734
 735         /*
 736          * Allocate the bytes in the current block if we can do so while still
 737          * leaving enough space for a PAD tlv.
 738          */
 739         remaining = bsize - EXT4_FC_TAG_BASE_LEN - off;
 740         if (len <= remaining) {
 741                 sbi->s_fc_bytes += len;
 742                 return dst;
 743         }
 744
 745         /*
 746          * Else, terminate the current block with a PAD tlv, then allocate a new
 747          * block and allocate the bytes at the start of that new block.
 748          */
 749
 750         tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_PAD);
 751         tl.fc_len = cpu_to_le16(remaining);
 752         ext4_fc_memcpy(sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, crc);
 753         ext4_fc_memzero(sb, dst + EXT4_FC_TAG_BASE_LEN, remaining, crc);
 754
 755         ext4_fc_submit_bh(sb, false);
 756
 757         ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh);
 758         if (ret)
 759                 return NULL;
 760         sbi->s_fc_bh = bh;
 761         sbi->s_fc_bytes += bsize - off + len;
 762         return sbi->s_fc_bh->b_data;
 763 }
 764
 765 /*
 766  * Complete a fast commit by writing tail tag.
 767  *
 768  * Writing tail tag marks the end of a fast commit. In order to guarantee
 769  * atomicity, after writing tail tag, even if there's space remaining
 770  * in the block, next commit shouldn't use it. That's why tail tag
 771  * has the length as that of the remaining space on the block.
 772  */
 773 static int ext4_fc_write_tail(struct super_block *sb, u32 crc)
 774 {
 775         struct ext4_sb_info *sbi = EXT4_SB(sb);
 776         struct ext4_fc_tl tl;
 777         struct ext4_fc_tail tail;
 778         int off, bsize = sbi->s_journal->j_blocksize;
 779         u8 *dst;
 780
 781         /*
 782          * ext4_fc_reserve_space takes care of allocating an extra block if
 783          * there's no enough space on this block for accommodating this tail.
 784          */
 785         dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + sizeof(tail), &crc);
 786         if (!dst)
 787                 return -ENOSPC;
 788
 789         off = sbi->s_fc_bytes % bsize;
 790
 791         tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_TAIL);
 792         tl.fc_len = cpu_to_le16(bsize - off + sizeof(struct ext4_fc_tail));
 793         sbi->s_fc_bytes = round_up(sbi->s_fc_bytes, bsize);
 794
 795         ext4_fc_memcpy(sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, &crc);
 796         dst += EXT4_FC_TAG_BASE_LEN;
 797         tail.fc_tid = cpu_to_le32(sbi->s_journal->j_running_transaction->t_tid);
 798         ext4_fc_memcpy(sb, dst, &tail.fc_tid, sizeof(tail.fc_tid), &crc);
 799         dst += sizeof(tail.fc_tid);
 800         tail.fc_crc = cpu_to_le32(crc);
 801         ext4_fc_memcpy(sb, dst, &tail.fc_crc, sizeof(tail.fc_crc), NULL);
 802         dst += sizeof(tail.fc_crc);
 803         memset(dst, 0, bsize - off); /* Don't leak uninitialized memory. */
 804
 805         ext4_fc_submit_bh(sb, true);
 806
 807         return 0;
 808 }
 809
 810 /*
 811  * Adds tag, length, value and updates CRC. Returns true if tlv was added.
 812  * Returns false if there's not enough space.
 813  */
 814 static bool ext4_fc_add_tlv(struct super_block *sb, u16 tag, u16 len, u8 *val,
 815                            u32 *crc)
 816 {
 817         struct ext4_fc_tl tl;
 818         u8 *dst;
 819
 820         dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + len, crc);
 821         if (!dst)
 822                 return false;
 823
 824         tl.fc_tag = cpu_to_le16(tag);
 825         tl.fc_len = cpu_to_le16(len);
 826
 827         ext4_fc_memcpy(sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, crc);
 828         ext4_fc_memcpy(sb, dst + EXT4_FC_TAG_BASE_LEN, val, len, crc);
 829
 830         return true;
 831 }
 832
 833 /* Same as above, but adds dentry tlv. */
 834 static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc,
 835                                    struct ext4_fc_dentry_update *fc_dentry)
 836 {
 837         struct ext4_fc_dentry_info fcd;
 838         struct ext4_fc_tl tl;
 839         int dlen = fc_dentry->fcd_name.len;
 840         u8 *dst = ext4_fc_reserve_space(sb,
 841                         EXT4_FC_TAG_BASE_LEN + sizeof(fcd) + dlen, crc);
 842
 843         if (!dst)
 844                 return false;
 845
 846         fcd.fc_parent_ino = cpu_to_le32(fc_dentry->fcd_parent);
 847         fcd.fc_ino = cpu_to_le32(fc_dentry->fcd_ino);
 848         tl.fc_tag = cpu_to_le16(fc_dentry->fcd_op);
 849         tl.fc_len = cpu_to_le16(sizeof(fcd) + dlen);
 850         ext4_fc_memcpy(sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, crc);
 851         dst += EXT4_FC_TAG_BASE_LEN;
 852         ext4_fc_memcpy(sb, dst, &fcd, sizeof(fcd), crc);
 853         dst += sizeof(fcd);
 854         ext4_fc_memcpy(sb, dst, fc_dentry->fcd_name.name, dlen, crc);
 855
 856         return true;
 857 }
 858
 859 /*
 860  * Writes inode in the fast commit space under TLV with tag @tag.
 861  * Returns 0 on success, error on failure.
 862  */
 863 static int ext4_fc_write_inode(struct inode *inode, u32 *crc)
 864 {
 865         struct ext4_inode_info *ei = EXT4_I(inode);
 866         int inode_len = EXT4_GOOD_OLD_INODE_SIZE;
 867         int ret;
 868         struct ext4_iloc iloc;
 869         struct ext4_fc_inode fc_inode;
 870         struct ext4_fc_tl tl;
 871         u8 *dst;
 872
 873         ret = ext4_get_inode_loc(inode, &iloc);
 874         if (ret)
 875                 return ret;
 876
 877         if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA))
 878                 inode_len = EXT4_INODE_SIZE(inode->i_sb);
 879         else if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE)
 880                 inode_len += ei->i_extra_isize;
 881
 882         fc_inode.fc_ino = cpu_to_le32(inode->i_ino);
 883         tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_INODE);
 884         tl.fc_len = cpu_to_le16(inode_len + sizeof(fc_inode.fc_ino));
 885
 886         ret = -ECANCELED;
 887         dst = ext4_fc_reserve_space(inode->i_sb,
 888                 EXT4_FC_TAG_BASE_LEN + inode_len + sizeof(fc_inode.fc_ino), crc);
 889         if (!dst)
 890                 goto err;
 891
 892         if (!ext4_fc_memcpy(inode->i_sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, crc))
 893                 goto err;
 894         dst += EXT4_FC_TAG_BASE_LEN;
 895         if (!ext4_fc_memcpy(inode->i_sb, dst, &fc_inode, sizeof(fc_inode), crc))
 896                 goto err;
 897         dst += sizeof(fc_inode);
 898         if (!ext4_fc_memcpy(inode->i_sb, dst, (u8 *)ext4_raw_inode(&iloc),
 899                                         inode_len, crc))
 900                 goto err;
 901         ret = 0;
 902 err:
 903         brelse(iloc.bh);
 904         return ret;
 905 }
 906
 907 /*
 908  * Writes updated data ranges for the inode in question. Updates CRC.
 909  * Returns 0 on success, error otherwise.
 910  */
 911 static int ext4_fc_write_inode_data(struct inode *inode, u32 *crc)
 912 {
 913         ext4_lblk_t old_blk_size, cur_lblk_off, new_blk_size;
 914         struct ext4_inode_info *ei = EXT4_I(inode);
 915         struct ext4_map_blocks map;
 916         struct ext4_fc_add_range fc_ext;
 917         struct ext4_fc_del_range lrange;
 918         struct ext4_extent *ex;
 919         int ret;
 920
 921         mutex_lock(&ei->i_fc_lock);
 922         if (ei->i_fc_lblk_len == 0) {
 923                 mutex_unlock(&ei->i_fc_lock);
 924                 return 0;
 925         }
 926         old_blk_size = ei->i_fc_lblk_start;
 927         new_blk_size = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1;
 928         ei->i_fc_lblk_len = 0;
 929         mutex_unlock(&ei->i_fc_lock);
 930
 931         cur_lblk_off = old_blk_size;
 932         ext4_debug("will try writing %d to %d for inode %ld\n",
 933                    cur_lblk_off, new_blk_size, inode->i_ino);
 934
 935         while (cur_lblk_off <= new_blk_size) {
 936                 map.m_lblk = cur_lblk_off;
 937                 map.m_len = new_blk_size - cur_lblk_off + 1;
 938                 ret = ext4_map_blocks(NULL, inode, &map, 0);
 939                 if (ret < 0)
 940                         return -ECANCELED;
 941
 942                 if (map.m_len == 0) {
 943                         cur_lblk_off++;
 944                         continue;
 945                 }
 946
 947                 if (ret == 0) {
 948                         lrange.fc_ino = cpu_to_le32(inode->i_ino);
 949                         lrange.fc_lblk = cpu_to_le32(map.m_lblk);
 950                         lrange.fc_len = cpu_to_le32(map.m_len);
 951                         if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_DEL_RANGE,
 952                                             sizeof(lrange), (u8 *)&lrange, crc))
 953                                 return -ENOSPC;
 954                 } else {
 955                         unsigned int max = (map.m_flags & EXT4_MAP_UNWRITTEN) ?
 956                                 EXT_UNWRITTEN_MAX_LEN : EXT_INIT_MAX_LEN;
 957
 958                         /* Limit the number of blocks in one extent */
 959                         map.m_len = min(max, map.m_len);
 960
 961                         fc_ext.fc_ino = cpu_to_le32(inode->i_ino);
 962                         ex = (struct ext4_extent *)&fc_ext.fc_ex;
 963                         ex->ee_block = cpu_to_le32(map.m_lblk);
 964                         ex->ee_len = cpu_to_le16(map.m_len);
 965                         ext4_ext_store_pblock(ex, map.m_pblk);
 966                         if (map.m_flags & EXT4_MAP_UNWRITTEN)
 967                                 ext4_ext_mark_unwritten(ex);
 968                         else
 969                                 ext4_ext_mark_initialized(ex);
 970                         if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_ADD_RANGE,
 971                                             sizeof(fc_ext), (u8 *)&fc_ext, crc))
 972                                 return -ENOSPC;
 973                 }
 974
 975                 cur_lblk_off += map.m_len;
 976         }
 977
 978         return 0;
 979 }
 980
 981
 982 /* Submit data for all the fast commit inodes */
 983 static int ext4_fc_submit_inode_data_all(journal_t *journal)
 984 {
 985         struct super_block *sb = journal->j_private;
 986         struct ext4_sb_info *sbi = EXT4_SB(sb);
 987         struct ext4_inode_info *ei;
 988         int ret = 0;
 989
 990         spin_lock(&sbi->s_fc_lock);
 991         list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
 992                 ext4_set_inode_state(&ei->vfs_inode, EXT4_STATE_FC_COMMITTING);
 993                 while (atomic_read(&ei->i_fc_updates)) {
 994                         DEFINE_WAIT(wait);
 995
 996                         prepare_to_wait(&ei->i_fc_wait, &wait,
 997                                                 TASK_UNINTERRUPTIBLE);
 998                         if (atomic_read(&ei->i_fc_updates)) {
 999                                 spin_unlock(&sbi->s_fc_lock);
1000                                 schedule();
1001                                 spin_lock(&sbi->s_fc_lock);
1002                         }
1003                         finish_wait(&ei->i_fc_wait, &wait);
1004                 }
1005                 spin_unlock(&sbi->s_fc_lock);
1006                 ret = jbd2_submit_inode_data(ei->jinode);
1007                 if (ret)
1008                         return ret;
1009                 spin_lock(&sbi->s_fc_lock);
1010         }
1011         spin_unlock(&sbi->s_fc_lock);
1012
1013         return ret;
1014 }
1015
1016 /* Wait for completion of data for all the fast commit inodes */
1017 static int ext4_fc_wait_inode_data_all(journal_t *journal)
1018 {
1019         struct super_block *sb = journal->j_private;
1020         struct ext4_sb_info *sbi = EXT4_SB(sb);
1021         struct ext4_inode_info *pos, *n;
1022         int ret = 0;
1023
1024         spin_lock(&sbi->s_fc_lock);
1025         list_for_each_entry_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
1026                 if (!ext4_test_inode_state(&pos->vfs_inode,
1027                                            EXT4_STATE_FC_COMMITTING))
1028                         continue;
1029                 spin_unlock(&sbi->s_fc_lock);
1030
1031                 ret = jbd2_wait_inode_data(journal, pos->jinode);
1032                 if (ret)
1033                         return ret;
1034                 spin_lock(&sbi->s_fc_lock);
1035         }
1036         spin_unlock(&sbi->s_fc_lock);
1037
1038         return 0;
1039 }
1040
1041 /* Commit all the directory entry updates */
1042 static int ext4_fc_commit_dentry_updates(journal_t *journal, u32 *crc)
1043 __acquires(&sbi->s_fc_lock)
1044 __releases(&sbi->s_fc_lock)
1045 {
1046         struct super_block *sb = journal->j_private;
1047         struct ext4_sb_info *sbi = EXT4_SB(sb);
1048         struct ext4_fc_dentry_update *fc_dentry, *fc_dentry_n;
1049         struct inode *inode;
1050         struct ext4_inode_info *ei;
1051         int ret;
1052
1053         if (list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN]))
1054                 return 0;
1055         list_for_each_entry_safe(fc_dentry, fc_dentry_n,
1056                                  &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) {
1057                 if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) {
1058                         spin_unlock(&sbi->s_fc_lock);
1059                         if (!ext4_fc_add_dentry_tlv(sb, crc, fc_dentry)) {
1060                                 ret = -ENOSPC;
1061                                 goto lock_and_exit;
1062                         }
1063                         spin_lock(&sbi->s_fc_lock);
1064                         continue;
1065                 }
1066                 /*
1067                  * With fcd_dilist we need not loop in sbi->s_fc_q to get the
1068                  * corresponding inode pointer
1069                  */
1070                 WARN_ON(list_empty(&fc_dentry->fcd_dilist));
1071                 ei = list_first_entry(&fc_dentry->fcd_dilist,
1072                                 struct ext4_inode_info, i_fc_dilist);
1073                 inode = &ei->vfs_inode;
1074                 WARN_ON(inode->i_ino != fc_dentry->fcd_ino);
1075
1076                 spin_unlock(&sbi->s_fc_lock);
1077
1078                 /*
1079                  * We first write the inode and then the create dirent. This
1080                  * allows the recovery code to create an unnamed inode first
1081                  * and then link it to a directory entry. This allows us
1082                  * to use namei.c routines almost as is and simplifies
1083                  * the recovery code.
1084                  */
1085                 ret = ext4_fc_write_inode(inode, crc);
1086                 if (ret)
1087                         goto lock_and_exit;
1088
1089                 ret = ext4_fc_write_inode_data(inode, crc);
1090                 if (ret)
1091                         goto lock_and_exit;
1092
1093                 if (!ext4_fc_add_dentry_tlv(sb, crc, fc_dentry)) {
1094                         ret = -ENOSPC;
1095                         goto lock_and_exit;
1096                 }
1097
1098                 spin_lock(&sbi->s_fc_lock);
1099         }
1100         return 0;
1101 lock_and_exit:
1102         spin_lock(&sbi->s_fc_lock);
1103         return ret;
1104 }
1105
1106 static int ext4_fc_perform_commit(journal_t *journal)
1107 {
1108         struct super_block *sb = journal->j_private;
1109         struct ext4_sb_info *sbi = EXT4_SB(sb);
1110         struct ext4_inode_info *iter;
1111         struct ext4_fc_head head;
1112         struct inode *inode;
1113         struct blk_plug plug;
1114         int ret = 0;
1115         u32 crc = 0;
1116
1117         ret = ext4_fc_submit_inode_data_all(journal);
1118         if (ret)
1119                 return ret;
1120
1121         ret = ext4_fc_wait_inode_data_all(journal);
1122         if (ret)
1123                 return ret;
1124
1125         /*
1126          * If file system device is different from journal device, issue a cache
1127          * flush before we start writing fast commit blocks.
1128          */
1129         if (journal->j_fs_dev != journal->j_dev)
1130                 blkdev_issue_flush(journal->j_fs_dev);
1131
1132         blk_start_plug(&plug);
1133         if (sbi->s_fc_bytes == 0) {
1134                 /*
1135                  * Add a head tag only if this is the first fast commit
1136                  * in this TID.
1137                  */
1138                 head.fc_features = cpu_to_le32(EXT4_FC_SUPPORTED_FEATURES);
1139                 head.fc_tid = cpu_to_le32(
1140                         sbi->s_journal->j_running_transaction->t_tid);
1141                 if (!ext4_fc_add_tlv(sb, EXT4_FC_TAG_HEAD, sizeof(head),
1142                         (u8 *)&head, &crc)) {
1143                         ret = -ENOSPC;
1144                         goto out;
1145                 }
1146         }
1147
1148         spin_lock(&sbi->s_fc_lock);
1149         ret = ext4_fc_commit_dentry_updates(journal, &crc);
1150         if (ret) {
1151                 spin_unlock(&sbi->s_fc_lock);
1152                 goto out;
1153         }
1154
1155         list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
1156                 inode = &iter->vfs_inode;
1157                 if (!ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING))
1158                         continue;
1159
1160                 spin_unlock(&sbi->s_fc_lock);
1161                 ret = ext4_fc_write_inode_data(inode, &crc);
1162                 if (ret)
1163                         goto out;
1164                 ret = ext4_fc_write_inode(inode, &crc);
1165                 if (ret)
1166                         goto out;
1167                 spin_lock(&sbi->s_fc_lock);
1168         }
1169         spin_unlock(&sbi->s_fc_lock);
1170
1171         ret = ext4_fc_write_tail(sb, crc);
1172
1173 out:
1174         blk_finish_plug(&plug);
1175         return ret;
1176 }
1177
1178 static void ext4_fc_update_stats(struct super_block *sb, int status,
1179                                  u64 commit_time, int nblks, tid_t commit_tid)
1180 {
1181         struct ext4_fc_stats *stats = &EXT4_SB(sb)->s_fc_stats;
1182
1183         ext4_debug("Fast commit ended with status = %d for tid %u",
1184                         status, commit_tid);
1185         if (status == EXT4_FC_STATUS_OK) {
1186                 stats->fc_num_commits++;
1187                 stats->fc_numblks += nblks;
1188                 if (likely(stats->s_fc_avg_commit_time))
1189                         stats->s_fc_avg_commit_time =
1190                                 (commit_time +
1191                                  stats->s_fc_avg_commit_time * 3) / 4;
1192                 else
1193                         stats->s_fc_avg_commit_time = commit_time;
1194         } else if (status == EXT4_FC_STATUS_FAILED ||
1195                    status == EXT4_FC_STATUS_INELIGIBLE) {
1196                 if (status == EXT4_FC_STATUS_FAILED)
1197                         stats->fc_failed_commits++;
1198                 stats->fc_ineligible_commits++;
1199         } else {
1200                 stats->fc_skipped_commits++;
1201         }
1202         trace_ext4_fc_commit_stop(sb, nblks, status, commit_tid);
1203 }
1204
1205 /*
1206  * The main commit entry point. Performs a fast commit for transaction
1207  * commit_tid if needed. If it's not possible to perform a fast commit
1208  * due to various reasons, we fall back to full commit. Returns 0
1209  * on success, error otherwise.
1210  */
1211 int ext4_fc_commit(journal_t *journal, tid_t commit_tid)
1212 {
1213         struct super_block *sb = journal->j_private;
1214         struct ext4_sb_info *sbi = EXT4_SB(sb);
1215         int nblks = 0, ret, bsize = journal->j_blocksize;
1216         int subtid = atomic_read(&sbi->s_fc_subtid);
1217         int status = EXT4_FC_STATUS_OK, fc_bufs_before = 0;
1218         ktime_t start_time, commit_time;
1219
1220         if (!test_opt2(sb, JOURNAL_FAST_COMMIT))
1221                 return jbd2_complete_transaction(journal, commit_tid);
1222
1223         trace_ext4_fc_commit_start(sb, commit_tid);
1224
1225         start_time = ktime_get();
1226
1227 restart_fc:
1228         ret = jbd2_fc_begin_commit(journal, commit_tid);
1229         if (ret == -EALREADY) {
1230                 /* There was an ongoing commit, check if we need to restart */
1231                 if (atomic_read(&sbi->s_fc_subtid) <= subtid &&
1232                         commit_tid > journal->j_commit_sequence)
1233                         goto restart_fc;
1234                 ext4_fc_update_stats(sb, EXT4_FC_STATUS_SKIPPED, 0, 0,
1235                                 commit_tid);
1236                 return 0;
1237         } else if (ret) {
1238                 /*
1239                  * Commit couldn't start. Just update stats and perform a
1240                  * full commit.
1241                  */
1242                 ext4_fc_update_stats(sb, EXT4_FC_STATUS_FAILED, 0, 0,
1243                                 commit_tid);
1244                 return jbd2_complete_transaction(journal, commit_tid);
1245         }
1246
1247         /*
1248          * After establishing journal barrier via jbd2_fc_begin_commit(), check
1249          * if we are fast commit ineligible.
1250          */
1251         if (ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE)) {
1252                 status = EXT4_FC_STATUS_INELIGIBLE;
1253                 goto fallback;
1254         }
1255
1256         fc_bufs_before = (sbi->s_fc_bytes + bsize - 1) / bsize;
1257         ret = ext4_fc_perform_commit(journal);
1258         if (ret < 0) {
1259                 status = EXT4_FC_STATUS_FAILED;
1260                 goto fallback;
1261         }
1262         nblks = (sbi->s_fc_bytes + bsize - 1) / bsize - fc_bufs_before;
1263         ret = jbd2_fc_wait_bufs(journal, nblks);
1264         if (ret < 0) {
1265                 status = EXT4_FC_STATUS_FAILED;
1266                 goto fallback;
1267         }
1268         atomic_inc(&sbi->s_fc_subtid);
1269         ret = jbd2_fc_end_commit(journal);
1270         /*
1271          * weight the commit time higher than the average time so we
1272          * don't react too strongly to vast changes in the commit time
1273          */
1274         commit_time = ktime_to_ns(ktime_sub(ktime_get(), start_time));
1275         ext4_fc_update_stats(sb, status, commit_time, nblks, commit_tid);
1276         return ret;
1277
1278 fallback:
1279         ret = jbd2_fc_end_commit_fallback(journal);
1280         ext4_fc_update_stats(sb, status, 0, 0, commit_tid);
1281         return ret;
1282 }
1283
1284 /*
1285  * Fast commit cleanup routine. This is called after every fast commit and
1286  * full commit. full is true if we are called after a full commit.
1287  */
1288 static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid)
1289 {
1290         struct super_block *sb = journal->j_private;
1291         struct ext4_sb_info *sbi = EXT4_SB(sb);
1292         struct ext4_inode_info *iter, *iter_n;
1293         struct ext4_fc_dentry_update *fc_dentry;
1294
1295         if (full && sbi->s_fc_bh)
1296                 sbi->s_fc_bh = NULL;
1297
1298         trace_ext4_fc_cleanup(journal, full, tid);
1299         jbd2_fc_release_bufs(journal);
1300
1301         spin_lock(&sbi->s_fc_lock);
1302         list_for_each_entry_safe(iter, iter_n, &sbi->s_fc_q[FC_Q_MAIN],
1303                                  i_fc_list) {
1304                 list_del_init(&iter->i_fc_list);
1305                 ext4_clear_inode_state(&iter->vfs_inode,
1306                                        EXT4_STATE_FC_COMMITTING);
1307                 if (iter->i_sync_tid <= tid)
1308                         ext4_fc_reset_inode(&iter->vfs_inode);
1309                 /* Make sure EXT4_STATE_FC_COMMITTING bit is clear */
1310                 smp_mb();
1311 #if (BITS_PER_LONG < 64)
1312                 wake_up_bit(&iter->i_state_flags, EXT4_STATE_FC_COMMITTING);
1313 #else
1314                 wake_up_bit(&iter->i_flags, EXT4_STATE_FC_COMMITTING);
1315 #endif
1316         }
1317
1318         while (!list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) {
1319                 fc_dentry = list_first_entry(&sbi->s_fc_dentry_q[FC_Q_MAIN],
1320                                              struct ext4_fc_dentry_update,
1321                                              fcd_list);
1322                 list_del_init(&fc_dentry->fcd_list);
1323                 list_del_init(&fc_dentry->fcd_dilist);
1324                 spin_unlock(&sbi->s_fc_lock);
1325
1326                 if (fc_dentry->fcd_name.name &&
1327                         fc_dentry->fcd_name.len > DNAME_INLINE_LEN)
1328                         kfree(fc_dentry->fcd_name.name);
1329                 kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry);
1330                 spin_lock(&sbi->s_fc_lock);
1331         }
1332
1333         list_splice_init(&sbi->s_fc_dentry_q[FC_Q_STAGING],
1334                                 &sbi->s_fc_dentry_q[FC_Q_MAIN]);
1335         list_splice_init(&sbi->s_fc_q[FC_Q_STAGING],
1336                                 &sbi->s_fc_q[FC_Q_MAIN]);
1337
1338         if (tid >= sbi->s_fc_ineligible_tid) {
1339                 sbi->s_fc_ineligible_tid = 0;
1340                 ext4_clear_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
1341         }
1342
1343         if (full)
1344                 sbi->s_fc_bytes = 0;
1345         spin_unlock(&sbi->s_fc_lock);
1346         trace_ext4_fc_stats(sb);
1347 }
1348
1349 /* Ext4 Replay Path Routines */
1350
1351 /* Helper struct for dentry replay routines */
1352 struct dentry_info_args {
1353         int parent_ino, dname_len, ino, inode_len;
1354         char *dname;
1355 };
1356
1357 /* Same as struct ext4_fc_tl, but uses native endianness fields */
1358 struct ext4_fc_tl_mem {
1359         u16 fc_tag;
1360         u16 fc_len;
1361 };
1362
1363 static inline void tl_to_darg(struct dentry_info_args *darg,
1364                               struct ext4_fc_tl_mem *tl, u8 *val)
1365 {
1366         struct ext4_fc_dentry_info fcd;
1367
1368         memcpy(&fcd, val, sizeof(fcd));
1369
1370         darg->parent_ino = le32_to_cpu(fcd.fc_parent_ino);
1371         darg->ino = le32_to_cpu(fcd.fc_ino);
1372         darg->dname = val + offsetof(struct ext4_fc_dentry_info, fc_dname);
1373         darg->dname_len = tl->fc_len - sizeof(struct ext4_fc_dentry_info);
1374 }
1375
1376 static inline void ext4_fc_get_tl(struct ext4_fc_tl_mem *tl, u8 *val)
1377 {
1378         struct ext4_fc_tl tl_disk;
1379
1380         memcpy(&tl_disk, val, EXT4_FC_TAG_BASE_LEN);
1381         tl->fc_len = le16_to_cpu(tl_disk.fc_len);
1382         tl->fc_tag = le16_to_cpu(tl_disk.fc_tag);
1383 }
1384
1385 /* Unlink replay function */
1386 static int ext4_fc_replay_unlink(struct super_block *sb,
1387                                  struct ext4_fc_tl_mem *tl, u8 *val)
1388 {
1389         struct inode *inode, *old_parent;
1390         struct qstr entry;
1391         struct dentry_info_args darg;
1392         int ret = 0;
1393
1394         tl_to_darg(&darg, tl, val);
1395
1396         trace_ext4_fc_replay(sb, EXT4_FC_TAG_UNLINK, darg.ino,
1397                         darg.parent_ino, darg.dname_len);
1398
1399         entry.name = darg.dname;
1400         entry.len = darg.dname_len;
1401         inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
1402
1403         if (IS_ERR(inode)) {
1404                 ext4_debug("Inode %d not found", darg.ino);
1405                 return 0;
1406         }
1407
1408         old_parent = ext4_iget(sb, darg.parent_ino,
1409                                 EXT4_IGET_NORMAL);
1410         if (IS_ERR(old_parent)) {
1411                 ext4_debug("Dir with inode %d not found", darg.parent_ino);
1412                 iput(inode);
1413                 return 0;
1414         }
1415
1416         ret = __ext4_unlink(old_parent, &entry, inode, NULL);
1417         /* -ENOENT ok coz it might not exist anymore. */
1418         if (ret == -ENOENT)
1419                 ret = 0;
1420         iput(old_parent);
1421         iput(inode);
1422         return ret;
1423 }
1424
1425 static int ext4_fc_replay_link_internal(struct super_block *sb,
1426                                 struct dentry_info_args *darg,
1427                                 struct inode *inode)
1428 {
1429         struct inode *dir = NULL;
1430         struct dentry *dentry_dir = NULL, *dentry_inode = NULL;
1431         struct qstr qstr_dname = QSTR_INIT(darg->dname, darg->dname_len);
1432         int ret = 0;
1433
1434         dir = ext4_iget(sb, darg->parent_ino, EXT4_IGET_NORMAL);
1435         if (IS_ERR(dir)) {
1436                 ext4_debug("Dir with inode %d not found.", darg->parent_ino);
1437                 dir = NULL;
1438                 goto out;
1439         }
1440
1441         dentry_dir = d_obtain_alias(dir);
1442         if (IS_ERR(dentry_dir)) {
1443                 ext4_debug("Failed to obtain dentry");
1444                 dentry_dir = NULL;
1445                 goto out;
1446         }
1447
1448         dentry_inode = d_alloc(dentry_dir, &qstr_dname);
1449         if (!dentry_inode) {
1450                 ext4_debug("Inode dentry not created.");
1451                 ret = -ENOMEM;
1452                 goto out;
1453         }
1454
1455         ret = __ext4_link(dir, inode, dentry_inode);
1456         /*
1457          * It's possible that link already existed since data blocks
1458          * for the dir in question got persisted before we crashed OR
1459          * we replayed this tag and crashed before the entire replay
1460          * could complete.
1461          */
1462         if (ret && ret != -EEXIST) {
1463                 ext4_debug("Failed to link\n");
1464                 goto out;
1465         }
1466
1467         ret = 0;
1468 out:
1469         if (dentry_dir) {
1470                 d_drop(dentry_dir);
1471                 dput(dentry_dir);
1472         } else if (dir) {
1473                 iput(dir);
1474         }
1475         if (dentry_inode) {
1476                 d_drop(dentry_inode);
1477                 dput(dentry_inode);
1478         }
1479
1480         return ret;
1481 }
1482
1483 /* Link replay function */
1484 static int ext4_fc_replay_link(struct super_block *sb,
1485                                struct ext4_fc_tl_mem *tl, u8 *val)
1486 {
1487         struct inode *inode;
1488         struct dentry_info_args darg;
1489         int ret = 0;
1490
1491         tl_to_darg(&darg, tl, val);
1492         trace_ext4_fc_replay(sb, EXT4_FC_TAG_LINK, darg.ino,
1493                         darg.parent_ino, darg.dname_len);
1494
1495         inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
1496         if (IS_ERR(inode)) {
1497                 ext4_debug("Inode not found.");
1498                 return 0;
1499         }
1500
1501         ret = ext4_fc_replay_link_internal(sb, &darg, inode);
1502         iput(inode);
1503         return ret;
1504 }
1505
1506 /*
1507  * Record all the modified inodes during replay. We use this later to setup
1508  * block bitmaps correctly.
1509  */
1510 static int ext4_fc_record_modified_inode(struct super_block *sb, int ino)
1511 {
1512         struct ext4_fc_replay_state *state;
1513         int i;
1514
1515         state = &EXT4_SB(sb)->s_fc_replay_state;
1516         for (i = 0; i < state->fc_modified_inodes_used; i++)
1517                 if (state->fc_modified_inodes[i] == ino)
1518                         return 0;
1519         if (state->fc_modified_inodes_used == state->fc_modified_inodes_size) {
1520                 int *fc_modified_inodes;
1521
1522                 fc_modified_inodes = krealloc(state->fc_modified_inodes,
1523                                 sizeof(int) * (state->fc_modified_inodes_size +
1524                                 EXT4_FC_REPLAY_REALLOC_INCREMENT),
1525                                 GFP_KERNEL);
1526                 if (!fc_modified_inodes)
1527                         return -ENOMEM;
1528                 state->fc_modified_inodes = fc_modified_inodes;
1529                 state->fc_modified_inodes_size +=
1530                         EXT4_FC_REPLAY_REALLOC_INCREMENT;
1531         }
1532         state->fc_modified_inodes[state->fc_modified_inodes_used++] = ino;
1533         return 0;
1534 }
1535
1536 /*
1537  * Inode replay function
1538  */
1539 static int ext4_fc_replay_inode(struct super_block *sb,
1540                                 struct ext4_fc_tl_mem *tl, u8 *val)
1541 {
1542         struct ext4_fc_inode fc_inode;
1543         struct ext4_inode *raw_inode;
1544         struct ext4_inode *raw_fc_inode;
1545         struct inode *inode = NULL;
1546         struct ext4_iloc iloc;
1547         int inode_len, ino, ret, tag = tl->fc_tag;
1548         struct ext4_extent_header *eh;
1549         size_t off_gen = offsetof(struct ext4_inode, i_generation);
1550
1551         memcpy(&fc_inode, val, sizeof(fc_inode));
1552
1553         ino = le32_to_cpu(fc_inode.fc_ino);
1554         trace_ext4_fc_replay(sb, tag, ino, 0, 0);
1555
1556         inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL);
1557         if (!IS_ERR(inode)) {
1558                 ext4_ext_clear_bb(inode);
1559                 iput(inode);
1560         }
1561         inode = NULL;
1562
1563         ret = ext4_fc_record_modified_inode(sb, ino);
1564         if (ret)
1565                 goto out;
1566
1567         raw_fc_inode = (struct ext4_inode *)
1568                 (val + offsetof(struct ext4_fc_inode, fc_raw_inode));
1569         ret = ext4_get_fc_inode_loc(sb, ino, &iloc);
1570         if (ret)
1571                 goto out;
1572
1573         inode_len = tl->fc_len - sizeof(struct ext4_fc_inode);
1574         raw_inode = ext4_raw_inode(&iloc);
1575
1576         memcpy(raw_inode, raw_fc_inode, offsetof(struct ext4_inode, i_block));
1577         memcpy((u8 *)raw_inode + off_gen, (u8 *)raw_fc_inode + off_gen,
1578                inode_len - off_gen);
1579         if (le32_to_cpu(raw_inode->i_flags) & EXT4_EXTENTS_FL) {
1580                 eh = (struct ext4_extent_header *)(&raw_inode->i_block[0]);
1581                 if (eh->eh_magic != EXT4_EXT_MAGIC) {
1582                         memset(eh, 0, sizeof(*eh));
1583                         eh->eh_magic = EXT4_EXT_MAGIC;
1584                         eh->eh_max = cpu_to_le16(
1585                                 (sizeof(raw_inode->i_block) -
1586                                  sizeof(struct ext4_extent_header))
1587                                  / sizeof(struct ext4_extent));
1588                 }
1589         } else if (le32_to_cpu(raw_inode->i_flags) & EXT4_INLINE_DATA_FL) {
1590                 memcpy(raw_inode->i_block, raw_fc_inode->i_block,
1591                         sizeof(raw_inode->i_block));
1592         }
1593
1594         /* Immediately update the inode on disk. */
1595         ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh);
1596         if (ret)
1597                 goto out;
1598         ret = sync_dirty_buffer(iloc.bh);
1599         if (ret)
1600                 goto out;
1601         ret = ext4_mark_inode_used(sb, ino);
1602         if (ret)
1603                 goto out;
1604
1605         /* Given that we just wrote the inode on disk, this SHOULD succeed. */
1606         inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL);
1607         if (IS_ERR(inode)) {
1608                 ext4_debug("Inode not found.");
1609                 return -EFSCORRUPTED;
1610         }
1611
1612         /*
1613          * Our allocator could have made different decisions than before
1614          * crashing. This should be fixed but until then, we calculate
1615          * the number of blocks the inode.
1616          */
1617         if (!ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA))
1618                 ext4_ext_replay_set_iblocks(inode);
1619
1620         inode->i_generation = le32_to_cpu(ext4_raw_inode(&iloc)->i_generation);
1621         ext4_reset_inode_seed(inode);
1622
1623         ext4_inode_csum_set(inode, ext4_raw_inode(&iloc), EXT4_I(inode));
1624         ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh);
1625         sync_dirty_buffer(iloc.bh);
1626         brelse(iloc.bh);
1627 out:
1628         iput(inode);
1629         if (!ret)
1630                 blkdev_issue_flush(sb->s_bdev);
1631
1632         return 0;
1633 }
1634
1635 /*
1636  * Dentry create replay function.
1637  *
1638  * EXT4_FC_TAG_CREAT is preceded by EXT4_FC_TAG_INODE_FULL. Which means, the
1639  * inode for which we are trying to create a dentry here, should already have
1640  * been replayed before we start here.
1641  */
1642 static int ext4_fc_replay_create(struct super_block *sb,
1643                                  struct ext4_fc_tl_mem *tl, u8 *val)
1644 {
1645         int ret = 0;
1646         struct inode *inode = NULL;
1647         struct inode *dir = NULL;
1648         struct dentry_info_args darg;
1649
1650         tl_to_darg(&darg, tl, val);
1651
1652         trace_ext4_fc_replay(sb, EXT4_FC_TAG_CREAT, darg.ino,
1653                         darg.parent_ino, darg.dname_len);
1654
1655         /* This takes care of update group descriptor and other metadata */
1656         ret = ext4_mark_inode_used(sb, darg.ino);
1657         if (ret)
1658                 goto out;
1659
1660         inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
1661         if (IS_ERR(inode)) {
1662                 ext4_debug("inode %d not found.", darg.ino);
1663                 inode = NULL;
1664                 ret = -EINVAL;
1665                 goto out;
1666         }
1667
1668         if (S_ISDIR(inode->i_mode)) {
1669                 /*
1670                  * If we are creating a directory, we need to make sure that the
1671                  * dot and dot dot dirents are setup properly.
1672                  */
1673                 dir = ext4_iget(sb, darg.parent_ino, EXT4_IGET_NORMAL);
1674                 if (IS_ERR(dir)) {
1675                         ext4_debug("Dir %d not found.", darg.ino);
1676                         goto out;
1677                 }
1678                 ret = ext4_init_new_dir(NULL, dir, inode);
1679                 iput(dir);
1680                 if (ret) {
1681                         ret = 0;
1682                         goto out;
1683                 }
1684         }
1685         ret = ext4_fc_replay_link_internal(sb, &darg, inode);
1686         if (ret)
1687                 goto out;
1688         set_nlink(inode, 1);
1689         ext4_mark_inode_dirty(NULL, inode);
1690 out:
1691         iput(inode);
1692         return ret;
1693 }
1694
1695 /*
1696  * Record physical disk regions which are in use as per fast commit area,
1697  * and used by inodes during replay phase. Our simple replay phase
1698  * allocator excludes these regions from allocation.
1699  */
1700 int ext4_fc_record_regions(struct super_block *sb, int ino,
1701                 ext4_lblk_t lblk, ext4_fsblk_t pblk, int len, int replay)
1702 {
1703         struct ext4_fc_replay_state *state;
1704         struct ext4_fc_alloc_region *region;
1705
1706         state = &EXT4_SB(sb)->s_fc_replay_state;
1707         /*
1708          * during replay phase, the fc_regions_valid may not same as
1709          * fc_regions_used, update it when do new additions.
1710          */
1711         if (replay && state->fc_regions_used != state->fc_regions_valid)
1712                 state->fc_regions_used = state->fc_regions_valid;
1713         if (state->fc_regions_used == state->fc_regions_size) {
1714                 struct ext4_fc_alloc_region *fc_regions;
1715
1716                 fc_regions = krealloc(state->fc_regions,
1717                                       sizeof(struct ext4_fc_alloc_region) *
1718                                       (state->fc_regions_size +
1719                                        EXT4_FC_REPLAY_REALLOC_INCREMENT),
1720                                       GFP_KERNEL);
1721                 if (!fc_regions)
1722                         return -ENOMEM;
1723                 state->fc_regions_size +=
1724                         EXT4_FC_REPLAY_REALLOC_INCREMENT;
1725                 state->fc_regions = fc_regions;
1726         }
1727         region = &state->fc_regions[state->fc_regions_used++];
1728         region->ino = ino;
1729         region->lblk = lblk;
1730         region->pblk = pblk;
1731         region->len = len;
1732
1733         if (replay)
1734                 state->fc_regions_valid++;
1735
1736         return 0;
1737 }
1738
1739 /* Replay add range tag */
1740 static int ext4_fc_replay_add_range(struct super_block *sb,
1741                                     struct ext4_fc_tl_mem *tl, u8 *val)
1742 {
1743         struct ext4_fc_add_range fc_add_ex;
1744         struct ext4_extent newex, *ex;
1745         struct inode *inode;
1746         ext4_lblk_t start, cur;
1747         int remaining, len;
1748         ext4_fsblk_t start_pblk;
1749         struct ext4_map_blocks map;
1750         struct ext4_ext_path *path = NULL;
1751         int ret;
1752
1753         memcpy(&fc_add_ex, val, sizeof(fc_add_ex));
1754         ex = (struct ext4_extent *)&fc_add_ex.fc_ex;
1755
1756         trace_ext4_fc_replay(sb, EXT4_FC_TAG_ADD_RANGE,
1757                 le32_to_cpu(fc_add_ex.fc_ino), le32_to_cpu(ex->ee_block),
1758                 ext4_ext_get_actual_len(ex));
1759
1760         inode = ext4_iget(sb, le32_to_cpu(fc_add_ex.fc_ino), EXT4_IGET_NORMAL);
1761         if (IS_ERR(inode)) {
1762                 ext4_debug("Inode not found.");
1763                 return 0;
1764         }
1765
1766         ret = ext4_fc_record_modified_inode(sb, inode->i_ino);
1767         if (ret)
1768                 goto out;
1769
1770         start = le32_to_cpu(ex->ee_block);
1771         start_pblk = ext4_ext_pblock(ex);
1772         len = ext4_ext_get_actual_len(ex);
1773
1774         cur = start;
1775         remaining = len;
1776         ext4_debug("ADD_RANGE, lblk %d, pblk %lld, len %d, unwritten %d, inode %ld\n",
1777                   start, start_pblk, len, ext4_ext_is_unwritten(ex),
1778                   inode->i_ino);
1779
1780         while (remaining > 0) {
1781                 map.m_lblk = cur;
1782                 map.m_len = remaining;
1783                 map.m_pblk = 0;
1784                 ret = ext4_map_blocks(NULL, inode, &map, 0);
1785
1786                 if (ret < 0)
1787                         goto out;
1788
1789                 if (ret == 0) {
1790                         /* Range is not mapped */
1791                         path = ext4_find_extent(inode, cur, NULL, 0);
1792                         if (IS_ERR(path))
1793                                 goto out;
1794                         memset(&newex, 0, sizeof(newex));
1795                         newex.ee_block = cpu_to_le32(cur);
1796                         ext4_ext_store_pblock(
1797                                 &newex, start_pblk + cur - start);
1798                         newex.ee_len = cpu_to_le16(map.m_len);
1799                         if (ext4_ext_is_unwritten(ex))
1800                                 ext4_ext_mark_unwritten(&newex);
1801                         down_write(&EXT4_I(inode)->i_data_sem);
1802                         ret = ext4_ext_insert_extent(
1803                                 NULL, inode, &path, &newex, 0);
1804                         up_write((&EXT4_I(inode)->i_data_sem));
1805                         ext4_free_ext_path(path);
1806                         if (ret)
1807                                 goto out;
1808                         goto next;
1809                 }
1810
1811                 if (start_pblk + cur - start != map.m_pblk) {
1812                         /*
1813                          * Logical to physical mapping changed. This can happen
1814                          * if this range was removed and then reallocated to
1815                          * map to new physical blocks during a fast commit.
1816                          */
1817                         ret = ext4_ext_replay_update_ex(inode, cur, map.m_len,
1818                                         ext4_ext_is_unwritten(ex),
1819                                         start_pblk + cur - start);
1820                         if (ret)
1821                                 goto out;
1822                         /*
1823                          * Mark the old blocks as free since they aren't used
1824                          * anymore. We maintain an array of all the modified
1825                          * inodes. In case these blocks are still used at either
1826                          * a different logical range in the same inode or in
1827                          * some different inode, we will mark them as allocated
1828                          * at the end of the FC replay using our array of
1829                          * modified inodes.
1830                          */
1831                         ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, 0);
1832                         goto next;
1833                 }
1834
1835                 /* Range is mapped and needs a state change */
1836                 ext4_debug("Converting from %ld to %d %lld",
1837                                 map.m_flags & EXT4_MAP_UNWRITTEN,
1838                         ext4_ext_is_unwritten(ex), map.m_pblk);
1839                 ret = ext4_ext_replay_update_ex(inode, cur, map.m_len,
1840                                         ext4_ext_is_unwritten(ex), map.m_pblk);
1841                 if (ret)
1842                         goto out;
1843                 /*
1844                  * We may have split the extent tree while toggling the state.
1845                  * Try to shrink the extent tree now.
1846                  */
1847                 ext4_ext_replay_shrink_inode(inode, start + len);
1848 next:
1849                 cur += map.m_len;
1850                 remaining -= map.m_len;
1851         }
1852         ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >>
1853                                         sb->s_blocksize_bits);
1854 out:
1855         iput(inode);
1856         return 0;
1857 }
1858
1859 /* Replay DEL_RANGE tag */
1860 static int
1861 ext4_fc_replay_del_range(struct super_block *sb,
1862                          struct ext4_fc_tl_mem *tl, u8 *val)
1863 {
1864         struct inode *inode;
1865         struct ext4_fc_del_range lrange;
1866         struct ext4_map_blocks map;
1867         ext4_lblk_t cur, remaining;
1868         int ret;
1869
1870         memcpy(&lrange, val, sizeof(lrange));
1871         cur = le32_to_cpu(lrange.fc_lblk);
1872         remaining = le32_to_cpu(lrange.fc_len);
1873
1874         trace_ext4_fc_replay(sb, EXT4_FC_TAG_DEL_RANGE,
1875                 le32_to_cpu(lrange.fc_ino), cur, remaining);
1876
1877         inode = ext4_iget(sb, le32_to_cpu(lrange.fc_ino), EXT4_IGET_NORMAL);
1878         if (IS_ERR(inode)) {
1879                 ext4_debug("Inode %d not found", le32_to_cpu(lrange.fc_ino));
1880                 return 0;
1881         }
1882
1883         ret = ext4_fc_record_modified_inode(sb, inode->i_ino);
1884         if (ret)
1885                 goto out;
1886
1887         ext4_debug("DEL_RANGE, inode %ld, lblk %d, len %d\n",
1888                         inode->i_ino, le32_to_cpu(lrange.fc_lblk),
1889                         le32_to_cpu(lrange.fc_len));
1890         while (remaining > 0) {
1891                 map.m_lblk = cur;
1892                 map.m_len = remaining;
1893
1894                 ret = ext4_map_blocks(NULL, inode, &map, 0);
1895                 if (ret < 0)
1896                         goto out;
1897                 if (ret > 0) {
1898                         remaining -= ret;
1899                         cur += ret;
1900                         ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, 0);
1901                 } else {
1902                         remaining -= map.m_len;
1903                         cur += map.m_len;
1904                 }
1905         }
1906
1907         down_write(&EXT4_I(inode)->i_data_sem);
1908         ret = ext4_ext_remove_space(inode, le32_to_cpu(lrange.fc_lblk),
1909                                 le32_to_cpu(lrange.fc_lblk) +
1910                                 le32_to_cpu(lrange.fc_len) - 1);
1911         up_write(&EXT4_I(inode)->i_data_sem);
1912         if (ret)
1913                 goto out;
1914         ext4_ext_replay_shrink_inode(inode,
1915                 i_size_read(inode) >> sb->s_blocksize_bits);
1916         ext4_mark_inode_dirty(NULL, inode);
1917 out:
1918         iput(inode);
1919         return 0;
1920 }
1921
1922 static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb)
1923 {
1924         struct ext4_fc_replay_state *state;
1925         struct inode *inode;
1926         struct ext4_ext_path *path = NULL;
1927         struct ext4_map_blocks map;
1928         int i, ret, j;
1929         ext4_lblk_t cur, end;
1930
1931         state = &EXT4_SB(sb)->s_fc_replay_state;
1932         for (i = 0; i < state->fc_modified_inodes_used; i++) {
1933                 inode = ext4_iget(sb, state->fc_modified_inodes[i],
1934                         EXT4_IGET_NORMAL);
1935                 if (IS_ERR(inode)) {
1936                         ext4_debug("Inode %d not found.",
1937                                 state->fc_modified_inodes[i]);
1938                         continue;
1939                 }
1940                 cur = 0;
1941                 end = EXT_MAX_BLOCKS;
1942                 if (ext4_test_inode_flag(inode, EXT4_INODE_INLINE_DATA)) {
1943                         iput(inode);
1944                         continue;
1945                 }
1946                 while (cur < end) {
1947                         map.m_lblk = cur;
1948                         map.m_len = end - cur;
1949
1950                         ret = ext4_map_blocks(NULL, inode, &map, 0);
1951                         if (ret < 0)
1952                                 break;
1953
1954                         if (ret > 0) {
1955                                 path = ext4_find_extent(inode, map.m_lblk, NULL, 0);
1956                                 if (!IS_ERR(path)) {
1957                                         for (j = 0; j < path->p_depth; j++)
1958                                                 ext4_mb_mark_bb(inode->i_sb,
1959                                                         path[j].p_block, 1, 1);
1960                                         ext4_free_ext_path(path);
1961                                 }
1962                                 cur += ret;
1963                                 ext4_mb_mark_bb(inode->i_sb, map.m_pblk,
1964                                                         map.m_len, 1);
1965                         } else {
1966                                 cur = cur + (map.m_len ? map.m_len : 1);
1967                         }
1968                 }
1969                 iput(inode);
1970         }
1971 }
1972
1973 /*
1974  * Check if block is in excluded regions for block allocation. The simple
1975  * allocator that runs during replay phase is calls this function to see
1976  * if it is okay to use a block.
1977  */
1978 bool ext4_fc_replay_check_excluded(struct super_block *sb, ext4_fsblk_t blk)
1979 {
1980         int i;
1981         struct ext4_fc_replay_state *state;
1982
1983         state = &EXT4_SB(sb)->s_fc_replay_state;
1984         for (i = 0; i < state->fc_regions_valid; i++) {
1985                 if (state->fc_regions[i].ino == 0 ||
1986                         state->fc_regions[i].len == 0)
1987                         continue;
1988                 if (in_range(blk, state->fc_regions[i].pblk,
1989                                         state->fc_regions[i].len))
1990                         return true;
1991         }
1992         return false;
1993 }
1994
1995 /* Cleanup function called after replay */
1996 void ext4_fc_replay_cleanup(struct super_block *sb)
1997 {
1998         struct ext4_sb_info *sbi = EXT4_SB(sb);
1999
2000         sbi->s_mount_state &= ~EXT4_FC_REPLAY;
2001         kfree(sbi->s_fc_replay_state.fc_regions);
2002         kfree(sbi->s_fc_replay_state.fc_modified_inodes);
2003 }
2004
2005 static bool ext4_fc_value_len_isvalid(struct ext4_sb_info *sbi,
2006                                       int tag, int len)
2007 {
2008         switch (tag) {
2009         case EXT4_FC_TAG_ADD_RANGE:
2010                 return len == sizeof(struct ext4_fc_add_range);
2011         case EXT4_FC_TAG_DEL_RANGE:
2012                 return len == sizeof(struct ext4_fc_del_range);
2013         case EXT4_FC_TAG_CREAT:
2014         case EXT4_FC_TAG_LINK:
2015         case EXT4_FC_TAG_UNLINK:
2016                 len -= sizeof(struct ext4_fc_dentry_info);
2017                 return len >= 1 && len <= EXT4_NAME_LEN;
2018         case EXT4_FC_TAG_INODE:
2019                 len -= sizeof(struct ext4_fc_inode);
2020                 return len >= EXT4_GOOD_OLD_INODE_SIZE &&
2021                         len <= sbi->s_inode_size;
2022         case EXT4_FC_TAG_PAD:
2023                 return true; /* padding can have any length */
2024         case EXT4_FC_TAG_TAIL:
2025                 return len >= sizeof(struct ext4_fc_tail);
2026         case EXT4_FC_TAG_HEAD:
2027                 return len == sizeof(struct ext4_fc_head);
2028         }
2029         return false;
2030 }
2031
2032 /*
2033  * Recovery Scan phase handler
2034  *
2035  * This function is called during the scan phase and is responsible
2036  * for doing following things:
2037  * - Make sure the fast commit area has valid tags for replay
2038  * - Count number of tags that need to be replayed by the replay handler
2039  * - Verify CRC
2040  * - Create a list of excluded blocks for allocation during replay phase
2041  *
2042  * This function returns JBD2_FC_REPLAY_CONTINUE to indicate that SCAN is
2043  * incomplete and JBD2 should send more blocks. It returns JBD2_FC_REPLAY_STOP
2044  * to indicate that scan has finished and JBD2 can now start replay phase.
2045  * It returns a negative error to indicate that there was an error. At the end
2046  * of a successful scan phase, sbi->s_fc_replay_state.fc_replay_num_tags is set
2047  * to indicate the number of tags that need to replayed during the replay phase.
2048  */
2049 static int ext4_fc_replay_scan(journal_t *journal,
2050                                 struct buffer_head *bh, int off,
2051                                 tid_t expected_tid)
2052 {
2053         struct super_block *sb = journal->j_private;
2054         struct ext4_sb_info *sbi = EXT4_SB(sb);
2055         struct ext4_fc_replay_state *state;
2056         int ret = JBD2_FC_REPLAY_CONTINUE;
2057         struct ext4_fc_add_range ext;
2058         struct ext4_fc_tl_mem tl;
2059         struct ext4_fc_tail tail;
2060         __u8 *start, *end, *cur, *val;
2061         struct ext4_fc_head head;
2062         struct ext4_extent *ex;
2063
2064         state = &sbi->s_fc_replay_state;
2065
2066         start = (u8 *)bh->b_data;
2067         end = start + journal->j_blocksize;
2068
2069         if (state->fc_replay_expected_off == 0) {
2070                 state->fc_cur_tag = 0;
2071                 state->fc_replay_num_tags = 0;
2072                 state->fc_crc = 0;
2073                 state->fc_regions = NULL;
2074                 state->fc_regions_valid = state->fc_regions_used =
2075                         state->fc_regions_size = 0;
2076                 /* Check if we can stop early */
2077                 if (le16_to_cpu(((struct ext4_fc_tl *)start)->fc_tag)
2078                         != EXT4_FC_TAG_HEAD)
2079                         return 0;
2080         }
2081
2082         if (off != state->fc_replay_expected_off) {
2083                 ret = -EFSCORRUPTED;
2084                 goto out_err;
2085         }
2086
2087         state->fc_replay_expected_off++;
2088         for (cur = start; cur <= end - EXT4_FC_TAG_BASE_LEN;
2089              cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) {
2090                 ext4_fc_get_tl(&tl, cur);
2091                 val = cur + EXT4_FC_TAG_BASE_LEN;
2092                 if (tl.fc_len > end - val ||
2093                     !ext4_fc_value_len_isvalid(sbi, tl.fc_tag, tl.fc_len)) {
2094                         ret = state->fc_replay_num_tags ?
2095                                 JBD2_FC_REPLAY_STOP : -ECANCELED;
2096                         goto out_err;
2097                 }
2098                 ext4_debug("Scan phase, tag:%s, blk %lld\n",
2099                            tag2str(tl.fc_tag), bh->b_blocknr);
2100                 switch (tl.fc_tag) {
2101                 case EXT4_FC_TAG_ADD_RANGE:
2102                         memcpy(&ext, val, sizeof(ext));
2103                         ex = (struct ext4_extent *)&ext.fc_ex;
2104                         ret = ext4_fc_record_regions(sb,
2105                                 le32_to_cpu(ext.fc_ino),
2106                                 le32_to_cpu(ex->ee_block), ext4_ext_pblock(ex),
2107                                 ext4_ext_get_actual_len(ex), 0);
2108                         if (ret < 0)
2109                                 break;
2110                         ret = JBD2_FC_REPLAY_CONTINUE;
2111                         fallthrough;
2112                 case EXT4_FC_TAG_DEL_RANGE:
2113                 case EXT4_FC_TAG_LINK:
2114                 case EXT4_FC_TAG_UNLINK:
2115                 case EXT4_FC_TAG_CREAT:
2116                 case EXT4_FC_TAG_INODE:
2117                 case EXT4_FC_TAG_PAD:
2118                         state->fc_cur_tag++;
2119                         state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur,
2120                                 EXT4_FC_TAG_BASE_LEN + tl.fc_len);
2121                         break;
2122                 case EXT4_FC_TAG_TAIL:
2123                         state->fc_cur_tag++;
2124                         memcpy(&tail, val, sizeof(tail));
2125                         state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur,
2126                                                 EXT4_FC_TAG_BASE_LEN +
2127                                                 offsetof(struct ext4_fc_tail,
2128                                                 fc_crc));
2129                         if (le32_to_cpu(tail.fc_tid) == expected_tid &&
2130                                 le32_to_cpu(tail.fc_crc) == state->fc_crc) {
2131                                 state->fc_replay_num_tags = state->fc_cur_tag;
2132                                 state->fc_regions_valid =
2133                                         state->fc_regions_used;
2134                         } else {
2135                                 ret = state->fc_replay_num_tags ?
2136                                         JBD2_FC_REPLAY_STOP : -EFSBADCRC;
2137                         }
2138                         state->fc_crc = 0;
2139                         break;
2140                 case EXT4_FC_TAG_HEAD:
2141                         memcpy(&head, val, sizeof(head));
2142                         if (le32_to_cpu(head.fc_features) &
2143                                 ~EXT4_FC_SUPPORTED_FEATURES) {
2144                                 ret = -EOPNOTSUPP;
2145                                 break;
2146                         }
2147                         if (le32_to_cpu(head.fc_tid) != expected_tid) {
2148                                 ret = JBD2_FC_REPLAY_STOP;
2149                                 break;
2150                         }
2151                         state->fc_cur_tag++;
2152                         state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur,
2153                                 EXT4_FC_TAG_BASE_LEN + tl.fc_len);
2154                         break;
2155                 default:
2156                         ret = state->fc_replay_num_tags ?
2157                                 JBD2_FC_REPLAY_STOP : -ECANCELED;
2158                 }
2159                 if (ret < 0 || ret == JBD2_FC_REPLAY_STOP)
2160                         break;
2161         }
2162
2163 out_err:
2164         trace_ext4_fc_replay_scan(sb, ret, off);
2165         return ret;
2166 }
2167
2168 /*
2169  * Main recovery path entry point.
2170  * The meaning of return codes is similar as above.
2171  */
2172 static int ext4_fc_replay(journal_t *journal, struct buffer_head *bh,
2173                                 enum passtype pass, int off, tid_t expected_tid)
2174 {
2175         struct super_block *sb = journal->j_private;
2176         struct ext4_sb_info *sbi = EXT4_SB(sb);
2177         struct ext4_fc_tl_mem tl;
2178         __u8 *start, *end, *cur, *val;
2179         int ret = JBD2_FC_REPLAY_CONTINUE;
2180         struct ext4_fc_replay_state *state = &sbi->s_fc_replay_state;
2181         struct ext4_fc_tail tail;
2182
2183         if (pass == PASS_SCAN) {
2184                 state->fc_current_pass = PASS_SCAN;
2185                 return ext4_fc_replay_scan(journal, bh, off, expected_tid);
2186         }
2187
2188         if (state->fc_current_pass != pass) {
2189                 state->fc_current_pass = pass;
2190                 sbi->s_mount_state |= EXT4_FC_REPLAY;
2191         }
2192         if (!sbi->s_fc_replay_state.fc_replay_num_tags) {
2193                 ext4_debug("Replay stops\n");
2194                 ext4_fc_set_bitmaps_and_counters(sb);
2195                 return 0;
2196         }
2197
2198 #ifdef CONFIG_EXT4_DEBUG
2199         if (sbi->s_fc_debug_max_replay && off >= sbi->s_fc_debug_max_replay) {
2200                 pr_warn("Dropping fc block %d because max_replay set\n", off);
2201                 return JBD2_FC_REPLAY_STOP;
2202         }
2203 #endif
2204
2205         start = (u8 *)bh->b_data;
2206         end = start + journal->j_blocksize;
2207
2208         for (cur = start; cur <= end - EXT4_FC_TAG_BASE_LEN;
2209              cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) {
2210                 ext4_fc_get_tl(&tl, cur);
2211                 val = cur + EXT4_FC_TAG_BASE_LEN;
2212
2213                 if (state->fc_replay_num_tags == 0) {
2214                         ret = JBD2_FC_REPLAY_STOP;
2215                         ext4_fc_set_bitmaps_and_counters(sb);
2216                         break;
2217                 }
2218
2219                 ext4_debug("Replay phase, tag:%s\n", tag2str(tl.fc_tag));
2220                 state->fc_replay_num_tags--;
2221                 switch (tl.fc_tag) {
2222                 case EXT4_FC_TAG_LINK:
2223                         ret = ext4_fc_replay_link(sb, &tl, val);
2224                         break;
2225                 case EXT4_FC_TAG_UNLINK:
2226                         ret = ext4_fc_replay_unlink(sb, &tl, val);
2227                         break;
2228                 case EXT4_FC_TAG_ADD_RANGE:
2229                         ret = ext4_fc_replay_add_range(sb, &tl, val);
2230                         break;
2231                 case EXT4_FC_TAG_CREAT:
2232                         ret = ext4_fc_replay_create(sb, &tl, val);
2233                         break;
2234                 case EXT4_FC_TAG_DEL_RANGE:
2235                         ret = ext4_fc_replay_del_range(sb, &tl, val);
2236                         break;
2237                 case EXT4_FC_TAG_INODE:
2238                         ret = ext4_fc_replay_inode(sb, &tl, val);
2239                         break;
2240                 case EXT4_FC_TAG_PAD:
2241                         trace_ext4_fc_replay(sb, EXT4_FC_TAG_PAD, 0,
2242                                              tl.fc_len, 0);
2243                         break;
2244                 case EXT4_FC_TAG_TAIL:
2245                         trace_ext4_fc_replay(sb, EXT4_FC_TAG_TAIL,
2246                                              0, tl.fc_len, 0);
2247                         memcpy(&tail, val, sizeof(tail));
2248                         WARN_ON(le32_to_cpu(tail.fc_tid) != expected_tid);
2249                         break;
2250                 case EXT4_FC_TAG_HEAD:
2251                         break;
2252                 default:
2253                         trace_ext4_fc_replay(sb, tl.fc_tag, 0, tl.fc_len, 0);
2254                         ret = -ECANCELED;
2255                         break;
2256                 }
2257                 if (ret < 0)
2258                         break;
2259                 ret = JBD2_FC_REPLAY_CONTINUE;
2260         }
2261         return ret;
2262 }
2263
2264 void ext4_fc_init(struct super_block *sb, journal_t *journal)
2265 {
2266         /*
2267          * We set replay callback even if fast commit disabled because we may
2268          * could still have fast commit blocks that need to be replayed even if
2269          * fast commit has now been turned off.
2270          */
2271         journal->j_fc_replay_callback = ext4_fc_replay;
2272         if (!test_opt2(sb, JOURNAL_FAST_COMMIT))
2273                 return;
2274         journal->j_fc_cleanup_callback = ext4_fc_cleanup;
2275 }
2276
2277 static const char * const fc_ineligible_reasons[] = {
2278         [EXT4_FC_REASON_XATTR] = "Extended attributes changed",
2279         [EXT4_FC_REASON_CROSS_RENAME] = "Cross rename",
2280         [EXT4_FC_REASON_JOURNAL_FLAG_CHANGE] = "Journal flag changed",
2281         [EXT4_FC_REASON_NOMEM] = "Insufficient memory",
2282         [EXT4_FC_REASON_SWAP_BOOT] = "Swap boot",
2283         [EXT4_FC_REASON_RESIZE] = "Resize",
2284         [EXT4_FC_REASON_RENAME_DIR] = "Dir renamed",
2285         [EXT4_FC_REASON_FALLOC_RANGE] = "Falloc range op",
2286         [EXT4_FC_REASON_INODE_JOURNAL_DATA] = "Data journalling",
2287         [EXT4_FC_REASON_ENCRYPTED_FILENAME] = "Encrypted filename",
2288 };
2289
2290 int ext4_fc_info_show(struct seq_file *seq, void *v)
2291 {
2292         struct ext4_sb_info *sbi = EXT4_SB((struct super_block *)seq->private);
2293         struct ext4_fc_stats *stats = &sbi->s_fc_stats;
2294         int i;
2295
2296         if (v != SEQ_START_TOKEN)
2297                 return 0;
2298
2299         seq_printf(seq,
2300                 "fc stats:\n%ld commits\n%ld ineligible\n%ld numblks\n%lluus avg_commit_time\n",
2301                    stats->fc_num_commits, stats->fc_ineligible_commits,
2302                    stats->fc_numblks,
2303                    div_u64(stats->s_fc_avg_commit_time, 1000));
2304         seq_puts(seq, "Ineligible reasons:\n");
2305         for (i = 0; i < EXT4_FC_REASON_MAX; i++)
2306                 seq_printf(seq, "\"%s\":\t%d\n", fc_ineligible_reasons[i],
2307                         stats->fc_ineligible_reason_count[i]);
2308
2309         return 0;
2310 }
2311
2312 int __init ext4_fc_init_dentry_cache(void)
2313 {
2314         ext4_fc_dentry_cachep = KMEM_CACHE(ext4_fc_dentry_update,
2315                                            SLAB_RECLAIM_ACCOUNT);
2316
2317         if (ext4_fc_dentry_cachep == NULL)
2318                 return -ENOMEM;
2319
2320         return 0;
2321 }
2322
2323 void ext4_fc_destroy_dentry_cache(void)
2324 {
2325         kmem_cache_destroy(ext4_fc_dentry_cachep);
2326 }