1 // SPDX-License-Identifier: GPL-2.0
4 * fs/ext4/fast_commit.c
6 * Written by Harshad Shirwadkar <harshadshirwadkar@gmail.com>
8 * Ext4 fast commits routines.
11 #include "ext4_jbd2.h"
12 #include "ext4_extents.h"
19 * Ext4 fast commits implement fine grained journalling for Ext4.
21 * Fast commits are organized as a log of tag-length-value (TLV) structs. (See
22 * struct ext4_fc_tl). Each TLV contains some delta that is replayed TLV by
23 * TLV during the recovery phase. For the scenarios for which we currently
24 * don't have replay code, fast commit falls back to full commits.
25 * Fast commits record delta in one of the following three categories.
27 * (A) Directory entry updates:
29 * - EXT4_FC_TAG_UNLINK - records directory entry unlink
30 * - EXT4_FC_TAG_LINK - records directory entry link
31 * - EXT4_FC_TAG_CREAT - records inode and directory entry creation
33 * (B) File specific data range updates:
35 * - EXT4_FC_TAG_ADD_RANGE - records addition of new blocks to an inode
36 * - EXT4_FC_TAG_DEL_RANGE - records deletion of blocks from an inode
38 * (C) Inode metadata (mtime / ctime etc):
40 * - EXT4_FC_TAG_INODE - record the inode that should be replayed
41 * during recovery. Note that iblocks field is
42 * not replayed and instead derived during
46 * With fast commits, we maintain all the directory entry operations in the
47 * order in which they are issued in an in-memory queue. This queue is flushed
48 * to disk during the commit operation. We also maintain a list of inodes
49 * that need to be committed during a fast commit in another in memory queue of
50 * inodes. During the commit operation, we commit in the following order:
52 * [1] Lock inodes for any further data updates by setting COMMITTING state
53 * [2] Submit data buffers of all the inodes
54 * [3] Wait for [2] to complete
55 * [4] Commit all the directory entry updates in the fast commit space
56 * [5] Commit all the changed inode structures
57 * [6] Write tail tag (this tag ensures the atomicity, please read the following
58 * section for more details).
59 * [7] Wait for [4], [5] and [6] to complete.
61 * All the inode updates must call ext4_fc_start_update() before starting an
62 * update. If such an ongoing update is present, fast commit waits for it to
63 * complete. The completion of such an update is marked by
64 * ext4_fc_stop_update().
66 * Fast Commit Ineligibility
67 * -------------------------
69 * Not all operations are supported by fast commits today (e.g extended
70 * attributes). Fast commit ineligibility is marked by calling
71 * ext4_fc_mark_ineligible(): This makes next fast commit operation to fall back
74 * Atomicity of commits
75 * --------------------
76 * In order to guarantee atomicity during the commit operation, fast commit
77 * uses "EXT4_FC_TAG_TAIL" tag that marks a fast commit as complete. Tail
78 * tag contains CRC of the contents and TID of the transaction after which
79 * this fast commit should be applied. Recovery code replays fast commit
80 * logs only if there's at least 1 valid tail present. For every fast commit
81 * operation, there is 1 tail. This means, we may end up with multiple tails
82 * in the fast commit space. Here's an example:
84 * - Create a new file A and remove existing file B
86 * - Append contents to file A
90 * The fast commit space at the end of above operations would look like this:
91 * [HEAD] [CREAT A] [UNLINK B] [TAIL] [ADD_RANGE A] [DEL_RANGE A] [TAIL]
92 * |<--- Fast Commit 1 --->|<--- Fast Commit 2 ---->|
94 * Replay code should thus check for all the valid tails in the FC area.
96 * Fast Commit Replay Idempotence
97 * ------------------------------
99 * Fast commits tags are idempotent in nature provided the recovery code follows
100 * certain rules. The guiding principle that the commit path follows while
101 * committing is that it stores the result of a particular operation instead of
102 * storing the procedure.
104 * Let's consider this rename operation: 'mv /a /b'. Let's assume dirent '/a'
105 * was associated with inode 10. During fast commit, instead of storing this
106 * operation as a procedure "rename a to b", we store the resulting file system
107 * state as a "series" of outcomes:
109 * - Link dirent b to inode 10
111 * - Inode <10> with valid refcount
113 * Now when recovery code runs, it needs "enforce" this state on the file
114 * system. This is what guarantees idempotence of fast commit replay.
116 * Let's take an example of a procedure that is not idempotent and see how fast
117 * commits make it idempotent. Consider following sequence of operations:
119 * rm A; mv B A; read A
122 * (x), (y) and (z) are the points at which we can crash. If we store this
123 * sequence of operations as is then the replay is not idempotent. Let's say
124 * while in replay, we crash at (z). During the second replay, file A (which was
125 * actually created as a result of "mv B A" operation) would get deleted. Thus,
126 * file named A would be absent when we try to read A. So, this sequence of
127 * operations is not idempotent. However, as mentioned above, instead of storing
128 * the procedure fast commits store the outcome of each procedure. Thus the fast
129 * commit log for above procedure would be as follows:
131 * (Let's assume dirent A was linked to inode 10 and dirent B was linked to
132 * inode 11 before the replay)
134 * [Unlink A] [Link A to inode 11] [Unlink B] [Inode 11]
137 * If we crash at (z), we will have file A linked to inode 11. During the second
138 * replay, we will remove file A (inode 11). But we will create it back and make
139 * it point to inode 11. We won't find B, so we'll just skip that step. At this
140 * point, the refcount for inode 11 is not reliable, but that gets fixed by the
141 * replay of last inode 11 tag. Crashes at points (w), (x) and (y) get handled
142 * similarly. Thus, by converting a non-idempotent procedure into a series of
143 * idempotent outcomes, fast commits ensured idempotence during the replay.
148 * 0) Fast commit replay path hardening: Fast commit replay code should use
149 * journal handles to make sure all the updates it does during the replay
150 * path are atomic. With that if we crash during fast commit replay, after
151 * trying to do recovery again, we will find a file system where fast commit
152 * area is invalid (because new full commit would be found). In order to deal
153 * with that, fast commit replay code should ensure that the "FC_REPLAY"
154 * superblock state is persisted before starting the replay, so that after
155 * the crash, fast commit recovery code can look at that flag and perform
156 * fast commit recovery even if that area is invalidated by later full
159 * 1) Make fast commit atomic updates more fine grained. Today, a fast commit
160 * eligible update must be protected within ext4_fc_start_update() and
161 * ext4_fc_stop_update(). These routines are called at much higher
162 * routines. This can be made more fine grained by combining with
163 * ext4_journal_start().
165 * 2) Same above for ext4_fc_start_ineligible() and ext4_fc_stop_ineligible()
167 * 3) Handle more ineligible cases.
170 #include <trace/events/ext4.h>
171 static struct kmem_cache *ext4_fc_dentry_cachep;
173 static void ext4_end_buffer_io_sync(struct buffer_head *bh, int uptodate)
175 BUFFER_TRACE(bh, "");
177 ext4_debug("%s: Block %lld up-to-date",
178 __func__, bh->b_blocknr);
179 set_buffer_uptodate(bh);
181 ext4_debug("%s: Block %lld not up-to-date",
182 __func__, bh->b_blocknr);
183 clear_buffer_uptodate(bh);
189 static inline void ext4_fc_reset_inode(struct inode *inode)
191 struct ext4_inode_info *ei = EXT4_I(inode);
193 ei->i_fc_lblk_start = 0;
194 ei->i_fc_lblk_len = 0;
197 void ext4_fc_init_inode(struct inode *inode)
199 struct ext4_inode_info *ei = EXT4_I(inode);
201 ext4_fc_reset_inode(inode);
202 ext4_clear_inode_state(inode, EXT4_STATE_FC_COMMITTING);
203 INIT_LIST_HEAD(&ei->i_fc_list);
204 init_waitqueue_head(&ei->i_fc_wait);
205 atomic_set(&ei->i_fc_updates, 0);
208 /* This function must be called with sbi->s_fc_lock held. */
209 static void ext4_fc_wait_committing_inode(struct inode *inode)
210 __releases(&EXT4_SB(inode->i_sb)->s_fc_lock)
212 wait_queue_head_t *wq;
213 struct ext4_inode_info *ei = EXT4_I(inode);
215 #if (BITS_PER_LONG < 64)
216 DEFINE_WAIT_BIT(wait, &ei->i_state_flags,
217 EXT4_STATE_FC_COMMITTING);
218 wq = bit_waitqueue(&ei->i_state_flags,
219 EXT4_STATE_FC_COMMITTING);
221 DEFINE_WAIT_BIT(wait, &ei->i_flags,
222 EXT4_STATE_FC_COMMITTING);
223 wq = bit_waitqueue(&ei->i_flags,
224 EXT4_STATE_FC_COMMITTING);
226 lockdep_assert_held(&EXT4_SB(inode->i_sb)->s_fc_lock);
227 prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
228 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
230 finish_wait(wq, &wait.wq_entry);
234 * Inform Ext4's fast about start of an inode update
236 * This function is called by the high level call VFS callbacks before
237 * performing any inode update. This function blocks if there's an ongoing
238 * fast commit on the inode in question.
240 void ext4_fc_start_update(struct inode *inode)
242 struct ext4_inode_info *ei = EXT4_I(inode);
244 if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) ||
245 (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY))
249 spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock);
250 if (list_empty(&ei->i_fc_list))
253 if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) {
254 ext4_fc_wait_committing_inode(inode);
258 atomic_inc(&ei->i_fc_updates);
259 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
263 * Stop inode update and wake up waiting fast commits if any.
265 void ext4_fc_stop_update(struct inode *inode)
267 struct ext4_inode_info *ei = EXT4_I(inode);
269 if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) ||
270 (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY))
273 if (atomic_dec_and_test(&ei->i_fc_updates))
274 wake_up_all(&ei->i_fc_wait);
278 * Remove inode from fast commit list. If the inode is being committed
279 * we wait until inode commit is done.
281 void ext4_fc_del(struct inode *inode)
283 struct ext4_inode_info *ei = EXT4_I(inode);
285 if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) ||
286 (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY))
290 spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock);
291 if (list_empty(&ei->i_fc_list)) {
292 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
296 if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) {
297 ext4_fc_wait_committing_inode(inode);
300 list_del_init(&ei->i_fc_list);
301 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
305 * Mark file system as fast commit ineligible, and record latest
306 * ineligible transaction tid. This means until the recorded
307 * transaction, commit operation would result in a full jbd2 commit.
309 void ext4_fc_mark_ineligible(struct super_block *sb, int reason, handle_t *handle)
311 struct ext4_sb_info *sbi = EXT4_SB(sb);
314 if (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
315 (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))
318 ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
319 if (handle && !IS_ERR(handle))
320 tid = handle->h_transaction->t_tid;
322 read_lock(&sbi->s_journal->j_state_lock);
323 tid = sbi->s_journal->j_running_transaction ?
324 sbi->s_journal->j_running_transaction->t_tid : 0;
325 read_unlock(&sbi->s_journal->j_state_lock);
327 spin_lock(&sbi->s_fc_lock);
328 if (sbi->s_fc_ineligible_tid < tid)
329 sbi->s_fc_ineligible_tid = tid;
330 spin_unlock(&sbi->s_fc_lock);
331 WARN_ON(reason >= EXT4_FC_REASON_MAX);
332 sbi->s_fc_stats.fc_ineligible_reason_count[reason]++;
336 * Generic fast commit tracking function. If this is the first time this we are
337 * called after a full commit, we initialize fast commit fields and then call
338 * __fc_track_fn() with update = 0. If we have already been called after a full
339 * commit, we pass update = 1. Based on that, the track function can determine
340 * if it needs to track a field for the first time or if it needs to just
341 * update the previously tracked value.
343 * If enqueue is set, this function enqueues the inode in fast commit list.
345 static int ext4_fc_track_template(
346 handle_t *handle, struct inode *inode,
347 int (*__fc_track_fn)(struct inode *, void *, bool),
348 void *args, int enqueue)
351 struct ext4_inode_info *ei = EXT4_I(inode);
352 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
356 if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) ||
357 (sbi->s_mount_state & EXT4_FC_REPLAY))
360 if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_INELIGIBLE))
363 tid = handle->h_transaction->t_tid;
364 mutex_lock(&ei->i_fc_lock);
365 if (tid == ei->i_sync_tid) {
368 ext4_fc_reset_inode(inode);
369 ei->i_sync_tid = tid;
371 ret = __fc_track_fn(inode, args, update);
372 mutex_unlock(&ei->i_fc_lock);
377 spin_lock(&sbi->s_fc_lock);
378 if (list_empty(&EXT4_I(inode)->i_fc_list))
379 list_add_tail(&EXT4_I(inode)->i_fc_list,
380 (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING ||
381 sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING) ?
382 &sbi->s_fc_q[FC_Q_STAGING] :
383 &sbi->s_fc_q[FC_Q_MAIN]);
384 spin_unlock(&sbi->s_fc_lock);
389 struct __track_dentry_update_args {
390 struct dentry *dentry;
394 /* __track_fn for directory entry updates. Called with ei->i_fc_lock. */
395 static int __track_dentry_update(struct inode *inode, void *arg, bool update)
397 struct ext4_fc_dentry_update *node;
398 struct ext4_inode_info *ei = EXT4_I(inode);
399 struct __track_dentry_update_args *dentry_update =
400 (struct __track_dentry_update_args *)arg;
401 struct dentry *dentry = dentry_update->dentry;
402 struct inode *dir = dentry->d_parent->d_inode;
403 struct super_block *sb = inode->i_sb;
404 struct ext4_sb_info *sbi = EXT4_SB(sb);
406 mutex_unlock(&ei->i_fc_lock);
408 if (IS_ENCRYPTED(dir)) {
409 ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_ENCRYPTED_FILENAME,
411 mutex_lock(&ei->i_fc_lock);
415 node = kmem_cache_alloc(ext4_fc_dentry_cachep, GFP_NOFS);
417 ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_NOMEM, NULL);
418 mutex_lock(&ei->i_fc_lock);
422 node->fcd_op = dentry_update->op;
423 node->fcd_parent = dir->i_ino;
424 node->fcd_ino = inode->i_ino;
425 if (dentry->d_name.len > DNAME_INLINE_LEN) {
426 node->fcd_name.name = kmalloc(dentry->d_name.len, GFP_NOFS);
427 if (!node->fcd_name.name) {
428 kmem_cache_free(ext4_fc_dentry_cachep, node);
429 ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_NOMEM, NULL);
430 mutex_lock(&ei->i_fc_lock);
433 memcpy((u8 *)node->fcd_name.name, dentry->d_name.name,
436 memcpy(node->fcd_iname, dentry->d_name.name,
438 node->fcd_name.name = node->fcd_iname;
440 node->fcd_name.len = dentry->d_name.len;
442 spin_lock(&sbi->s_fc_lock);
443 if (sbi->s_journal->j_flags & JBD2_FULL_COMMIT_ONGOING ||
444 sbi->s_journal->j_flags & JBD2_FAST_COMMIT_ONGOING)
445 list_add_tail(&node->fcd_list,
446 &sbi->s_fc_dentry_q[FC_Q_STAGING]);
448 list_add_tail(&node->fcd_list, &sbi->s_fc_dentry_q[FC_Q_MAIN]);
449 spin_unlock(&sbi->s_fc_lock);
450 mutex_lock(&ei->i_fc_lock);
455 void __ext4_fc_track_unlink(handle_t *handle,
456 struct inode *inode, struct dentry *dentry)
458 struct __track_dentry_update_args args;
461 args.dentry = dentry;
462 args.op = EXT4_FC_TAG_UNLINK;
464 ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
466 trace_ext4_fc_track_unlink(inode, dentry, ret);
469 void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry)
471 __ext4_fc_track_unlink(handle, d_inode(dentry), dentry);
474 void __ext4_fc_track_link(handle_t *handle,
475 struct inode *inode, struct dentry *dentry)
477 struct __track_dentry_update_args args;
480 args.dentry = dentry;
481 args.op = EXT4_FC_TAG_LINK;
483 ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
485 trace_ext4_fc_track_link(inode, dentry, ret);
488 void ext4_fc_track_link(handle_t *handle, struct dentry *dentry)
490 __ext4_fc_track_link(handle, d_inode(dentry), dentry);
493 void __ext4_fc_track_create(handle_t *handle, struct inode *inode,
494 struct dentry *dentry)
496 struct __track_dentry_update_args args;
499 args.dentry = dentry;
500 args.op = EXT4_FC_TAG_CREAT;
502 ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
504 trace_ext4_fc_track_create(inode, dentry, ret);
507 void ext4_fc_track_create(handle_t *handle, struct dentry *dentry)
509 __ext4_fc_track_create(handle, d_inode(dentry), dentry);
512 /* __track_fn for inode tracking */
513 static int __track_inode(struct inode *inode, void *arg, bool update)
518 EXT4_I(inode)->i_fc_lblk_len = 0;
523 void ext4_fc_track_inode(handle_t *handle, struct inode *inode)
527 if (S_ISDIR(inode->i_mode))
530 if (ext4_should_journal_data(inode)) {
531 ext4_fc_mark_ineligible(inode->i_sb,
532 EXT4_FC_REASON_INODE_JOURNAL_DATA, handle);
536 ret = ext4_fc_track_template(handle, inode, __track_inode, NULL, 1);
537 trace_ext4_fc_track_inode(inode, ret);
540 struct __track_range_args {
541 ext4_lblk_t start, end;
544 /* __track_fn for tracking data updates */
545 static int __track_range(struct inode *inode, void *arg, bool update)
547 struct ext4_inode_info *ei = EXT4_I(inode);
548 ext4_lblk_t oldstart;
549 struct __track_range_args *__arg =
550 (struct __track_range_args *)arg;
552 if (inode->i_ino < EXT4_FIRST_INO(inode->i_sb)) {
553 ext4_debug("Special inode %ld being modified\n", inode->i_ino);
557 oldstart = ei->i_fc_lblk_start;
559 if (update && ei->i_fc_lblk_len > 0) {
560 ei->i_fc_lblk_start = min(ei->i_fc_lblk_start, __arg->start);
562 max(oldstart + ei->i_fc_lblk_len - 1, __arg->end) -
563 ei->i_fc_lblk_start + 1;
565 ei->i_fc_lblk_start = __arg->start;
566 ei->i_fc_lblk_len = __arg->end - __arg->start + 1;
572 void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t start,
575 struct __track_range_args args;
578 if (S_ISDIR(inode->i_mode))
584 ret = ext4_fc_track_template(handle, inode, __track_range, &args, 1);
586 trace_ext4_fc_track_range(inode, start, end, ret);
589 static void ext4_fc_submit_bh(struct super_block *sb, bool is_tail)
591 int write_flags = REQ_SYNC;
592 struct buffer_head *bh = EXT4_SB(sb)->s_fc_bh;
594 /* Add REQ_FUA | REQ_PREFLUSH only its tail */
595 if (test_opt(sb, BARRIER) && is_tail)
596 write_flags |= REQ_FUA | REQ_PREFLUSH;
598 set_buffer_dirty(bh);
599 set_buffer_uptodate(bh);
600 bh->b_end_io = ext4_end_buffer_io_sync;
601 submit_bh(REQ_OP_WRITE, write_flags, bh);
602 EXT4_SB(sb)->s_fc_bh = NULL;
605 /* Ext4 commit path routines */
607 /* memcpy to fc reserved space and update CRC */
608 static void *ext4_fc_memcpy(struct super_block *sb, void *dst, const void *src,
612 *crc = ext4_chksum(EXT4_SB(sb), *crc, src, len);
613 return memcpy(dst, src, len);
616 /* memzero and update CRC */
617 static void *ext4_fc_memzero(struct super_block *sb, void *dst, int len,
622 ret = memset(dst, 0, len);
624 *crc = ext4_chksum(EXT4_SB(sb), *crc, dst, len);
629 * Allocate len bytes on a fast commit buffer.
631 * During the commit time this function is used to manage fast commit
632 * block space. We don't split a fast commit log onto different
633 * blocks. So this function makes sure that if there's not enough space
634 * on the current block, the remaining space in the current block is
635 * marked as unused by adding EXT4_FC_TAG_PAD tag. In that case,
636 * new block is from jbd2 and CRC is updated to reflect the padding
639 static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc)
641 struct ext4_fc_tl tl;
642 struct ext4_sb_info *sbi = EXT4_SB(sb);
643 struct buffer_head *bh;
644 int bsize = sbi->s_journal->j_blocksize;
645 int ret, off = sbi->s_fc_bytes % bsize;
650 * If 'len' is too long to fit in any block alongside a PAD tlv, then we
651 * cannot fulfill the request.
653 if (len > bsize - EXT4_FC_TAG_BASE_LEN)
657 ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh);
662 dst = sbi->s_fc_bh->b_data + off;
665 * Allocate the bytes in the current block if we can do so while still
666 * leaving enough space for a PAD tlv.
668 remaining = bsize - EXT4_FC_TAG_BASE_LEN - off;
669 if (len <= remaining) {
670 sbi->s_fc_bytes += len;
675 * Else, terminate the current block with a PAD tlv, then allocate a new
676 * block and allocate the bytes at the start of that new block.
679 tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_PAD);
680 tl.fc_len = cpu_to_le16(remaining);
681 ext4_fc_memcpy(sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, crc);
682 ext4_fc_memzero(sb, dst + EXT4_FC_TAG_BASE_LEN, remaining, crc);
684 ext4_fc_submit_bh(sb, false);
686 ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh);
690 sbi->s_fc_bytes += bsize - off + len;
691 return sbi->s_fc_bh->b_data;
695 * Complete a fast commit by writing tail tag.
697 * Writing tail tag marks the end of a fast commit. In order to guarantee
698 * atomicity, after writing tail tag, even if there's space remaining
699 * in the block, next commit shouldn't use it. That's why tail tag
700 * has the length as that of the remaining space on the block.
702 static int ext4_fc_write_tail(struct super_block *sb, u32 crc)
704 struct ext4_sb_info *sbi = EXT4_SB(sb);
705 struct ext4_fc_tl tl;
706 struct ext4_fc_tail tail;
707 int off, bsize = sbi->s_journal->j_blocksize;
711 * ext4_fc_reserve_space takes care of allocating an extra block if
712 * there's no enough space on this block for accommodating this tail.
714 dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + sizeof(tail), &crc);
718 off = sbi->s_fc_bytes % bsize;
720 tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_TAIL);
721 tl.fc_len = cpu_to_le16(bsize - off + sizeof(struct ext4_fc_tail));
722 sbi->s_fc_bytes = round_up(sbi->s_fc_bytes, bsize);
724 ext4_fc_memcpy(sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, &crc);
725 dst += EXT4_FC_TAG_BASE_LEN;
726 tail.fc_tid = cpu_to_le32(sbi->s_journal->j_running_transaction->t_tid);
727 ext4_fc_memcpy(sb, dst, &tail.fc_tid, sizeof(tail.fc_tid), &crc);
728 dst += sizeof(tail.fc_tid);
729 tail.fc_crc = cpu_to_le32(crc);
730 ext4_fc_memcpy(sb, dst, &tail.fc_crc, sizeof(tail.fc_crc), NULL);
731 dst += sizeof(tail.fc_crc);
732 memset(dst, 0, bsize - off); /* Don't leak uninitialized memory. */
734 ext4_fc_submit_bh(sb, true);
740 * Adds tag, length, value and updates CRC. Returns true if tlv was added.
741 * Returns false if there's not enough space.
743 static bool ext4_fc_add_tlv(struct super_block *sb, u16 tag, u16 len, u8 *val,
746 struct ext4_fc_tl tl;
749 dst = ext4_fc_reserve_space(sb, EXT4_FC_TAG_BASE_LEN + len, crc);
753 tl.fc_tag = cpu_to_le16(tag);
754 tl.fc_len = cpu_to_le16(len);
756 ext4_fc_memcpy(sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, crc);
757 ext4_fc_memcpy(sb, dst + EXT4_FC_TAG_BASE_LEN, val, len, crc);
762 /* Same as above, but adds dentry tlv. */
763 static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u32 *crc,
764 struct ext4_fc_dentry_update *fc_dentry)
766 struct ext4_fc_dentry_info fcd;
767 struct ext4_fc_tl tl;
768 int dlen = fc_dentry->fcd_name.len;
769 u8 *dst = ext4_fc_reserve_space(sb,
770 EXT4_FC_TAG_BASE_LEN + sizeof(fcd) + dlen, crc);
775 fcd.fc_parent_ino = cpu_to_le32(fc_dentry->fcd_parent);
776 fcd.fc_ino = cpu_to_le32(fc_dentry->fcd_ino);
777 tl.fc_tag = cpu_to_le16(fc_dentry->fcd_op);
778 tl.fc_len = cpu_to_le16(sizeof(fcd) + dlen);
779 ext4_fc_memcpy(sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, crc);
780 dst += EXT4_FC_TAG_BASE_LEN;
781 ext4_fc_memcpy(sb, dst, &fcd, sizeof(fcd), crc);
783 ext4_fc_memcpy(sb, dst, fc_dentry->fcd_name.name, dlen, crc);
790 * Writes inode in the fast commit space under TLV with tag @tag.
791 * Returns 0 on success, error on failure.
793 static int ext4_fc_write_inode(struct inode *inode, u32 *crc)
795 struct ext4_inode_info *ei = EXT4_I(inode);
796 int inode_len = EXT4_GOOD_OLD_INODE_SIZE;
798 struct ext4_iloc iloc;
799 struct ext4_fc_inode fc_inode;
800 struct ext4_fc_tl tl;
803 ret = ext4_get_inode_loc(inode, &iloc);
807 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE)
808 inode_len += ei->i_extra_isize;
810 fc_inode.fc_ino = cpu_to_le32(inode->i_ino);
811 tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_INODE);
812 tl.fc_len = cpu_to_le16(inode_len + sizeof(fc_inode.fc_ino));
815 dst = ext4_fc_reserve_space(inode->i_sb,
816 EXT4_FC_TAG_BASE_LEN + inode_len + sizeof(fc_inode.fc_ino), crc);
820 if (!ext4_fc_memcpy(inode->i_sb, dst, &tl, EXT4_FC_TAG_BASE_LEN, crc))
822 dst += EXT4_FC_TAG_BASE_LEN;
823 if (!ext4_fc_memcpy(inode->i_sb, dst, &fc_inode, sizeof(fc_inode), crc))
825 dst += sizeof(fc_inode);
826 if (!ext4_fc_memcpy(inode->i_sb, dst, (u8 *)ext4_raw_inode(&iloc),
836 * Writes updated data ranges for the inode in question. Updates CRC.
837 * Returns 0 on success, error otherwise.
839 static int ext4_fc_write_inode_data(struct inode *inode, u32 *crc)
841 ext4_lblk_t old_blk_size, cur_lblk_off, new_blk_size;
842 struct ext4_inode_info *ei = EXT4_I(inode);
843 struct ext4_map_blocks map;
844 struct ext4_fc_add_range fc_ext;
845 struct ext4_fc_del_range lrange;
846 struct ext4_extent *ex;
849 mutex_lock(&ei->i_fc_lock);
850 if (ei->i_fc_lblk_len == 0) {
851 mutex_unlock(&ei->i_fc_lock);
854 old_blk_size = ei->i_fc_lblk_start;
855 new_blk_size = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1;
856 ei->i_fc_lblk_len = 0;
857 mutex_unlock(&ei->i_fc_lock);
859 cur_lblk_off = old_blk_size;
860 ext4_debug("will try writing %d to %d for inode %ld\n",
861 cur_lblk_off, new_blk_size, inode->i_ino);
863 while (cur_lblk_off <= new_blk_size) {
864 map.m_lblk = cur_lblk_off;
865 map.m_len = new_blk_size - cur_lblk_off + 1;
866 ret = ext4_map_blocks(NULL, inode, &map, 0);
870 if (map.m_len == 0) {
876 lrange.fc_ino = cpu_to_le32(inode->i_ino);
877 lrange.fc_lblk = cpu_to_le32(map.m_lblk);
878 lrange.fc_len = cpu_to_le32(map.m_len);
879 if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_DEL_RANGE,
880 sizeof(lrange), (u8 *)&lrange, crc))
883 unsigned int max = (map.m_flags & EXT4_MAP_UNWRITTEN) ?
884 EXT_UNWRITTEN_MAX_LEN : EXT_INIT_MAX_LEN;
886 /* Limit the number of blocks in one extent */
887 map.m_len = min(max, map.m_len);
889 fc_ext.fc_ino = cpu_to_le32(inode->i_ino);
890 ex = (struct ext4_extent *)&fc_ext.fc_ex;
891 ex->ee_block = cpu_to_le32(map.m_lblk);
892 ex->ee_len = cpu_to_le16(map.m_len);
893 ext4_ext_store_pblock(ex, map.m_pblk);
894 if (map.m_flags & EXT4_MAP_UNWRITTEN)
895 ext4_ext_mark_unwritten(ex);
897 ext4_ext_mark_initialized(ex);
898 if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_ADD_RANGE,
899 sizeof(fc_ext), (u8 *)&fc_ext, crc))
903 cur_lblk_off += map.m_len;
910 /* Submit data for all the fast commit inodes */
911 static int ext4_fc_submit_inode_data_all(journal_t *journal)
913 struct super_block *sb = (struct super_block *)(journal->j_private);
914 struct ext4_sb_info *sbi = EXT4_SB(sb);
915 struct ext4_inode_info *ei;
918 spin_lock(&sbi->s_fc_lock);
919 list_for_each_entry(ei, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
920 ext4_set_inode_state(&ei->vfs_inode, EXT4_STATE_FC_COMMITTING);
921 while (atomic_read(&ei->i_fc_updates)) {
924 prepare_to_wait(&ei->i_fc_wait, &wait,
925 TASK_UNINTERRUPTIBLE);
926 if (atomic_read(&ei->i_fc_updates)) {
927 spin_unlock(&sbi->s_fc_lock);
929 spin_lock(&sbi->s_fc_lock);
931 finish_wait(&ei->i_fc_wait, &wait);
933 spin_unlock(&sbi->s_fc_lock);
934 ret = jbd2_submit_inode_data(ei->jinode);
937 spin_lock(&sbi->s_fc_lock);
939 spin_unlock(&sbi->s_fc_lock);
944 /* Wait for completion of data for all the fast commit inodes */
945 static int ext4_fc_wait_inode_data_all(journal_t *journal)
947 struct super_block *sb = (struct super_block *)(journal->j_private);
948 struct ext4_sb_info *sbi = EXT4_SB(sb);
949 struct ext4_inode_info *pos, *n;
952 spin_lock(&sbi->s_fc_lock);
953 list_for_each_entry_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
954 if (!ext4_test_inode_state(&pos->vfs_inode,
955 EXT4_STATE_FC_COMMITTING))
957 spin_unlock(&sbi->s_fc_lock);
959 ret = jbd2_wait_inode_data(journal, pos->jinode);
962 spin_lock(&sbi->s_fc_lock);
964 spin_unlock(&sbi->s_fc_lock);
969 /* Commit all the directory entry updates */
970 static int ext4_fc_commit_dentry_updates(journal_t *journal, u32 *crc)
971 __acquires(&sbi->s_fc_lock)
972 __releases(&sbi->s_fc_lock)
974 struct super_block *sb = (struct super_block *)(journal->j_private);
975 struct ext4_sb_info *sbi = EXT4_SB(sb);
976 struct ext4_fc_dentry_update *fc_dentry, *fc_dentry_n;
978 struct ext4_inode_info *ei, *ei_n;
981 if (list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN]))
983 list_for_each_entry_safe(fc_dentry, fc_dentry_n,
984 &sbi->s_fc_dentry_q[FC_Q_MAIN], fcd_list) {
985 if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) {
986 spin_unlock(&sbi->s_fc_lock);
987 if (!ext4_fc_add_dentry_tlv(sb, crc, fc_dentry)) {
991 spin_lock(&sbi->s_fc_lock);
996 list_for_each_entry_safe(ei, ei_n, &sbi->s_fc_q[FC_Q_MAIN],
998 if (ei->vfs_inode.i_ino == fc_dentry->fcd_ino) {
999 inode = &ei->vfs_inode;
1004 * If we don't find inode in our list, then it was deleted,
1005 * in which case, we don't need to record it's create tag.
1009 spin_unlock(&sbi->s_fc_lock);
1012 * We first write the inode and then the create dirent. This
1013 * allows the recovery code to create an unnamed inode first
1014 * and then link it to a directory entry. This allows us
1015 * to use namei.c routines almost as is and simplifies
1016 * the recovery code.
1018 ret = ext4_fc_write_inode(inode, crc);
1022 ret = ext4_fc_write_inode_data(inode, crc);
1026 if (!ext4_fc_add_dentry_tlv(sb, crc, fc_dentry)) {
1031 spin_lock(&sbi->s_fc_lock);
1035 spin_lock(&sbi->s_fc_lock);
1039 static int ext4_fc_perform_commit(journal_t *journal)
1041 struct super_block *sb = (struct super_block *)(journal->j_private);
1042 struct ext4_sb_info *sbi = EXT4_SB(sb);
1043 struct ext4_inode_info *iter;
1044 struct ext4_fc_head head;
1045 struct inode *inode;
1046 struct blk_plug plug;
1050 ret = ext4_fc_submit_inode_data_all(journal);
1054 ret = ext4_fc_wait_inode_data_all(journal);
1059 * If file system device is different from journal device, issue a cache
1060 * flush before we start writing fast commit blocks.
1062 if (journal->j_fs_dev != journal->j_dev)
1063 blkdev_issue_flush(journal->j_fs_dev);
1065 blk_start_plug(&plug);
1066 if (sbi->s_fc_bytes == 0) {
1068 * Add a head tag only if this is the first fast commit
1071 head.fc_features = cpu_to_le32(EXT4_FC_SUPPORTED_FEATURES);
1072 head.fc_tid = cpu_to_le32(
1073 sbi->s_journal->j_running_transaction->t_tid);
1074 if (!ext4_fc_add_tlv(sb, EXT4_FC_TAG_HEAD, sizeof(head),
1075 (u8 *)&head, &crc)) {
1081 spin_lock(&sbi->s_fc_lock);
1082 ret = ext4_fc_commit_dentry_updates(journal, &crc);
1084 spin_unlock(&sbi->s_fc_lock);
1088 list_for_each_entry(iter, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
1089 inode = &iter->vfs_inode;
1090 if (!ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING))
1093 spin_unlock(&sbi->s_fc_lock);
1094 ret = ext4_fc_write_inode_data(inode, &crc);
1097 ret = ext4_fc_write_inode(inode, &crc);
1100 spin_lock(&sbi->s_fc_lock);
1102 spin_unlock(&sbi->s_fc_lock);
1104 ret = ext4_fc_write_tail(sb, crc);
1107 blk_finish_plug(&plug);
1111 static void ext4_fc_update_stats(struct super_block *sb, int status,
1112 u64 commit_time, int nblks)
1114 struct ext4_fc_stats *stats = &EXT4_SB(sb)->s_fc_stats;
1116 ext4_debug("Fast commit ended with status = %d", status);
1117 if (status == EXT4_FC_STATUS_OK) {
1118 stats->fc_num_commits++;
1119 stats->fc_numblks += nblks;
1120 if (likely(stats->s_fc_avg_commit_time))
1121 stats->s_fc_avg_commit_time =
1123 stats->s_fc_avg_commit_time * 3) / 4;
1125 stats->s_fc_avg_commit_time = commit_time;
1126 } else if (status == EXT4_FC_STATUS_FAILED ||
1127 status == EXT4_FC_STATUS_INELIGIBLE) {
1128 if (status == EXT4_FC_STATUS_FAILED)
1129 stats->fc_failed_commits++;
1130 stats->fc_ineligible_commits++;
1132 stats->fc_skipped_commits++;
1134 trace_ext4_fc_commit_stop(sb, nblks, status);
1138 * The main commit entry point. Performs a fast commit for transaction
1139 * commit_tid if needed. If it's not possible to perform a fast commit
1140 * due to various reasons, we fall back to full commit. Returns 0
1141 * on success, error otherwise.
1143 int ext4_fc_commit(journal_t *journal, tid_t commit_tid)
1145 struct super_block *sb = (struct super_block *)(journal->j_private);
1146 struct ext4_sb_info *sbi = EXT4_SB(sb);
1147 int nblks = 0, ret, bsize = journal->j_blocksize;
1148 int subtid = atomic_read(&sbi->s_fc_subtid);
1149 int status = EXT4_FC_STATUS_OK, fc_bufs_before = 0;
1150 ktime_t start_time, commit_time;
1152 trace_ext4_fc_commit_start(sb);
1154 start_time = ktime_get();
1156 if (!test_opt2(sb, JOURNAL_FAST_COMMIT))
1157 return jbd2_complete_transaction(journal, commit_tid);
1160 ret = jbd2_fc_begin_commit(journal, commit_tid);
1161 if (ret == -EALREADY) {
1162 /* There was an ongoing commit, check if we need to restart */
1163 if (atomic_read(&sbi->s_fc_subtid) <= subtid &&
1164 commit_tid > journal->j_commit_sequence)
1166 ext4_fc_update_stats(sb, EXT4_FC_STATUS_SKIPPED, 0, 0);
1170 * Commit couldn't start. Just update stats and perform a
1173 ext4_fc_update_stats(sb, EXT4_FC_STATUS_FAILED, 0, 0);
1174 return jbd2_complete_transaction(journal, commit_tid);
1178 * After establishing journal barrier via jbd2_fc_begin_commit(), check
1179 * if we are fast commit ineligible.
1181 if (ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE)) {
1182 status = EXT4_FC_STATUS_INELIGIBLE;
1186 fc_bufs_before = (sbi->s_fc_bytes + bsize - 1) / bsize;
1187 ret = ext4_fc_perform_commit(journal);
1189 status = EXT4_FC_STATUS_FAILED;
1192 nblks = (sbi->s_fc_bytes + bsize - 1) / bsize - fc_bufs_before;
1193 ret = jbd2_fc_wait_bufs(journal, nblks);
1195 status = EXT4_FC_STATUS_FAILED;
1198 atomic_inc(&sbi->s_fc_subtid);
1199 ret = jbd2_fc_end_commit(journal);
1201 * weight the commit time higher than the average time so we
1202 * don't react too strongly to vast changes in the commit time
1204 commit_time = ktime_to_ns(ktime_sub(ktime_get(), start_time));
1205 ext4_fc_update_stats(sb, status, commit_time, nblks);
1209 ret = jbd2_fc_end_commit_fallback(journal);
1210 ext4_fc_update_stats(sb, status, 0, 0);
1215 * Fast commit cleanup routine. This is called after every fast commit and
1216 * full commit. full is true if we are called after a full commit.
1218 static void ext4_fc_cleanup(journal_t *journal, int full, tid_t tid)
1220 struct super_block *sb = journal->j_private;
1221 struct ext4_sb_info *sbi = EXT4_SB(sb);
1222 struct ext4_inode_info *iter, *iter_n;
1223 struct ext4_fc_dentry_update *fc_dentry;
1225 if (full && sbi->s_fc_bh)
1226 sbi->s_fc_bh = NULL;
1228 jbd2_fc_release_bufs(journal);
1230 spin_lock(&sbi->s_fc_lock);
1231 list_for_each_entry_safe(iter, iter_n, &sbi->s_fc_q[FC_Q_MAIN],
1233 list_del_init(&iter->i_fc_list);
1234 ext4_clear_inode_state(&iter->vfs_inode,
1235 EXT4_STATE_FC_COMMITTING);
1236 if (iter->i_sync_tid <= tid)
1237 ext4_fc_reset_inode(&iter->vfs_inode);
1238 /* Make sure EXT4_STATE_FC_COMMITTING bit is clear */
1240 #if (BITS_PER_LONG < 64)
1241 wake_up_bit(&iter->i_state_flags, EXT4_STATE_FC_COMMITTING);
1243 wake_up_bit(&iter->i_flags, EXT4_STATE_FC_COMMITTING);
1247 while (!list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) {
1248 fc_dentry = list_first_entry(&sbi->s_fc_dentry_q[FC_Q_MAIN],
1249 struct ext4_fc_dentry_update,
1251 list_del_init(&fc_dentry->fcd_list);
1252 spin_unlock(&sbi->s_fc_lock);
1254 if (fc_dentry->fcd_name.name &&
1255 fc_dentry->fcd_name.len > DNAME_INLINE_LEN)
1256 kfree(fc_dentry->fcd_name.name);
1257 kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry);
1258 spin_lock(&sbi->s_fc_lock);
1261 list_splice_init(&sbi->s_fc_dentry_q[FC_Q_STAGING],
1262 &sbi->s_fc_dentry_q[FC_Q_MAIN]);
1263 list_splice_init(&sbi->s_fc_q[FC_Q_STAGING],
1264 &sbi->s_fc_q[FC_Q_MAIN]);
1266 if (tid >= sbi->s_fc_ineligible_tid) {
1267 sbi->s_fc_ineligible_tid = 0;
1268 ext4_clear_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
1272 sbi->s_fc_bytes = 0;
1273 spin_unlock(&sbi->s_fc_lock);
1274 trace_ext4_fc_stats(sb);
1277 /* Ext4 Replay Path Routines */
1279 /* Helper struct for dentry replay routines */
1280 struct dentry_info_args {
1281 int parent_ino, dname_len, ino, inode_len;
1285 /* Same as struct ext4_fc_tl, but uses native endianness fields */
1286 struct ext4_fc_tl_mem {
1291 static inline void tl_to_darg(struct dentry_info_args *darg,
1292 struct ext4_fc_tl_mem *tl, u8 *val)
1294 struct ext4_fc_dentry_info fcd;
1296 memcpy(&fcd, val, sizeof(fcd));
1298 darg->parent_ino = le32_to_cpu(fcd.fc_parent_ino);
1299 darg->ino = le32_to_cpu(fcd.fc_ino);
1300 darg->dname = val + offsetof(struct ext4_fc_dentry_info, fc_dname);
1301 darg->dname_len = tl->fc_len - sizeof(struct ext4_fc_dentry_info);
1304 static inline void ext4_fc_get_tl(struct ext4_fc_tl_mem *tl, u8 *val)
1306 struct ext4_fc_tl tl_disk;
1308 memcpy(&tl_disk, val, EXT4_FC_TAG_BASE_LEN);
1309 tl->fc_len = le16_to_cpu(tl_disk.fc_len);
1310 tl->fc_tag = le16_to_cpu(tl_disk.fc_tag);
1313 /* Unlink replay function */
1314 static int ext4_fc_replay_unlink(struct super_block *sb,
1315 struct ext4_fc_tl_mem *tl, u8 *val)
1317 struct inode *inode, *old_parent;
1319 struct dentry_info_args darg;
1322 tl_to_darg(&darg, tl, val);
1324 trace_ext4_fc_replay(sb, EXT4_FC_TAG_UNLINK, darg.ino,
1325 darg.parent_ino, darg.dname_len);
1327 entry.name = darg.dname;
1328 entry.len = darg.dname_len;
1329 inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
1331 if (IS_ERR(inode)) {
1332 ext4_debug("Inode %d not found", darg.ino);
1336 old_parent = ext4_iget(sb, darg.parent_ino,
1338 if (IS_ERR(old_parent)) {
1339 ext4_debug("Dir with inode %d not found", darg.parent_ino);
1344 ret = __ext4_unlink(old_parent, &entry, inode, NULL);
1345 /* -ENOENT ok coz it might not exist anymore. */
1353 static int ext4_fc_replay_link_internal(struct super_block *sb,
1354 struct dentry_info_args *darg,
1355 struct inode *inode)
1357 struct inode *dir = NULL;
1358 struct dentry *dentry_dir = NULL, *dentry_inode = NULL;
1359 struct qstr qstr_dname = QSTR_INIT(darg->dname, darg->dname_len);
1362 dir = ext4_iget(sb, darg->parent_ino, EXT4_IGET_NORMAL);
1364 ext4_debug("Dir with inode %d not found.", darg->parent_ino);
1369 dentry_dir = d_obtain_alias(dir);
1370 if (IS_ERR(dentry_dir)) {
1371 ext4_debug("Failed to obtain dentry");
1376 dentry_inode = d_alloc(dentry_dir, &qstr_dname);
1377 if (!dentry_inode) {
1378 ext4_debug("Inode dentry not created.");
1383 ret = __ext4_link(dir, inode, dentry_inode);
1385 * It's possible that link already existed since data blocks
1386 * for the dir in question got persisted before we crashed OR
1387 * we replayed this tag and crashed before the entire replay
1390 if (ret && ret != -EEXIST) {
1391 ext4_debug("Failed to link\n");
1404 d_drop(dentry_inode);
1411 /* Link replay function */
1412 static int ext4_fc_replay_link(struct super_block *sb,
1413 struct ext4_fc_tl_mem *tl, u8 *val)
1415 struct inode *inode;
1416 struct dentry_info_args darg;
1419 tl_to_darg(&darg, tl, val);
1420 trace_ext4_fc_replay(sb, EXT4_FC_TAG_LINK, darg.ino,
1421 darg.parent_ino, darg.dname_len);
1423 inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
1424 if (IS_ERR(inode)) {
1425 ext4_debug("Inode not found.");
1429 ret = ext4_fc_replay_link_internal(sb, &darg, inode);
1435 * Record all the modified inodes during replay. We use this later to setup
1436 * block bitmaps correctly.
1438 static int ext4_fc_record_modified_inode(struct super_block *sb, int ino)
1440 struct ext4_fc_replay_state *state;
1443 state = &EXT4_SB(sb)->s_fc_replay_state;
1444 for (i = 0; i < state->fc_modified_inodes_used; i++)
1445 if (state->fc_modified_inodes[i] == ino)
1447 if (state->fc_modified_inodes_used == state->fc_modified_inodes_size) {
1448 int *fc_modified_inodes;
1450 fc_modified_inodes = krealloc(state->fc_modified_inodes,
1451 sizeof(int) * (state->fc_modified_inodes_size +
1452 EXT4_FC_REPLAY_REALLOC_INCREMENT),
1454 if (!fc_modified_inodes)
1456 state->fc_modified_inodes = fc_modified_inodes;
1457 state->fc_modified_inodes_size +=
1458 EXT4_FC_REPLAY_REALLOC_INCREMENT;
1460 state->fc_modified_inodes[state->fc_modified_inodes_used++] = ino;
1465 * Inode replay function
1467 static int ext4_fc_replay_inode(struct super_block *sb,
1468 struct ext4_fc_tl_mem *tl, u8 *val)
1470 struct ext4_fc_inode fc_inode;
1471 struct ext4_inode *raw_inode;
1472 struct ext4_inode *raw_fc_inode;
1473 struct inode *inode = NULL;
1474 struct ext4_iloc iloc;
1475 int inode_len, ino, ret, tag = tl->fc_tag;
1476 struct ext4_extent_header *eh;
1478 memcpy(&fc_inode, val, sizeof(fc_inode));
1480 ino = le32_to_cpu(fc_inode.fc_ino);
1481 trace_ext4_fc_replay(sb, tag, ino, 0, 0);
1483 inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL);
1484 if (!IS_ERR(inode)) {
1485 ext4_ext_clear_bb(inode);
1490 ret = ext4_fc_record_modified_inode(sb, ino);
1494 raw_fc_inode = (struct ext4_inode *)
1495 (val + offsetof(struct ext4_fc_inode, fc_raw_inode));
1496 ret = ext4_get_fc_inode_loc(sb, ino, &iloc);
1500 inode_len = tl->fc_len - sizeof(struct ext4_fc_inode);
1501 raw_inode = ext4_raw_inode(&iloc);
1503 memcpy(raw_inode, raw_fc_inode, offsetof(struct ext4_inode, i_block));
1504 memcpy(&raw_inode->i_generation, &raw_fc_inode->i_generation,
1505 inode_len - offsetof(struct ext4_inode, i_generation));
1506 if (le32_to_cpu(raw_inode->i_flags) & EXT4_EXTENTS_FL) {
1507 eh = (struct ext4_extent_header *)(&raw_inode->i_block[0]);
1508 if (eh->eh_magic != EXT4_EXT_MAGIC) {
1509 memset(eh, 0, sizeof(*eh));
1510 eh->eh_magic = EXT4_EXT_MAGIC;
1511 eh->eh_max = cpu_to_le16(
1512 (sizeof(raw_inode->i_block) -
1513 sizeof(struct ext4_extent_header))
1514 / sizeof(struct ext4_extent));
1516 } else if (le32_to_cpu(raw_inode->i_flags) & EXT4_INLINE_DATA_FL) {
1517 memcpy(raw_inode->i_block, raw_fc_inode->i_block,
1518 sizeof(raw_inode->i_block));
1521 /* Immediately update the inode on disk. */
1522 ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh);
1525 ret = sync_dirty_buffer(iloc.bh);
1528 ret = ext4_mark_inode_used(sb, ino);
1532 /* Given that we just wrote the inode on disk, this SHOULD succeed. */
1533 inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL);
1534 if (IS_ERR(inode)) {
1535 ext4_debug("Inode not found.");
1536 return -EFSCORRUPTED;
1540 * Our allocator could have made different decisions than before
1541 * crashing. This should be fixed but until then, we calculate
1542 * the number of blocks the inode.
1544 ext4_ext_replay_set_iblocks(inode);
1546 inode->i_generation = le32_to_cpu(ext4_raw_inode(&iloc)->i_generation);
1547 ext4_reset_inode_seed(inode);
1549 ext4_inode_csum_set(inode, ext4_raw_inode(&iloc), EXT4_I(inode));
1550 ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh);
1551 sync_dirty_buffer(iloc.bh);
1556 blkdev_issue_flush(sb->s_bdev);
1562 * Dentry create replay function.
1564 * EXT4_FC_TAG_CREAT is preceded by EXT4_FC_TAG_INODE_FULL. Which means, the
1565 * inode for which we are trying to create a dentry here, should already have
1566 * been replayed before we start here.
1568 static int ext4_fc_replay_create(struct super_block *sb,
1569 struct ext4_fc_tl_mem *tl, u8 *val)
1572 struct inode *inode = NULL;
1573 struct inode *dir = NULL;
1574 struct dentry_info_args darg;
1576 tl_to_darg(&darg, tl, val);
1578 trace_ext4_fc_replay(sb, EXT4_FC_TAG_CREAT, darg.ino,
1579 darg.parent_ino, darg.dname_len);
1581 /* This takes care of update group descriptor and other metadata */
1582 ret = ext4_mark_inode_used(sb, darg.ino);
1586 inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
1587 if (IS_ERR(inode)) {
1588 ext4_debug("inode %d not found.", darg.ino);
1594 if (S_ISDIR(inode->i_mode)) {
1596 * If we are creating a directory, we need to make sure that the
1597 * dot and dot dot dirents are setup properly.
1599 dir = ext4_iget(sb, darg.parent_ino, EXT4_IGET_NORMAL);
1601 ext4_debug("Dir %d not found.", darg.ino);
1604 ret = ext4_init_new_dir(NULL, dir, inode);
1611 ret = ext4_fc_replay_link_internal(sb, &darg, inode);
1614 set_nlink(inode, 1);
1615 ext4_mark_inode_dirty(NULL, inode);
1623 * Record physical disk regions which are in use as per fast commit area,
1624 * and used by inodes during replay phase. Our simple replay phase
1625 * allocator excludes these regions from allocation.
1627 int ext4_fc_record_regions(struct super_block *sb, int ino,
1628 ext4_lblk_t lblk, ext4_fsblk_t pblk, int len, int replay)
1630 struct ext4_fc_replay_state *state;
1631 struct ext4_fc_alloc_region *region;
1633 state = &EXT4_SB(sb)->s_fc_replay_state;
1635 * during replay phase, the fc_regions_valid may not same as
1636 * fc_regions_used, update it when do new additions.
1638 if (replay && state->fc_regions_used != state->fc_regions_valid)
1639 state->fc_regions_used = state->fc_regions_valid;
1640 if (state->fc_regions_used == state->fc_regions_size) {
1641 struct ext4_fc_alloc_region *fc_regions;
1643 fc_regions = krealloc(state->fc_regions,
1644 sizeof(struct ext4_fc_alloc_region) *
1645 (state->fc_regions_size +
1646 EXT4_FC_REPLAY_REALLOC_INCREMENT),
1650 state->fc_regions_size +=
1651 EXT4_FC_REPLAY_REALLOC_INCREMENT;
1652 state->fc_regions = fc_regions;
1654 region = &state->fc_regions[state->fc_regions_used++];
1656 region->lblk = lblk;
1657 region->pblk = pblk;
1661 state->fc_regions_valid++;
1666 /* Replay add range tag */
1667 static int ext4_fc_replay_add_range(struct super_block *sb,
1668 struct ext4_fc_tl_mem *tl, u8 *val)
1670 struct ext4_fc_add_range fc_add_ex;
1671 struct ext4_extent newex, *ex;
1672 struct inode *inode;
1673 ext4_lblk_t start, cur;
1675 ext4_fsblk_t start_pblk;
1676 struct ext4_map_blocks map;
1677 struct ext4_ext_path *path = NULL;
1680 memcpy(&fc_add_ex, val, sizeof(fc_add_ex));
1681 ex = (struct ext4_extent *)&fc_add_ex.fc_ex;
1683 trace_ext4_fc_replay(sb, EXT4_FC_TAG_ADD_RANGE,
1684 le32_to_cpu(fc_add_ex.fc_ino), le32_to_cpu(ex->ee_block),
1685 ext4_ext_get_actual_len(ex));
1687 inode = ext4_iget(sb, le32_to_cpu(fc_add_ex.fc_ino), EXT4_IGET_NORMAL);
1688 if (IS_ERR(inode)) {
1689 ext4_debug("Inode not found.");
1693 ret = ext4_fc_record_modified_inode(sb, inode->i_ino);
1697 start = le32_to_cpu(ex->ee_block);
1698 start_pblk = ext4_ext_pblock(ex);
1699 len = ext4_ext_get_actual_len(ex);
1703 ext4_debug("ADD_RANGE, lblk %d, pblk %lld, len %d, unwritten %d, inode %ld\n",
1704 start, start_pblk, len, ext4_ext_is_unwritten(ex),
1707 while (remaining > 0) {
1709 map.m_len = remaining;
1711 ret = ext4_map_blocks(NULL, inode, &map, 0);
1717 /* Range is not mapped */
1718 path = ext4_find_extent(inode, cur, NULL, 0);
1721 memset(&newex, 0, sizeof(newex));
1722 newex.ee_block = cpu_to_le32(cur);
1723 ext4_ext_store_pblock(
1724 &newex, start_pblk + cur - start);
1725 newex.ee_len = cpu_to_le16(map.m_len);
1726 if (ext4_ext_is_unwritten(ex))
1727 ext4_ext_mark_unwritten(&newex);
1728 down_write(&EXT4_I(inode)->i_data_sem);
1729 ret = ext4_ext_insert_extent(
1730 NULL, inode, &path, &newex, 0);
1731 up_write((&EXT4_I(inode)->i_data_sem));
1732 ext4_ext_drop_refs(path);
1739 if (start_pblk + cur - start != map.m_pblk) {
1741 * Logical to physical mapping changed. This can happen
1742 * if this range was removed and then reallocated to
1743 * map to new physical blocks during a fast commit.
1745 ret = ext4_ext_replay_update_ex(inode, cur, map.m_len,
1746 ext4_ext_is_unwritten(ex),
1747 start_pblk + cur - start);
1751 * Mark the old blocks as free since they aren't used
1752 * anymore. We maintain an array of all the modified
1753 * inodes. In case these blocks are still used at either
1754 * a different logical range in the same inode or in
1755 * some different inode, we will mark them as allocated
1756 * at the end of the FC replay using our array of
1759 ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, 0);
1763 /* Range is mapped and needs a state change */
1764 ext4_debug("Converting from %ld to %d %lld",
1765 map.m_flags & EXT4_MAP_UNWRITTEN,
1766 ext4_ext_is_unwritten(ex), map.m_pblk);
1767 ret = ext4_ext_replay_update_ex(inode, cur, map.m_len,
1768 ext4_ext_is_unwritten(ex), map.m_pblk);
1772 * We may have split the extent tree while toggling the state.
1773 * Try to shrink the extent tree now.
1775 ext4_ext_replay_shrink_inode(inode, start + len);
1778 remaining -= map.m_len;
1780 ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >>
1781 sb->s_blocksize_bits);
1787 /* Replay DEL_RANGE tag */
1789 ext4_fc_replay_del_range(struct super_block *sb,
1790 struct ext4_fc_tl_mem *tl, u8 *val)
1792 struct inode *inode;
1793 struct ext4_fc_del_range lrange;
1794 struct ext4_map_blocks map;
1795 ext4_lblk_t cur, remaining;
1798 memcpy(&lrange, val, sizeof(lrange));
1799 cur = le32_to_cpu(lrange.fc_lblk);
1800 remaining = le32_to_cpu(lrange.fc_len);
1802 trace_ext4_fc_replay(sb, EXT4_FC_TAG_DEL_RANGE,
1803 le32_to_cpu(lrange.fc_ino), cur, remaining);
1805 inode = ext4_iget(sb, le32_to_cpu(lrange.fc_ino), EXT4_IGET_NORMAL);
1806 if (IS_ERR(inode)) {
1807 ext4_debug("Inode %d not found", le32_to_cpu(lrange.fc_ino));
1811 ret = ext4_fc_record_modified_inode(sb, inode->i_ino);
1815 ext4_debug("DEL_RANGE, inode %ld, lblk %d, len %d\n",
1816 inode->i_ino, le32_to_cpu(lrange.fc_lblk),
1817 le32_to_cpu(lrange.fc_len));
1818 while (remaining > 0) {
1820 map.m_len = remaining;
1822 ret = ext4_map_blocks(NULL, inode, &map, 0);
1828 ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, 0);
1830 remaining -= map.m_len;
1835 down_write(&EXT4_I(inode)->i_data_sem);
1836 ret = ext4_ext_remove_space(inode, le32_to_cpu(lrange.fc_lblk),
1837 le32_to_cpu(lrange.fc_lblk) +
1838 le32_to_cpu(lrange.fc_len) - 1);
1839 up_write(&EXT4_I(inode)->i_data_sem);
1842 ext4_ext_replay_shrink_inode(inode,
1843 i_size_read(inode) >> sb->s_blocksize_bits);
1844 ext4_mark_inode_dirty(NULL, inode);
1850 static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb)
1852 struct ext4_fc_replay_state *state;
1853 struct inode *inode;
1854 struct ext4_ext_path *path = NULL;
1855 struct ext4_map_blocks map;
1857 ext4_lblk_t cur, end;
1859 state = &EXT4_SB(sb)->s_fc_replay_state;
1860 for (i = 0; i < state->fc_modified_inodes_used; i++) {
1861 inode = ext4_iget(sb, state->fc_modified_inodes[i],
1863 if (IS_ERR(inode)) {
1864 ext4_debug("Inode %d not found.",
1865 state->fc_modified_inodes[i]);
1869 end = EXT_MAX_BLOCKS;
1872 map.m_len = end - cur;
1874 ret = ext4_map_blocks(NULL, inode, &map, 0);
1879 path = ext4_find_extent(inode, map.m_lblk, NULL, 0);
1880 if (!IS_ERR(path)) {
1881 for (j = 0; j < path->p_depth; j++)
1882 ext4_mb_mark_bb(inode->i_sb,
1883 path[j].p_block, 1, 1);
1884 ext4_ext_drop_refs(path);
1888 ext4_mb_mark_bb(inode->i_sb, map.m_pblk,
1891 cur = cur + (map.m_len ? map.m_len : 1);
1899 * Check if block is in excluded regions for block allocation. The simple
1900 * allocator that runs during replay phase is calls this function to see
1901 * if it is okay to use a block.
1903 bool ext4_fc_replay_check_excluded(struct super_block *sb, ext4_fsblk_t blk)
1906 struct ext4_fc_replay_state *state;
1908 state = &EXT4_SB(sb)->s_fc_replay_state;
1909 for (i = 0; i < state->fc_regions_valid; i++) {
1910 if (state->fc_regions[i].ino == 0 ||
1911 state->fc_regions[i].len == 0)
1913 if (blk >= state->fc_regions[i].pblk &&
1914 blk < state->fc_regions[i].pblk + state->fc_regions[i].len)
1920 /* Cleanup function called after replay */
1921 void ext4_fc_replay_cleanup(struct super_block *sb)
1923 struct ext4_sb_info *sbi = EXT4_SB(sb);
1925 sbi->s_mount_state &= ~EXT4_FC_REPLAY;
1926 kfree(sbi->s_fc_replay_state.fc_regions);
1927 kfree(sbi->s_fc_replay_state.fc_modified_inodes);
1930 static bool ext4_fc_value_len_isvalid(struct ext4_sb_info *sbi,
1934 case EXT4_FC_TAG_ADD_RANGE:
1935 return len == sizeof(struct ext4_fc_add_range);
1936 case EXT4_FC_TAG_DEL_RANGE:
1937 return len == sizeof(struct ext4_fc_del_range);
1938 case EXT4_FC_TAG_CREAT:
1939 case EXT4_FC_TAG_LINK:
1940 case EXT4_FC_TAG_UNLINK:
1941 len -= sizeof(struct ext4_fc_dentry_info);
1942 return len >= 1 && len <= EXT4_NAME_LEN;
1943 case EXT4_FC_TAG_INODE:
1944 len -= sizeof(struct ext4_fc_inode);
1945 return len >= EXT4_GOOD_OLD_INODE_SIZE &&
1946 len <= sbi->s_inode_size;
1947 case EXT4_FC_TAG_PAD:
1948 return true; /* padding can have any length */
1949 case EXT4_FC_TAG_TAIL:
1950 return len >= sizeof(struct ext4_fc_tail);
1951 case EXT4_FC_TAG_HEAD:
1952 return len == sizeof(struct ext4_fc_head);
1958 * Recovery Scan phase handler
1960 * This function is called during the scan phase and is responsible
1961 * for doing following things:
1962 * - Make sure the fast commit area has valid tags for replay
1963 * - Count number of tags that need to be replayed by the replay handler
1965 * - Create a list of excluded blocks for allocation during replay phase
1967 * This function returns JBD2_FC_REPLAY_CONTINUE to indicate that SCAN is
1968 * incomplete and JBD2 should send more blocks. It returns JBD2_FC_REPLAY_STOP
1969 * to indicate that scan has finished and JBD2 can now start replay phase.
1970 * It returns a negative error to indicate that there was an error. At the end
1971 * of a successful scan phase, sbi->s_fc_replay_state.fc_replay_num_tags is set
1972 * to indicate the number of tags that need to replayed during the replay phase.
1974 static int ext4_fc_replay_scan(journal_t *journal,
1975 struct buffer_head *bh, int off,
1978 struct super_block *sb = journal->j_private;
1979 struct ext4_sb_info *sbi = EXT4_SB(sb);
1980 struct ext4_fc_replay_state *state;
1981 int ret = JBD2_FC_REPLAY_CONTINUE;
1982 struct ext4_fc_add_range ext;
1983 struct ext4_fc_tl_mem tl;
1984 struct ext4_fc_tail tail;
1985 __u8 *start, *end, *cur, *val;
1986 struct ext4_fc_head head;
1987 struct ext4_extent *ex;
1989 state = &sbi->s_fc_replay_state;
1991 start = (u8 *)bh->b_data;
1992 end = start + journal->j_blocksize;
1994 if (state->fc_replay_expected_off == 0) {
1995 state->fc_cur_tag = 0;
1996 state->fc_replay_num_tags = 0;
1998 state->fc_regions = NULL;
1999 state->fc_regions_valid = state->fc_regions_used =
2000 state->fc_regions_size = 0;
2001 /* Check if we can stop early */
2002 if (le16_to_cpu(((struct ext4_fc_tl *)start)->fc_tag)
2003 != EXT4_FC_TAG_HEAD)
2007 if (off != state->fc_replay_expected_off) {
2008 ret = -EFSCORRUPTED;
2012 state->fc_replay_expected_off++;
2013 for (cur = start; cur <= end - EXT4_FC_TAG_BASE_LEN;
2014 cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) {
2015 ext4_fc_get_tl(&tl, cur);
2016 val = cur + EXT4_FC_TAG_BASE_LEN;
2017 if (tl.fc_len > end - val ||
2018 !ext4_fc_value_len_isvalid(sbi, tl.fc_tag, tl.fc_len)) {
2019 ret = state->fc_replay_num_tags ?
2020 JBD2_FC_REPLAY_STOP : -ECANCELED;
2023 ext4_debug("Scan phase, tag:%s, blk %lld\n",
2024 tag2str(tl.fc_tag), bh->b_blocknr);
2025 switch (tl.fc_tag) {
2026 case EXT4_FC_TAG_ADD_RANGE:
2027 memcpy(&ext, val, sizeof(ext));
2028 ex = (struct ext4_extent *)&ext.fc_ex;
2029 ret = ext4_fc_record_regions(sb,
2030 le32_to_cpu(ext.fc_ino),
2031 le32_to_cpu(ex->ee_block), ext4_ext_pblock(ex),
2032 ext4_ext_get_actual_len(ex), 0);
2035 ret = JBD2_FC_REPLAY_CONTINUE;
2037 case EXT4_FC_TAG_DEL_RANGE:
2038 case EXT4_FC_TAG_LINK:
2039 case EXT4_FC_TAG_UNLINK:
2040 case EXT4_FC_TAG_CREAT:
2041 case EXT4_FC_TAG_INODE:
2042 case EXT4_FC_TAG_PAD:
2043 state->fc_cur_tag++;
2044 state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur,
2045 EXT4_FC_TAG_BASE_LEN + tl.fc_len);
2047 case EXT4_FC_TAG_TAIL:
2048 state->fc_cur_tag++;
2049 memcpy(&tail, val, sizeof(tail));
2050 state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur,
2051 EXT4_FC_TAG_BASE_LEN +
2052 offsetof(struct ext4_fc_tail,
2054 if (le32_to_cpu(tail.fc_tid) == expected_tid &&
2055 le32_to_cpu(tail.fc_crc) == state->fc_crc) {
2056 state->fc_replay_num_tags = state->fc_cur_tag;
2057 state->fc_regions_valid =
2058 state->fc_regions_used;
2060 ret = state->fc_replay_num_tags ?
2061 JBD2_FC_REPLAY_STOP : -EFSBADCRC;
2065 case EXT4_FC_TAG_HEAD:
2066 memcpy(&head, val, sizeof(head));
2067 if (le32_to_cpu(head.fc_features) &
2068 ~EXT4_FC_SUPPORTED_FEATURES) {
2072 if (le32_to_cpu(head.fc_tid) != expected_tid) {
2073 ret = JBD2_FC_REPLAY_STOP;
2076 state->fc_cur_tag++;
2077 state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur,
2078 EXT4_FC_TAG_BASE_LEN + tl.fc_len);
2081 ret = state->fc_replay_num_tags ?
2082 JBD2_FC_REPLAY_STOP : -ECANCELED;
2084 if (ret < 0 || ret == JBD2_FC_REPLAY_STOP)
2089 trace_ext4_fc_replay_scan(sb, ret, off);
2094 * Main recovery path entry point.
2095 * The meaning of return codes is similar as above.
2097 static int ext4_fc_replay(journal_t *journal, struct buffer_head *bh,
2098 enum passtype pass, int off, tid_t expected_tid)
2100 struct super_block *sb = journal->j_private;
2101 struct ext4_sb_info *sbi = EXT4_SB(sb);
2102 struct ext4_fc_tl_mem tl;
2103 __u8 *start, *end, *cur, *val;
2104 int ret = JBD2_FC_REPLAY_CONTINUE;
2105 struct ext4_fc_replay_state *state = &sbi->s_fc_replay_state;
2106 struct ext4_fc_tail tail;
2108 if (pass == PASS_SCAN) {
2109 state->fc_current_pass = PASS_SCAN;
2110 return ext4_fc_replay_scan(journal, bh, off, expected_tid);
2113 if (state->fc_current_pass != pass) {
2114 state->fc_current_pass = pass;
2115 sbi->s_mount_state |= EXT4_FC_REPLAY;
2117 if (!sbi->s_fc_replay_state.fc_replay_num_tags) {
2118 ext4_debug("Replay stops\n");
2119 ext4_fc_set_bitmaps_and_counters(sb);
2123 #ifdef CONFIG_EXT4_DEBUG
2124 if (sbi->s_fc_debug_max_replay && off >= sbi->s_fc_debug_max_replay) {
2125 pr_warn("Dropping fc block %d because max_replay set\n", off);
2126 return JBD2_FC_REPLAY_STOP;
2130 start = (u8 *)bh->b_data;
2131 end = start + journal->j_blocksize;
2133 for (cur = start; cur <= end - EXT4_FC_TAG_BASE_LEN;
2134 cur = cur + EXT4_FC_TAG_BASE_LEN + tl.fc_len) {
2135 ext4_fc_get_tl(&tl, cur);
2136 val = cur + EXT4_FC_TAG_BASE_LEN;
2138 if (state->fc_replay_num_tags == 0) {
2139 ret = JBD2_FC_REPLAY_STOP;
2140 ext4_fc_set_bitmaps_and_counters(sb);
2144 ext4_debug("Replay phase, tag:%s\n", tag2str(tl.fc_tag));
2145 state->fc_replay_num_tags--;
2146 switch (tl.fc_tag) {
2147 case EXT4_FC_TAG_LINK:
2148 ret = ext4_fc_replay_link(sb, &tl, val);
2150 case EXT4_FC_TAG_UNLINK:
2151 ret = ext4_fc_replay_unlink(sb, &tl, val);
2153 case EXT4_FC_TAG_ADD_RANGE:
2154 ret = ext4_fc_replay_add_range(sb, &tl, val);
2156 case EXT4_FC_TAG_CREAT:
2157 ret = ext4_fc_replay_create(sb, &tl, val);
2159 case EXT4_FC_TAG_DEL_RANGE:
2160 ret = ext4_fc_replay_del_range(sb, &tl, val);
2162 case EXT4_FC_TAG_INODE:
2163 ret = ext4_fc_replay_inode(sb, &tl, val);
2165 case EXT4_FC_TAG_PAD:
2166 trace_ext4_fc_replay(sb, EXT4_FC_TAG_PAD, 0,
2169 case EXT4_FC_TAG_TAIL:
2170 trace_ext4_fc_replay(sb, EXT4_FC_TAG_TAIL,
2172 memcpy(&tail, val, sizeof(tail));
2173 WARN_ON(le32_to_cpu(tail.fc_tid) != expected_tid);
2175 case EXT4_FC_TAG_HEAD:
2178 trace_ext4_fc_replay(sb, tl.fc_tag, 0, tl.fc_len, 0);
2184 ret = JBD2_FC_REPLAY_CONTINUE;
2189 void ext4_fc_init(struct super_block *sb, journal_t *journal)
2192 * We set replay callback even if fast commit disabled because we may
2193 * could still have fast commit blocks that need to be replayed even if
2194 * fast commit has now been turned off.
2196 journal->j_fc_replay_callback = ext4_fc_replay;
2197 if (!test_opt2(sb, JOURNAL_FAST_COMMIT))
2199 journal->j_fc_cleanup_callback = ext4_fc_cleanup;
2202 static const char * const fc_ineligible_reasons[] = {
2203 [EXT4_FC_REASON_XATTR] = "Extended attributes changed",
2204 [EXT4_FC_REASON_CROSS_RENAME] = "Cross rename",
2205 [EXT4_FC_REASON_JOURNAL_FLAG_CHANGE] = "Journal flag changed",
2206 [EXT4_FC_REASON_NOMEM] = "Insufficient memory",
2207 [EXT4_FC_REASON_SWAP_BOOT] = "Swap boot",
2208 [EXT4_FC_REASON_RESIZE] = "Resize",
2209 [EXT4_FC_REASON_RENAME_DIR] = "Dir renamed",
2210 [EXT4_FC_REASON_FALLOC_RANGE] = "Falloc range op",
2211 [EXT4_FC_REASON_INODE_JOURNAL_DATA] = "Data journalling",
2212 [EXT4_FC_REASON_ENCRYPTED_FILENAME] = "Encrypted filename",
2215 int ext4_fc_info_show(struct seq_file *seq, void *v)
2217 struct ext4_sb_info *sbi = EXT4_SB((struct super_block *)seq->private);
2218 struct ext4_fc_stats *stats = &sbi->s_fc_stats;
2221 if (v != SEQ_START_TOKEN)
2225 "fc stats:\n%ld commits\n%ld ineligible\n%ld numblks\n%lluus avg_commit_time\n",
2226 stats->fc_num_commits, stats->fc_ineligible_commits,
2228 div_u64(stats->s_fc_avg_commit_time, 1000));
2229 seq_puts(seq, "Ineligible reasons:\n");
2230 for (i = 0; i < EXT4_FC_REASON_MAX; i++)
2231 seq_printf(seq, "\"%s\":\t%d\n", fc_ineligible_reasons[i],
2232 stats->fc_ineligible_reason_count[i]);
2237 int __init ext4_fc_init_dentry_cache(void)
2239 ext4_fc_dentry_cachep = KMEM_CACHE(ext4_fc_dentry_update,
2240 SLAB_RECLAIM_ACCOUNT);
2242 if (ext4_fc_dentry_cachep == NULL)
2248 void ext4_fc_destroy_dentry_cache(void)
2250 kmem_cache_destroy(ext4_fc_dentry_cachep);