1 // SPDX-License-Identifier: GPL-2.0
4 * fs/ext4/fast_commit.c
6 * Written by Harshad Shirwadkar <harshadshirwadkar@gmail.com>
8 * Ext4 fast commits routines.
11 #include "ext4_jbd2.h"
12 #include "ext4_extents.h"
19 * Ext4 fast commits implement fine grained journalling for Ext4.
21 * Fast commits are organized as a log of tag-length-value (TLV) structs. (See
22 * struct ext4_fc_tl). Each TLV contains some delta that is replayed TLV by
23 * TLV during the recovery phase. For the scenarios for which we currently
24 * don't have replay code, fast commit falls back to full commits.
25 * Fast commits record delta in one of the following three categories.
27 * (A) Directory entry updates:
29 * - EXT4_FC_TAG_UNLINK - records directory entry unlink
30 * - EXT4_FC_TAG_LINK - records directory entry link
31 * - EXT4_FC_TAG_CREAT - records inode and directory entry creation
33 * (B) File specific data range updates:
35 * - EXT4_FC_TAG_ADD_RANGE - records addition of new blocks to an inode
36 * - EXT4_FC_TAG_DEL_RANGE - records deletion of blocks from an inode
38 * (C) Inode metadata (mtime / ctime etc):
40 * - EXT4_FC_TAG_INODE - record the inode that should be replayed
41 * during recovery. Note that iblocks field is
42 * not replayed and instead derived during
46 * With fast commits, we maintain all the directory entry operations in the
47 * order in which they are issued in an in-memory queue. This queue is flushed
48 * to disk during the commit operation. We also maintain a list of inodes
49 * that need to be committed during a fast commit in another in memory queue of
50 * inodes. During the commit operation, we commit in the following order:
52 * [1] Lock inodes for any further data updates by setting COMMITTING state
53 * [2] Submit data buffers of all the inodes
54 * [3] Wait for [2] to complete
55 * [4] Commit all the directory entry updates in the fast commit space
56 * [5] Commit all the changed inode structures
57 * [6] Write tail tag (this tag ensures the atomicity, please read the following
58 * section for more details).
59 * [7] Wait for [4], [5] and [6] to complete.
61 * All the inode updates must call ext4_fc_start_update() before starting an
62 * update. If such an ongoing update is present, fast commit waits for it to
63 * complete. The completion of such an update is marked by
64 * ext4_fc_stop_update().
66 * Fast Commit Ineligibility
67 * -------------------------
68 * Not all operations are supported by fast commits today (e.g extended
69 * attributes). Fast commit ineligibility is marked by calling one of the
70 * two following functions:
72 * - ext4_fc_mark_ineligible(): This makes next fast commit operation to fall
73 * back to full commit. This is useful in case of transient errors.
75 * - ext4_fc_start_ineligible() and ext4_fc_stop_ineligible() - This makes all
76 * the fast commits happening between ext4_fc_start_ineligible() and
77 * ext4_fc_stop_ineligible() and one fast commit after the call to
78 * ext4_fc_stop_ineligible() to fall back to full commits. It is important to
79 * make one more fast commit to fall back to full commit after stop call so
80 * that it guaranteed that the fast commit ineligible operation contained
81 * within ext4_fc_start_ineligible() and ext4_fc_stop_ineligible() is
82 * followed by at least 1 full commit.
84 * Atomicity of commits
85 * --------------------
86 * In order to guarantee atomicity during the commit operation, fast commit
87 * uses "EXT4_FC_TAG_TAIL" tag that marks a fast commit as complete. Tail
88 * tag contains CRC of the contents and TID of the transaction after which
89 * this fast commit should be applied. Recovery code replays fast commit
90 * logs only if there's at least 1 valid tail present. For every fast commit
91 * operation, there is 1 tail. This means, we may end up with multiple tails
92 * in the fast commit space. Here's an example:
94 * - Create a new file A and remove existing file B
96 * - Append contents to file A
100 * The fast commit space at the end of above operations would look like this:
101 * [HEAD] [CREAT A] [UNLINK B] [TAIL] [ADD_RANGE A] [DEL_RANGE A] [TAIL]
102 * |<--- Fast Commit 1 --->|<--- Fast Commit 2 ---->|
104 * Replay code should thus check for all the valid tails in the FC area.
108 * 1) Make fast commit atomic updates more fine grained. Today, a fast commit
109 * eligible update must be protected within ext4_fc_start_update() and
110 * ext4_fc_stop_update(). These routines are called at much higher
111 * routines. This can be made more fine grained by combining with
112 * ext4_journal_start().
114 * 2) Same above for ext4_fc_start_ineligible() and ext4_fc_stop_ineligible()
116 * 3) Handle more ineligible cases.
119 #include <trace/events/ext4.h>
120 static struct kmem_cache *ext4_fc_dentry_cachep;
122 static void ext4_end_buffer_io_sync(struct buffer_head *bh, int uptodate)
124 BUFFER_TRACE(bh, "");
126 ext4_debug("%s: Block %lld up-to-date",
127 __func__, bh->b_blocknr);
128 set_buffer_uptodate(bh);
130 ext4_debug("%s: Block %lld not up-to-date",
131 __func__, bh->b_blocknr);
132 clear_buffer_uptodate(bh);
138 static inline void ext4_fc_reset_inode(struct inode *inode)
140 struct ext4_inode_info *ei = EXT4_I(inode);
142 ei->i_fc_lblk_start = 0;
143 ei->i_fc_lblk_len = 0;
146 void ext4_fc_init_inode(struct inode *inode)
148 struct ext4_inode_info *ei = EXT4_I(inode);
150 ext4_fc_reset_inode(inode);
151 ext4_clear_inode_state(inode, EXT4_STATE_FC_COMMITTING);
152 INIT_LIST_HEAD(&ei->i_fc_list);
153 init_waitqueue_head(&ei->i_fc_wait);
154 atomic_set(&ei->i_fc_updates, 0);
157 /* This function must be called with sbi->s_fc_lock held. */
158 static void ext4_fc_wait_committing_inode(struct inode *inode)
159 __releases(&EXT4_SB(inode->i_sb)->s_fc_lock)
161 wait_queue_head_t *wq;
162 struct ext4_inode_info *ei = EXT4_I(inode);
164 #if (BITS_PER_LONG < 64)
165 DEFINE_WAIT_BIT(wait, &ei->i_state_flags,
166 EXT4_STATE_FC_COMMITTING);
167 wq = bit_waitqueue(&ei->i_state_flags,
168 EXT4_STATE_FC_COMMITTING);
170 DEFINE_WAIT_BIT(wait, &ei->i_flags,
171 EXT4_STATE_FC_COMMITTING);
172 wq = bit_waitqueue(&ei->i_flags,
173 EXT4_STATE_FC_COMMITTING);
175 lockdep_assert_held(&EXT4_SB(inode->i_sb)->s_fc_lock);
176 prepare_to_wait(wq, &wait.wq_entry, TASK_UNINTERRUPTIBLE);
177 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
179 finish_wait(wq, &wait.wq_entry);
183 * Inform Ext4's fast about start of an inode update
185 * This function is called by the high level call VFS callbacks before
186 * performing any inode update. This function blocks if there's an ongoing
187 * fast commit on the inode in question.
189 void ext4_fc_start_update(struct inode *inode)
191 struct ext4_inode_info *ei = EXT4_I(inode);
193 if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) ||
194 (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY))
198 spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock);
199 if (list_empty(&ei->i_fc_list))
202 if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) {
203 ext4_fc_wait_committing_inode(inode);
207 atomic_inc(&ei->i_fc_updates);
208 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
212 * Stop inode update and wake up waiting fast commits if any.
214 void ext4_fc_stop_update(struct inode *inode)
216 struct ext4_inode_info *ei = EXT4_I(inode);
218 if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) ||
219 (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY))
222 if (atomic_dec_and_test(&ei->i_fc_updates))
223 wake_up_all(&ei->i_fc_wait);
227 * Remove inode from fast commit list. If the inode is being committed
228 * we wait until inode commit is done.
230 void ext4_fc_del(struct inode *inode)
232 struct ext4_inode_info *ei = EXT4_I(inode);
234 if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) ||
235 (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY))
239 spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock);
240 if (list_empty(&ei->i_fc_list)) {
241 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
245 if (ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING)) {
246 ext4_fc_wait_committing_inode(inode);
249 list_del_init(&ei->i_fc_list);
250 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock);
254 * Mark file system as fast commit ineligible. This means that next commit
255 * operation would result in a full jbd2 commit.
257 void ext4_fc_mark_ineligible(struct super_block *sb, int reason)
259 struct ext4_sb_info *sbi = EXT4_SB(sb);
261 if (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
262 (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))
265 ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
266 WARN_ON(reason >= EXT4_FC_REASON_MAX);
267 sbi->s_fc_stats.fc_ineligible_reason_count[reason]++;
271 * Start a fast commit ineligible update. Any commits that happen while
272 * such an operation is in progress fall back to full commits.
274 void ext4_fc_start_ineligible(struct super_block *sb, int reason)
276 struct ext4_sb_info *sbi = EXT4_SB(sb);
278 if (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
279 (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))
282 WARN_ON(reason >= EXT4_FC_REASON_MAX);
283 sbi->s_fc_stats.fc_ineligible_reason_count[reason]++;
284 atomic_inc(&sbi->s_fc_ineligible_updates);
288 * Stop a fast commit ineligible update. We set EXT4_MF_FC_INELIGIBLE flag here
289 * to ensure that after stopping the ineligible update, at least one full
290 * commit takes place.
292 void ext4_fc_stop_ineligible(struct super_block *sb)
294 if (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
295 (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))
298 ext4_set_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
299 atomic_dec(&EXT4_SB(sb)->s_fc_ineligible_updates);
302 static inline int ext4_fc_is_ineligible(struct super_block *sb)
304 return (ext4_test_mount_flag(sb, EXT4_MF_FC_INELIGIBLE) ||
305 atomic_read(&EXT4_SB(sb)->s_fc_ineligible_updates));
309 * Generic fast commit tracking function. If this is the first time this we are
310 * called after a full commit, we initialize fast commit fields and then call
311 * __fc_track_fn() with update = 0. If we have already been called after a full
312 * commit, we pass update = 1. Based on that, the track function can determine
313 * if it needs to track a field for the first time or if it needs to just
314 * update the previously tracked value.
316 * If enqueue is set, this function enqueues the inode in fast commit list.
318 static int ext4_fc_track_template(
319 handle_t *handle, struct inode *inode,
320 int (*__fc_track_fn)(struct inode *, void *, bool),
321 void *args, int enqueue)
324 struct ext4_inode_info *ei = EXT4_I(inode);
325 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
329 if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) ||
330 (sbi->s_mount_state & EXT4_FC_REPLAY))
333 if (ext4_fc_is_ineligible(inode->i_sb))
336 tid = handle->h_transaction->t_tid;
337 mutex_lock(&ei->i_fc_lock);
338 if (tid == ei->i_sync_tid) {
341 ext4_fc_reset_inode(inode);
342 ei->i_sync_tid = tid;
344 ret = __fc_track_fn(inode, args, update);
345 mutex_unlock(&ei->i_fc_lock);
350 spin_lock(&sbi->s_fc_lock);
351 if (list_empty(&EXT4_I(inode)->i_fc_list))
352 list_add_tail(&EXT4_I(inode)->i_fc_list,
353 (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_COMMITTING)) ?
354 &sbi->s_fc_q[FC_Q_STAGING] :
355 &sbi->s_fc_q[FC_Q_MAIN]);
356 spin_unlock(&sbi->s_fc_lock);
361 struct __track_dentry_update_args {
362 struct dentry *dentry;
366 /* __track_fn for directory entry updates. Called with ei->i_fc_lock. */
367 static int __track_dentry_update(struct inode *inode, void *arg, bool update)
369 struct ext4_fc_dentry_update *node;
370 struct ext4_inode_info *ei = EXT4_I(inode);
371 struct __track_dentry_update_args *dentry_update =
372 (struct __track_dentry_update_args *)arg;
373 struct dentry *dentry = dentry_update->dentry;
374 struct inode *dir = dentry->d_parent->d_inode;
375 struct super_block *sb = inode->i_sb;
376 struct ext4_sb_info *sbi = EXT4_SB(sb);
378 mutex_unlock(&ei->i_fc_lock);
380 if (IS_ENCRYPTED(dir)) {
381 ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_ENCRYPTED_FILENAME);
382 mutex_lock(&ei->i_fc_lock);
386 node = kmem_cache_alloc(ext4_fc_dentry_cachep, GFP_NOFS);
388 ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_NOMEM);
389 mutex_lock(&ei->i_fc_lock);
393 node->fcd_op = dentry_update->op;
394 node->fcd_parent = dir->i_ino;
395 node->fcd_ino = inode->i_ino;
396 if (dentry->d_name.len > DNAME_INLINE_LEN) {
397 node->fcd_name.name = kmalloc(dentry->d_name.len, GFP_NOFS);
398 if (!node->fcd_name.name) {
399 kmem_cache_free(ext4_fc_dentry_cachep, node);
400 ext4_fc_mark_ineligible(sb, EXT4_FC_REASON_NOMEM);
401 mutex_lock(&ei->i_fc_lock);
404 memcpy((u8 *)node->fcd_name.name, dentry->d_name.name,
407 memcpy(node->fcd_iname, dentry->d_name.name,
409 node->fcd_name.name = node->fcd_iname;
411 node->fcd_name.len = dentry->d_name.len;
413 spin_lock(&sbi->s_fc_lock);
414 if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_COMMITTING))
415 list_add_tail(&node->fcd_list,
416 &sbi->s_fc_dentry_q[FC_Q_STAGING]);
418 list_add_tail(&node->fcd_list, &sbi->s_fc_dentry_q[FC_Q_MAIN]);
419 spin_unlock(&sbi->s_fc_lock);
420 mutex_lock(&ei->i_fc_lock);
425 void __ext4_fc_track_unlink(handle_t *handle,
426 struct inode *inode, struct dentry *dentry)
428 struct __track_dentry_update_args args;
431 args.dentry = dentry;
432 args.op = EXT4_FC_TAG_UNLINK;
434 ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
436 trace_ext4_fc_track_unlink(inode, dentry, ret);
439 void ext4_fc_track_unlink(handle_t *handle, struct dentry *dentry)
441 __ext4_fc_track_unlink(handle, d_inode(dentry), dentry);
444 void __ext4_fc_track_link(handle_t *handle,
445 struct inode *inode, struct dentry *dentry)
447 struct __track_dentry_update_args args;
450 args.dentry = dentry;
451 args.op = EXT4_FC_TAG_LINK;
453 ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
455 trace_ext4_fc_track_link(inode, dentry, ret);
458 void ext4_fc_track_link(handle_t *handle, struct dentry *dentry)
460 __ext4_fc_track_link(handle, d_inode(dentry), dentry);
463 void __ext4_fc_track_create(handle_t *handle, struct inode *inode,
464 struct dentry *dentry)
466 struct __track_dentry_update_args args;
469 args.dentry = dentry;
470 args.op = EXT4_FC_TAG_CREAT;
472 ret = ext4_fc_track_template(handle, inode, __track_dentry_update,
474 trace_ext4_fc_track_create(inode, dentry, ret);
477 void ext4_fc_track_create(handle_t *handle, struct dentry *dentry)
479 __ext4_fc_track_create(handle, d_inode(dentry), dentry);
482 /* __track_fn for inode tracking */
483 static int __track_inode(struct inode *inode, void *arg, bool update)
488 EXT4_I(inode)->i_fc_lblk_len = 0;
493 void ext4_fc_track_inode(handle_t *handle, struct inode *inode)
497 if (S_ISDIR(inode->i_mode))
500 if (ext4_should_journal_data(inode)) {
501 ext4_fc_mark_ineligible(inode->i_sb,
502 EXT4_FC_REASON_INODE_JOURNAL_DATA);
506 ret = ext4_fc_track_template(handle, inode, __track_inode, NULL, 1);
507 trace_ext4_fc_track_inode(inode, ret);
510 struct __track_range_args {
511 ext4_lblk_t start, end;
514 /* __track_fn for tracking data updates */
515 static int __track_range(struct inode *inode, void *arg, bool update)
517 struct ext4_inode_info *ei = EXT4_I(inode);
518 ext4_lblk_t oldstart;
519 struct __track_range_args *__arg =
520 (struct __track_range_args *)arg;
522 if (inode->i_ino < EXT4_FIRST_INO(inode->i_sb)) {
523 ext4_debug("Special inode %ld being modified\n", inode->i_ino);
527 oldstart = ei->i_fc_lblk_start;
529 if (update && ei->i_fc_lblk_len > 0) {
530 ei->i_fc_lblk_start = min(ei->i_fc_lblk_start, __arg->start);
532 max(oldstart + ei->i_fc_lblk_len - 1, __arg->end) -
533 ei->i_fc_lblk_start + 1;
535 ei->i_fc_lblk_start = __arg->start;
536 ei->i_fc_lblk_len = __arg->end - __arg->start + 1;
542 void ext4_fc_track_range(handle_t *handle, struct inode *inode, ext4_lblk_t start,
545 struct __track_range_args args;
548 if (S_ISDIR(inode->i_mode))
554 ret = ext4_fc_track_template(handle, inode, __track_range, &args, 1);
556 trace_ext4_fc_track_range(inode, start, end, ret);
559 static void ext4_fc_submit_bh(struct super_block *sb)
561 int write_flags = REQ_SYNC;
562 struct buffer_head *bh = EXT4_SB(sb)->s_fc_bh;
564 /* TODO: REQ_FUA | REQ_PREFLUSH is unnecessarily expensive. */
565 if (test_opt(sb, BARRIER))
566 write_flags |= REQ_FUA | REQ_PREFLUSH;
568 set_buffer_dirty(bh);
569 set_buffer_uptodate(bh);
570 bh->b_end_io = ext4_end_buffer_io_sync;
571 submit_bh(REQ_OP_WRITE, write_flags, bh);
572 EXT4_SB(sb)->s_fc_bh = NULL;
575 /* Ext4 commit path routines */
577 /* memzero and update CRC */
578 static void *ext4_fc_memzero(struct super_block *sb, void *dst, int len,
583 ret = memset(dst, 0, len);
585 *crc = ext4_chksum(EXT4_SB(sb), *crc, dst, len);
590 * Allocate len bytes on a fast commit buffer.
592 * During the commit time this function is used to manage fast commit
593 * block space. We don't split a fast commit log onto different
594 * blocks. So this function makes sure that if there's not enough space
595 * on the current block, the remaining space in the current block is
596 * marked as unused by adding EXT4_FC_TAG_PAD tag. In that case,
597 * new block is from jbd2 and CRC is updated to reflect the padding
600 static u8 *ext4_fc_reserve_space(struct super_block *sb, int len, u32 *crc)
602 struct ext4_fc_tl *tl;
603 struct ext4_sb_info *sbi = EXT4_SB(sb);
604 struct buffer_head *bh;
605 int bsize = sbi->s_journal->j_blocksize;
606 int ret, off = sbi->s_fc_bytes % bsize;
610 * After allocating len, we should have space at least for a 0 byte
613 if (len + sizeof(struct ext4_fc_tl) > bsize)
616 if (bsize - off - 1 > len + sizeof(struct ext4_fc_tl)) {
618 * Only allocate from current buffer if we have enough space for
619 * this request AND we have space to add a zero byte padding.
622 ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh);
627 sbi->s_fc_bytes += len;
628 return sbi->s_fc_bh->b_data + off;
630 /* Need to add PAD tag */
631 tl = (struct ext4_fc_tl *)(sbi->s_fc_bh->b_data + off);
632 tl->fc_tag = cpu_to_le16(EXT4_FC_TAG_PAD);
633 pad_len = bsize - off - 1 - sizeof(struct ext4_fc_tl);
634 tl->fc_len = cpu_to_le16(pad_len);
636 *crc = ext4_chksum(sbi, *crc, tl, sizeof(*tl));
638 ext4_fc_memzero(sb, tl + 1, pad_len, crc);
639 /* Don't leak uninitialized memory in the unused last byte. */
640 *((u8 *)(tl + 1) + pad_len) = 0;
642 ext4_fc_submit_bh(sb);
644 ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh);
648 sbi->s_fc_bytes = (sbi->s_fc_bytes / bsize + 1) * bsize + len;
649 return sbi->s_fc_bh->b_data;
652 /* memcpy to fc reserved space and update CRC */
653 static void *ext4_fc_memcpy(struct super_block *sb, void *dst, const void *src,
657 *crc = ext4_chksum(EXT4_SB(sb), *crc, src, len);
658 return memcpy(dst, src, len);
662 * Complete a fast commit by writing tail tag.
664 * Writing tail tag marks the end of a fast commit. In order to guarantee
665 * atomicity, after writing tail tag, even if there's space remaining
666 * in the block, next commit shouldn't use it. That's why tail tag
667 * has the length as that of the remaining space on the block.
669 static int ext4_fc_write_tail(struct super_block *sb, u32 crc)
671 struct ext4_sb_info *sbi = EXT4_SB(sb);
672 struct ext4_fc_tl tl;
673 struct ext4_fc_tail tail;
674 int off, bsize = sbi->s_journal->j_blocksize;
678 * ext4_fc_reserve_space takes care of allocating an extra block if
679 * there's no enough space on this block for accommodating this tail.
681 dst = ext4_fc_reserve_space(sb, sizeof(tl) + sizeof(tail), &crc);
685 off = sbi->s_fc_bytes % bsize;
687 tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_TAIL);
688 tl.fc_len = cpu_to_le16(bsize - off - 1 + sizeof(struct ext4_fc_tail));
689 sbi->s_fc_bytes = round_up(sbi->s_fc_bytes, bsize);
691 ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), &crc);
693 tail.fc_tid = cpu_to_le32(sbi->s_journal->j_running_transaction->t_tid);
694 ext4_fc_memcpy(sb, dst, &tail.fc_tid, sizeof(tail.fc_tid), &crc);
695 dst += sizeof(tail.fc_tid);
696 tail.fc_crc = cpu_to_le32(crc);
697 ext4_fc_memcpy(sb, dst, &tail.fc_crc, sizeof(tail.fc_crc), NULL);
698 dst += sizeof(tail.fc_crc);
699 memset(dst, 0, bsize - off); /* Don't leak uninitialized memory. */
701 ext4_fc_submit_bh(sb);
707 * Adds tag, length, value and updates CRC. Returns true if tlv was added.
708 * Returns false if there's not enough space.
710 static bool ext4_fc_add_tlv(struct super_block *sb, u16 tag, u16 len, u8 *val,
713 struct ext4_fc_tl tl;
716 dst = ext4_fc_reserve_space(sb, sizeof(tl) + len, crc);
720 tl.fc_tag = cpu_to_le16(tag);
721 tl.fc_len = cpu_to_le16(len);
723 ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), crc);
724 ext4_fc_memcpy(sb, dst + sizeof(tl), val, len, crc);
729 /* Same as above, but adds dentry tlv. */
730 static bool ext4_fc_add_dentry_tlv(struct super_block *sb, u16 tag,
731 int parent_ino, int ino, int dlen,
732 const unsigned char *dname,
735 struct ext4_fc_dentry_info fcd;
736 struct ext4_fc_tl tl;
737 u8 *dst = ext4_fc_reserve_space(sb, sizeof(tl) + sizeof(fcd) + dlen,
743 fcd.fc_parent_ino = cpu_to_le32(parent_ino);
744 fcd.fc_ino = cpu_to_le32(ino);
745 tl.fc_tag = cpu_to_le16(tag);
746 tl.fc_len = cpu_to_le16(sizeof(fcd) + dlen);
747 ext4_fc_memcpy(sb, dst, &tl, sizeof(tl), crc);
749 ext4_fc_memcpy(sb, dst, &fcd, sizeof(fcd), crc);
751 ext4_fc_memcpy(sb, dst, dname, dlen, crc);
758 * Writes inode in the fast commit space under TLV with tag @tag.
759 * Returns 0 on success, error on failure.
761 static int ext4_fc_write_inode(struct inode *inode, u32 *crc)
763 struct ext4_inode_info *ei = EXT4_I(inode);
764 int inode_len = EXT4_GOOD_OLD_INODE_SIZE;
766 struct ext4_iloc iloc;
767 struct ext4_fc_inode fc_inode;
768 struct ext4_fc_tl tl;
771 ret = ext4_get_inode_loc(inode, &iloc);
775 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE)
776 inode_len += ei->i_extra_isize;
778 fc_inode.fc_ino = cpu_to_le32(inode->i_ino);
779 tl.fc_tag = cpu_to_le16(EXT4_FC_TAG_INODE);
780 tl.fc_len = cpu_to_le16(inode_len + sizeof(fc_inode.fc_ino));
783 dst = ext4_fc_reserve_space(inode->i_sb,
784 sizeof(tl) + inode_len + sizeof(fc_inode.fc_ino), crc);
788 if (!ext4_fc_memcpy(inode->i_sb, dst, &tl, sizeof(tl), crc))
791 if (!ext4_fc_memcpy(inode->i_sb, dst, &fc_inode, sizeof(fc_inode), crc))
793 dst += sizeof(fc_inode);
794 if (!ext4_fc_memcpy(inode->i_sb, dst, (u8 *)ext4_raw_inode(&iloc),
804 * Writes updated data ranges for the inode in question. Updates CRC.
805 * Returns 0 on success, error otherwise.
807 static int ext4_fc_write_inode_data(struct inode *inode, u32 *crc)
809 ext4_lblk_t old_blk_size, cur_lblk_off, new_blk_size;
810 struct ext4_inode_info *ei = EXT4_I(inode);
811 struct ext4_map_blocks map;
812 struct ext4_fc_add_range fc_ext;
813 struct ext4_fc_del_range lrange;
814 struct ext4_extent *ex;
817 mutex_lock(&ei->i_fc_lock);
818 if (ei->i_fc_lblk_len == 0) {
819 mutex_unlock(&ei->i_fc_lock);
822 old_blk_size = ei->i_fc_lblk_start;
823 new_blk_size = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1;
824 ei->i_fc_lblk_len = 0;
825 mutex_unlock(&ei->i_fc_lock);
827 cur_lblk_off = old_blk_size;
828 jbd_debug(1, "%s: will try writing %d to %d for inode %ld\n",
829 __func__, cur_lblk_off, new_blk_size, inode->i_ino);
831 while (cur_lblk_off <= new_blk_size) {
832 map.m_lblk = cur_lblk_off;
833 map.m_len = new_blk_size - cur_lblk_off + 1;
834 ret = ext4_map_blocks(NULL, inode, &map, 0);
838 if (map.m_len == 0) {
844 lrange.fc_ino = cpu_to_le32(inode->i_ino);
845 lrange.fc_lblk = cpu_to_le32(map.m_lblk);
846 lrange.fc_len = cpu_to_le32(map.m_len);
847 if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_DEL_RANGE,
848 sizeof(lrange), (u8 *)&lrange, crc))
851 unsigned int max = (map.m_flags & EXT4_MAP_UNWRITTEN) ?
852 EXT_UNWRITTEN_MAX_LEN : EXT_INIT_MAX_LEN;
854 /* Limit the number of blocks in one extent */
855 map.m_len = min(max, map.m_len);
857 fc_ext.fc_ino = cpu_to_le32(inode->i_ino);
858 ex = (struct ext4_extent *)&fc_ext.fc_ex;
859 ex->ee_block = cpu_to_le32(map.m_lblk);
860 ex->ee_len = cpu_to_le16(map.m_len);
861 ext4_ext_store_pblock(ex, map.m_pblk);
862 if (map.m_flags & EXT4_MAP_UNWRITTEN)
863 ext4_ext_mark_unwritten(ex);
865 ext4_ext_mark_initialized(ex);
866 if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_ADD_RANGE,
867 sizeof(fc_ext), (u8 *)&fc_ext, crc))
871 cur_lblk_off += map.m_len;
878 /* Submit data for all the fast commit inodes */
879 static int ext4_fc_submit_inode_data_all(journal_t *journal)
881 struct super_block *sb = (struct super_block *)(journal->j_private);
882 struct ext4_sb_info *sbi = EXT4_SB(sb);
883 struct ext4_inode_info *ei;
884 struct list_head *pos;
887 spin_lock(&sbi->s_fc_lock);
888 ext4_set_mount_flag(sb, EXT4_MF_FC_COMMITTING);
889 list_for_each(pos, &sbi->s_fc_q[FC_Q_MAIN]) {
890 ei = list_entry(pos, struct ext4_inode_info, i_fc_list);
891 ext4_set_inode_state(&ei->vfs_inode, EXT4_STATE_FC_COMMITTING);
892 while (atomic_read(&ei->i_fc_updates)) {
895 prepare_to_wait(&ei->i_fc_wait, &wait,
896 TASK_UNINTERRUPTIBLE);
897 if (atomic_read(&ei->i_fc_updates)) {
898 spin_unlock(&sbi->s_fc_lock);
900 spin_lock(&sbi->s_fc_lock);
902 finish_wait(&ei->i_fc_wait, &wait);
904 spin_unlock(&sbi->s_fc_lock);
905 ret = jbd2_submit_inode_data(ei->jinode);
908 spin_lock(&sbi->s_fc_lock);
910 spin_unlock(&sbi->s_fc_lock);
915 /* Wait for completion of data for all the fast commit inodes */
916 static int ext4_fc_wait_inode_data_all(journal_t *journal)
918 struct super_block *sb = (struct super_block *)(journal->j_private);
919 struct ext4_sb_info *sbi = EXT4_SB(sb);
920 struct ext4_inode_info *pos, *n;
923 spin_lock(&sbi->s_fc_lock);
924 list_for_each_entry_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) {
925 if (!ext4_test_inode_state(&pos->vfs_inode,
926 EXT4_STATE_FC_COMMITTING))
928 spin_unlock(&sbi->s_fc_lock);
930 ret = jbd2_wait_inode_data(journal, pos->jinode);
933 spin_lock(&sbi->s_fc_lock);
935 spin_unlock(&sbi->s_fc_lock);
940 /* Commit all the directory entry updates */
941 static int ext4_fc_commit_dentry_updates(journal_t *journal, u32 *crc)
942 __acquires(&sbi->s_fc_lock)
943 __releases(&sbi->s_fc_lock)
945 struct super_block *sb = (struct super_block *)(journal->j_private);
946 struct ext4_sb_info *sbi = EXT4_SB(sb);
947 struct ext4_fc_dentry_update *fc_dentry;
949 struct list_head *pos, *n, *fcd_pos, *fcd_n;
950 struct ext4_inode_info *ei;
953 if (list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN]))
955 list_for_each_safe(fcd_pos, fcd_n, &sbi->s_fc_dentry_q[FC_Q_MAIN]) {
956 fc_dentry = list_entry(fcd_pos, struct ext4_fc_dentry_update,
958 if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) {
959 spin_unlock(&sbi->s_fc_lock);
960 if (!ext4_fc_add_dentry_tlv(
961 sb, fc_dentry->fcd_op,
962 fc_dentry->fcd_parent, fc_dentry->fcd_ino,
963 fc_dentry->fcd_name.len,
964 fc_dentry->fcd_name.name, crc)) {
968 spin_lock(&sbi->s_fc_lock);
973 list_for_each_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN]) {
974 ei = list_entry(pos, struct ext4_inode_info, i_fc_list);
975 if (ei->vfs_inode.i_ino == fc_dentry->fcd_ino) {
976 inode = &ei->vfs_inode;
981 * If we don't find inode in our list, then it was deleted,
982 * in which case, we don't need to record it's create tag.
986 spin_unlock(&sbi->s_fc_lock);
989 * We first write the inode and then the create dirent. This
990 * allows the recovery code to create an unnamed inode first
991 * and then link it to a directory entry. This allows us
992 * to use namei.c routines almost as is and simplifies
995 ret = ext4_fc_write_inode(inode, crc);
999 ret = ext4_fc_write_inode_data(inode, crc);
1003 if (!ext4_fc_add_dentry_tlv(
1004 sb, fc_dentry->fcd_op,
1005 fc_dentry->fcd_parent, fc_dentry->fcd_ino,
1006 fc_dentry->fcd_name.len,
1007 fc_dentry->fcd_name.name, crc)) {
1012 spin_lock(&sbi->s_fc_lock);
1016 spin_lock(&sbi->s_fc_lock);
1020 static int ext4_fc_perform_commit(journal_t *journal)
1022 struct super_block *sb = (struct super_block *)(journal->j_private);
1023 struct ext4_sb_info *sbi = EXT4_SB(sb);
1024 struct ext4_inode_info *iter;
1025 struct ext4_fc_head head;
1026 struct list_head *pos;
1027 struct inode *inode;
1028 struct blk_plug plug;
1032 ret = ext4_fc_submit_inode_data_all(journal);
1036 ret = ext4_fc_wait_inode_data_all(journal);
1041 * If file system device is different from journal device, issue a cache
1042 * flush before we start writing fast commit blocks.
1044 if (journal->j_fs_dev != journal->j_dev)
1045 blkdev_issue_flush(journal->j_fs_dev, GFP_NOFS);
1047 blk_start_plug(&plug);
1048 if (sbi->s_fc_bytes == 0) {
1050 * Add a head tag only if this is the first fast commit
1053 head.fc_features = cpu_to_le32(EXT4_FC_SUPPORTED_FEATURES);
1054 head.fc_tid = cpu_to_le32(
1055 sbi->s_journal->j_running_transaction->t_tid);
1056 if (!ext4_fc_add_tlv(sb, EXT4_FC_TAG_HEAD, sizeof(head),
1057 (u8 *)&head, &crc)) {
1063 spin_lock(&sbi->s_fc_lock);
1064 ret = ext4_fc_commit_dentry_updates(journal, &crc);
1066 spin_unlock(&sbi->s_fc_lock);
1070 list_for_each(pos, &sbi->s_fc_q[FC_Q_MAIN]) {
1071 iter = list_entry(pos, struct ext4_inode_info, i_fc_list);
1072 inode = &iter->vfs_inode;
1073 if (!ext4_test_inode_state(inode, EXT4_STATE_FC_COMMITTING))
1076 spin_unlock(&sbi->s_fc_lock);
1077 ret = ext4_fc_write_inode_data(inode, &crc);
1080 ret = ext4_fc_write_inode(inode, &crc);
1083 spin_lock(&sbi->s_fc_lock);
1085 spin_unlock(&sbi->s_fc_lock);
1087 ret = ext4_fc_write_tail(sb, crc);
1090 blk_finish_plug(&plug);
1095 * The main commit entry point. Performs a fast commit for transaction
1096 * commit_tid if needed. If it's not possible to perform a fast commit
1097 * due to various reasons, we fall back to full commit. Returns 0
1098 * on success, error otherwise.
1100 int ext4_fc_commit(journal_t *journal, tid_t commit_tid)
1102 struct super_block *sb = (struct super_block *)(journal->j_private);
1103 struct ext4_sb_info *sbi = EXT4_SB(sb);
1104 int nblks = 0, ret, bsize = journal->j_blocksize;
1105 int subtid = atomic_read(&sbi->s_fc_subtid);
1106 int reason = EXT4_FC_REASON_OK, fc_bufs_before = 0;
1107 ktime_t start_time, commit_time;
1109 trace_ext4_fc_commit_start(sb);
1111 start_time = ktime_get();
1113 if (!test_opt2(sb, JOURNAL_FAST_COMMIT) ||
1114 (ext4_fc_is_ineligible(sb))) {
1115 reason = EXT4_FC_REASON_INELIGIBLE;
1120 ret = jbd2_fc_begin_commit(journal, commit_tid);
1121 if (ret == -EALREADY) {
1122 /* There was an ongoing commit, check if we need to restart */
1123 if (atomic_read(&sbi->s_fc_subtid) <= subtid &&
1124 commit_tid > journal->j_commit_sequence)
1126 reason = EXT4_FC_REASON_ALREADY_COMMITTED;
1129 sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++;
1130 reason = EXT4_FC_REASON_FC_START_FAILED;
1134 fc_bufs_before = (sbi->s_fc_bytes + bsize - 1) / bsize;
1135 ret = ext4_fc_perform_commit(journal);
1137 sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++;
1138 reason = EXT4_FC_REASON_FC_FAILED;
1141 nblks = (sbi->s_fc_bytes + bsize - 1) / bsize - fc_bufs_before;
1142 ret = jbd2_fc_wait_bufs(journal, nblks);
1144 sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++;
1145 reason = EXT4_FC_REASON_FC_FAILED;
1148 atomic_inc(&sbi->s_fc_subtid);
1149 jbd2_fc_end_commit(journal);
1151 /* Has any ineligible update happened since we started? */
1152 if (reason == EXT4_FC_REASON_OK && ext4_fc_is_ineligible(sb)) {
1153 sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++;
1154 reason = EXT4_FC_REASON_INELIGIBLE;
1157 spin_lock(&sbi->s_fc_lock);
1158 if (reason != EXT4_FC_REASON_OK &&
1159 reason != EXT4_FC_REASON_ALREADY_COMMITTED) {
1160 sbi->s_fc_stats.fc_ineligible_commits++;
1162 sbi->s_fc_stats.fc_num_commits++;
1163 sbi->s_fc_stats.fc_numblks += nblks;
1165 spin_unlock(&sbi->s_fc_lock);
1166 nblks = (reason == EXT4_FC_REASON_OK) ? nblks : 0;
1167 trace_ext4_fc_commit_stop(sb, nblks, reason);
1168 commit_time = ktime_to_ns(ktime_sub(ktime_get(), start_time));
1170 * weight the commit time higher than the average time so we don't
1171 * react too strongly to vast changes in the commit time
1173 if (likely(sbi->s_fc_avg_commit_time))
1174 sbi->s_fc_avg_commit_time = (commit_time +
1175 sbi->s_fc_avg_commit_time * 3) / 4;
1177 sbi->s_fc_avg_commit_time = commit_time;
1179 "Fast commit ended with blks = %d, reason = %d, subtid - %d",
1180 nblks, reason, subtid);
1181 if (reason == EXT4_FC_REASON_FC_FAILED)
1182 return jbd2_fc_end_commit_fallback(journal);
1183 if (reason == EXT4_FC_REASON_FC_START_FAILED ||
1184 reason == EXT4_FC_REASON_INELIGIBLE)
1185 return jbd2_complete_transaction(journal, commit_tid);
1190 * Fast commit cleanup routine. This is called after every fast commit and
1191 * full commit. full is true if we are called after a full commit.
1193 static void ext4_fc_cleanup(journal_t *journal, int full)
1195 struct super_block *sb = journal->j_private;
1196 struct ext4_sb_info *sbi = EXT4_SB(sb);
1197 struct ext4_inode_info *iter;
1198 struct ext4_fc_dentry_update *fc_dentry;
1199 struct list_head *pos, *n;
1201 if (full && sbi->s_fc_bh)
1202 sbi->s_fc_bh = NULL;
1204 jbd2_fc_release_bufs(journal);
1206 spin_lock(&sbi->s_fc_lock);
1207 list_for_each_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN]) {
1208 iter = list_entry(pos, struct ext4_inode_info, i_fc_list);
1209 list_del_init(&iter->i_fc_list);
1210 ext4_clear_inode_state(&iter->vfs_inode,
1211 EXT4_STATE_FC_COMMITTING);
1212 ext4_fc_reset_inode(&iter->vfs_inode);
1213 /* Make sure EXT4_STATE_FC_COMMITTING bit is clear */
1215 #if (BITS_PER_LONG < 64)
1216 wake_up_bit(&iter->i_state_flags, EXT4_STATE_FC_COMMITTING);
1218 wake_up_bit(&iter->i_flags, EXT4_STATE_FC_COMMITTING);
1222 while (!list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) {
1223 fc_dentry = list_first_entry(&sbi->s_fc_dentry_q[FC_Q_MAIN],
1224 struct ext4_fc_dentry_update,
1226 list_del_init(&fc_dentry->fcd_list);
1227 spin_unlock(&sbi->s_fc_lock);
1229 if (fc_dentry->fcd_name.name &&
1230 fc_dentry->fcd_name.len > DNAME_INLINE_LEN)
1231 kfree(fc_dentry->fcd_name.name);
1232 kmem_cache_free(ext4_fc_dentry_cachep, fc_dentry);
1233 spin_lock(&sbi->s_fc_lock);
1236 list_splice_init(&sbi->s_fc_dentry_q[FC_Q_STAGING],
1237 &sbi->s_fc_dentry_q[FC_Q_MAIN]);
1238 list_splice_init(&sbi->s_fc_q[FC_Q_STAGING],
1239 &sbi->s_fc_q[FC_Q_MAIN]);
1241 ext4_clear_mount_flag(sb, EXT4_MF_FC_COMMITTING);
1242 ext4_clear_mount_flag(sb, EXT4_MF_FC_INELIGIBLE);
1245 sbi->s_fc_bytes = 0;
1246 spin_unlock(&sbi->s_fc_lock);
1247 trace_ext4_fc_stats(sb);
1250 /* Ext4 Replay Path Routines */
1252 /* Helper struct for dentry replay routines */
1253 struct dentry_info_args {
1254 int parent_ino, dname_len, ino, inode_len;
1258 static inline void tl_to_darg(struct dentry_info_args *darg,
1259 struct ext4_fc_tl *tl, u8 *val)
1261 struct ext4_fc_dentry_info fcd;
1263 memcpy(&fcd, val, sizeof(fcd));
1265 darg->parent_ino = le32_to_cpu(fcd.fc_parent_ino);
1266 darg->ino = le32_to_cpu(fcd.fc_ino);
1267 darg->dname = val + offsetof(struct ext4_fc_dentry_info, fc_dname);
1268 darg->dname_len = le16_to_cpu(tl->fc_len) -
1269 sizeof(struct ext4_fc_dentry_info);
1272 /* Unlink replay function */
1273 static int ext4_fc_replay_unlink(struct super_block *sb, struct ext4_fc_tl *tl,
1276 struct inode *inode, *old_parent;
1278 struct dentry_info_args darg;
1281 tl_to_darg(&darg, tl, val);
1283 trace_ext4_fc_replay(sb, EXT4_FC_TAG_UNLINK, darg.ino,
1284 darg.parent_ino, darg.dname_len);
1286 entry.name = darg.dname;
1287 entry.len = darg.dname_len;
1288 inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
1290 if (IS_ERR(inode)) {
1291 jbd_debug(1, "Inode %d not found", darg.ino);
1295 old_parent = ext4_iget(sb, darg.parent_ino,
1297 if (IS_ERR(old_parent)) {
1298 jbd_debug(1, "Dir with inode %d not found", darg.parent_ino);
1303 ret = __ext4_unlink(old_parent, &entry, inode, NULL);
1304 /* -ENOENT ok coz it might not exist anymore. */
1312 static int ext4_fc_replay_link_internal(struct super_block *sb,
1313 struct dentry_info_args *darg,
1314 struct inode *inode)
1316 struct inode *dir = NULL;
1317 struct dentry *dentry_dir = NULL, *dentry_inode = NULL;
1318 struct qstr qstr_dname = QSTR_INIT(darg->dname, darg->dname_len);
1321 dir = ext4_iget(sb, darg->parent_ino, EXT4_IGET_NORMAL);
1323 jbd_debug(1, "Dir with inode %d not found.", darg->parent_ino);
1328 dentry_dir = d_obtain_alias(dir);
1329 if (IS_ERR(dentry_dir)) {
1330 jbd_debug(1, "Failed to obtain dentry");
1335 dentry_inode = d_alloc(dentry_dir, &qstr_dname);
1336 if (!dentry_inode) {
1337 jbd_debug(1, "Inode dentry not created.");
1342 ret = __ext4_link(dir, inode, dentry_inode);
1344 * It's possible that link already existed since data blocks
1345 * for the dir in question got persisted before we crashed OR
1346 * we replayed this tag and crashed before the entire replay
1349 if (ret && ret != -EEXIST) {
1350 jbd_debug(1, "Failed to link\n");
1363 d_drop(dentry_inode);
1370 /* Link replay function */
1371 static int ext4_fc_replay_link(struct super_block *sb, struct ext4_fc_tl *tl,
1374 struct inode *inode;
1375 struct dentry_info_args darg;
1378 tl_to_darg(&darg, tl, val);
1379 trace_ext4_fc_replay(sb, EXT4_FC_TAG_LINK, darg.ino,
1380 darg.parent_ino, darg.dname_len);
1382 inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
1383 if (IS_ERR(inode)) {
1384 jbd_debug(1, "Inode not found.");
1388 ret = ext4_fc_replay_link_internal(sb, &darg, inode);
1394 * Record all the modified inodes during replay. We use this later to setup
1395 * block bitmaps correctly.
1397 static int ext4_fc_record_modified_inode(struct super_block *sb, int ino)
1399 struct ext4_fc_replay_state *state;
1402 state = &EXT4_SB(sb)->s_fc_replay_state;
1403 for (i = 0; i < state->fc_modified_inodes_used; i++)
1404 if (state->fc_modified_inodes[i] == ino)
1406 if (state->fc_modified_inodes_used == state->fc_modified_inodes_size) {
1407 int *fc_modified_inodes;
1409 fc_modified_inodes = krealloc(state->fc_modified_inodes,
1410 sizeof(int) * (state->fc_modified_inodes_size +
1411 EXT4_FC_REPLAY_REALLOC_INCREMENT),
1413 if (!fc_modified_inodes)
1415 state->fc_modified_inodes = fc_modified_inodes;
1416 state->fc_modified_inodes_size +=
1417 EXT4_FC_REPLAY_REALLOC_INCREMENT;
1419 state->fc_modified_inodes[state->fc_modified_inodes_used++] = ino;
1424 * Inode replay function
1426 static int ext4_fc_replay_inode(struct super_block *sb, struct ext4_fc_tl *tl,
1429 struct ext4_fc_inode fc_inode;
1430 struct ext4_inode *raw_inode;
1431 struct ext4_inode *raw_fc_inode;
1432 struct inode *inode = NULL;
1433 struct ext4_iloc iloc;
1434 int inode_len, ino, ret, tag = le16_to_cpu(tl->fc_tag);
1435 struct ext4_extent_header *eh;
1437 memcpy(&fc_inode, val, sizeof(fc_inode));
1439 ino = le32_to_cpu(fc_inode.fc_ino);
1440 trace_ext4_fc_replay(sb, tag, ino, 0, 0);
1442 inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL);
1443 if (!IS_ERR(inode)) {
1444 ext4_ext_clear_bb(inode);
1449 ret = ext4_fc_record_modified_inode(sb, ino);
1453 raw_fc_inode = (struct ext4_inode *)
1454 (val + offsetof(struct ext4_fc_inode, fc_raw_inode));
1455 ret = ext4_get_fc_inode_loc(sb, ino, &iloc);
1459 inode_len = le16_to_cpu(tl->fc_len) - sizeof(struct ext4_fc_inode);
1460 raw_inode = ext4_raw_inode(&iloc);
1462 memcpy(raw_inode, raw_fc_inode, offsetof(struct ext4_inode, i_block));
1463 memcpy(&raw_inode->i_generation, &raw_fc_inode->i_generation,
1464 inode_len - offsetof(struct ext4_inode, i_generation));
1465 if (le32_to_cpu(raw_inode->i_flags) & EXT4_EXTENTS_FL) {
1466 eh = (struct ext4_extent_header *)(&raw_inode->i_block[0]);
1467 if (eh->eh_magic != EXT4_EXT_MAGIC) {
1468 memset(eh, 0, sizeof(*eh));
1469 eh->eh_magic = EXT4_EXT_MAGIC;
1470 eh->eh_max = cpu_to_le16(
1471 (sizeof(raw_inode->i_block) -
1472 sizeof(struct ext4_extent_header))
1473 / sizeof(struct ext4_extent));
1475 } else if (le32_to_cpu(raw_inode->i_flags) & EXT4_INLINE_DATA_FL) {
1476 memcpy(raw_inode->i_block, raw_fc_inode->i_block,
1477 sizeof(raw_inode->i_block));
1480 /* Immediately update the inode on disk. */
1481 ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh);
1484 ret = sync_dirty_buffer(iloc.bh);
1487 ret = ext4_mark_inode_used(sb, ino);
1491 /* Given that we just wrote the inode on disk, this SHOULD succeed. */
1492 inode = ext4_iget(sb, ino, EXT4_IGET_NORMAL);
1493 if (IS_ERR(inode)) {
1494 jbd_debug(1, "Inode not found.");
1495 return -EFSCORRUPTED;
1499 * Our allocator could have made different decisions than before
1500 * crashing. This should be fixed but until then, we calculate
1501 * the number of blocks the inode.
1503 ext4_ext_replay_set_iblocks(inode);
1505 inode->i_generation = le32_to_cpu(ext4_raw_inode(&iloc)->i_generation);
1506 ext4_reset_inode_seed(inode);
1508 ext4_inode_csum_set(inode, ext4_raw_inode(&iloc), EXT4_I(inode));
1509 ret = ext4_handle_dirty_metadata(NULL, NULL, iloc.bh);
1510 sync_dirty_buffer(iloc.bh);
1515 blkdev_issue_flush(sb->s_bdev, GFP_KERNEL);
1521 * Dentry create replay function.
1523 * EXT4_FC_TAG_CREAT is preceded by EXT4_FC_TAG_INODE_FULL. Which means, the
1524 * inode for which we are trying to create a dentry here, should already have
1525 * been replayed before we start here.
1527 static int ext4_fc_replay_create(struct super_block *sb, struct ext4_fc_tl *tl,
1531 struct inode *inode = NULL;
1532 struct inode *dir = NULL;
1533 struct dentry_info_args darg;
1535 tl_to_darg(&darg, tl, val);
1537 trace_ext4_fc_replay(sb, EXT4_FC_TAG_CREAT, darg.ino,
1538 darg.parent_ino, darg.dname_len);
1540 /* This takes care of update group descriptor and other metadata */
1541 ret = ext4_mark_inode_used(sb, darg.ino);
1545 inode = ext4_iget(sb, darg.ino, EXT4_IGET_NORMAL);
1546 if (IS_ERR(inode)) {
1547 jbd_debug(1, "inode %d not found.", darg.ino);
1553 if (S_ISDIR(inode->i_mode)) {
1555 * If we are creating a directory, we need to make sure that the
1556 * dot and dot dot dirents are setup properly.
1558 dir = ext4_iget(sb, darg.parent_ino, EXT4_IGET_NORMAL);
1560 jbd_debug(1, "Dir %d not found.", darg.ino);
1563 ret = ext4_init_new_dir(NULL, dir, inode);
1570 ret = ext4_fc_replay_link_internal(sb, &darg, inode);
1573 set_nlink(inode, 1);
1574 ext4_mark_inode_dirty(NULL, inode);
1582 * Record physical disk regions which are in use as per fast commit area,
1583 * and used by inodes during replay phase. Our simple replay phase
1584 * allocator excludes these regions from allocation.
1586 int ext4_fc_record_regions(struct super_block *sb, int ino,
1587 ext4_lblk_t lblk, ext4_fsblk_t pblk, int len, int replay)
1589 struct ext4_fc_replay_state *state;
1590 struct ext4_fc_alloc_region *region;
1592 state = &EXT4_SB(sb)->s_fc_replay_state;
1594 * during replay phase, the fc_regions_valid may not same as
1595 * fc_regions_used, update it when do new additions.
1597 if (replay && state->fc_regions_used != state->fc_regions_valid)
1598 state->fc_regions_used = state->fc_regions_valid;
1599 if (state->fc_regions_used == state->fc_regions_size) {
1600 struct ext4_fc_alloc_region *fc_regions;
1602 fc_regions = krealloc(state->fc_regions,
1603 sizeof(struct ext4_fc_alloc_region) *
1604 (state->fc_regions_size +
1605 EXT4_FC_REPLAY_REALLOC_INCREMENT),
1609 state->fc_regions_size +=
1610 EXT4_FC_REPLAY_REALLOC_INCREMENT;
1611 state->fc_regions = fc_regions;
1613 region = &state->fc_regions[state->fc_regions_used++];
1615 region->lblk = lblk;
1616 region->pblk = pblk;
1620 state->fc_regions_valid++;
1625 /* Replay add range tag */
1626 static int ext4_fc_replay_add_range(struct super_block *sb,
1627 struct ext4_fc_tl *tl, u8 *val)
1629 struct ext4_fc_add_range fc_add_ex;
1630 struct ext4_extent newex, *ex;
1631 struct inode *inode;
1632 ext4_lblk_t start, cur;
1634 ext4_fsblk_t start_pblk;
1635 struct ext4_map_blocks map;
1636 struct ext4_ext_path *path = NULL;
1639 memcpy(&fc_add_ex, val, sizeof(fc_add_ex));
1640 ex = (struct ext4_extent *)&fc_add_ex.fc_ex;
1642 trace_ext4_fc_replay(sb, EXT4_FC_TAG_ADD_RANGE,
1643 le32_to_cpu(fc_add_ex.fc_ino), le32_to_cpu(ex->ee_block),
1644 ext4_ext_get_actual_len(ex));
1646 inode = ext4_iget(sb, le32_to_cpu(fc_add_ex.fc_ino), EXT4_IGET_NORMAL);
1647 if (IS_ERR(inode)) {
1648 jbd_debug(1, "Inode not found.");
1652 ret = ext4_fc_record_modified_inode(sb, inode->i_ino);
1656 start = le32_to_cpu(ex->ee_block);
1657 start_pblk = ext4_ext_pblock(ex);
1658 len = ext4_ext_get_actual_len(ex);
1662 jbd_debug(1, "ADD_RANGE, lblk %d, pblk %lld, len %d, unwritten %d, inode %ld\n",
1663 start, start_pblk, len, ext4_ext_is_unwritten(ex),
1666 while (remaining > 0) {
1668 map.m_len = remaining;
1670 ret = ext4_map_blocks(NULL, inode, &map, 0);
1676 /* Range is not mapped */
1677 path = ext4_find_extent(inode, cur, NULL, 0);
1680 memset(&newex, 0, sizeof(newex));
1681 newex.ee_block = cpu_to_le32(cur);
1682 ext4_ext_store_pblock(
1683 &newex, start_pblk + cur - start);
1684 newex.ee_len = cpu_to_le16(map.m_len);
1685 if (ext4_ext_is_unwritten(ex))
1686 ext4_ext_mark_unwritten(&newex);
1687 down_write(&EXT4_I(inode)->i_data_sem);
1688 ret = ext4_ext_insert_extent(
1689 NULL, inode, &path, &newex, 0);
1690 up_write((&EXT4_I(inode)->i_data_sem));
1691 ext4_ext_drop_refs(path);
1698 if (start_pblk + cur - start != map.m_pblk) {
1700 * Logical to physical mapping changed. This can happen
1701 * if this range was removed and then reallocated to
1702 * map to new physical blocks during a fast commit.
1704 ret = ext4_ext_replay_update_ex(inode, cur, map.m_len,
1705 ext4_ext_is_unwritten(ex),
1706 start_pblk + cur - start);
1710 * Mark the old blocks as free since they aren't used
1711 * anymore. We maintain an array of all the modified
1712 * inodes. In case these blocks are still used at either
1713 * a different logical range in the same inode or in
1714 * some different inode, we will mark them as allocated
1715 * at the end of the FC replay using our array of
1718 ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, 0);
1722 /* Range is mapped and needs a state change */
1723 jbd_debug(1, "Converting from %ld to %d %lld",
1724 map.m_flags & EXT4_MAP_UNWRITTEN,
1725 ext4_ext_is_unwritten(ex), map.m_pblk);
1726 ret = ext4_ext_replay_update_ex(inode, cur, map.m_len,
1727 ext4_ext_is_unwritten(ex), map.m_pblk);
1731 * We may have split the extent tree while toggling the state.
1732 * Try to shrink the extent tree now.
1734 ext4_ext_replay_shrink_inode(inode, start + len);
1737 remaining -= map.m_len;
1739 ext4_ext_replay_shrink_inode(inode, i_size_read(inode) >>
1740 sb->s_blocksize_bits);
1746 /* Replay DEL_RANGE tag */
1748 ext4_fc_replay_del_range(struct super_block *sb, struct ext4_fc_tl *tl,
1751 struct inode *inode;
1752 struct ext4_fc_del_range lrange;
1753 struct ext4_map_blocks map;
1754 ext4_lblk_t cur, remaining;
1757 memcpy(&lrange, val, sizeof(lrange));
1758 cur = le32_to_cpu(lrange.fc_lblk);
1759 remaining = le32_to_cpu(lrange.fc_len);
1761 trace_ext4_fc_replay(sb, EXT4_FC_TAG_DEL_RANGE,
1762 le32_to_cpu(lrange.fc_ino), cur, remaining);
1764 inode = ext4_iget(sb, le32_to_cpu(lrange.fc_ino), EXT4_IGET_NORMAL);
1765 if (IS_ERR(inode)) {
1766 jbd_debug(1, "Inode %d not found", le32_to_cpu(lrange.fc_ino));
1770 ret = ext4_fc_record_modified_inode(sb, inode->i_ino);
1774 jbd_debug(1, "DEL_RANGE, inode %ld, lblk %d, len %d\n",
1775 inode->i_ino, le32_to_cpu(lrange.fc_lblk),
1776 le32_to_cpu(lrange.fc_len));
1777 while (remaining > 0) {
1779 map.m_len = remaining;
1781 ret = ext4_map_blocks(NULL, inode, &map, 0);
1787 ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, 0);
1789 remaining -= map.m_len;
1794 down_write(&EXT4_I(inode)->i_data_sem);
1795 ret = ext4_ext_remove_space(inode, le32_to_cpu(lrange.fc_lblk),
1796 le32_to_cpu(lrange.fc_lblk) +
1797 le32_to_cpu(lrange.fc_len) - 1);
1798 up_write(&EXT4_I(inode)->i_data_sem);
1801 ext4_ext_replay_shrink_inode(inode,
1802 i_size_read(inode) >> sb->s_blocksize_bits);
1803 ext4_mark_inode_dirty(NULL, inode);
1809 static inline const char *tag2str(u16 tag)
1812 case EXT4_FC_TAG_LINK:
1813 return "TAG_ADD_ENTRY";
1814 case EXT4_FC_TAG_UNLINK:
1815 return "TAG_DEL_ENTRY";
1816 case EXT4_FC_TAG_ADD_RANGE:
1817 return "TAG_ADD_RANGE";
1818 case EXT4_FC_TAG_CREAT:
1819 return "TAG_CREAT_DENTRY";
1820 case EXT4_FC_TAG_DEL_RANGE:
1821 return "TAG_DEL_RANGE";
1822 case EXT4_FC_TAG_INODE:
1824 case EXT4_FC_TAG_PAD:
1826 case EXT4_FC_TAG_TAIL:
1828 case EXT4_FC_TAG_HEAD:
1835 static void ext4_fc_set_bitmaps_and_counters(struct super_block *sb)
1837 struct ext4_fc_replay_state *state;
1838 struct inode *inode;
1839 struct ext4_ext_path *path = NULL;
1840 struct ext4_map_blocks map;
1842 ext4_lblk_t cur, end;
1844 state = &EXT4_SB(sb)->s_fc_replay_state;
1845 for (i = 0; i < state->fc_modified_inodes_used; i++) {
1846 inode = ext4_iget(sb, state->fc_modified_inodes[i],
1848 if (IS_ERR(inode)) {
1849 jbd_debug(1, "Inode %d not found.",
1850 state->fc_modified_inodes[i]);
1854 end = EXT_MAX_BLOCKS;
1857 map.m_len = end - cur;
1859 ret = ext4_map_blocks(NULL, inode, &map, 0);
1864 path = ext4_find_extent(inode, map.m_lblk, NULL, 0);
1865 if (!IS_ERR(path)) {
1866 for (j = 0; j < path->p_depth; j++)
1867 ext4_mb_mark_bb(inode->i_sb,
1868 path[j].p_block, 1, 1);
1869 ext4_ext_drop_refs(path);
1873 ext4_mb_mark_bb(inode->i_sb, map.m_pblk,
1876 cur = cur + (map.m_len ? map.m_len : 1);
1884 * Check if block is in excluded regions for block allocation. The simple
1885 * allocator that runs during replay phase is calls this function to see
1886 * if it is okay to use a block.
1888 bool ext4_fc_replay_check_excluded(struct super_block *sb, ext4_fsblk_t blk)
1891 struct ext4_fc_replay_state *state;
1893 state = &EXT4_SB(sb)->s_fc_replay_state;
1894 for (i = 0; i < state->fc_regions_valid; i++) {
1895 if (state->fc_regions[i].ino == 0 ||
1896 state->fc_regions[i].len == 0)
1898 if (blk >= state->fc_regions[i].pblk &&
1899 blk < state->fc_regions[i].pblk + state->fc_regions[i].len)
1905 /* Cleanup function called after replay */
1906 void ext4_fc_replay_cleanup(struct super_block *sb)
1908 struct ext4_sb_info *sbi = EXT4_SB(sb);
1910 sbi->s_mount_state &= ~EXT4_FC_REPLAY;
1911 kfree(sbi->s_fc_replay_state.fc_regions);
1912 kfree(sbi->s_fc_replay_state.fc_modified_inodes);
1916 * Recovery Scan phase handler
1918 * This function is called during the scan phase and is responsible
1919 * for doing following things:
1920 * - Make sure the fast commit area has valid tags for replay
1921 * - Count number of tags that need to be replayed by the replay handler
1923 * - Create a list of excluded blocks for allocation during replay phase
1925 * This function returns JBD2_FC_REPLAY_CONTINUE to indicate that SCAN is
1926 * incomplete and JBD2 should send more blocks. It returns JBD2_FC_REPLAY_STOP
1927 * to indicate that scan has finished and JBD2 can now start replay phase.
1928 * It returns a negative error to indicate that there was an error. At the end
1929 * of a successful scan phase, sbi->s_fc_replay_state.fc_replay_num_tags is set
1930 * to indicate the number of tags that need to replayed during the replay phase.
1932 static int ext4_fc_replay_scan(journal_t *journal,
1933 struct buffer_head *bh, int off,
1936 struct super_block *sb = journal->j_private;
1937 struct ext4_sb_info *sbi = EXT4_SB(sb);
1938 struct ext4_fc_replay_state *state;
1939 int ret = JBD2_FC_REPLAY_CONTINUE;
1940 struct ext4_fc_add_range ext;
1941 struct ext4_fc_tl tl;
1942 struct ext4_fc_tail tail;
1943 __u8 *start, *end, *cur, *val;
1944 struct ext4_fc_head head;
1945 struct ext4_extent *ex;
1947 state = &sbi->s_fc_replay_state;
1949 start = (u8 *)bh->b_data;
1950 end = (__u8 *)bh->b_data + journal->j_blocksize - 1;
1952 if (state->fc_replay_expected_off == 0) {
1953 state->fc_cur_tag = 0;
1954 state->fc_replay_num_tags = 0;
1956 state->fc_regions = NULL;
1957 state->fc_regions_valid = state->fc_regions_used =
1958 state->fc_regions_size = 0;
1959 /* Check if we can stop early */
1960 if (le16_to_cpu(((struct ext4_fc_tl *)start)->fc_tag)
1961 != EXT4_FC_TAG_HEAD)
1965 if (off != state->fc_replay_expected_off) {
1966 ret = -EFSCORRUPTED;
1970 state->fc_replay_expected_off++;
1971 for (cur = start; cur < end; cur = cur + sizeof(tl) + le16_to_cpu(tl.fc_len)) {
1972 memcpy(&tl, cur, sizeof(tl));
1973 val = cur + sizeof(tl);
1974 jbd_debug(3, "Scan phase, tag:%s, blk %lld\n",
1975 tag2str(le16_to_cpu(tl.fc_tag)), bh->b_blocknr);
1976 switch (le16_to_cpu(tl.fc_tag)) {
1977 case EXT4_FC_TAG_ADD_RANGE:
1978 memcpy(&ext, val, sizeof(ext));
1979 ex = (struct ext4_extent *)&ext.fc_ex;
1980 ret = ext4_fc_record_regions(sb,
1981 le32_to_cpu(ext.fc_ino),
1982 le32_to_cpu(ex->ee_block), ext4_ext_pblock(ex),
1983 ext4_ext_get_actual_len(ex), 0);
1986 ret = JBD2_FC_REPLAY_CONTINUE;
1988 case EXT4_FC_TAG_DEL_RANGE:
1989 case EXT4_FC_TAG_LINK:
1990 case EXT4_FC_TAG_UNLINK:
1991 case EXT4_FC_TAG_CREAT:
1992 case EXT4_FC_TAG_INODE:
1993 case EXT4_FC_TAG_PAD:
1994 state->fc_cur_tag++;
1995 state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur,
1996 sizeof(tl) + le16_to_cpu(tl.fc_len));
1998 case EXT4_FC_TAG_TAIL:
1999 state->fc_cur_tag++;
2000 memcpy(&tail, val, sizeof(tail));
2001 state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur,
2003 offsetof(struct ext4_fc_tail,
2005 if (le32_to_cpu(tail.fc_tid) == expected_tid &&
2006 le32_to_cpu(tail.fc_crc) == state->fc_crc) {
2007 state->fc_replay_num_tags = state->fc_cur_tag;
2008 state->fc_regions_valid =
2009 state->fc_regions_used;
2011 ret = state->fc_replay_num_tags ?
2012 JBD2_FC_REPLAY_STOP : -EFSBADCRC;
2016 case EXT4_FC_TAG_HEAD:
2017 memcpy(&head, val, sizeof(head));
2018 if (le32_to_cpu(head.fc_features) &
2019 ~EXT4_FC_SUPPORTED_FEATURES) {
2023 if (le32_to_cpu(head.fc_tid) != expected_tid) {
2024 ret = JBD2_FC_REPLAY_STOP;
2027 state->fc_cur_tag++;
2028 state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur,
2029 sizeof(tl) + le16_to_cpu(tl.fc_len));
2032 ret = state->fc_replay_num_tags ?
2033 JBD2_FC_REPLAY_STOP : -ECANCELED;
2035 if (ret < 0 || ret == JBD2_FC_REPLAY_STOP)
2040 trace_ext4_fc_replay_scan(sb, ret, off);
2045 * Main recovery path entry point.
2046 * The meaning of return codes is similar as above.
2048 static int ext4_fc_replay(journal_t *journal, struct buffer_head *bh,
2049 enum passtype pass, int off, tid_t expected_tid)
2051 struct super_block *sb = journal->j_private;
2052 struct ext4_sb_info *sbi = EXT4_SB(sb);
2053 struct ext4_fc_tl tl;
2054 __u8 *start, *end, *cur, *val;
2055 int ret = JBD2_FC_REPLAY_CONTINUE;
2056 struct ext4_fc_replay_state *state = &sbi->s_fc_replay_state;
2057 struct ext4_fc_tail tail;
2059 if (pass == PASS_SCAN) {
2060 state->fc_current_pass = PASS_SCAN;
2061 return ext4_fc_replay_scan(journal, bh, off, expected_tid);
2064 if (state->fc_current_pass != pass) {
2065 state->fc_current_pass = pass;
2066 sbi->s_mount_state |= EXT4_FC_REPLAY;
2068 if (!sbi->s_fc_replay_state.fc_replay_num_tags) {
2069 jbd_debug(1, "Replay stops\n");
2070 ext4_fc_set_bitmaps_and_counters(sb);
2074 #ifdef CONFIG_EXT4_DEBUG
2075 if (sbi->s_fc_debug_max_replay && off >= sbi->s_fc_debug_max_replay) {
2076 pr_warn("Dropping fc block %d because max_replay set\n", off);
2077 return JBD2_FC_REPLAY_STOP;
2081 start = (u8 *)bh->b_data;
2082 end = (__u8 *)bh->b_data + journal->j_blocksize - 1;
2084 for (cur = start; cur < end; cur = cur + sizeof(tl) + le16_to_cpu(tl.fc_len)) {
2085 memcpy(&tl, cur, sizeof(tl));
2086 val = cur + sizeof(tl);
2088 if (state->fc_replay_num_tags == 0) {
2089 ret = JBD2_FC_REPLAY_STOP;
2090 ext4_fc_set_bitmaps_and_counters(sb);
2093 jbd_debug(3, "Replay phase, tag:%s\n",
2094 tag2str(le16_to_cpu(tl.fc_tag)));
2095 state->fc_replay_num_tags--;
2096 switch (le16_to_cpu(tl.fc_tag)) {
2097 case EXT4_FC_TAG_LINK:
2098 ret = ext4_fc_replay_link(sb, &tl, val);
2100 case EXT4_FC_TAG_UNLINK:
2101 ret = ext4_fc_replay_unlink(sb, &tl, val);
2103 case EXT4_FC_TAG_ADD_RANGE:
2104 ret = ext4_fc_replay_add_range(sb, &tl, val);
2106 case EXT4_FC_TAG_CREAT:
2107 ret = ext4_fc_replay_create(sb, &tl, val);
2109 case EXT4_FC_TAG_DEL_RANGE:
2110 ret = ext4_fc_replay_del_range(sb, &tl, val);
2112 case EXT4_FC_TAG_INODE:
2113 ret = ext4_fc_replay_inode(sb, &tl, val);
2115 case EXT4_FC_TAG_PAD:
2116 trace_ext4_fc_replay(sb, EXT4_FC_TAG_PAD, 0,
2117 le16_to_cpu(tl.fc_len), 0);
2119 case EXT4_FC_TAG_TAIL:
2120 trace_ext4_fc_replay(sb, EXT4_FC_TAG_TAIL, 0,
2121 le16_to_cpu(tl.fc_len), 0);
2122 memcpy(&tail, val, sizeof(tail));
2123 WARN_ON(le32_to_cpu(tail.fc_tid) != expected_tid);
2125 case EXT4_FC_TAG_HEAD:
2128 trace_ext4_fc_replay(sb, le16_to_cpu(tl.fc_tag), 0,
2129 le16_to_cpu(tl.fc_len), 0);
2135 ret = JBD2_FC_REPLAY_CONTINUE;
2140 void ext4_fc_init(struct super_block *sb, journal_t *journal)
2143 * We set replay callback even if fast commit disabled because we may
2144 * could still have fast commit blocks that need to be replayed even if
2145 * fast commit has now been turned off.
2147 journal->j_fc_replay_callback = ext4_fc_replay;
2148 if (!test_opt2(sb, JOURNAL_FAST_COMMIT))
2150 journal->j_fc_cleanup_callback = ext4_fc_cleanup;
2153 static const char * const fc_ineligible_reasons[] = {
2154 [EXT4_FC_REASON_XATTR] = "Extended attributes changed",
2155 [EXT4_FC_REASON_CROSS_RENAME] = "Cross rename",
2156 [EXT4_FC_REASON_JOURNAL_FLAG_CHANGE] = "Journal flag changed",
2157 [EXT4_FC_REASON_NOMEM] = "Insufficient memory",
2158 [EXT4_FC_REASON_SWAP_BOOT] = "Swap boot",
2159 [EXT4_FC_REASON_RESIZE] = "Resize",
2160 [EXT4_FC_REASON_RENAME_DIR] = "Dir renamed",
2161 [EXT4_FC_REASON_FALLOC_RANGE] = "Falloc range op",
2162 [EXT4_FC_REASON_INODE_JOURNAL_DATA] = "Data journalling",
2163 [EXT4_FC_REASON_ENCRYPTED_FILENAME] = "Encrypted filename",
2166 int ext4_fc_info_show(struct seq_file *seq, void *v)
2168 struct ext4_sb_info *sbi = EXT4_SB((struct super_block *)seq->private);
2169 struct ext4_fc_stats *stats = &sbi->s_fc_stats;
2172 if (v != SEQ_START_TOKEN)
2176 "fc stats:\n%ld commits\n%ld ineligible\n%ld numblks\n%lluus avg_commit_time\n",
2177 stats->fc_num_commits, stats->fc_ineligible_commits,
2179 div_u64(sbi->s_fc_avg_commit_time, 1000));
2180 seq_puts(seq, "Ineligible reasons:\n");
2181 for (i = 0; i < EXT4_FC_REASON_MAX; i++)
2182 seq_printf(seq, "\"%s\":\t%d\n", fc_ineligible_reasons[i],
2183 stats->fc_ineligible_reason_count[i]);
2188 int __init ext4_fc_init_dentry_cache(void)
2190 ext4_fc_dentry_cachep = KMEM_CACHE(ext4_fc_dentry_update,
2191 SLAB_RECLAIM_ACCOUNT);
2193 if (ext4_fc_dentry_cachep == NULL)
2199 void ext4_fc_destroy_dentry_cache(void)
2201 kmem_cache_destroy(ext4_fc_dentry_cachep);