1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright (c) 2013, Intel Corporation
5 * Authors: Huajun Li <huajun.li@intel.com>
6 * Haicheng Li <haicheng.li@intel.com>
10 #include <linux/f2fs_fs.h>
15 bool f2fs_may_inline_data(struct inode *inode)
17 if (f2fs_is_atomic_file(inode))
20 if (!S_ISREG(inode->i_mode) && !S_ISLNK(inode->i_mode))
23 if (i_size_read(inode) > MAX_INLINE_DATA(inode))
26 if (f2fs_post_read_required(inode))
32 bool f2fs_may_inline_dentry(struct inode *inode)
34 if (!test_opt(F2FS_I_SB(inode), INLINE_DENTRY))
37 if (!S_ISDIR(inode->i_mode))
43 void f2fs_do_read_inline_data(struct page *page, struct page *ipage)
45 struct inode *inode = page->mapping->host;
47 if (PageUptodate(page))
50 f2fs_bug_on(F2FS_P_SB(page), page->index);
52 zero_user_segment(page, MAX_INLINE_DATA(inode), PAGE_SIZE);
54 /* Copy the whole inline data block */
55 memcpy_to_page(page, 0, inline_data_addr(inode, ipage),
56 MAX_INLINE_DATA(inode));
57 if (!PageUptodate(page))
58 SetPageUptodate(page);
61 void f2fs_truncate_inline_inode(struct inode *inode,
62 struct page *ipage, u64 from)
66 if (from >= MAX_INLINE_DATA(inode))
69 addr = inline_data_addr(inode, ipage);
71 f2fs_wait_on_page_writeback(ipage, NODE, true, true);
72 memset(addr + from, 0, MAX_INLINE_DATA(inode) - from);
73 set_page_dirty(ipage);
76 clear_inode_flag(inode, FI_DATA_EXIST);
79 int f2fs_read_inline_data(struct inode *inode, struct page *page)
83 ipage = f2fs_get_node_page(F2FS_I_SB(inode), inode->i_ino);
86 return PTR_ERR(ipage);
89 if (!f2fs_has_inline_data(inode)) {
90 f2fs_put_page(ipage, 1);
95 zero_user_segment(page, 0, PAGE_SIZE);
97 f2fs_do_read_inline_data(page, ipage);
99 if (!PageUptodate(page))
100 SetPageUptodate(page);
101 f2fs_put_page(ipage, 1);
106 int f2fs_convert_inline_page(struct dnode_of_data *dn, struct page *page)
108 struct f2fs_io_info fio = {
109 .sbi = F2FS_I_SB(dn->inode),
110 .ino = dn->inode->i_ino,
113 .op_flags = REQ_SYNC | REQ_PRIO,
115 .encrypted_page = NULL,
116 .io_type = FS_DATA_IO,
121 if (!f2fs_exist_data(dn->inode))
124 err = f2fs_reserve_block(dn, 0);
128 err = f2fs_get_node_info(fio.sbi, dn->nid, &ni);
130 f2fs_truncate_data_blocks_range(dn, 1);
135 fio.version = ni.version;
137 if (unlikely(dn->data_blkaddr != NEW_ADDR)) {
139 set_sbi_flag(fio.sbi, SBI_NEED_FSCK);
140 f2fs_warn(fio.sbi, "%s: corrupted inline inode ino=%lx, i_addr[0]:0x%x, run fsck to fix.",
141 __func__, dn->inode->i_ino, dn->data_blkaddr);
142 return -EFSCORRUPTED;
145 f2fs_bug_on(F2FS_P_SB(page), PageWriteback(page));
147 f2fs_do_read_inline_data(page, dn->inode_page);
148 set_page_dirty(page);
150 /* clear dirty state */
151 dirty = clear_page_dirty_for_io(page);
153 /* write data page to try to make data consistent */
154 set_page_writeback(page);
155 ClearPageError(page);
156 fio.old_blkaddr = dn->data_blkaddr;
157 set_inode_flag(dn->inode, FI_HOT_DATA);
158 f2fs_outplace_write_data(dn, &fio);
159 f2fs_wait_on_page_writeback(page, DATA, true, true);
161 inode_dec_dirty_pages(dn->inode);
162 f2fs_remove_dirty_inode(dn->inode);
165 /* this converted inline_data should be recovered. */
166 set_inode_flag(dn->inode, FI_APPEND_WRITE);
168 /* clear inline data and flag after data writeback */
169 f2fs_truncate_inline_inode(dn->inode, dn->inode_page, 0);
170 clear_inline_node(dn->inode_page);
172 stat_dec_inline_inode(dn->inode);
173 clear_inode_flag(dn->inode, FI_INLINE_DATA);
178 int f2fs_convert_inline_inode(struct inode *inode)
180 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
181 struct dnode_of_data dn;
182 struct page *ipage, *page;
185 if (!f2fs_has_inline_data(inode))
188 err = dquot_initialize(inode);
192 page = f2fs_grab_cache_page(inode->i_mapping, 0, false);
198 ipage = f2fs_get_node_page(sbi, inode->i_ino);
200 err = PTR_ERR(ipage);
204 set_new_dnode(&dn, inode, ipage, ipage, 0);
206 if (f2fs_has_inline_data(inode))
207 err = f2fs_convert_inline_page(&dn, page);
213 f2fs_put_page(page, 1);
216 f2fs_balance_fs(sbi, dn.node_changed);
221 int f2fs_write_inline_data(struct inode *inode, struct page *page)
223 struct dnode_of_data dn;
226 set_new_dnode(&dn, inode, NULL, NULL, 0);
227 err = f2fs_get_dnode_of_data(&dn, 0, LOOKUP_NODE);
231 if (!f2fs_has_inline_data(inode)) {
236 f2fs_bug_on(F2FS_I_SB(inode), page->index);
238 f2fs_wait_on_page_writeback(dn.inode_page, NODE, true, true);
239 memcpy_from_page(inline_data_addr(inode, dn.inode_page),
240 page, 0, MAX_INLINE_DATA(inode));
241 set_page_dirty(dn.inode_page);
243 f2fs_clear_page_cache_dirty_tag(page);
245 set_inode_flag(inode, FI_APPEND_WRITE);
246 set_inode_flag(inode, FI_DATA_EXIST);
248 clear_inline_node(dn.inode_page);
253 int f2fs_recover_inline_data(struct inode *inode, struct page *npage)
255 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
256 struct f2fs_inode *ri = NULL;
257 void *src_addr, *dst_addr;
261 * The inline_data recovery policy is as follows.
262 * [prev.] [next] of inline_data flag
263 * o o -> recover inline_data
264 * o x -> remove inline_data, and then recover data blocks
265 * x o -> remove inline_data, and then recover inline_data
266 * x x -> recover data blocks
269 ri = F2FS_INODE(npage);
271 if (f2fs_has_inline_data(inode) &&
272 ri && (ri->i_inline & F2FS_INLINE_DATA)) {
274 ipage = f2fs_get_node_page(sbi, inode->i_ino);
276 return PTR_ERR(ipage);
278 f2fs_wait_on_page_writeback(ipage, NODE, true, true);
280 src_addr = inline_data_addr(inode, npage);
281 dst_addr = inline_data_addr(inode, ipage);
282 memcpy(dst_addr, src_addr, MAX_INLINE_DATA(inode));
284 set_inode_flag(inode, FI_INLINE_DATA);
285 set_inode_flag(inode, FI_DATA_EXIST);
287 set_page_dirty(ipage);
288 f2fs_put_page(ipage, 1);
292 if (f2fs_has_inline_data(inode)) {
293 ipage = f2fs_get_node_page(sbi, inode->i_ino);
295 return PTR_ERR(ipage);
296 f2fs_truncate_inline_inode(inode, ipage, 0);
297 clear_inode_flag(inode, FI_INLINE_DATA);
298 f2fs_put_page(ipage, 1);
299 } else if (ri && (ri->i_inline & F2FS_INLINE_DATA)) {
302 ret = f2fs_truncate_blocks(inode, 0, false);
310 struct f2fs_dir_entry *f2fs_find_in_inline_dir(struct inode *dir,
311 struct fscrypt_name *fname, struct page **res_page)
313 struct f2fs_sb_info *sbi = F2FS_SB(dir->i_sb);
314 struct qstr name = FSTR_TO_QSTR(&fname->disk_name);
315 struct f2fs_dir_entry *de;
316 struct f2fs_dentry_ptr d;
319 f2fs_hash_t namehash;
321 ipage = f2fs_get_node_page(sbi, dir->i_ino);
327 namehash = f2fs_dentry_hash(dir, &name, fname);
329 inline_dentry = inline_data_addr(dir, ipage);
331 make_dentry_ptr_inline(dir, &d, inline_dentry);
332 de = f2fs_find_target_dentry(fname, namehash, NULL, &d);
337 f2fs_put_page(ipage, 0);
342 int f2fs_make_empty_inline_dir(struct inode *inode, struct inode *parent,
345 struct f2fs_dentry_ptr d;
348 inline_dentry = inline_data_addr(inode, ipage);
350 make_dentry_ptr_inline(inode, &d, inline_dentry);
351 f2fs_do_make_empty_dir(inode, parent, &d);
353 set_page_dirty(ipage);
355 /* update i_size to MAX_INLINE_DATA */
356 if (i_size_read(inode) < MAX_INLINE_DATA(inode))
357 f2fs_i_size_write(inode, MAX_INLINE_DATA(inode));
362 * NOTE: ipage is grabbed by caller, but if any error occurs, we should
363 * release ipage in this function.
365 static int f2fs_move_inline_dirents(struct inode *dir, struct page *ipage,
369 struct dnode_of_data dn;
370 struct f2fs_dentry_block *dentry_blk;
371 struct f2fs_dentry_ptr src, dst;
374 page = f2fs_grab_cache_page(dir->i_mapping, 0, false);
376 f2fs_put_page(ipage, 1);
380 set_new_dnode(&dn, dir, ipage, NULL, 0);
381 err = f2fs_reserve_block(&dn, 0);
385 if (unlikely(dn.data_blkaddr != NEW_ADDR)) {
387 set_sbi_flag(F2FS_P_SB(page), SBI_NEED_FSCK);
388 f2fs_warn(F2FS_P_SB(page), "%s: corrupted inline inode ino=%lx, i_addr[0]:0x%x, run fsck to fix.",
389 __func__, dir->i_ino, dn.data_blkaddr);
394 f2fs_wait_on_page_writeback(page, DATA, true, true);
396 dentry_blk = page_address(page);
399 * Start by zeroing the full block, to ensure that all unused space is
400 * zeroed and no uninitialized memory is leaked to disk.
402 memset(dentry_blk, 0, F2FS_BLKSIZE);
404 make_dentry_ptr_inline(dir, &src, inline_dentry);
405 make_dentry_ptr_block(dir, &dst, dentry_blk);
407 /* copy data from inline dentry block to new dentry block */
408 memcpy(dst.bitmap, src.bitmap, src.nr_bitmap);
409 memcpy(dst.dentry, src.dentry, SIZE_OF_DIR_ENTRY * src.max);
410 memcpy(dst.filename, src.filename, src.max * F2FS_SLOT_LEN);
412 if (!PageUptodate(page))
413 SetPageUptodate(page);
414 set_page_dirty(page);
416 /* clear inline dir and flag after data writeback */
417 f2fs_truncate_inline_inode(dir, ipage, 0);
419 stat_dec_inline_dir(dir);
420 clear_inode_flag(dir, FI_INLINE_DENTRY);
423 * should retrieve reserved space which was used to keep
424 * inline_dentry's structure for backward compatibility.
426 if (!f2fs_sb_has_flexible_inline_xattr(F2FS_I_SB(dir)) &&
427 !f2fs_has_inline_xattr(dir))
428 F2FS_I(dir)->i_inline_xattr_size = 0;
430 f2fs_i_depth_write(dir, 1);
431 if (i_size_read(dir) < PAGE_SIZE)
432 f2fs_i_size_write(dir, PAGE_SIZE);
434 f2fs_put_page(page, 1);
438 static int f2fs_add_inline_entries(struct inode *dir, void *inline_dentry)
440 struct f2fs_dentry_ptr d;
441 unsigned long bit_pos = 0;
444 make_dentry_ptr_inline(dir, &d, inline_dentry);
446 while (bit_pos < d.max) {
447 struct f2fs_dir_entry *de;
448 struct qstr new_name;
452 if (!test_bit_le(bit_pos, d.bitmap)) {
457 de = &d.dentry[bit_pos];
459 if (unlikely(!de->name_len)) {
464 new_name.name = d.filename[bit_pos];
465 new_name.len = le16_to_cpu(de->name_len);
467 ino = le32_to_cpu(de->ino);
468 fake_mode = f2fs_get_de_type(de) << S_SHIFT;
470 err = f2fs_add_regular_entry(dir, &new_name, NULL, NULL,
473 goto punch_dentry_pages;
475 bit_pos += GET_DENTRY_SLOTS(le16_to_cpu(de->name_len));
479 truncate_inode_pages(&dir->i_data, 0);
480 f2fs_truncate_blocks(dir, 0, false);
481 f2fs_remove_dirty_inode(dir);
485 static int f2fs_move_rehashed_dirents(struct inode *dir, struct page *ipage,
491 backup_dentry = f2fs_kmalloc(F2FS_I_SB(dir),
492 MAX_INLINE_DATA(dir), GFP_F2FS_ZERO);
493 if (!backup_dentry) {
494 f2fs_put_page(ipage, 1);
498 memcpy(backup_dentry, inline_dentry, MAX_INLINE_DATA(dir));
499 f2fs_truncate_inline_inode(dir, ipage, 0);
503 err = f2fs_add_inline_entries(dir, backup_dentry);
509 stat_dec_inline_dir(dir);
510 clear_inode_flag(dir, FI_INLINE_DENTRY);
513 * should retrieve reserved space which was used to keep
514 * inline_dentry's structure for backward compatibility.
516 if (!f2fs_sb_has_flexible_inline_xattr(F2FS_I_SB(dir)) &&
517 !f2fs_has_inline_xattr(dir))
518 F2FS_I(dir)->i_inline_xattr_size = 0;
520 kvfree(backup_dentry);
524 f2fs_wait_on_page_writeback(ipage, NODE, true, true);
525 memcpy(inline_dentry, backup_dentry, MAX_INLINE_DATA(dir));
526 f2fs_i_depth_write(dir, 0);
527 f2fs_i_size_write(dir, MAX_INLINE_DATA(dir));
528 set_page_dirty(ipage);
529 f2fs_put_page(ipage, 1);
531 kvfree(backup_dentry);
535 static int f2fs_convert_inline_dir(struct inode *dir, struct page *ipage,
538 if (!F2FS_I(dir)->i_dir_level)
539 return f2fs_move_inline_dirents(dir, ipage, inline_dentry);
541 return f2fs_move_rehashed_dirents(dir, ipage, inline_dentry);
544 int f2fs_add_inline_entry(struct inode *dir, const struct qstr *new_name,
545 const struct qstr *orig_name,
546 struct inode *inode, nid_t ino, umode_t mode)
548 struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
550 unsigned int bit_pos;
551 f2fs_hash_t name_hash;
552 void *inline_dentry = NULL;
553 struct f2fs_dentry_ptr d;
554 int slots = GET_DENTRY_SLOTS(new_name->len);
555 struct page *page = NULL;
558 ipage = f2fs_get_node_page(sbi, dir->i_ino);
560 return PTR_ERR(ipage);
562 inline_dentry = inline_data_addr(dir, ipage);
563 make_dentry_ptr_inline(dir, &d, inline_dentry);
565 bit_pos = f2fs_room_for_filename(d.bitmap, slots, d.max);
566 if (bit_pos >= d.max) {
567 err = f2fs_convert_inline_dir(dir, ipage, inline_dentry);
575 down_write(&F2FS_I(inode)->i_sem);
576 page = f2fs_init_inode_metadata(inode, dir, new_name,
584 f2fs_wait_on_page_writeback(ipage, NODE, true, true);
586 name_hash = f2fs_dentry_hash(dir, new_name, NULL);
587 f2fs_update_dentry(ino, mode, &d, new_name, name_hash, bit_pos);
589 set_page_dirty(ipage);
591 /* we don't need to mark_inode_dirty now */
593 f2fs_i_pino_write(inode, dir->i_ino);
595 /* synchronize inode page's data from inode cache */
596 if (is_inode_flag_set(inode, FI_NEW_INODE))
597 f2fs_update_inode(inode, page);
599 f2fs_put_page(page, 1);
602 f2fs_update_parent_metadata(dir, inode, 0);
605 up_write(&F2FS_I(inode)->i_sem);
607 f2fs_put_page(ipage, 1);
611 void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry, struct page *page,
612 struct inode *dir, struct inode *inode)
614 struct f2fs_dentry_ptr d;
616 int slots = GET_DENTRY_SLOTS(le16_to_cpu(dentry->name_len));
617 unsigned int bit_pos;
621 f2fs_wait_on_page_writeback(page, NODE, true, true);
623 inline_dentry = inline_data_addr(dir, page);
624 make_dentry_ptr_inline(dir, &d, inline_dentry);
626 bit_pos = dentry - d.dentry;
627 for (i = 0; i < slots; i++)
628 __clear_bit_le(bit_pos + i, d.bitmap);
630 set_page_dirty(page);
631 f2fs_put_page(page, 1);
633 dir->i_ctime = dir->i_mtime = current_time(dir);
634 f2fs_mark_inode_dirty_sync(dir, false);
637 f2fs_drop_nlink(dir, inode);
640 bool f2fs_empty_inline_dir(struct inode *dir)
642 struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
644 unsigned int bit_pos = 2;
646 struct f2fs_dentry_ptr d;
648 ipage = f2fs_get_node_page(sbi, dir->i_ino);
652 inline_dentry = inline_data_addr(dir, ipage);
653 make_dentry_ptr_inline(dir, &d, inline_dentry);
655 bit_pos = find_next_bit_le(d.bitmap, d.max, bit_pos);
657 f2fs_put_page(ipage, 1);
665 int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx,
666 struct fscrypt_str *fstr)
668 struct inode *inode = file_inode(file);
669 struct page *ipage = NULL;
670 struct f2fs_dentry_ptr d;
671 void *inline_dentry = NULL;
674 make_dentry_ptr_inline(inode, &d, inline_dentry);
676 if (ctx->pos == d.max)
679 ipage = f2fs_get_node_page(F2FS_I_SB(inode), inode->i_ino);
681 return PTR_ERR(ipage);
684 * f2fs_readdir was protected by inode.i_rwsem, it is safe to access
685 * ipage without page's lock held.
689 inline_dentry = inline_data_addr(inode, ipage);
691 make_dentry_ptr_inline(inode, &d, inline_dentry);
693 err = f2fs_fill_dentries(ctx, &d, 0, fstr);
697 f2fs_put_page(ipage, 0);
698 return err < 0 ? err : 0;
701 int f2fs_inline_data_fiemap(struct inode *inode,
702 struct fiemap_extent_info *fieinfo, __u64 start, __u64 len)
704 __u64 byteaddr, ilen;
705 __u32 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED |
711 ipage = f2fs_get_node_page(F2FS_I_SB(inode), inode->i_ino);
713 return PTR_ERR(ipage);
715 if ((S_ISREG(inode->i_mode) || S_ISLNK(inode->i_mode)) &&
716 !f2fs_has_inline_data(inode)) {
721 if (S_ISDIR(inode->i_mode) && !f2fs_has_inline_dentry(inode)) {
726 ilen = min_t(size_t, MAX_INLINE_DATA(inode), i_size_read(inode));
729 if (start + len < ilen)
733 err = f2fs_get_node_info(F2FS_I_SB(inode), inode->i_ino, &ni);
737 byteaddr = (__u64)ni.blk_addr << inode->i_sb->s_blocksize_bits;
738 byteaddr += (char *)inline_data_addr(inode, ipage) -
739 (char *)F2FS_INODE(ipage);
740 err = fiemap_fill_next_extent(fieinfo, start, byteaddr, ilen, flags);
742 f2fs_put_page(ipage, 1);