2479508deab3b9d256d3bd1b2dd9065eeb4a9e9c
[releases.git] / inode.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  linux/fs/ext4/inode.c
4  *
5  * Copyright (C) 1992, 1993, 1994, 1995
6  * Remy Card (card@masi.ibp.fr)
7  * Laboratoire MASI - Institut Blaise Pascal
8  * Universite Pierre et Marie Curie (Paris VI)
9  *
10  *  from
11  *
12  *  linux/fs/minix/inode.c
13  *
14  *  Copyright (C) 1991, 1992  Linus Torvalds
15  *
16  *  64-bit file support on 64-bit platforms by Jakub Jelinek
17  *      (jj@sunsite.ms.mff.cuni.cz)
18  *
19  *  Assorted race fixes, rewrite of ext4_get_block() by Al Viro, 2000
20  */
21
22 #include <linux/fs.h>
23 #include <linux/mount.h>
24 #include <linux/time.h>
25 #include <linux/highuid.h>
26 #include <linux/pagemap.h>
27 #include <linux/dax.h>
28 #include <linux/quotaops.h>
29 #include <linux/string.h>
30 #include <linux/buffer_head.h>
31 #include <linux/writeback.h>
32 #include <linux/pagevec.h>
33 #include <linux/mpage.h>
34 #include <linux/namei.h>
35 #include <linux/uio.h>
36 #include <linux/bio.h>
37 #include <linux/workqueue.h>
38 #include <linux/kernel.h>
39 #include <linux/printk.h>
40 #include <linux/slab.h>
41 #include <linux/bitops.h>
42 #include <linux/iomap.h>
43 #include <linux/iversion.h>
44
45 #include "ext4_jbd2.h"
46 #include "xattr.h"
47 #include "acl.h"
48 #include "truncate.h"
49
50 #include <trace/events/ext4.h>
51
52 static __u32 ext4_inode_csum(struct inode *inode, struct ext4_inode *raw,
53                               struct ext4_inode_info *ei)
54 {
55         struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
56         __u32 csum;
57         __u16 dummy_csum = 0;
58         int offset = offsetof(struct ext4_inode, i_checksum_lo);
59         unsigned int csum_size = sizeof(dummy_csum);
60
61         csum = ext4_chksum(sbi, ei->i_csum_seed, (__u8 *)raw, offset);
62         csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum, csum_size);
63         offset += csum_size;
64         csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,
65                            EXT4_GOOD_OLD_INODE_SIZE - offset);
66
67         if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
68                 offset = offsetof(struct ext4_inode, i_checksum_hi);
69                 csum = ext4_chksum(sbi, csum, (__u8 *)raw +
70                                    EXT4_GOOD_OLD_INODE_SIZE,
71                                    offset - EXT4_GOOD_OLD_INODE_SIZE);
72                 if (EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi)) {
73                         csum = ext4_chksum(sbi, csum, (__u8 *)&dummy_csum,
74                                            csum_size);
75                         offset += csum_size;
76                 }
77                 csum = ext4_chksum(sbi, csum, (__u8 *)raw + offset,
78                                    EXT4_INODE_SIZE(inode->i_sb) - offset);
79         }
80
81         return csum;
82 }
83
84 static int ext4_inode_csum_verify(struct inode *inode, struct ext4_inode *raw,
85                                   struct ext4_inode_info *ei)
86 {
87         __u32 provided, calculated;
88
89         if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
90             cpu_to_le32(EXT4_OS_LINUX) ||
91             !ext4_has_metadata_csum(inode->i_sb))
92                 return 1;
93
94         provided = le16_to_cpu(raw->i_checksum_lo);
95         calculated = ext4_inode_csum(inode, raw, ei);
96         if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
97             EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
98                 provided |= ((__u32)le16_to_cpu(raw->i_checksum_hi)) << 16;
99         else
100                 calculated &= 0xFFFF;
101
102         return provided == calculated;
103 }
104
105 void ext4_inode_csum_set(struct inode *inode, struct ext4_inode *raw,
106                          struct ext4_inode_info *ei)
107 {
108         __u32 csum;
109
110         if (EXT4_SB(inode->i_sb)->s_es->s_creator_os !=
111             cpu_to_le32(EXT4_OS_LINUX) ||
112             !ext4_has_metadata_csum(inode->i_sb))
113                 return;
114
115         csum = ext4_inode_csum(inode, raw, ei);
116         raw->i_checksum_lo = cpu_to_le16(csum & 0xFFFF);
117         if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
118             EXT4_FITS_IN_INODE(raw, ei, i_checksum_hi))
119                 raw->i_checksum_hi = cpu_to_le16(csum >> 16);
120 }
121
122 static inline int ext4_begin_ordered_truncate(struct inode *inode,
123                                               loff_t new_size)
124 {
125         trace_ext4_begin_ordered_truncate(inode, new_size);
126         /*
127          * If jinode is zero, then we never opened the file for
128          * writing, so there's no need to call
129          * jbd2_journal_begin_ordered_truncate() since there's no
130          * outstanding writes we need to flush.
131          */
132         if (!EXT4_I(inode)->jinode)
133                 return 0;
134         return jbd2_journal_begin_ordered_truncate(EXT4_JOURNAL(inode),
135                                                    EXT4_I(inode)->jinode,
136                                                    new_size);
137 }
138
139 static int __ext4_journalled_writepage(struct page *page, unsigned int len);
140 static int ext4_meta_trans_blocks(struct inode *inode, int lblocks,
141                                   int pextents);
142
143 /*
144  * Test whether an inode is a fast symlink.
145  * A fast symlink has its symlink data stored in ext4_inode_info->i_data.
146  */
147 int ext4_inode_is_fast_symlink(struct inode *inode)
148 {
149         if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)) {
150                 int ea_blocks = EXT4_I(inode)->i_file_acl ?
151                                 EXT4_CLUSTER_SIZE(inode->i_sb) >> 9 : 0;
152
153                 if (ext4_has_inline_data(inode))
154                         return 0;
155
156                 return (S_ISLNK(inode->i_mode) && inode->i_blocks - ea_blocks == 0);
157         }
158         return S_ISLNK(inode->i_mode) && inode->i_size &&
159                (inode->i_size < EXT4_N_BLOCKS * 4);
160 }
161
162 /*
163  * Called at the last iput() if i_nlink is zero.
164  */
165 void ext4_evict_inode(struct inode *inode)
166 {
167         handle_t *handle;
168         int err;
169         /*
170          * Credits for final inode cleanup and freeing:
171          * sb + inode (ext4_orphan_del()), block bitmap, group descriptor
172          * (xattr block freeing), bitmap, group descriptor (inode freeing)
173          */
174         int extra_credits = 6;
175         struct ext4_xattr_inode_array *ea_inode_array = NULL;
176         bool freeze_protected = false;
177
178         trace_ext4_evict_inode(inode);
179
180         if (EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL)
181                 ext4_evict_ea_inode(inode);
182         if (inode->i_nlink) {
183                 /*
184                  * When journalling data dirty buffers are tracked only in the
185                  * journal. So although mm thinks everything is clean and
186                  * ready for reaping the inode might still have some pages to
187                  * write in the running transaction or waiting to be
188                  * checkpointed. Thus calling jbd2_journal_invalidate_folio()
189                  * (via truncate_inode_pages()) to discard these buffers can
190                  * cause data loss. Also even if we did not discard these
191                  * buffers, we would have no way to find them after the inode
192                  * is reaped and thus user could see stale data if he tries to
193                  * read them before the transaction is checkpointed. So be
194                  * careful and force everything to disk here... We use
195                  * ei->i_datasync_tid to store the newest transaction
196                  * containing inode's data.
197                  *
198                  * Note that directories do not have this problem because they
199                  * don't use page cache.
200                  */
201                 if (inode->i_ino != EXT4_JOURNAL_INO &&
202                     ext4_should_journal_data(inode) &&
203                     S_ISREG(inode->i_mode) && inode->i_data.nrpages) {
204                         journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
205                         tid_t commit_tid = EXT4_I(inode)->i_datasync_tid;
206
207                         jbd2_complete_transaction(journal, commit_tid);
208                         filemap_write_and_wait(&inode->i_data);
209                 }
210                 truncate_inode_pages_final(&inode->i_data);
211
212                 goto no_delete;
213         }
214
215         if (is_bad_inode(inode))
216                 goto no_delete;
217         dquot_initialize(inode);
218
219         if (ext4_should_order_data(inode))
220                 ext4_begin_ordered_truncate(inode, 0);
221         truncate_inode_pages_final(&inode->i_data);
222
223         /*
224          * For inodes with journalled data, transaction commit could have
225          * dirtied the inode. And for inodes with dioread_nolock, unwritten
226          * extents converting worker could merge extents and also have dirtied
227          * the inode. Flush worker is ignoring it because of I_FREEING flag but
228          * we still need to remove the inode from the writeback lists.
229          */
230         if (!list_empty_careful(&inode->i_io_list))
231                 inode_io_list_del(inode);
232
233         /*
234          * Protect us against freezing - iput() caller didn't have to have any
235          * protection against it. When we are in a running transaction though,
236          * we are already protected against freezing and we cannot grab further
237          * protection due to lock ordering constraints.
238          */
239         if (!ext4_journal_current_handle()) {
240                 sb_start_intwrite(inode->i_sb);
241                 freeze_protected = true;
242         }
243
244         if (!IS_NOQUOTA(inode))
245                 extra_credits += EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb);
246
247         /*
248          * Block bitmap, group descriptor, and inode are accounted in both
249          * ext4_blocks_for_truncate() and extra_credits. So subtract 3.
250          */
251         handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE,
252                          ext4_blocks_for_truncate(inode) + extra_credits - 3);
253         if (IS_ERR(handle)) {
254                 ext4_std_error(inode->i_sb, PTR_ERR(handle));
255                 /*
256                  * If we're going to skip the normal cleanup, we still need to
257                  * make sure that the in-core orphan linked list is properly
258                  * cleaned up.
259                  */
260                 ext4_orphan_del(NULL, inode);
261                 if (freeze_protected)
262                         sb_end_intwrite(inode->i_sb);
263                 goto no_delete;
264         }
265
266         if (IS_SYNC(inode))
267                 ext4_handle_sync(handle);
268
269         /*
270          * Set inode->i_size to 0 before calling ext4_truncate(). We need
271          * special handling of symlinks here because i_size is used to
272          * determine whether ext4_inode_info->i_data contains symlink data or
273          * block mappings. Setting i_size to 0 will remove its fast symlink
274          * status. Erase i_data so that it becomes a valid empty block map.
275          */
276         if (ext4_inode_is_fast_symlink(inode))
277                 memset(EXT4_I(inode)->i_data, 0, sizeof(EXT4_I(inode)->i_data));
278         inode->i_size = 0;
279         err = ext4_mark_inode_dirty(handle, inode);
280         if (err) {
281                 ext4_warning(inode->i_sb,
282                              "couldn't mark inode dirty (err %d)", err);
283                 goto stop_handle;
284         }
285         if (inode->i_blocks) {
286                 err = ext4_truncate(inode);
287                 if (err) {
288                         ext4_error_err(inode->i_sb, -err,
289                                        "couldn't truncate inode %lu (err %d)",
290                                        inode->i_ino, err);
291                         goto stop_handle;
292                 }
293         }
294
295         /* Remove xattr references. */
296         err = ext4_xattr_delete_inode(handle, inode, &ea_inode_array,
297                                       extra_credits);
298         if (err) {
299                 ext4_warning(inode->i_sb, "xattr delete (err %d)", err);
300 stop_handle:
301                 ext4_journal_stop(handle);
302                 ext4_orphan_del(NULL, inode);
303                 if (freeze_protected)
304                         sb_end_intwrite(inode->i_sb);
305                 ext4_xattr_inode_array_free(ea_inode_array);
306                 goto no_delete;
307         }
308
309         /*
310          * Kill off the orphan record which ext4_truncate created.
311          * AKPM: I think this can be inside the above `if'.
312          * Note that ext4_orphan_del() has to be able to cope with the
313          * deletion of a non-existent orphan - this is because we don't
314          * know if ext4_truncate() actually created an orphan record.
315          * (Well, we could do this if we need to, but heck - it works)
316          */
317         ext4_orphan_del(handle, inode);
318         EXT4_I(inode)->i_dtime  = (__u32)ktime_get_real_seconds();
319
320         /*
321          * One subtle ordering requirement: if anything has gone wrong
322          * (transaction abort, IO errors, whatever), then we can still
323          * do these next steps (the fs will already have been marked as
324          * having errors), but we can't free the inode if the mark_dirty
325          * fails.
326          */
327         if (ext4_mark_inode_dirty(handle, inode))
328                 /* If that failed, just do the required in-core inode clear. */
329                 ext4_clear_inode(inode);
330         else
331                 ext4_free_inode(handle, inode);
332         ext4_journal_stop(handle);
333         if (freeze_protected)
334                 sb_end_intwrite(inode->i_sb);
335         ext4_xattr_inode_array_free(ea_inode_array);
336         return;
337 no_delete:
338         /*
339          * Check out some where else accidentally dirty the evicting inode,
340          * which may probably cause inode use-after-free issues later.
341          */
342         WARN_ON_ONCE(!list_empty_careful(&inode->i_io_list));
343
344         if (!list_empty(&EXT4_I(inode)->i_fc_list))
345                 ext4_fc_mark_ineligible(inode->i_sb, EXT4_FC_REASON_NOMEM, NULL);
346         ext4_clear_inode(inode);        /* We must guarantee clearing of inode... */
347 }
348
349 #ifdef CONFIG_QUOTA
350 qsize_t *ext4_get_reserved_space(struct inode *inode)
351 {
352         return &EXT4_I(inode)->i_reserved_quota;
353 }
354 #endif
355
356 /*
357  * Called with i_data_sem down, which is important since we can call
358  * ext4_discard_preallocations() from here.
359  */
360 void ext4_da_update_reserve_space(struct inode *inode,
361                                         int used, int quota_claim)
362 {
363         struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
364         struct ext4_inode_info *ei = EXT4_I(inode);
365
366         spin_lock(&ei->i_block_reservation_lock);
367         trace_ext4_da_update_reserve_space(inode, used, quota_claim);
368         if (unlikely(used > ei->i_reserved_data_blocks)) {
369                 ext4_warning(inode->i_sb, "%s: ino %lu, used %d "
370                          "with only %d reserved data blocks",
371                          __func__, inode->i_ino, used,
372                          ei->i_reserved_data_blocks);
373                 WARN_ON(1);
374                 used = ei->i_reserved_data_blocks;
375         }
376
377         /* Update per-inode reservations */
378         ei->i_reserved_data_blocks -= used;
379         percpu_counter_sub(&sbi->s_dirtyclusters_counter, used);
380
381         spin_unlock(&ei->i_block_reservation_lock);
382
383         /* Update quota subsystem for data blocks */
384         if (quota_claim)
385                 dquot_claim_block(inode, EXT4_C2B(sbi, used));
386         else {
387                 /*
388                  * We did fallocate with an offset that is already delayed
389                  * allocated. So on delayed allocated writeback we should
390                  * not re-claim the quota for fallocated blocks.
391                  */
392                 dquot_release_reservation_block(inode, EXT4_C2B(sbi, used));
393         }
394
395         /*
396          * If we have done all the pending block allocations and if
397          * there aren't any writers on the inode, we can discard the
398          * inode's preallocations.
399          */
400         if ((ei->i_reserved_data_blocks == 0) &&
401             !inode_is_open_for_write(inode))
402                 ext4_discard_preallocations(inode, 0);
403 }
404
405 static int __check_block_validity(struct inode *inode, const char *func,
406                                 unsigned int line,
407                                 struct ext4_map_blocks *map)
408 {
409         if (ext4_has_feature_journal(inode->i_sb) &&
410             (inode->i_ino ==
411              le32_to_cpu(EXT4_SB(inode->i_sb)->s_es->s_journal_inum)))
412                 return 0;
413         if (!ext4_inode_block_valid(inode, map->m_pblk, map->m_len)) {
414                 ext4_error_inode(inode, func, line, map->m_pblk,
415                                  "lblock %lu mapped to illegal pblock %llu "
416                                  "(length %d)", (unsigned long) map->m_lblk,
417                                  map->m_pblk, map->m_len);
418                 return -EFSCORRUPTED;
419         }
420         return 0;
421 }
422
423 int ext4_issue_zeroout(struct inode *inode, ext4_lblk_t lblk, ext4_fsblk_t pblk,
424                        ext4_lblk_t len)
425 {
426         int ret;
427
428         if (IS_ENCRYPTED(inode) && S_ISREG(inode->i_mode))
429                 return fscrypt_zeroout_range(inode, lblk, pblk, len);
430
431         ret = sb_issue_zeroout(inode->i_sb, pblk, len, GFP_NOFS);
432         if (ret > 0)
433                 ret = 0;
434
435         return ret;
436 }
437
438 #define check_block_validity(inode, map)        \
439         __check_block_validity((inode), __func__, __LINE__, (map))
440
441 #ifdef ES_AGGRESSIVE_TEST
442 static void ext4_map_blocks_es_recheck(handle_t *handle,
443                                        struct inode *inode,
444                                        struct ext4_map_blocks *es_map,
445                                        struct ext4_map_blocks *map,
446                                        int flags)
447 {
448         int retval;
449
450         map->m_flags = 0;
451         /*
452          * There is a race window that the result is not the same.
453          * e.g. xfstests #223 when dioread_nolock enables.  The reason
454          * is that we lookup a block mapping in extent status tree with
455          * out taking i_data_sem.  So at the time the unwritten extent
456          * could be converted.
457          */
458         down_read(&EXT4_I(inode)->i_data_sem);
459         if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
460                 retval = ext4_ext_map_blocks(handle, inode, map, 0);
461         } else {
462                 retval = ext4_ind_map_blocks(handle, inode, map, 0);
463         }
464         up_read((&EXT4_I(inode)->i_data_sem));
465
466         /*
467          * We don't check m_len because extent will be collpased in status
468          * tree.  So the m_len might not equal.
469          */
470         if (es_map->m_lblk != map->m_lblk ||
471             es_map->m_flags != map->m_flags ||
472             es_map->m_pblk != map->m_pblk) {
473                 printk("ES cache assertion failed for inode: %lu "
474                        "es_cached ex [%d/%d/%llu/%x] != "
475                        "found ex [%d/%d/%llu/%x] retval %d flags %x\n",
476                        inode->i_ino, es_map->m_lblk, es_map->m_len,
477                        es_map->m_pblk, es_map->m_flags, map->m_lblk,
478                        map->m_len, map->m_pblk, map->m_flags,
479                        retval, flags);
480         }
481 }
482 #endif /* ES_AGGRESSIVE_TEST */
483
484 /*
485  * The ext4_map_blocks() function tries to look up the requested blocks,
486  * and returns if the blocks are already mapped.
487  *
488  * Otherwise it takes the write lock of the i_data_sem and allocate blocks
489  * and store the allocated blocks in the result buffer head and mark it
490  * mapped.
491  *
492  * If file type is extents based, it will call ext4_ext_map_blocks(),
493  * Otherwise, call with ext4_ind_map_blocks() to handle indirect mapping
494  * based files
495  *
496  * On success, it returns the number of blocks being mapped or allocated.  if
497  * create==0 and the blocks are pre-allocated and unwritten, the resulting @map
498  * is marked as unwritten. If the create == 1, it will mark @map as mapped.
499  *
500  * It returns 0 if plain look up failed (blocks have not been allocated), in
501  * that case, @map is returned as unmapped but we still do fill map->m_len to
502  * indicate the length of a hole starting at map->m_lblk.
503  *
504  * It returns the error in case of allocation failure.
505  */
506 int ext4_map_blocks(handle_t *handle, struct inode *inode,
507                     struct ext4_map_blocks *map, int flags)
508 {
509         struct extent_status es;
510         int retval;
511         int ret = 0;
512 #ifdef ES_AGGRESSIVE_TEST
513         struct ext4_map_blocks orig_map;
514
515         memcpy(&orig_map, map, sizeof(*map));
516 #endif
517
518         map->m_flags = 0;
519         ext_debug(inode, "flag 0x%x, max_blocks %u, logical block %lu\n",
520                   flags, map->m_len, (unsigned long) map->m_lblk);
521
522         /*
523          * ext4_map_blocks returns an int, and m_len is an unsigned int
524          */
525         if (unlikely(map->m_len > INT_MAX))
526                 map->m_len = INT_MAX;
527
528         /* We can handle the block number less than EXT_MAX_BLOCKS */
529         if (unlikely(map->m_lblk >= EXT_MAX_BLOCKS))
530                 return -EFSCORRUPTED;
531
532         /* Lookup extent status tree firstly */
533         if (!(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY) &&
534             ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es)) {
535                 if (ext4_es_is_written(&es) || ext4_es_is_unwritten(&es)) {
536                         map->m_pblk = ext4_es_pblock(&es) +
537                                         map->m_lblk - es.es_lblk;
538                         map->m_flags |= ext4_es_is_written(&es) ?
539                                         EXT4_MAP_MAPPED : EXT4_MAP_UNWRITTEN;
540                         retval = es.es_len - (map->m_lblk - es.es_lblk);
541                         if (retval > map->m_len)
542                                 retval = map->m_len;
543                         map->m_len = retval;
544                 } else if (ext4_es_is_delayed(&es) || ext4_es_is_hole(&es)) {
545                         map->m_pblk = 0;
546                         retval = es.es_len - (map->m_lblk - es.es_lblk);
547                         if (retval > map->m_len)
548                                 retval = map->m_len;
549                         map->m_len = retval;
550                         retval = 0;
551                 } else {
552                         BUG();
553                 }
554
555                 if (flags & EXT4_GET_BLOCKS_CACHED_NOWAIT)
556                         return retval;
557 #ifdef ES_AGGRESSIVE_TEST
558                 ext4_map_blocks_es_recheck(handle, inode, map,
559                                            &orig_map, flags);
560 #endif
561                 goto found;
562         }
563         /*
564          * In the query cache no-wait mode, nothing we can do more if we
565          * cannot find extent in the cache.
566          */
567         if (flags & EXT4_GET_BLOCKS_CACHED_NOWAIT)
568                 return 0;
569
570         /*
571          * Try to see if we can get the block without requesting a new
572          * file system block.
573          */
574         down_read(&EXT4_I(inode)->i_data_sem);
575         if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
576                 retval = ext4_ext_map_blocks(handle, inode, map, 0);
577         } else {
578                 retval = ext4_ind_map_blocks(handle, inode, map, 0);
579         }
580         if (retval > 0) {
581                 unsigned int status;
582
583                 if (unlikely(retval != map->m_len)) {
584                         ext4_warning(inode->i_sb,
585                                      "ES len assertion failed for inode "
586                                      "%lu: retval %d != map->m_len %d",
587                                      inode->i_ino, retval, map->m_len);
588                         WARN_ON(1);
589                 }
590
591                 status = map->m_flags & EXT4_MAP_UNWRITTEN ?
592                                 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
593                 if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) &&
594                     !(status & EXTENT_STATUS_WRITTEN) &&
595                     ext4_es_scan_range(inode, &ext4_es_is_delayed, map->m_lblk,
596                                        map->m_lblk + map->m_len - 1))
597                         status |= EXTENT_STATUS_DELAYED;
598                 ret = ext4_es_insert_extent(inode, map->m_lblk,
599                                             map->m_len, map->m_pblk, status);
600                 if (ret < 0)
601                         retval = ret;
602         }
603         up_read((&EXT4_I(inode)->i_data_sem));
604
605 found:
606         if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
607                 ret = check_block_validity(inode, map);
608                 if (ret != 0)
609                         return ret;
610         }
611
612         /* If it is only a block(s) look up */
613         if ((flags & EXT4_GET_BLOCKS_CREATE) == 0)
614                 return retval;
615
616         /*
617          * Returns if the blocks have already allocated
618          *
619          * Note that if blocks have been preallocated
620          * ext4_ext_get_block() returns the create = 0
621          * with buffer head unmapped.
622          */
623         if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED)
624                 /*
625                  * If we need to convert extent to unwritten
626                  * we continue and do the actual work in
627                  * ext4_ext_map_blocks()
628                  */
629                 if (!(flags & EXT4_GET_BLOCKS_CONVERT_UNWRITTEN))
630                         return retval;
631
632         /*
633          * Here we clear m_flags because after allocating an new extent,
634          * it will be set again.
635          */
636         map->m_flags &= ~EXT4_MAP_FLAGS;
637
638         /*
639          * New blocks allocate and/or writing to unwritten extent
640          * will possibly result in updating i_data, so we take
641          * the write lock of i_data_sem, and call get_block()
642          * with create == 1 flag.
643          */
644         down_write(&EXT4_I(inode)->i_data_sem);
645
646         /*
647          * We need to check for EXT4 here because migrate
648          * could have changed the inode type in between
649          */
650         if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)) {
651                 retval = ext4_ext_map_blocks(handle, inode, map, flags);
652         } else {
653                 retval = ext4_ind_map_blocks(handle, inode, map, flags);
654
655                 if (retval > 0 && map->m_flags & EXT4_MAP_NEW) {
656                         /*
657                          * We allocated new blocks which will result in
658                          * i_data's format changing.  Force the migrate
659                          * to fail by clearing migrate flags
660                          */
661                         ext4_clear_inode_state(inode, EXT4_STATE_EXT_MIGRATE);
662                 }
663         }
664
665         if (retval > 0) {
666                 unsigned int status;
667
668                 if (unlikely(retval != map->m_len)) {
669                         ext4_warning(inode->i_sb,
670                                      "ES len assertion failed for inode "
671                                      "%lu: retval %d != map->m_len %d",
672                                      inode->i_ino, retval, map->m_len);
673                         WARN_ON(1);
674                 }
675
676                 /*
677                  * We have to zeroout blocks before inserting them into extent
678                  * status tree. Otherwise someone could look them up there and
679                  * use them before they are really zeroed. We also have to
680                  * unmap metadata before zeroing as otherwise writeback can
681                  * overwrite zeros with stale data from block device.
682                  */
683                 if (flags & EXT4_GET_BLOCKS_ZERO &&
684                     map->m_flags & EXT4_MAP_MAPPED &&
685                     map->m_flags & EXT4_MAP_NEW) {
686                         ret = ext4_issue_zeroout(inode, map->m_lblk,
687                                                  map->m_pblk, map->m_len);
688                         if (ret) {
689                                 retval = ret;
690                                 goto out_sem;
691                         }
692                 }
693
694                 /*
695                  * If the extent has been zeroed out, we don't need to update
696                  * extent status tree.
697                  */
698                 if ((flags & EXT4_GET_BLOCKS_PRE_IO) &&
699                     ext4_es_lookup_extent(inode, map->m_lblk, NULL, &es)) {
700                         if (ext4_es_is_written(&es))
701                                 goto out_sem;
702                 }
703                 status = map->m_flags & EXT4_MAP_UNWRITTEN ?
704                                 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
705                 if (!(flags & EXT4_GET_BLOCKS_DELALLOC_RESERVE) &&
706                     !(status & EXTENT_STATUS_WRITTEN) &&
707                     ext4_es_scan_range(inode, &ext4_es_is_delayed, map->m_lblk,
708                                        map->m_lblk + map->m_len - 1))
709                         status |= EXTENT_STATUS_DELAYED;
710                 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
711                                             map->m_pblk, status);
712                 if (ret < 0) {
713                         retval = ret;
714                         goto out_sem;
715                 }
716         }
717
718 out_sem:
719         up_write((&EXT4_I(inode)->i_data_sem));
720         if (retval > 0 && map->m_flags & EXT4_MAP_MAPPED) {
721                 ret = check_block_validity(inode, map);
722                 if (ret != 0)
723                         return ret;
724
725                 /*
726                  * Inodes with freshly allocated blocks where contents will be
727                  * visible after transaction commit must be on transaction's
728                  * ordered data list.
729                  */
730                 if (map->m_flags & EXT4_MAP_NEW &&
731                     !(map->m_flags & EXT4_MAP_UNWRITTEN) &&
732                     !(flags & EXT4_GET_BLOCKS_ZERO) &&
733                     !ext4_is_quota_file(inode) &&
734                     ext4_should_order_data(inode)) {
735                         loff_t start_byte =
736                                 (loff_t)map->m_lblk << inode->i_blkbits;
737                         loff_t length = (loff_t)map->m_len << inode->i_blkbits;
738
739                         if (flags & EXT4_GET_BLOCKS_IO_SUBMIT)
740                                 ret = ext4_jbd2_inode_add_wait(handle, inode,
741                                                 start_byte, length);
742                         else
743                                 ret = ext4_jbd2_inode_add_write(handle, inode,
744                                                 start_byte, length);
745                         if (ret)
746                                 return ret;
747                 }
748         }
749         if (retval > 0 && (map->m_flags & EXT4_MAP_UNWRITTEN ||
750                                 map->m_flags & EXT4_MAP_MAPPED))
751                 ext4_fc_track_range(handle, inode, map->m_lblk,
752                                         map->m_lblk + map->m_len - 1);
753         if (retval < 0)
754                 ext_debug(inode, "failed with err %d\n", retval);
755         return retval;
756 }
757
758 /*
759  * Update EXT4_MAP_FLAGS in bh->b_state. For buffer heads attached to pages
760  * we have to be careful as someone else may be manipulating b_state as well.
761  */
762 static void ext4_update_bh_state(struct buffer_head *bh, unsigned long flags)
763 {
764         unsigned long old_state;
765         unsigned long new_state;
766
767         flags &= EXT4_MAP_FLAGS;
768
769         /* Dummy buffer_head? Set non-atomically. */
770         if (!bh->b_page) {
771                 bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | flags;
772                 return;
773         }
774         /*
775          * Someone else may be modifying b_state. Be careful! This is ugly but
776          * once we get rid of using bh as a container for mapping information
777          * to pass to / from get_block functions, this can go away.
778          */
779         do {
780                 old_state = READ_ONCE(bh->b_state);
781                 new_state = (old_state & ~EXT4_MAP_FLAGS) | flags;
782         } while (unlikely(
783                  cmpxchg(&bh->b_state, old_state, new_state) != old_state));
784 }
785
786 static int _ext4_get_block(struct inode *inode, sector_t iblock,
787                            struct buffer_head *bh, int flags)
788 {
789         struct ext4_map_blocks map;
790         int ret = 0;
791
792         if (ext4_has_inline_data(inode))
793                 return -ERANGE;
794
795         map.m_lblk = iblock;
796         map.m_len = bh->b_size >> inode->i_blkbits;
797
798         ret = ext4_map_blocks(ext4_journal_current_handle(), inode, &map,
799                               flags);
800         if (ret > 0) {
801                 map_bh(bh, inode->i_sb, map.m_pblk);
802                 ext4_update_bh_state(bh, map.m_flags);
803                 bh->b_size = inode->i_sb->s_blocksize * map.m_len;
804                 ret = 0;
805         } else if (ret == 0) {
806                 /* hole case, need to fill in bh->b_size */
807                 bh->b_size = inode->i_sb->s_blocksize * map.m_len;
808         }
809         return ret;
810 }
811
812 int ext4_get_block(struct inode *inode, sector_t iblock,
813                    struct buffer_head *bh, int create)
814 {
815         return _ext4_get_block(inode, iblock, bh,
816                                create ? EXT4_GET_BLOCKS_CREATE : 0);
817 }
818
819 /*
820  * Get block function used when preparing for buffered write if we require
821  * creating an unwritten extent if blocks haven't been allocated.  The extent
822  * will be converted to written after the IO is complete.
823  */
824 int ext4_get_block_unwritten(struct inode *inode, sector_t iblock,
825                              struct buffer_head *bh_result, int create)
826 {
827         int ret = 0;
828
829         ext4_debug("ext4_get_block_unwritten: inode %lu, create flag %d\n",
830                    inode->i_ino, create);
831         ret = _ext4_get_block(inode, iblock, bh_result,
832                                EXT4_GET_BLOCKS_CREATE_UNWRIT_EXT);
833
834         /*
835          * If the buffer is marked unwritten, mark it as new to make sure it is
836          * zeroed out correctly in case of partial writes. Otherwise, there is
837          * a chance of stale data getting exposed.
838          */
839         if (ret == 0 && buffer_unwritten(bh_result))
840                 set_buffer_new(bh_result);
841
842         return ret;
843 }
844
845 /* Maximum number of blocks we map for direct IO at once. */
846 #define DIO_MAX_BLOCKS 4096
847
848 /*
849  * `handle' can be NULL if create is zero
850  */
851 struct buffer_head *ext4_getblk(handle_t *handle, struct inode *inode,
852                                 ext4_lblk_t block, int map_flags)
853 {
854         struct ext4_map_blocks map;
855         struct buffer_head *bh;
856         int create = map_flags & EXT4_GET_BLOCKS_CREATE;
857         bool nowait = map_flags & EXT4_GET_BLOCKS_CACHED_NOWAIT;
858         int err;
859
860         ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
861                     || handle != NULL || create == 0);
862         ASSERT(create == 0 || !nowait);
863
864         map.m_lblk = block;
865         map.m_len = 1;
866         err = ext4_map_blocks(handle, inode, &map, map_flags);
867
868         if (err == 0)
869                 return create ? ERR_PTR(-ENOSPC) : NULL;
870         if (err < 0)
871                 return ERR_PTR(err);
872
873         if (nowait)
874                 return sb_find_get_block(inode->i_sb, map.m_pblk);
875
876         bh = sb_getblk(inode->i_sb, map.m_pblk);
877         if (unlikely(!bh))
878                 return ERR_PTR(-ENOMEM);
879         if (map.m_flags & EXT4_MAP_NEW) {
880                 ASSERT(create != 0);
881                 ASSERT((EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)
882                             || (handle != NULL));
883
884                 /*
885                  * Now that we do not always journal data, we should
886                  * keep in mind whether this should always journal the
887                  * new buffer as metadata.  For now, regular file
888                  * writes use ext4_get_block instead, so it's not a
889                  * problem.
890                  */
891                 lock_buffer(bh);
892                 BUFFER_TRACE(bh, "call get_create_access");
893                 err = ext4_journal_get_create_access(handle, inode->i_sb, bh,
894                                                      EXT4_JTR_NONE);
895                 if (unlikely(err)) {
896                         unlock_buffer(bh);
897                         goto errout;
898                 }
899                 if (!buffer_uptodate(bh)) {
900                         memset(bh->b_data, 0, inode->i_sb->s_blocksize);
901                         set_buffer_uptodate(bh);
902                 }
903                 unlock_buffer(bh);
904                 BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
905                 err = ext4_handle_dirty_metadata(handle, inode, bh);
906                 if (unlikely(err))
907                         goto errout;
908         } else
909                 BUFFER_TRACE(bh, "not a new buffer");
910         return bh;
911 errout:
912         brelse(bh);
913         return ERR_PTR(err);
914 }
915
916 struct buffer_head *ext4_bread(handle_t *handle, struct inode *inode,
917                                ext4_lblk_t block, int map_flags)
918 {
919         struct buffer_head *bh;
920         int ret;
921
922         bh = ext4_getblk(handle, inode, block, map_flags);
923         if (IS_ERR(bh))
924                 return bh;
925         if (!bh || ext4_buffer_uptodate(bh))
926                 return bh;
927
928         ret = ext4_read_bh_lock(bh, REQ_META | REQ_PRIO, true);
929         if (ret) {
930                 put_bh(bh);
931                 return ERR_PTR(ret);
932         }
933         return bh;
934 }
935
936 /* Read a contiguous batch of blocks. */
937 int ext4_bread_batch(struct inode *inode, ext4_lblk_t block, int bh_count,
938                      bool wait, struct buffer_head **bhs)
939 {
940         int i, err;
941
942         for (i = 0; i < bh_count; i++) {
943                 bhs[i] = ext4_getblk(NULL, inode, block + i, 0 /* map_flags */);
944                 if (IS_ERR(bhs[i])) {
945                         err = PTR_ERR(bhs[i]);
946                         bh_count = i;
947                         goto out_brelse;
948                 }
949         }
950
951         for (i = 0; i < bh_count; i++)
952                 /* Note that NULL bhs[i] is valid because of holes. */
953                 if (bhs[i] && !ext4_buffer_uptodate(bhs[i]))
954                         ext4_read_bh_lock(bhs[i], REQ_META | REQ_PRIO, false);
955
956         if (!wait)
957                 return 0;
958
959         for (i = 0; i < bh_count; i++)
960                 if (bhs[i])
961                         wait_on_buffer(bhs[i]);
962
963         for (i = 0; i < bh_count; i++) {
964                 if (bhs[i] && !buffer_uptodate(bhs[i])) {
965                         err = -EIO;
966                         goto out_brelse;
967                 }
968         }
969         return 0;
970
971 out_brelse:
972         for (i = 0; i < bh_count; i++) {
973                 brelse(bhs[i]);
974                 bhs[i] = NULL;
975         }
976         return err;
977 }
978
979 int ext4_walk_page_buffers(handle_t *handle, struct inode *inode,
980                            struct buffer_head *head,
981                            unsigned from,
982                            unsigned to,
983                            int *partial,
984                            int (*fn)(handle_t *handle, struct inode *inode,
985                                      struct buffer_head *bh))
986 {
987         struct buffer_head *bh;
988         unsigned block_start, block_end;
989         unsigned blocksize = head->b_size;
990         int err, ret = 0;
991         struct buffer_head *next;
992
993         for (bh = head, block_start = 0;
994              ret == 0 && (bh != head || !block_start);
995              block_start = block_end, bh = next) {
996                 next = bh->b_this_page;
997                 block_end = block_start + blocksize;
998                 if (block_end <= from || block_start >= to) {
999                         if (partial && !buffer_uptodate(bh))
1000                                 *partial = 1;
1001                         continue;
1002                 }
1003                 err = (*fn)(handle, inode, bh);
1004                 if (!ret)
1005                         ret = err;
1006         }
1007         return ret;
1008 }
1009
1010 /*
1011  * To preserve ordering, it is essential that the hole instantiation and
1012  * the data write be encapsulated in a single transaction.  We cannot
1013  * close off a transaction and start a new one between the ext4_get_block()
1014  * and the commit_write().  So doing the jbd2_journal_start at the start of
1015  * prepare_write() is the right place.
1016  *
1017  * Also, this function can nest inside ext4_writepage().  In that case, we
1018  * *know* that ext4_writepage() has generated enough buffer credits to do the
1019  * whole page.  So we won't block on the journal in that case, which is good,
1020  * because the caller may be PF_MEMALLOC.
1021  *
1022  * By accident, ext4 can be reentered when a transaction is open via
1023  * quota file writes.  If we were to commit the transaction while thus
1024  * reentered, there can be a deadlock - we would be holding a quota
1025  * lock, and the commit would never complete if another thread had a
1026  * transaction open and was blocking on the quota lock - a ranking
1027  * violation.
1028  *
1029  * So what we do is to rely on the fact that jbd2_journal_stop/journal_start
1030  * will _not_ run commit under these circumstances because handle->h_ref
1031  * is elevated.  We'll still have enough credits for the tiny quotafile
1032  * write.
1033  */
1034 int do_journal_get_write_access(handle_t *handle, struct inode *inode,
1035                                 struct buffer_head *bh)
1036 {
1037         int dirty = buffer_dirty(bh);
1038         int ret;
1039
1040         if (!buffer_mapped(bh) || buffer_freed(bh))
1041                 return 0;
1042         /*
1043          * __block_write_begin() could have dirtied some buffers. Clean
1044          * the dirty bit as jbd2_journal_get_write_access() could complain
1045          * otherwise about fs integrity issues. Setting of the dirty bit
1046          * by __block_write_begin() isn't a real problem here as we clear
1047          * the bit before releasing a page lock and thus writeback cannot
1048          * ever write the buffer.
1049          */
1050         if (dirty)
1051                 clear_buffer_dirty(bh);
1052         BUFFER_TRACE(bh, "get write access");
1053         ret = ext4_journal_get_write_access(handle, inode->i_sb, bh,
1054                                             EXT4_JTR_NONE);
1055         if (!ret && dirty)
1056                 ret = ext4_handle_dirty_metadata(handle, NULL, bh);
1057         return ret;
1058 }
1059
1060 #ifdef CONFIG_FS_ENCRYPTION
1061 static int ext4_block_write_begin(struct page *page, loff_t pos, unsigned len,
1062                                   get_block_t *get_block)
1063 {
1064         unsigned from = pos & (PAGE_SIZE - 1);
1065         unsigned to = from + len;
1066         struct inode *inode = page->mapping->host;
1067         unsigned block_start, block_end;
1068         sector_t block;
1069         int err = 0;
1070         unsigned blocksize = inode->i_sb->s_blocksize;
1071         unsigned bbits;
1072         struct buffer_head *bh, *head, *wait[2];
1073         int nr_wait = 0;
1074         int i;
1075
1076         BUG_ON(!PageLocked(page));
1077         BUG_ON(from > PAGE_SIZE);
1078         BUG_ON(to > PAGE_SIZE);
1079         BUG_ON(from > to);
1080
1081         if (!page_has_buffers(page))
1082                 create_empty_buffers(page, blocksize, 0);
1083         head = page_buffers(page);
1084         bbits = ilog2(blocksize);
1085         block = (sector_t)page->index << (PAGE_SHIFT - bbits);
1086
1087         for (bh = head, block_start = 0; bh != head || !block_start;
1088             block++, block_start = block_end, bh = bh->b_this_page) {
1089                 block_end = block_start + blocksize;
1090                 if (block_end <= from || block_start >= to) {
1091                         if (PageUptodate(page)) {
1092                                 set_buffer_uptodate(bh);
1093                         }
1094                         continue;
1095                 }
1096                 if (buffer_new(bh))
1097                         clear_buffer_new(bh);
1098                 if (!buffer_mapped(bh)) {
1099                         WARN_ON(bh->b_size != blocksize);
1100                         err = get_block(inode, block, bh, 1);
1101                         if (err)
1102                                 break;
1103                         if (buffer_new(bh)) {
1104                                 if (PageUptodate(page)) {
1105                                         clear_buffer_new(bh);
1106                                         set_buffer_uptodate(bh);
1107                                         mark_buffer_dirty(bh);
1108                                         continue;
1109                                 }
1110                                 if (block_end > to || block_start < from)
1111                                         zero_user_segments(page, to, block_end,
1112                                                            block_start, from);
1113                                 continue;
1114                         }
1115                 }
1116                 if (PageUptodate(page)) {
1117                         set_buffer_uptodate(bh);
1118                         continue;
1119                 }
1120                 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1121                     !buffer_unwritten(bh) &&
1122                     (block_start < from || block_end > to)) {
1123                         ext4_read_bh_lock(bh, 0, false);
1124                         wait[nr_wait++] = bh;
1125                 }
1126         }
1127         /*
1128          * If we issued read requests, let them complete.
1129          */
1130         for (i = 0; i < nr_wait; i++) {
1131                 wait_on_buffer(wait[i]);
1132                 if (!buffer_uptodate(wait[i]))
1133                         err = -EIO;
1134         }
1135         if (unlikely(err)) {
1136                 page_zero_new_buffers(page, from, to);
1137         } else if (fscrypt_inode_uses_fs_layer_crypto(inode)) {
1138                 for (i = 0; i < nr_wait; i++) {
1139                         int err2;
1140
1141                         err2 = fscrypt_decrypt_pagecache_blocks(page, blocksize,
1142                                                                 bh_offset(wait[i]));
1143                         if (err2) {
1144                                 clear_buffer_uptodate(wait[i]);
1145                                 err = err2;
1146                         }
1147                 }
1148         }
1149
1150         return err;
1151 }
1152 #endif
1153
1154 static int ext4_write_begin(struct file *file, struct address_space *mapping,
1155                             loff_t pos, unsigned len,
1156                             struct page **pagep, void **fsdata)
1157 {
1158         struct inode *inode = mapping->host;
1159         int ret, needed_blocks;
1160         handle_t *handle;
1161         int retries = 0;
1162         struct page *page;
1163         pgoff_t index;
1164         unsigned from, to;
1165
1166         if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
1167                 return -EIO;
1168
1169         trace_ext4_write_begin(inode, pos, len);
1170         /*
1171          * Reserve one block more for addition to orphan list in case
1172          * we allocate blocks but write fails for some reason
1173          */
1174         needed_blocks = ext4_writepage_trans_blocks(inode) + 1;
1175         index = pos >> PAGE_SHIFT;
1176         from = pos & (PAGE_SIZE - 1);
1177         to = from + len;
1178
1179         if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
1180                 ret = ext4_try_to_write_inline_data(mapping, inode, pos, len,
1181                                                     pagep);
1182                 if (ret < 0)
1183                         return ret;
1184                 if (ret == 1)
1185                         return 0;
1186         }
1187
1188         /*
1189          * grab_cache_page_write_begin() can take a long time if the
1190          * system is thrashing due to memory pressure, or if the page
1191          * is being written back.  So grab it first before we start
1192          * the transaction handle.  This also allows us to allocate
1193          * the page (if needed) without using GFP_NOFS.
1194          */
1195 retry_grab:
1196         page = grab_cache_page_write_begin(mapping, index);
1197         if (!page)
1198                 return -ENOMEM;
1199         /*
1200          * The same as page allocation, we prealloc buffer heads before
1201          * starting the handle.
1202          */
1203         if (!page_has_buffers(page))
1204                 create_empty_buffers(page, inode->i_sb->s_blocksize, 0);
1205
1206         unlock_page(page);
1207
1208 retry_journal:
1209         handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE, needed_blocks);
1210         if (IS_ERR(handle)) {
1211                 put_page(page);
1212                 return PTR_ERR(handle);
1213         }
1214
1215         lock_page(page);
1216         if (page->mapping != mapping) {
1217                 /* The page got truncated from under us */
1218                 unlock_page(page);
1219                 put_page(page);
1220                 ext4_journal_stop(handle);
1221                 goto retry_grab;
1222         }
1223         /* In case writeback began while the page was unlocked */
1224         wait_for_stable_page(page);
1225
1226 #ifdef CONFIG_FS_ENCRYPTION
1227         if (ext4_should_dioread_nolock(inode))
1228                 ret = ext4_block_write_begin(page, pos, len,
1229                                              ext4_get_block_unwritten);
1230         else
1231                 ret = ext4_block_write_begin(page, pos, len,
1232                                              ext4_get_block);
1233 #else
1234         if (ext4_should_dioread_nolock(inode))
1235                 ret = __block_write_begin(page, pos, len,
1236                                           ext4_get_block_unwritten);
1237         else
1238                 ret = __block_write_begin(page, pos, len, ext4_get_block);
1239 #endif
1240         if (!ret && ext4_should_journal_data(inode)) {
1241                 ret = ext4_walk_page_buffers(handle, inode,
1242                                              page_buffers(page), from, to, NULL,
1243                                              do_journal_get_write_access);
1244         }
1245
1246         if (ret) {
1247                 bool extended = (pos + len > inode->i_size) &&
1248                                 !ext4_verity_in_progress(inode);
1249
1250                 unlock_page(page);
1251                 /*
1252                  * __block_write_begin may have instantiated a few blocks
1253                  * outside i_size.  Trim these off again. Don't need
1254                  * i_size_read because we hold i_rwsem.
1255                  *
1256                  * Add inode to orphan list in case we crash before
1257                  * truncate finishes
1258                  */
1259                 if (extended && ext4_can_truncate(inode))
1260                         ext4_orphan_add(handle, inode);
1261
1262                 ext4_journal_stop(handle);
1263                 if (extended) {
1264                         ext4_truncate_failed_write(inode);
1265                         /*
1266                          * If truncate failed early the inode might
1267                          * still be on the orphan list; we need to
1268                          * make sure the inode is removed from the
1269                          * orphan list in that case.
1270                          */
1271                         if (inode->i_nlink)
1272                                 ext4_orphan_del(NULL, inode);
1273                 }
1274
1275                 if (ret == -ENOSPC &&
1276                     ext4_should_retry_alloc(inode->i_sb, &retries))
1277                         goto retry_journal;
1278                 put_page(page);
1279                 return ret;
1280         }
1281         *pagep = page;
1282         return ret;
1283 }
1284
1285 /* For write_end() in data=journal mode */
1286 static int write_end_fn(handle_t *handle, struct inode *inode,
1287                         struct buffer_head *bh)
1288 {
1289         int ret;
1290         if (!buffer_mapped(bh) || buffer_freed(bh))
1291                 return 0;
1292         set_buffer_uptodate(bh);
1293         ret = ext4_handle_dirty_metadata(handle, NULL, bh);
1294         clear_buffer_meta(bh);
1295         clear_buffer_prio(bh);
1296         return ret;
1297 }
1298
1299 /*
1300  * We need to pick up the new inode size which generic_commit_write gave us
1301  * `file' can be NULL - eg, when called from page_symlink().
1302  *
1303  * ext4 never places buffers on inode->i_mapping->private_list.  metadata
1304  * buffers are managed internally.
1305  */
1306 static int ext4_write_end(struct file *file,
1307                           struct address_space *mapping,
1308                           loff_t pos, unsigned len, unsigned copied,
1309                           struct page *page, void *fsdata)
1310 {
1311         handle_t *handle = ext4_journal_current_handle();
1312         struct inode *inode = mapping->host;
1313         loff_t old_size = inode->i_size;
1314         int ret = 0, ret2;
1315         int i_size_changed = 0;
1316         bool verity = ext4_verity_in_progress(inode);
1317
1318         trace_ext4_write_end(inode, pos, len, copied);
1319
1320         if (ext4_has_inline_data(inode) &&
1321             ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA))
1322                 return ext4_write_inline_data_end(inode, pos, len, copied, page);
1323
1324         copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
1325         /*
1326          * it's important to update i_size while still holding page lock:
1327          * page writeout could otherwise come in and zero beyond i_size.
1328          *
1329          * If FS_IOC_ENABLE_VERITY is running on this inode, then Merkle tree
1330          * blocks are being written past EOF, so skip the i_size update.
1331          */
1332         if (!verity)
1333                 i_size_changed = ext4_update_inode_size(inode, pos + copied);
1334         unlock_page(page);
1335         put_page(page);
1336
1337         if (old_size < pos && !verity)
1338                 pagecache_isize_extended(inode, old_size, pos);
1339         /*
1340          * Don't mark the inode dirty under page lock. First, it unnecessarily
1341          * makes the holding time of page lock longer. Second, it forces lock
1342          * ordering of page lock and transaction start for journaling
1343          * filesystems.
1344          */
1345         if (i_size_changed)
1346                 ret = ext4_mark_inode_dirty(handle, inode);
1347
1348         if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode))
1349                 /* if we have allocated more blocks and copied
1350                  * less. We will have blocks allocated outside
1351                  * inode->i_size. So truncate them
1352                  */
1353                 ext4_orphan_add(handle, inode);
1354
1355         ret2 = ext4_journal_stop(handle);
1356         if (!ret)
1357                 ret = ret2;
1358
1359         if (pos + len > inode->i_size && !verity) {
1360                 ext4_truncate_failed_write(inode);
1361                 /*
1362                  * If truncate failed early the inode might still be
1363                  * on the orphan list; we need to make sure the inode
1364                  * is removed from the orphan list in that case.
1365                  */
1366                 if (inode->i_nlink)
1367                         ext4_orphan_del(NULL, inode);
1368         }
1369
1370         return ret ? ret : copied;
1371 }
1372
1373 /*
1374  * This is a private version of page_zero_new_buffers() which doesn't
1375  * set the buffer to be dirty, since in data=journalled mode we need
1376  * to call ext4_handle_dirty_metadata() instead.
1377  */
1378 static void ext4_journalled_zero_new_buffers(handle_t *handle,
1379                                             struct inode *inode,
1380                                             struct page *page,
1381                                             unsigned from, unsigned to)
1382 {
1383         unsigned int block_start = 0, block_end;
1384         struct buffer_head *head, *bh;
1385
1386         bh = head = page_buffers(page);
1387         do {
1388                 block_end = block_start + bh->b_size;
1389                 if (buffer_new(bh)) {
1390                         if (block_end > from && block_start < to) {
1391                                 if (!PageUptodate(page)) {
1392                                         unsigned start, size;
1393
1394                                         start = max(from, block_start);
1395                                         size = min(to, block_end) - start;
1396
1397                                         zero_user(page, start, size);
1398                                         write_end_fn(handle, inode, bh);
1399                                 }
1400                                 clear_buffer_new(bh);
1401                         }
1402                 }
1403                 block_start = block_end;
1404                 bh = bh->b_this_page;
1405         } while (bh != head);
1406 }
1407
1408 static int ext4_journalled_write_end(struct file *file,
1409                                      struct address_space *mapping,
1410                                      loff_t pos, unsigned len, unsigned copied,
1411                                      struct page *page, void *fsdata)
1412 {
1413         handle_t *handle = ext4_journal_current_handle();
1414         struct inode *inode = mapping->host;
1415         loff_t old_size = inode->i_size;
1416         int ret = 0, ret2;
1417         int partial = 0;
1418         unsigned from, to;
1419         int size_changed = 0;
1420         bool verity = ext4_verity_in_progress(inode);
1421
1422         trace_ext4_journalled_write_end(inode, pos, len, copied);
1423         from = pos & (PAGE_SIZE - 1);
1424         to = from + len;
1425
1426         BUG_ON(!ext4_handle_valid(handle));
1427
1428         if (ext4_has_inline_data(inode))
1429                 return ext4_write_inline_data_end(inode, pos, len, copied, page);
1430
1431         if (unlikely(copied < len) && !PageUptodate(page)) {
1432                 copied = 0;
1433                 ext4_journalled_zero_new_buffers(handle, inode, page, from, to);
1434         } else {
1435                 if (unlikely(copied < len))
1436                         ext4_journalled_zero_new_buffers(handle, inode, page,
1437                                                          from + copied, to);
1438                 ret = ext4_walk_page_buffers(handle, inode, page_buffers(page),
1439                                              from, from + copied, &partial,
1440                                              write_end_fn);
1441                 if (!partial)
1442                         SetPageUptodate(page);
1443         }
1444         if (!verity)
1445                 size_changed = ext4_update_inode_size(inode, pos + copied);
1446         ext4_set_inode_state(inode, EXT4_STATE_JDATA);
1447         EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
1448         unlock_page(page);
1449         put_page(page);
1450
1451         if (old_size < pos && !verity)
1452                 pagecache_isize_extended(inode, old_size, pos);
1453
1454         if (size_changed) {
1455                 ret2 = ext4_mark_inode_dirty(handle, inode);
1456                 if (!ret)
1457                         ret = ret2;
1458         }
1459
1460         if (pos + len > inode->i_size && !verity && ext4_can_truncate(inode))
1461                 /* if we have allocated more blocks and copied
1462                  * less. We will have blocks allocated outside
1463                  * inode->i_size. So truncate them
1464                  */
1465                 ext4_orphan_add(handle, inode);
1466
1467         ret2 = ext4_journal_stop(handle);
1468         if (!ret)
1469                 ret = ret2;
1470         if (pos + len > inode->i_size && !verity) {
1471                 ext4_truncate_failed_write(inode);
1472                 /*
1473                  * If truncate failed early the inode might still be
1474                  * on the orphan list; we need to make sure the inode
1475                  * is removed from the orphan list in that case.
1476                  */
1477                 if (inode->i_nlink)
1478                         ext4_orphan_del(NULL, inode);
1479         }
1480
1481         return ret ? ret : copied;
1482 }
1483
1484 /*
1485  * Reserve space for a single cluster
1486  */
1487 static int ext4_da_reserve_space(struct inode *inode)
1488 {
1489         struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1490         struct ext4_inode_info *ei = EXT4_I(inode);
1491         int ret;
1492
1493         /*
1494          * We will charge metadata quota at writeout time; this saves
1495          * us from metadata over-estimation, though we may go over by
1496          * a small amount in the end.  Here we just reserve for data.
1497          */
1498         ret = dquot_reserve_block(inode, EXT4_C2B(sbi, 1));
1499         if (ret)
1500                 return ret;
1501
1502         spin_lock(&ei->i_block_reservation_lock);
1503         if (ext4_claim_free_clusters(sbi, 1, 0)) {
1504                 spin_unlock(&ei->i_block_reservation_lock);
1505                 dquot_release_reservation_block(inode, EXT4_C2B(sbi, 1));
1506                 return -ENOSPC;
1507         }
1508         ei->i_reserved_data_blocks++;
1509         trace_ext4_da_reserve_space(inode);
1510         spin_unlock(&ei->i_block_reservation_lock);
1511
1512         return 0;       /* success */
1513 }
1514
1515 void ext4_da_release_space(struct inode *inode, int to_free)
1516 {
1517         struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1518         struct ext4_inode_info *ei = EXT4_I(inode);
1519
1520         if (!to_free)
1521                 return;         /* Nothing to release, exit */
1522
1523         spin_lock(&EXT4_I(inode)->i_block_reservation_lock);
1524
1525         trace_ext4_da_release_space(inode, to_free);
1526         if (unlikely(to_free > ei->i_reserved_data_blocks)) {
1527                 /*
1528                  * if there aren't enough reserved blocks, then the
1529                  * counter is messed up somewhere.  Since this
1530                  * function is called from invalidate page, it's
1531                  * harmless to return without any action.
1532                  */
1533                 ext4_warning(inode->i_sb, "ext4_da_release_space: "
1534                          "ino %lu, to_free %d with only %d reserved "
1535                          "data blocks", inode->i_ino, to_free,
1536                          ei->i_reserved_data_blocks);
1537                 WARN_ON(1);
1538                 to_free = ei->i_reserved_data_blocks;
1539         }
1540         ei->i_reserved_data_blocks -= to_free;
1541
1542         /* update fs dirty data blocks counter */
1543         percpu_counter_sub(&sbi->s_dirtyclusters_counter, to_free);
1544
1545         spin_unlock(&EXT4_I(inode)->i_block_reservation_lock);
1546
1547         dquot_release_reservation_block(inode, EXT4_C2B(sbi, to_free));
1548 }
1549
1550 /*
1551  * Delayed allocation stuff
1552  */
1553
1554 struct mpage_da_data {
1555         struct inode *inode;
1556         struct writeback_control *wbc;
1557
1558         pgoff_t first_page;     /* The first page to write */
1559         pgoff_t next_page;      /* Current page to examine */
1560         pgoff_t last_page;      /* Last page to examine */
1561         /*
1562          * Extent to map - this can be after first_page because that can be
1563          * fully mapped. We somewhat abuse m_flags to store whether the extent
1564          * is delalloc or unwritten.
1565          */
1566         struct ext4_map_blocks map;
1567         struct ext4_io_submit io_submit;        /* IO submission data */
1568         unsigned int do_map:1;
1569         unsigned int scanned_until_end:1;
1570 };
1571
1572 static void mpage_release_unused_pages(struct mpage_da_data *mpd,
1573                                        bool invalidate)
1574 {
1575         unsigned nr, i;
1576         pgoff_t index, end;
1577         struct folio_batch fbatch;
1578         struct inode *inode = mpd->inode;
1579         struct address_space *mapping = inode->i_mapping;
1580
1581         /* This is necessary when next_page == 0. */
1582         if (mpd->first_page >= mpd->next_page)
1583                 return;
1584
1585         mpd->scanned_until_end = 0;
1586         index = mpd->first_page;
1587         end   = mpd->next_page - 1;
1588         if (invalidate) {
1589                 ext4_lblk_t start, last;
1590                 start = index << (PAGE_SHIFT - inode->i_blkbits);
1591                 last = end << (PAGE_SHIFT - inode->i_blkbits);
1592
1593                 /*
1594                  * avoid racing with extent status tree scans made by
1595                  * ext4_insert_delayed_block()
1596                  */
1597                 down_write(&EXT4_I(inode)->i_data_sem);
1598                 ext4_es_remove_extent(inode, start, last - start + 1);
1599                 up_write(&EXT4_I(inode)->i_data_sem);
1600         }
1601
1602         folio_batch_init(&fbatch);
1603         while (index <= end) {
1604                 nr = filemap_get_folios(mapping, &index, end, &fbatch);
1605                 if (nr == 0)
1606                         break;
1607                 for (i = 0; i < nr; i++) {
1608                         struct folio *folio = fbatch.folios[i];
1609
1610                         if (folio->index < mpd->first_page)
1611                                 continue;
1612                         if (folio->index + folio_nr_pages(folio) - 1 > end)
1613                                 continue;
1614                         BUG_ON(!folio_test_locked(folio));
1615                         BUG_ON(folio_test_writeback(folio));
1616                         if (invalidate) {
1617                                 if (folio_mapped(folio))
1618                                         folio_clear_dirty_for_io(folio);
1619                                 block_invalidate_folio(folio, 0,
1620                                                 folio_size(folio));
1621                                 folio_clear_uptodate(folio);
1622                         }
1623                         folio_unlock(folio);
1624                 }
1625                 folio_batch_release(&fbatch);
1626         }
1627 }
1628
1629 static void ext4_print_free_blocks(struct inode *inode)
1630 {
1631         struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1632         struct super_block *sb = inode->i_sb;
1633         struct ext4_inode_info *ei = EXT4_I(inode);
1634
1635         ext4_msg(sb, KERN_CRIT, "Total free blocks count %lld",
1636                EXT4_C2B(EXT4_SB(inode->i_sb),
1637                         ext4_count_free_clusters(sb)));
1638         ext4_msg(sb, KERN_CRIT, "Free/Dirty block details");
1639         ext4_msg(sb, KERN_CRIT, "free_blocks=%lld",
1640                (long long) EXT4_C2B(EXT4_SB(sb),
1641                 percpu_counter_sum(&sbi->s_freeclusters_counter)));
1642         ext4_msg(sb, KERN_CRIT, "dirty_blocks=%lld",
1643                (long long) EXT4_C2B(EXT4_SB(sb),
1644                 percpu_counter_sum(&sbi->s_dirtyclusters_counter)));
1645         ext4_msg(sb, KERN_CRIT, "Block reservation details");
1646         ext4_msg(sb, KERN_CRIT, "i_reserved_data_blocks=%u",
1647                  ei->i_reserved_data_blocks);
1648         return;
1649 }
1650
1651 static int ext4_bh_delay_or_unwritten(handle_t *handle, struct inode *inode,
1652                                       struct buffer_head *bh)
1653 {
1654         return (buffer_delay(bh) || buffer_unwritten(bh)) && buffer_dirty(bh);
1655 }
1656
1657 /*
1658  * ext4_insert_delayed_block - adds a delayed block to the extents status
1659  *                             tree, incrementing the reserved cluster/block
1660  *                             count or making a pending reservation
1661  *                             where needed
1662  *
1663  * @inode - file containing the newly added block
1664  * @lblk - logical block to be added
1665  *
1666  * Returns 0 on success, negative error code on failure.
1667  */
1668 static int ext4_insert_delayed_block(struct inode *inode, ext4_lblk_t lblk)
1669 {
1670         struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
1671         int ret;
1672         bool allocated = false;
1673         bool reserved = false;
1674
1675         /*
1676          * If the cluster containing lblk is shared with a delayed,
1677          * written, or unwritten extent in a bigalloc file system, it's
1678          * already been accounted for and does not need to be reserved.
1679          * A pending reservation must be made for the cluster if it's
1680          * shared with a written or unwritten extent and doesn't already
1681          * have one.  Written and unwritten extents can be purged from the
1682          * extents status tree if the system is under memory pressure, so
1683          * it's necessary to examine the extent tree if a search of the
1684          * extents status tree doesn't get a match.
1685          */
1686         if (sbi->s_cluster_ratio == 1) {
1687                 ret = ext4_da_reserve_space(inode);
1688                 if (ret != 0)   /* ENOSPC */
1689                         goto errout;
1690                 reserved = true;
1691         } else {   /* bigalloc */
1692                 if (!ext4_es_scan_clu(inode, &ext4_es_is_delonly, lblk)) {
1693                         if (!ext4_es_scan_clu(inode,
1694                                               &ext4_es_is_mapped, lblk)) {
1695                                 ret = ext4_clu_mapped(inode,
1696                                                       EXT4_B2C(sbi, lblk));
1697                                 if (ret < 0)
1698                                         goto errout;
1699                                 if (ret == 0) {
1700                                         ret = ext4_da_reserve_space(inode);
1701                                         if (ret != 0)   /* ENOSPC */
1702                                                 goto errout;
1703                                         reserved = true;
1704                                 } else {
1705                                         allocated = true;
1706                                 }
1707                         } else {
1708                                 allocated = true;
1709                         }
1710                 }
1711         }
1712
1713         ret = ext4_es_insert_delayed_block(inode, lblk, allocated);
1714         if (ret && reserved)
1715                 ext4_da_release_space(inode, 1);
1716
1717 errout:
1718         return ret;
1719 }
1720
1721 /*
1722  * This function is grabs code from the very beginning of
1723  * ext4_map_blocks, but assumes that the caller is from delayed write
1724  * time. This function looks up the requested blocks and sets the
1725  * buffer delay bit under the protection of i_data_sem.
1726  */
1727 static int ext4_da_map_blocks(struct inode *inode, sector_t iblock,
1728                               struct ext4_map_blocks *map,
1729                               struct buffer_head *bh)
1730 {
1731         struct extent_status es;
1732         int retval;
1733         sector_t invalid_block = ~((sector_t) 0xffff);
1734 #ifdef ES_AGGRESSIVE_TEST
1735         struct ext4_map_blocks orig_map;
1736
1737         memcpy(&orig_map, map, sizeof(*map));
1738 #endif
1739
1740         if (invalid_block < ext4_blocks_count(EXT4_SB(inode->i_sb)->s_es))
1741                 invalid_block = ~0;
1742
1743         map->m_flags = 0;
1744         ext_debug(inode, "max_blocks %u, logical block %lu\n", map->m_len,
1745                   (unsigned long) map->m_lblk);
1746
1747         /* Lookup extent status tree firstly */
1748         if (ext4_es_lookup_extent(inode, iblock, NULL, &es)) {
1749                 if (ext4_es_is_hole(&es)) {
1750                         retval = 0;
1751                         down_read(&EXT4_I(inode)->i_data_sem);
1752                         goto add_delayed;
1753                 }
1754
1755                 /*
1756                  * Delayed extent could be allocated by fallocate.
1757                  * So we need to check it.
1758                  */
1759                 if (ext4_es_is_delayed(&es) && !ext4_es_is_unwritten(&es)) {
1760                         map_bh(bh, inode->i_sb, invalid_block);
1761                         set_buffer_new(bh);
1762                         set_buffer_delay(bh);
1763                         return 0;
1764                 }
1765
1766                 map->m_pblk = ext4_es_pblock(&es) + iblock - es.es_lblk;
1767                 retval = es.es_len - (iblock - es.es_lblk);
1768                 if (retval > map->m_len)
1769                         retval = map->m_len;
1770                 map->m_len = retval;
1771                 if (ext4_es_is_written(&es))
1772                         map->m_flags |= EXT4_MAP_MAPPED;
1773                 else if (ext4_es_is_unwritten(&es))
1774                         map->m_flags |= EXT4_MAP_UNWRITTEN;
1775                 else
1776                         BUG();
1777
1778 #ifdef ES_AGGRESSIVE_TEST
1779                 ext4_map_blocks_es_recheck(NULL, inode, map, &orig_map, 0);
1780 #endif
1781                 return retval;
1782         }
1783
1784         /*
1785          * Try to see if we can get the block without requesting a new
1786          * file system block.
1787          */
1788         down_read(&EXT4_I(inode)->i_data_sem);
1789         if (ext4_has_inline_data(inode))
1790                 retval = 0;
1791         else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
1792                 retval = ext4_ext_map_blocks(NULL, inode, map, 0);
1793         else
1794                 retval = ext4_ind_map_blocks(NULL, inode, map, 0);
1795
1796 add_delayed:
1797         if (retval == 0) {
1798                 int ret;
1799
1800                 /*
1801                  * XXX: __block_prepare_write() unmaps passed block,
1802                  * is it OK?
1803                  */
1804
1805                 ret = ext4_insert_delayed_block(inode, map->m_lblk);
1806                 if (ret != 0) {
1807                         retval = ret;
1808                         goto out_unlock;
1809                 }
1810
1811                 map_bh(bh, inode->i_sb, invalid_block);
1812                 set_buffer_new(bh);
1813                 set_buffer_delay(bh);
1814         } else if (retval > 0) {
1815                 int ret;
1816                 unsigned int status;
1817
1818                 if (unlikely(retval != map->m_len)) {
1819                         ext4_warning(inode->i_sb,
1820                                      "ES len assertion failed for inode "
1821                                      "%lu: retval %d != map->m_len %d",
1822                                      inode->i_ino, retval, map->m_len);
1823                         WARN_ON(1);
1824                 }
1825
1826                 status = map->m_flags & EXT4_MAP_UNWRITTEN ?
1827                                 EXTENT_STATUS_UNWRITTEN : EXTENT_STATUS_WRITTEN;
1828                 ret = ext4_es_insert_extent(inode, map->m_lblk, map->m_len,
1829                                             map->m_pblk, status);
1830                 if (ret != 0)
1831                         retval = ret;
1832         }
1833
1834 out_unlock:
1835         up_read((&EXT4_I(inode)->i_data_sem));
1836
1837         return retval;
1838 }
1839
1840 /*
1841  * This is a special get_block_t callback which is used by
1842  * ext4_da_write_begin().  It will either return mapped block or
1843  * reserve space for a single block.
1844  *
1845  * For delayed buffer_head we have BH_Mapped, BH_New, BH_Delay set.
1846  * We also have b_blocknr = -1 and b_bdev initialized properly
1847  *
1848  * For unwritten buffer_head we have BH_Mapped, BH_New, BH_Unwritten set.
1849  * We also have b_blocknr = physicalblock mapping unwritten extent and b_bdev
1850  * initialized properly.
1851  */
1852 int ext4_da_get_block_prep(struct inode *inode, sector_t iblock,
1853                            struct buffer_head *bh, int create)
1854 {
1855         struct ext4_map_blocks map;
1856         int ret = 0;
1857
1858         BUG_ON(create == 0);
1859         BUG_ON(bh->b_size != inode->i_sb->s_blocksize);
1860
1861         map.m_lblk = iblock;
1862         map.m_len = 1;
1863
1864         /*
1865          * first, we need to know whether the block is allocated already
1866          * preallocated blocks are unmapped but should treated
1867          * the same as allocated blocks.
1868          */
1869         ret = ext4_da_map_blocks(inode, iblock, &map, bh);
1870         if (ret <= 0)
1871                 return ret;
1872
1873         map_bh(bh, inode->i_sb, map.m_pblk);
1874         ext4_update_bh_state(bh, map.m_flags);
1875
1876         if (buffer_unwritten(bh)) {
1877                 /* A delayed write to unwritten bh should be marked
1878                  * new and mapped.  Mapped ensures that we don't do
1879                  * get_block multiple times when we write to the same
1880                  * offset and new ensures that we do proper zero out
1881                  * for partial write.
1882                  */
1883                 set_buffer_new(bh);
1884                 set_buffer_mapped(bh);
1885         }
1886         return 0;
1887 }
1888
1889 static int __ext4_journalled_writepage(struct page *page,
1890                                        unsigned int len)
1891 {
1892         struct address_space *mapping = page->mapping;
1893         struct inode *inode = mapping->host;
1894         handle_t *handle = NULL;
1895         int ret = 0, err = 0;
1896         int inline_data = ext4_has_inline_data(inode);
1897         struct buffer_head *inode_bh = NULL;
1898         loff_t size;
1899
1900         ClearPageChecked(page);
1901
1902         if (inline_data) {
1903                 BUG_ON(page->index != 0);
1904                 BUG_ON(len > ext4_get_max_inline_size(inode));
1905                 inode_bh = ext4_journalled_write_inline_data(inode, len, page);
1906                 if (inode_bh == NULL)
1907                         goto out;
1908         }
1909         /*
1910          * We need to release the page lock before we start the
1911          * journal, so grab a reference so the page won't disappear
1912          * out from under us.
1913          */
1914         get_page(page);
1915         unlock_page(page);
1916
1917         handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
1918                                     ext4_writepage_trans_blocks(inode));
1919         if (IS_ERR(handle)) {
1920                 ret = PTR_ERR(handle);
1921                 put_page(page);
1922                 goto out_no_pagelock;
1923         }
1924         BUG_ON(!ext4_handle_valid(handle));
1925
1926         lock_page(page);
1927         put_page(page);
1928         size = i_size_read(inode);
1929         if (page->mapping != mapping || page_offset(page) > size) {
1930                 /* The page got truncated from under us */
1931                 ext4_journal_stop(handle);
1932                 ret = 0;
1933                 goto out;
1934         }
1935
1936         if (inline_data) {
1937                 ret = ext4_mark_inode_dirty(handle, inode);
1938         } else {
1939                 struct buffer_head *page_bufs = page_buffers(page);
1940
1941                 if (page->index == size >> PAGE_SHIFT)
1942                         len = size & ~PAGE_MASK;
1943                 else
1944                         len = PAGE_SIZE;
1945
1946                 ret = ext4_walk_page_buffers(handle, inode, page_bufs, 0, len,
1947                                              NULL, do_journal_get_write_access);
1948
1949                 err = ext4_walk_page_buffers(handle, inode, page_bufs, 0, len,
1950                                              NULL, write_end_fn);
1951         }
1952         if (ret == 0)
1953                 ret = err;
1954         err = ext4_jbd2_inode_add_write(handle, inode, page_offset(page), len);
1955         if (ret == 0)
1956                 ret = err;
1957         EXT4_I(inode)->i_datasync_tid = handle->h_transaction->t_tid;
1958         err = ext4_journal_stop(handle);
1959         if (!ret)
1960                 ret = err;
1961
1962         ext4_set_inode_state(inode, EXT4_STATE_JDATA);
1963 out:
1964         unlock_page(page);
1965 out_no_pagelock:
1966         brelse(inode_bh);
1967         return ret;
1968 }
1969
1970 /*
1971  * Note that we don't need to start a transaction unless we're journaling data
1972  * because we should have holes filled from ext4_page_mkwrite(). We even don't
1973  * need to file the inode to the transaction's list in ordered mode because if
1974  * we are writing back data added by write(), the inode is already there and if
1975  * we are writing back data modified via mmap(), no one guarantees in which
1976  * transaction the data will hit the disk. In case we are journaling data, we
1977  * cannot start transaction directly because transaction start ranks above page
1978  * lock so we have to do some magic.
1979  *
1980  * This function can get called via...
1981  *   - ext4_writepages after taking page lock (have journal handle)
1982  *   - journal_submit_inode_data_buffers (no journal handle)
1983  *   - shrink_page_list via the kswapd/direct reclaim (no journal handle)
1984  *   - grab_page_cache when doing write_begin (have journal handle)
1985  *
1986  * We don't do any block allocation in this function. If we have page with
1987  * multiple blocks we need to write those buffer_heads that are mapped. This
1988  * is important for mmaped based write. So if we do with blocksize 1K
1989  * truncate(f, 1024);
1990  * a = mmap(f, 0, 4096);
1991  * a[0] = 'a';
1992  * truncate(f, 4096);
1993  * we have in the page first buffer_head mapped via page_mkwrite call back
1994  * but other buffer_heads would be unmapped but dirty (dirty done via the
1995  * do_wp_page). So writepage should write the first block. If we modify
1996  * the mmap area beyond 1024 we will again get a page_fault and the
1997  * page_mkwrite callback will do the block allocation and mark the
1998  * buffer_heads mapped.
1999  *
2000  * We redirty the page if we have any buffer_heads that is either delay or
2001  * unwritten in the page.
2002  *
2003  * We can get recursively called as show below.
2004  *
2005  *      ext4_writepage() -> kmalloc() -> __alloc_pages() -> page_launder() ->
2006  *              ext4_writepage()
2007  *
2008  * But since we don't do any block allocation we should not deadlock.
2009  * Page also have the dirty flag cleared so we don't get recurive page_lock.
2010  */
2011 static int ext4_writepage(struct page *page,
2012                           struct writeback_control *wbc)
2013 {
2014         struct folio *folio = page_folio(page);
2015         int ret = 0;
2016         loff_t size;
2017         unsigned int len;
2018         struct buffer_head *page_bufs = NULL;
2019         struct inode *inode = page->mapping->host;
2020         struct ext4_io_submit io_submit;
2021         bool keep_towrite = false;
2022
2023         if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb)))) {
2024                 folio_invalidate(folio, 0, folio_size(folio));
2025                 folio_unlock(folio);
2026                 return -EIO;
2027         }
2028
2029         trace_ext4_writepage(page);
2030         size = i_size_read(inode);
2031         if (page->index == size >> PAGE_SHIFT &&
2032             !ext4_verity_in_progress(inode))
2033                 len = size & ~PAGE_MASK;
2034         else
2035                 len = PAGE_SIZE;
2036
2037         /* Should never happen but for bugs in other kernel subsystems */
2038         if (!page_has_buffers(page)) {
2039                 ext4_warning_inode(inode,
2040                    "page %lu does not have buffers attached", page->index);
2041                 ClearPageDirty(page);
2042                 unlock_page(page);
2043                 return 0;
2044         }
2045
2046         page_bufs = page_buffers(page);
2047         /*
2048          * We cannot do block allocation or other extent handling in this
2049          * function. If there are buffers needing that, we have to redirty
2050          * the page. But we may reach here when we do a journal commit via
2051          * journal_submit_inode_data_buffers() and in that case we must write
2052          * allocated buffers to achieve data=ordered mode guarantees.
2053          *
2054          * Also, if there is only one buffer per page (the fs block
2055          * size == the page size), if one buffer needs block
2056          * allocation or needs to modify the extent tree to clear the
2057          * unwritten flag, we know that the page can't be written at
2058          * all, so we might as well refuse the write immediately.
2059          * Unfortunately if the block size != page size, we can't as
2060          * easily detect this case using ext4_walk_page_buffers(), but
2061          * for the extremely common case, this is an optimization that
2062          * skips a useless round trip through ext4_bio_write_page().
2063          */
2064         if (ext4_walk_page_buffers(NULL, inode, page_bufs, 0, len, NULL,
2065                                    ext4_bh_delay_or_unwritten)) {
2066                 redirty_page_for_writepage(wbc, page);
2067                 if ((current->flags & PF_MEMALLOC) ||
2068                     (inode->i_sb->s_blocksize == PAGE_SIZE)) {
2069                         /*
2070                          * For memory cleaning there's no point in writing only
2071                          * some buffers. So just bail out. Warn if we came here
2072                          * from direct reclaim.
2073                          */
2074                         WARN_ON_ONCE((current->flags & (PF_MEMALLOC|PF_KSWAPD))
2075                                                         == PF_MEMALLOC);
2076                         unlock_page(page);
2077                         return 0;
2078                 }
2079                 keep_towrite = true;
2080         }
2081
2082         if (PageChecked(page) && ext4_should_journal_data(inode))
2083                 /*
2084                  * It's mmapped pagecache.  Add buffers and journal it.  There
2085                  * doesn't seem much point in redirtying the page here.
2086                  */
2087                 return __ext4_journalled_writepage(page, len);
2088
2089         ext4_io_submit_init(&io_submit, wbc);
2090         io_submit.io_end = ext4_init_io_end(inode, GFP_NOFS);
2091         if (!io_submit.io_end) {
2092                 redirty_page_for_writepage(wbc, page);
2093                 unlock_page(page);
2094                 return -ENOMEM;
2095         }
2096         ret = ext4_bio_write_page(&io_submit, page, len, keep_towrite);
2097         ext4_io_submit(&io_submit);
2098         /* Drop io_end reference we got from init */
2099         ext4_put_io_end_defer(io_submit.io_end);
2100         return ret;
2101 }
2102
2103 static int mpage_submit_page(struct mpage_da_data *mpd, struct page *page)
2104 {
2105         int len;
2106         loff_t size;
2107         int err;
2108
2109         BUG_ON(page->index != mpd->first_page);
2110         clear_page_dirty_for_io(page);
2111         /*
2112          * We have to be very careful here!  Nothing protects writeback path
2113          * against i_size changes and the page can be writeably mapped into
2114          * page tables. So an application can be growing i_size and writing
2115          * data through mmap while writeback runs. clear_page_dirty_for_io()
2116          * write-protects our page in page tables and the page cannot get
2117          * written to again until we release page lock. So only after
2118          * clear_page_dirty_for_io() we are safe to sample i_size for
2119          * ext4_bio_write_page() to zero-out tail of the written page. We rely
2120          * on the barrier provided by TestClearPageDirty in
2121          * clear_page_dirty_for_io() to make sure i_size is really sampled only
2122          * after page tables are updated.
2123          */
2124         size = i_size_read(mpd->inode);
2125         if (page->index == size >> PAGE_SHIFT &&
2126             !ext4_verity_in_progress(mpd->inode))
2127                 len = size & ~PAGE_MASK;
2128         else
2129                 len = PAGE_SIZE;
2130         err = ext4_bio_write_page(&mpd->io_submit, page, len, false);
2131         if (!err)
2132                 mpd->wbc->nr_to_write--;
2133         mpd->first_page++;
2134
2135         return err;
2136 }
2137
2138 #define BH_FLAGS (BIT(BH_Unwritten) | BIT(BH_Delay))
2139
2140 /*
2141  * mballoc gives us at most this number of blocks...
2142  * XXX: That seems to be only a limitation of ext4_mb_normalize_request().
2143  * The rest of mballoc seems to handle chunks up to full group size.
2144  */
2145 #define MAX_WRITEPAGES_EXTENT_LEN 2048
2146
2147 /*
2148  * mpage_add_bh_to_extent - try to add bh to extent of blocks to map
2149  *
2150  * @mpd - extent of blocks
2151  * @lblk - logical number of the block in the file
2152  * @bh - buffer head we want to add to the extent
2153  *
2154  * The function is used to collect contig. blocks in the same state. If the
2155  * buffer doesn't require mapping for writeback and we haven't started the
2156  * extent of buffers to map yet, the function returns 'true' immediately - the
2157  * caller can write the buffer right away. Otherwise the function returns true
2158  * if the block has been added to the extent, false if the block couldn't be
2159  * added.
2160  */
2161 static bool mpage_add_bh_to_extent(struct mpage_da_data *mpd, ext4_lblk_t lblk,
2162                                    struct buffer_head *bh)
2163 {
2164         struct ext4_map_blocks *map = &mpd->map;
2165
2166         /* Buffer that doesn't need mapping for writeback? */
2167         if (!buffer_dirty(bh) || !buffer_mapped(bh) ||
2168             (!buffer_delay(bh) && !buffer_unwritten(bh))) {
2169                 /* So far no extent to map => we write the buffer right away */
2170                 if (map->m_len == 0)
2171                         return true;
2172                 return false;
2173         }
2174
2175         /* First block in the extent? */
2176         if (map->m_len == 0) {
2177                 /* We cannot map unless handle is started... */
2178                 if (!mpd->do_map)
2179                         return false;
2180                 map->m_lblk = lblk;
2181                 map->m_len = 1;
2182                 map->m_flags = bh->b_state & BH_FLAGS;
2183                 return true;
2184         }
2185
2186         /* Don't go larger than mballoc is willing to allocate */
2187         if (map->m_len >= MAX_WRITEPAGES_EXTENT_LEN)
2188                 return false;
2189
2190         /* Can we merge the block to our big extent? */
2191         if (lblk == map->m_lblk + map->m_len &&
2192             (bh->b_state & BH_FLAGS) == map->m_flags) {
2193                 map->m_len++;
2194                 return true;
2195         }
2196         return false;
2197 }
2198
2199 /*
2200  * mpage_process_page_bufs - submit page buffers for IO or add them to extent
2201  *
2202  * @mpd - extent of blocks for mapping
2203  * @head - the first buffer in the page
2204  * @bh - buffer we should start processing from
2205  * @lblk - logical number of the block in the file corresponding to @bh
2206  *
2207  * Walk through page buffers from @bh upto @head (exclusive) and either submit
2208  * the page for IO if all buffers in this page were mapped and there's no
2209  * accumulated extent of buffers to map or add buffers in the page to the
2210  * extent of buffers to map. The function returns 1 if the caller can continue
2211  * by processing the next page, 0 if it should stop adding buffers to the
2212  * extent to map because we cannot extend it anymore. It can also return value
2213  * < 0 in case of error during IO submission.
2214  */
2215 static int mpage_process_page_bufs(struct mpage_da_data *mpd,
2216                                    struct buffer_head *head,
2217                                    struct buffer_head *bh,
2218                                    ext4_lblk_t lblk)
2219 {
2220         struct inode *inode = mpd->inode;
2221         int err;
2222         ext4_lblk_t blocks = (i_size_read(inode) + i_blocksize(inode) - 1)
2223                                                         >> inode->i_blkbits;
2224
2225         if (ext4_verity_in_progress(inode))
2226                 blocks = EXT_MAX_BLOCKS;
2227
2228         do {
2229                 BUG_ON(buffer_locked(bh));
2230
2231                 if (lblk >= blocks || !mpage_add_bh_to_extent(mpd, lblk, bh)) {
2232                         /* Found extent to map? */
2233                         if (mpd->map.m_len)
2234                                 return 0;
2235                         /* Buffer needs mapping and handle is not started? */
2236                         if (!mpd->do_map)
2237                                 return 0;
2238                         /* Everything mapped so far and we hit EOF */
2239                         break;
2240                 }
2241         } while (lblk++, (bh = bh->b_this_page) != head);
2242         /* So far everything mapped? Submit the page for IO. */
2243         if (mpd->map.m_len == 0) {
2244                 err = mpage_submit_page(mpd, head->b_page);
2245                 if (err < 0)
2246                         return err;
2247         }
2248         if (lblk >= blocks) {
2249                 mpd->scanned_until_end = 1;
2250                 return 0;
2251         }
2252         return 1;
2253 }
2254
2255 /*
2256  * mpage_process_page - update page buffers corresponding to changed extent and
2257  *                     may submit fully mapped page for IO
2258  *
2259  * @mpd         - description of extent to map, on return next extent to map
2260  * @m_lblk      - logical block mapping.
2261  * @m_pblk      - corresponding physical mapping.
2262  * @map_bh      - determines on return whether this page requires any further
2263  *                mapping or not.
2264  * Scan given page buffers corresponding to changed extent and update buffer
2265  * state according to new extent state.
2266  * We map delalloc buffers to their physical location, clear unwritten bits.
2267  * If the given page is not fully mapped, we update @map to the next extent in
2268  * the given page that needs mapping & return @map_bh as true.
2269  */
2270 static int mpage_process_page(struct mpage_da_data *mpd, struct page *page,
2271                               ext4_lblk_t *m_lblk, ext4_fsblk_t *m_pblk,
2272                               bool *map_bh)
2273 {
2274         struct buffer_head *head, *bh;
2275         ext4_io_end_t *io_end = mpd->io_submit.io_end;
2276         ext4_lblk_t lblk = *m_lblk;
2277         ext4_fsblk_t pblock = *m_pblk;
2278         int err = 0;
2279         int blkbits = mpd->inode->i_blkbits;
2280         ssize_t io_end_size = 0;
2281         struct ext4_io_end_vec *io_end_vec = ext4_last_io_end_vec(io_end);
2282
2283         bh = head = page_buffers(page);
2284         do {
2285                 if (lblk < mpd->map.m_lblk)
2286                         continue;
2287                 if (lblk >= mpd->map.m_lblk + mpd->map.m_len) {
2288                         /*
2289                          * Buffer after end of mapped extent.
2290                          * Find next buffer in the page to map.
2291                          */
2292                         mpd->map.m_len = 0;
2293                         mpd->map.m_flags = 0;
2294                         io_end_vec->size += io_end_size;
2295
2296                         err = mpage_process_page_bufs(mpd, head, bh, lblk);
2297                         if (err > 0)
2298                                 err = 0;
2299                         if (!err && mpd->map.m_len && mpd->map.m_lblk > lblk) {
2300                                 io_end_vec = ext4_alloc_io_end_vec(io_end);
2301                                 if (IS_ERR(io_end_vec)) {
2302                                         err = PTR_ERR(io_end_vec);
2303                                         goto out;
2304                                 }
2305                                 io_end_vec->offset = (loff_t)mpd->map.m_lblk << blkbits;
2306                         }
2307                         *map_bh = true;
2308                         goto out;
2309                 }
2310                 if (buffer_delay(bh)) {
2311                         clear_buffer_delay(bh);
2312                         bh->b_blocknr = pblock++;
2313                 }
2314                 clear_buffer_unwritten(bh);
2315                 io_end_size += (1 << blkbits);
2316         } while (lblk++, (bh = bh->b_this_page) != head);
2317
2318         io_end_vec->size += io_end_size;
2319         *map_bh = false;
2320 out:
2321         *m_lblk = lblk;
2322         *m_pblk = pblock;
2323         return err;
2324 }
2325
2326 /*
2327  * mpage_map_buffers - update buffers corresponding to changed extent and
2328  *                     submit fully mapped pages for IO
2329  *
2330  * @mpd - description of extent to map, on return next extent to map
2331  *
2332  * Scan buffers corresponding to changed extent (we expect corresponding pages
2333  * to be already locked) and update buffer state according to new extent state.
2334  * We map delalloc buffers to their physical location, clear unwritten bits,
2335  * and mark buffers as uninit when we perform writes to unwritten extents
2336  * and do extent conversion after IO is finished. If the last page is not fully
2337  * mapped, we update @map to the next extent in the last page that needs
2338  * mapping. Otherwise we submit the page for IO.
2339  */
2340 static int mpage_map_and_submit_buffers(struct mpage_da_data *mpd)
2341 {
2342         struct folio_batch fbatch;
2343         unsigned nr, i;
2344         struct inode *inode = mpd->inode;
2345         int bpp_bits = PAGE_SHIFT - inode->i_blkbits;
2346         pgoff_t start, end;
2347         ext4_lblk_t lblk;
2348         ext4_fsblk_t pblock;
2349         int err;
2350         bool map_bh = false;
2351
2352         start = mpd->map.m_lblk >> bpp_bits;
2353         end = (mpd->map.m_lblk + mpd->map.m_len - 1) >> bpp_bits;
2354         lblk = start << bpp_bits;
2355         pblock = mpd->map.m_pblk;
2356
2357         folio_batch_init(&fbatch);
2358         while (start <= end) {
2359                 nr = filemap_get_folios(inode->i_mapping, &start, end, &fbatch);
2360                 if (nr == 0)
2361                         break;
2362                 for (i = 0; i < nr; i++) {
2363                         struct page *page = &fbatch.folios[i]->page;
2364
2365                         err = mpage_process_page(mpd, page, &lblk, &pblock,
2366                                                  &map_bh);
2367                         /*
2368                          * If map_bh is true, means page may require further bh
2369                          * mapping, or maybe the page was submitted for IO.
2370                          * So we return to call further extent mapping.
2371                          */
2372                         if (err < 0 || map_bh)
2373                                 goto out;
2374                         /* Page fully mapped - let IO run! */
2375                         err = mpage_submit_page(mpd, page);
2376                         if (err < 0)
2377                                 goto out;
2378                 }
2379                 folio_batch_release(&fbatch);
2380         }
2381         /* Extent fully mapped and matches with page boundary. We are done. */
2382         mpd->map.m_len = 0;
2383         mpd->map.m_flags = 0;
2384         return 0;
2385 out:
2386         folio_batch_release(&fbatch);
2387         return err;
2388 }
2389
2390 static int mpage_map_one_extent(handle_t *handle, struct mpage_da_data *mpd)
2391 {
2392         struct inode *inode = mpd->inode;
2393         struct ext4_map_blocks *map = &mpd->map;
2394         int get_blocks_flags;
2395         int err, dioread_nolock;
2396
2397         trace_ext4_da_write_pages_extent(inode, map);
2398         /*
2399          * Call ext4_map_blocks() to allocate any delayed allocation blocks, or
2400          * to convert an unwritten extent to be initialized (in the case
2401          * where we have written into one or more preallocated blocks).  It is
2402          * possible that we're going to need more metadata blocks than
2403          * previously reserved. However we must not fail because we're in
2404          * writeback and there is nothing we can do about it so it might result
2405          * in data loss.  So use reserved blocks to allocate metadata if
2406          * possible.
2407          *
2408          * We pass in the magic EXT4_GET_BLOCKS_DELALLOC_RESERVE if
2409          * the blocks in question are delalloc blocks.  This indicates
2410          * that the blocks and quotas has already been checked when
2411          * the data was copied into the page cache.
2412          */
2413         get_blocks_flags = EXT4_GET_BLOCKS_CREATE |
2414                            EXT4_GET_BLOCKS_METADATA_NOFAIL |
2415                            EXT4_GET_BLOCKS_IO_SUBMIT;
2416         dioread_nolock = ext4_should_dioread_nolock(inode);
2417         if (dioread_nolock)
2418                 get_blocks_flags |= EXT4_GET_BLOCKS_IO_CREATE_EXT;
2419         if (map->m_flags & BIT(BH_Delay))
2420                 get_blocks_flags |= EXT4_GET_BLOCKS_DELALLOC_RESERVE;
2421
2422         err = ext4_map_blocks(handle, inode, map, get_blocks_flags);
2423         if (err < 0)
2424                 return err;
2425         if (dioread_nolock && (map->m_flags & EXT4_MAP_UNWRITTEN)) {
2426                 if (!mpd->io_submit.io_end->handle &&
2427                     ext4_handle_valid(handle)) {
2428                         mpd->io_submit.io_end->handle = handle->h_rsv_handle;
2429                         handle->h_rsv_handle = NULL;
2430                 }
2431                 ext4_set_io_unwritten_flag(inode, mpd->io_submit.io_end);
2432         }
2433
2434         BUG_ON(map->m_len == 0);
2435         return 0;
2436 }
2437
2438 /*
2439  * mpage_map_and_submit_extent - map extent starting at mpd->lblk of length
2440  *                               mpd->len and submit pages underlying it for IO
2441  *
2442  * @handle - handle for journal operations
2443  * @mpd - extent to map
2444  * @give_up_on_write - we set this to true iff there is a fatal error and there
2445  *                     is no hope of writing the data. The caller should discard
2446  *                     dirty pages to avoid infinite loops.
2447  *
2448  * The function maps extent starting at mpd->lblk of length mpd->len. If it is
2449  * delayed, blocks are allocated, if it is unwritten, we may need to convert
2450  * them to initialized or split the described range from larger unwritten
2451  * extent. Note that we need not map all the described range since allocation
2452  * can return less blocks or the range is covered by more unwritten extents. We
2453  * cannot map more because we are limited by reserved transaction credits. On
2454  * the other hand we always make sure that the last touched page is fully
2455  * mapped so that it can be written out (and thus forward progress is
2456  * guaranteed). After mapping we submit all mapped pages for IO.
2457  */
2458 static int mpage_map_and_submit_extent(handle_t *handle,
2459                                        struct mpage_da_data *mpd,
2460                                        bool *give_up_on_write)
2461 {
2462         struct inode *inode = mpd->inode;
2463         struct ext4_map_blocks *map = &mpd->map;
2464         int err;
2465         loff_t disksize;
2466         int progress = 0;
2467         ext4_io_end_t *io_end = mpd->io_submit.io_end;
2468         struct ext4_io_end_vec *io_end_vec;
2469
2470         io_end_vec = ext4_alloc_io_end_vec(io_end);
2471         if (IS_ERR(io_end_vec))
2472                 return PTR_ERR(io_end_vec);
2473         io_end_vec->offset = ((loff_t)map->m_lblk) << inode->i_blkbits;
2474         do {
2475                 err = mpage_map_one_extent(handle, mpd);
2476                 if (err < 0) {
2477                         struct super_block *sb = inode->i_sb;
2478
2479                         if (ext4_forced_shutdown(EXT4_SB(sb)) ||
2480                             ext4_test_mount_flag(sb, EXT4_MF_FS_ABORTED))
2481                                 goto invalidate_dirty_pages;
2482                         /*
2483                          * Let the uper layers retry transient errors.
2484                          * In the case of ENOSPC, if ext4_count_free_blocks()
2485                          * is non-zero, a commit should free up blocks.
2486                          */
2487                         if ((err == -ENOMEM) ||
2488                             (err == -ENOSPC && ext4_count_free_clusters(sb))) {
2489                                 if (progress)
2490                                         goto update_disksize;
2491                                 return err;
2492                         }
2493                         ext4_msg(sb, KERN_CRIT,
2494                                  "Delayed block allocation failed for "
2495                                  "inode %lu at logical offset %llu with"
2496                                  " max blocks %u with error %d",
2497                                  inode->i_ino,
2498                                  (unsigned long long)map->m_lblk,
2499                                  (unsigned)map->m_len, -err);
2500                         ext4_msg(sb, KERN_CRIT,
2501                                  "This should not happen!! Data will "
2502                                  "be lost\n");
2503                         if (err == -ENOSPC)
2504                                 ext4_print_free_blocks(inode);
2505                 invalidate_dirty_pages:
2506                         *give_up_on_write = true;
2507                         return err;
2508                 }
2509                 progress = 1;
2510                 /*
2511                  * Update buffer state, submit mapped pages, and get us new
2512                  * extent to map
2513                  */
2514                 err = mpage_map_and_submit_buffers(mpd);
2515                 if (err < 0)
2516                         goto update_disksize;
2517         } while (map->m_len);
2518
2519 update_disksize:
2520         /*
2521          * Update on-disk size after IO is submitted.  Races with
2522          * truncate are avoided by checking i_size under i_data_sem.
2523          */
2524         disksize = ((loff_t)mpd->first_page) << PAGE_SHIFT;
2525         if (disksize > READ_ONCE(EXT4_I(inode)->i_disksize)) {
2526                 int err2;
2527                 loff_t i_size;
2528
2529                 down_write(&EXT4_I(inode)->i_data_sem);
2530                 i_size = i_size_read(inode);
2531                 if (disksize > i_size)
2532                         disksize = i_size;
2533                 if (disksize > EXT4_I(inode)->i_disksize)
2534                         EXT4_I(inode)->i_disksize = disksize;
2535                 up_write(&EXT4_I(inode)->i_data_sem);
2536                 err2 = ext4_mark_inode_dirty(handle, inode);
2537                 if (err2) {
2538                         ext4_error_err(inode->i_sb, -err2,
2539                                        "Failed to mark inode %lu dirty",
2540                                        inode->i_ino);
2541                 }
2542                 if (!err)
2543                         err = err2;
2544         }
2545         return err;
2546 }
2547
2548 /*
2549  * Calculate the total number of credits to reserve for one writepages
2550  * iteration. This is called from ext4_writepages(). We map an extent of
2551  * up to MAX_WRITEPAGES_EXTENT_LEN blocks and then we go on and finish mapping
2552  * the last partial page. So in total we can map MAX_WRITEPAGES_EXTENT_LEN +
2553  * bpp - 1 blocks in bpp different extents.
2554  */
2555 static int ext4_da_writepages_trans_blocks(struct inode *inode)
2556 {
2557         int bpp = ext4_journal_blocks_per_page(inode);
2558
2559         return ext4_meta_trans_blocks(inode,
2560                                 MAX_WRITEPAGES_EXTENT_LEN + bpp - 1, bpp);
2561 }
2562
2563 /*
2564  * mpage_prepare_extent_to_map - find & lock contiguous range of dirty pages
2565  *                               and underlying extent to map
2566  *
2567  * @mpd - where to look for pages
2568  *
2569  * Walk dirty pages in the mapping. If they are fully mapped, submit them for
2570  * IO immediately. When we find a page which isn't mapped we start accumulating
2571  * extent of buffers underlying these pages that needs mapping (formed by
2572  * either delayed or unwritten buffers). We also lock the pages containing
2573  * these buffers. The extent found is returned in @mpd structure (starting at
2574  * mpd->lblk with length mpd->len blocks).
2575  *
2576  * Note that this function can attach bios to one io_end structure which are
2577  * neither logically nor physically contiguous. Although it may seem as an
2578  * unnecessary complication, it is actually inevitable in blocksize < pagesize
2579  * case as we need to track IO to all buffers underlying a page in one io_end.
2580  */
2581 static int mpage_prepare_extent_to_map(struct mpage_da_data *mpd)
2582 {
2583         struct address_space *mapping = mpd->inode->i_mapping;
2584         struct pagevec pvec;
2585         unsigned int nr_pages;
2586         long left = mpd->wbc->nr_to_write;
2587         pgoff_t index = mpd->first_page;
2588         pgoff_t end = mpd->last_page;
2589         xa_mark_t tag;
2590         int i, err = 0;
2591         int blkbits = mpd->inode->i_blkbits;
2592         ext4_lblk_t lblk;
2593         struct buffer_head *head;
2594
2595         if (mpd->wbc->sync_mode == WB_SYNC_ALL || mpd->wbc->tagged_writepages)
2596                 tag = PAGECACHE_TAG_TOWRITE;
2597         else
2598                 tag = PAGECACHE_TAG_DIRTY;
2599
2600         pagevec_init(&pvec);
2601         mpd->map.m_len = 0;
2602         mpd->next_page = index;
2603         while (index <= end) {
2604                 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end,
2605                                 tag);
2606                 if (nr_pages == 0)
2607                         break;
2608
2609                 for (i = 0; i < nr_pages; i++) {
2610                         struct page *page = pvec.pages[i];
2611
2612                         /*
2613                          * Accumulated enough dirty pages? This doesn't apply
2614                          * to WB_SYNC_ALL mode. For integrity sync we have to
2615                          * keep going because someone may be concurrently
2616                          * dirtying pages, and we might have synced a lot of
2617                          * newly appeared dirty pages, but have not synced all
2618                          * of the old dirty pages.
2619                          */
2620                         if (mpd->wbc->sync_mode == WB_SYNC_NONE && left <= 0)
2621                                 goto out;
2622
2623                         /* If we can't merge this page, we are done. */
2624                         if (mpd->map.m_len > 0 && mpd->next_page != page->index)
2625                                 goto out;
2626
2627                         lock_page(page);
2628                         /*
2629                          * If the page is no longer dirty, or its mapping no
2630                          * longer corresponds to inode we are writing (which
2631                          * means it has been truncated or invalidated), or the
2632                          * page is already under writeback and we are not doing
2633                          * a data integrity writeback, skip the page
2634                          */
2635                         if (!PageDirty(page) ||
2636                             (PageWriteback(page) &&
2637                              (mpd->wbc->sync_mode == WB_SYNC_NONE)) ||
2638                             unlikely(page->mapping != mapping)) {
2639                                 unlock_page(page);
2640                                 continue;
2641                         }
2642
2643                         wait_on_page_writeback(page);
2644                         BUG_ON(PageWriteback(page));
2645
2646                         /*
2647                          * Should never happen but for buggy code in
2648                          * other subsystems that call
2649                          * set_page_dirty() without properly warning
2650                          * the file system first.  See [1] for more
2651                          * information.
2652                          *
2653                          * [1] https://lore.kernel.org/linux-mm/20180103100430.GE4911@quack2.suse.cz
2654                          */
2655                         if (!page_has_buffers(page)) {
2656                                 ext4_warning_inode(mpd->inode, "page %lu does not have buffers attached", page->index);
2657                                 ClearPageDirty(page);
2658                                 unlock_page(page);
2659                                 continue;
2660                         }
2661
2662                         if (mpd->map.m_len == 0)
2663                                 mpd->first_page = page->index;
2664                         mpd->next_page = page->index + 1;
2665                         /* Add all dirty buffers to mpd */
2666                         lblk = ((ext4_lblk_t)page->index) <<
2667                                 (PAGE_SHIFT - blkbits);
2668                         head = page_buffers(page);
2669                         err = mpage_process_page_bufs(mpd, head, head, lblk);
2670                         if (err <= 0)
2671                                 goto out;
2672                         err = 0;
2673                         left--;
2674                 }
2675                 pagevec_release(&pvec);
2676                 cond_resched();
2677         }
2678         mpd->scanned_until_end = 1;
2679         return 0;
2680 out:
2681         pagevec_release(&pvec);
2682         return err;
2683 }
2684
2685 static int ext4_writepages(struct address_space *mapping,
2686                            struct writeback_control *wbc)
2687 {
2688         pgoff_t writeback_index = 0;
2689         long nr_to_write = wbc->nr_to_write;
2690         int range_whole = 0;
2691         int cycled = 1;
2692         handle_t *handle = NULL;
2693         struct mpage_da_data mpd;
2694         struct inode *inode = mapping->host;
2695         int needed_blocks, rsv_blocks = 0, ret = 0;
2696         struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
2697         struct blk_plug plug;
2698         bool give_up_on_write = false;
2699
2700         if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
2701                 return -EIO;
2702
2703         percpu_down_read(&sbi->s_writepages_rwsem);
2704         trace_ext4_writepages(inode, wbc);
2705
2706         /*
2707          * No pages to write? This is mainly a kludge to avoid starting
2708          * a transaction for special inodes like journal inode on last iput()
2709          * because that could violate lock ordering on umount
2710          */
2711         if (!mapping->nrpages || !mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
2712                 goto out_writepages;
2713
2714         if (ext4_should_journal_data(inode)) {
2715                 ret = generic_writepages(mapping, wbc);
2716                 goto out_writepages;
2717         }
2718
2719         /*
2720          * If the filesystem has aborted, it is read-only, so return
2721          * right away instead of dumping stack traces later on that
2722          * will obscure the real source of the problem.  We test
2723          * EXT4_MF_FS_ABORTED instead of sb->s_flag's SB_RDONLY because
2724          * the latter could be true if the filesystem is mounted
2725          * read-only, and in that case, ext4_writepages should
2726          * *never* be called, so if that ever happens, we would want
2727          * the stack trace.
2728          */
2729         if (unlikely(ext4_forced_shutdown(EXT4_SB(mapping->host->i_sb)) ||
2730                      ext4_test_mount_flag(inode->i_sb, EXT4_MF_FS_ABORTED))) {
2731                 ret = -EROFS;
2732                 goto out_writepages;
2733         }
2734
2735         /*
2736          * If we have inline data and arrive here, it means that
2737          * we will soon create the block for the 1st page, so
2738          * we'd better clear the inline data here.
2739          */
2740         if (ext4_has_inline_data(inode)) {
2741                 /* Just inode will be modified... */
2742                 handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
2743                 if (IS_ERR(handle)) {
2744                         ret = PTR_ERR(handle);
2745                         goto out_writepages;
2746                 }
2747                 BUG_ON(ext4_test_inode_state(inode,
2748                                 EXT4_STATE_MAY_INLINE_DATA));
2749                 ext4_destroy_inline_data(handle, inode);
2750                 ext4_journal_stop(handle);
2751         }
2752
2753         if (ext4_should_dioread_nolock(inode)) {
2754                 /*
2755                  * We may need to convert up to one extent per block in
2756                  * the page and we may dirty the inode.
2757                  */
2758                 rsv_blocks = 1 + ext4_chunk_trans_blocks(inode,
2759                                                 PAGE_SIZE >> inode->i_blkbits);
2760         }
2761
2762         if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2763                 range_whole = 1;
2764
2765         if (wbc->range_cyclic) {
2766                 writeback_index = mapping->writeback_index;
2767                 if (writeback_index)
2768                         cycled = 0;
2769                 mpd.first_page = writeback_index;
2770                 mpd.last_page = -1;
2771         } else {
2772                 mpd.first_page = wbc->range_start >> PAGE_SHIFT;
2773                 mpd.last_page = wbc->range_end >> PAGE_SHIFT;
2774         }
2775
2776         mpd.inode = inode;
2777         mpd.wbc = wbc;
2778         ext4_io_submit_init(&mpd.io_submit, wbc);
2779 retry:
2780         if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2781                 tag_pages_for_writeback(mapping, mpd.first_page, mpd.last_page);
2782         blk_start_plug(&plug);
2783
2784         /*
2785          * First writeback pages that don't need mapping - we can avoid
2786          * starting a transaction unnecessarily and also avoid being blocked
2787          * in the block layer on device congestion while having transaction
2788          * started.
2789          */
2790         mpd.do_map = 0;
2791         mpd.scanned_until_end = 0;
2792         mpd.io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
2793         if (!mpd.io_submit.io_end) {
2794                 ret = -ENOMEM;
2795                 goto unplug;
2796         }
2797         ret = mpage_prepare_extent_to_map(&mpd);
2798         /* Unlock pages we didn't use */
2799         mpage_release_unused_pages(&mpd, false);
2800         /* Submit prepared bio */
2801         ext4_io_submit(&mpd.io_submit);
2802         ext4_put_io_end_defer(mpd.io_submit.io_end);
2803         mpd.io_submit.io_end = NULL;
2804         if (ret < 0)
2805                 goto unplug;
2806
2807         while (!mpd.scanned_until_end && wbc->nr_to_write > 0) {
2808                 /* For each extent of pages we use new io_end */
2809                 mpd.io_submit.io_end = ext4_init_io_end(inode, GFP_KERNEL);
2810                 if (!mpd.io_submit.io_end) {
2811                         ret = -ENOMEM;
2812                         break;
2813                 }
2814
2815                 /*
2816                  * We have two constraints: We find one extent to map and we
2817                  * must always write out whole page (makes a difference when
2818                  * blocksize < pagesize) so that we don't block on IO when we
2819                  * try to write out the rest of the page. Journalled mode is
2820                  * not supported by delalloc.
2821                  */
2822                 BUG_ON(ext4_should_journal_data(inode));
2823                 needed_blocks = ext4_da_writepages_trans_blocks(inode);
2824
2825                 /* start a new transaction */
2826                 handle = ext4_journal_start_with_reserve(inode,
2827                                 EXT4_HT_WRITE_PAGE, needed_blocks, rsv_blocks);
2828                 if (IS_ERR(handle)) {
2829                         ret = PTR_ERR(handle);
2830                         ext4_msg(inode->i_sb, KERN_CRIT, "%s: jbd2_start: "
2831                                "%ld pages, ino %lu; err %d", __func__,
2832                                 wbc->nr_to_write, inode->i_ino, ret);
2833                         /* Release allocated io_end */
2834                         ext4_put_io_end(mpd.io_submit.io_end);
2835                         mpd.io_submit.io_end = NULL;
2836                         break;
2837                 }
2838                 mpd.do_map = 1;
2839
2840                 trace_ext4_da_write_pages(inode, mpd.first_page, mpd.wbc);
2841                 ret = mpage_prepare_extent_to_map(&mpd);
2842                 if (!ret && mpd.map.m_len)
2843                         ret = mpage_map_and_submit_extent(handle, &mpd,
2844                                         &give_up_on_write);
2845                 /*
2846                  * Caution: If the handle is synchronous,
2847                  * ext4_journal_stop() can wait for transaction commit
2848                  * to finish which may depend on writeback of pages to
2849                  * complete or on page lock to be released.  In that
2850                  * case, we have to wait until after we have
2851                  * submitted all the IO, released page locks we hold,
2852                  * and dropped io_end reference (for extent conversion
2853                  * to be able to complete) before stopping the handle.
2854                  */
2855                 if (!ext4_handle_valid(handle) || handle->h_sync == 0) {
2856                         ext4_journal_stop(handle);
2857                         handle = NULL;
2858                         mpd.do_map = 0;
2859                 }
2860                 /* Unlock pages we didn't use */
2861                 mpage_release_unused_pages(&mpd, give_up_on_write);
2862                 /* Submit prepared bio */
2863                 ext4_io_submit(&mpd.io_submit);
2864
2865                 /*
2866                  * Drop our io_end reference we got from init. We have
2867                  * to be careful and use deferred io_end finishing if
2868                  * we are still holding the transaction as we can
2869                  * release the last reference to io_end which may end
2870                  * up doing unwritten extent conversion.
2871                  */
2872                 if (handle) {
2873                         ext4_put_io_end_defer(mpd.io_submit.io_end);
2874                         ext4_journal_stop(handle);
2875                 } else
2876                         ext4_put_io_end(mpd.io_submit.io_end);
2877                 mpd.io_submit.io_end = NULL;
2878
2879                 if (ret == -ENOSPC && sbi->s_journal) {
2880                         /*
2881                          * Commit the transaction which would
2882                          * free blocks released in the transaction
2883                          * and try again
2884                          */
2885                         jbd2_journal_force_commit_nested(sbi->s_journal);
2886                         ret = 0;
2887                         continue;
2888                 }
2889                 /* Fatal error - ENOMEM, EIO... */
2890                 if (ret)
2891                         break;
2892         }
2893 unplug:
2894         blk_finish_plug(&plug);
2895         if (!ret && !cycled && wbc->nr_to_write > 0) {
2896                 cycled = 1;
2897                 mpd.last_page = writeback_index - 1;
2898                 mpd.first_page = 0;
2899                 goto retry;
2900         }
2901
2902         /* Update index */
2903         if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
2904                 /*
2905                  * Set the writeback_index so that range_cyclic
2906                  * mode will write it back later
2907                  */
2908                 mapping->writeback_index = mpd.first_page;
2909
2910 out_writepages:
2911         trace_ext4_writepages_result(inode, wbc, ret,
2912                                      nr_to_write - wbc->nr_to_write);
2913         percpu_up_read(&sbi->s_writepages_rwsem);
2914         return ret;
2915 }
2916
2917 static int ext4_dax_writepages(struct address_space *mapping,
2918                                struct writeback_control *wbc)
2919 {
2920         int ret;
2921         long nr_to_write = wbc->nr_to_write;
2922         struct inode *inode = mapping->host;
2923         struct ext4_sb_info *sbi = EXT4_SB(mapping->host->i_sb);
2924
2925         if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
2926                 return -EIO;
2927
2928         percpu_down_read(&sbi->s_writepages_rwsem);
2929         trace_ext4_writepages(inode, wbc);
2930
2931         ret = dax_writeback_mapping_range(mapping, sbi->s_daxdev, wbc);
2932         trace_ext4_writepages_result(inode, wbc, ret,
2933                                      nr_to_write - wbc->nr_to_write);
2934         percpu_up_read(&sbi->s_writepages_rwsem);
2935         return ret;
2936 }
2937
2938 static int ext4_nonda_switch(struct super_block *sb)
2939 {
2940         s64 free_clusters, dirty_clusters;
2941         struct ext4_sb_info *sbi = EXT4_SB(sb);
2942
2943         /*
2944          * switch to non delalloc mode if we are running low
2945          * on free block. The free block accounting via percpu
2946          * counters can get slightly wrong with percpu_counter_batch getting
2947          * accumulated on each CPU without updating global counters
2948          * Delalloc need an accurate free block accounting. So switch
2949          * to non delalloc when we are near to error range.
2950          */
2951         free_clusters =
2952                 percpu_counter_read_positive(&sbi->s_freeclusters_counter);
2953         dirty_clusters =
2954                 percpu_counter_read_positive(&sbi->s_dirtyclusters_counter);
2955         /*
2956          * Start pushing delalloc when 1/2 of free blocks are dirty.
2957          */
2958         if (dirty_clusters && (free_clusters < 2 * dirty_clusters))
2959                 try_to_writeback_inodes_sb(sb, WB_REASON_FS_FREE_SPACE);
2960
2961         if (2 * free_clusters < 3 * dirty_clusters ||
2962             free_clusters < (dirty_clusters + EXT4_FREECLUSTERS_WATERMARK)) {
2963                 /*
2964                  * free block count is less than 150% of dirty blocks
2965                  * or free blocks is less than watermark
2966                  */
2967                 return 1;
2968         }
2969         return 0;
2970 }
2971
2972 static int ext4_da_write_begin(struct file *file, struct address_space *mapping,
2973                                loff_t pos, unsigned len,
2974                                struct page **pagep, void **fsdata)
2975 {
2976         int ret, retries = 0;
2977         struct page *page;
2978         pgoff_t index;
2979         struct inode *inode = mapping->host;
2980
2981         if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
2982                 return -EIO;
2983
2984         index = pos >> PAGE_SHIFT;
2985
2986         if (ext4_nonda_switch(inode->i_sb) || ext4_verity_in_progress(inode)) {
2987                 *fsdata = (void *)FALL_BACK_TO_NONDELALLOC;
2988                 return ext4_write_begin(file, mapping, pos,
2989                                         len, pagep, fsdata);
2990         }
2991         *fsdata = (void *)0;
2992         trace_ext4_da_write_begin(inode, pos, len);
2993
2994         if (ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA)) {
2995                 ret = ext4_da_write_inline_data_begin(mapping, inode, pos, len,
2996                                                       pagep, fsdata);
2997                 if (ret < 0)
2998                         return ret;
2999                 if (ret == 1)
3000                         return 0;
3001         }
3002
3003 retry:
3004         page = grab_cache_page_write_begin(mapping, index);
3005         if (!page)
3006                 return -ENOMEM;
3007
3008         /* In case writeback began while the page was unlocked */
3009         wait_for_stable_page(page);
3010
3011 #ifdef CONFIG_FS_ENCRYPTION
3012         ret = ext4_block_write_begin(page, pos, len,
3013                                      ext4_da_get_block_prep);
3014 #else
3015         ret = __block_write_begin(page, pos, len, ext4_da_get_block_prep);
3016 #endif
3017         if (ret < 0) {
3018                 unlock_page(page);
3019                 put_page(page);
3020                 /*
3021                  * block_write_begin may have instantiated a few blocks
3022                  * outside i_size.  Trim these off again. Don't need
3023                  * i_size_read because we hold inode lock.
3024                  */
3025                 if (pos + len > inode->i_size)
3026                         ext4_truncate_failed_write(inode);
3027
3028                 if (ret == -ENOSPC &&
3029                     ext4_should_retry_alloc(inode->i_sb, &retries))
3030                         goto retry;
3031                 return ret;
3032         }
3033
3034         *pagep = page;
3035         return ret;
3036 }
3037
3038 /*
3039  * Check if we should update i_disksize
3040  * when write to the end of file but not require block allocation
3041  */
3042 static int ext4_da_should_update_i_disksize(struct page *page,
3043                                             unsigned long offset)
3044 {
3045         struct buffer_head *bh;
3046         struct inode *inode = page->mapping->host;
3047         unsigned int idx;
3048         int i;
3049
3050         bh = page_buffers(page);
3051         idx = offset >> inode->i_blkbits;
3052
3053         for (i = 0; i < idx; i++)
3054                 bh = bh->b_this_page;
3055
3056         if (!buffer_mapped(bh) || (buffer_delay(bh)) || buffer_unwritten(bh))
3057                 return 0;
3058         return 1;
3059 }
3060
3061 static int ext4_da_write_end(struct file *file,
3062                              struct address_space *mapping,
3063                              loff_t pos, unsigned len, unsigned copied,
3064                              struct page *page, void *fsdata)
3065 {
3066         struct inode *inode = mapping->host;
3067         loff_t new_i_size;
3068         unsigned long start, end;
3069         int write_mode = (int)(unsigned long)fsdata;
3070
3071         if (write_mode == FALL_BACK_TO_NONDELALLOC)
3072                 return ext4_write_end(file, mapping, pos,
3073                                       len, copied, page, fsdata);
3074
3075         trace_ext4_da_write_end(inode, pos, len, copied);
3076
3077         if (write_mode != CONVERT_INLINE_DATA &&
3078             ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA) &&
3079             ext4_has_inline_data(inode))
3080                 return ext4_write_inline_data_end(inode, pos, len, copied, page);
3081
3082         if (unlikely(copied < len) && !PageUptodate(page))
3083                 copied = 0;
3084
3085         start = pos & (PAGE_SIZE - 1);
3086         end = start + copied - 1;
3087
3088         /*
3089          * Since we are holding inode lock, we are sure i_disksize <=
3090          * i_size. We also know that if i_disksize < i_size, there are
3091          * delalloc writes pending in the range upto i_size. If the end of
3092          * the current write is <= i_size, there's no need to touch
3093          * i_disksize since writeback will push i_disksize upto i_size
3094          * eventually. If the end of the current write is > i_size and
3095          * inside an allocated block (ext4_da_should_update_i_disksize()
3096          * check), we need to update i_disksize here as neither
3097          * ext4_writepage() nor certain ext4_writepages() paths not
3098          * allocating blocks update i_disksize.
3099          *
3100          * Note that we defer inode dirtying to generic_write_end() /
3101          * ext4_da_write_inline_data_end().
3102          */
3103         new_i_size = pos + copied;
3104         if (copied && new_i_size > inode->i_size &&
3105             ext4_da_should_update_i_disksize(page, end))
3106                 ext4_update_i_disksize(inode, new_i_size);
3107
3108         return generic_write_end(file, mapping, pos, len, copied, page, fsdata);
3109 }
3110
3111 /*
3112  * Force all delayed allocation blocks to be allocated for a given inode.
3113  */
3114 int ext4_alloc_da_blocks(struct inode *inode)
3115 {
3116         trace_ext4_alloc_da_blocks(inode);
3117
3118         if (!EXT4_I(inode)->i_reserved_data_blocks)
3119                 return 0;
3120
3121         /*
3122          * We do something simple for now.  The filemap_flush() will
3123          * also start triggering a write of the data blocks, which is
3124          * not strictly speaking necessary (and for users of
3125          * laptop_mode, not even desirable).  However, to do otherwise
3126          * would require replicating code paths in:
3127          *
3128          * ext4_writepages() ->
3129          *    write_cache_pages() ---> (via passed in callback function)
3130          *        __mpage_da_writepage() -->
3131          *           mpage_add_bh_to_extent()
3132          *           mpage_da_map_blocks()
3133          *
3134          * The problem is that write_cache_pages(), located in
3135          * mm/page-writeback.c, marks pages clean in preparation for
3136          * doing I/O, which is not desirable if we're not planning on
3137          * doing I/O at all.
3138          *
3139          * We could call write_cache_pages(), and then redirty all of
3140          * the pages by calling redirty_page_for_writepage() but that
3141          * would be ugly in the extreme.  So instead we would need to
3142          * replicate parts of the code in the above functions,
3143          * simplifying them because we wouldn't actually intend to
3144          * write out the pages, but rather only collect contiguous
3145          * logical block extents, call the multi-block allocator, and
3146          * then update the buffer heads with the block allocations.
3147          *
3148          * For now, though, we'll cheat by calling filemap_flush(),
3149          * which will map the blocks, and start the I/O, but not
3150          * actually wait for the I/O to complete.
3151          */
3152         return filemap_flush(inode->i_mapping);
3153 }
3154
3155 /*
3156  * bmap() is special.  It gets used by applications such as lilo and by
3157  * the swapper to find the on-disk block of a specific piece of data.
3158  *
3159  * Naturally, this is dangerous if the block concerned is still in the
3160  * journal.  If somebody makes a swapfile on an ext4 data-journaling
3161  * filesystem and enables swap, then they may get a nasty shock when the
3162  * data getting swapped to that swapfile suddenly gets overwritten by
3163  * the original zero's written out previously to the journal and
3164  * awaiting writeback in the kernel's buffer cache.
3165  *
3166  * So, if we see any bmap calls here on a modified, data-journaled file,
3167  * take extra steps to flush any blocks which might be in the cache.
3168  */
3169 static sector_t ext4_bmap(struct address_space *mapping, sector_t block)
3170 {
3171         struct inode *inode = mapping->host;
3172         journal_t *journal;
3173         sector_t ret = 0;
3174         int err;
3175
3176         inode_lock_shared(inode);
3177         /*
3178          * We can get here for an inline file via the FIBMAP ioctl
3179          */
3180         if (ext4_has_inline_data(inode))
3181                 goto out;
3182
3183         if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY) &&
3184                         test_opt(inode->i_sb, DELALLOC)) {
3185                 /*
3186                  * With delalloc we want to sync the file
3187                  * so that we can make sure we allocate
3188                  * blocks for file
3189                  */
3190                 filemap_write_and_wait(mapping);
3191         }
3192
3193         if (EXT4_JOURNAL(inode) &&
3194             ext4_test_inode_state(inode, EXT4_STATE_JDATA)) {
3195                 /*
3196                  * This is a REALLY heavyweight approach, but the use of
3197                  * bmap on dirty files is expected to be extremely rare:
3198                  * only if we run lilo or swapon on a freshly made file
3199                  * do we expect this to happen.
3200                  *
3201                  * (bmap requires CAP_SYS_RAWIO so this does not
3202                  * represent an unprivileged user DOS attack --- we'd be
3203                  * in trouble if mortal users could trigger this path at
3204                  * will.)
3205                  *
3206                  * NB. EXT4_STATE_JDATA is not set on files other than
3207                  * regular files.  If somebody wants to bmap a directory
3208                  * or symlink and gets confused because the buffer
3209                  * hasn't yet been flushed to disk, they deserve
3210                  * everything they get.
3211                  */
3212
3213                 ext4_clear_inode_state(inode, EXT4_STATE_JDATA);
3214                 journal = EXT4_JOURNAL(inode);
3215                 jbd2_journal_lock_updates(journal);
3216                 err = jbd2_journal_flush(journal, 0);
3217                 jbd2_journal_unlock_updates(journal);
3218
3219                 if (err)
3220                         goto out;
3221         }
3222
3223         ret = iomap_bmap(mapping, block, &ext4_iomap_ops);
3224
3225 out:
3226         inode_unlock_shared(inode);
3227         return ret;
3228 }
3229
3230 static int ext4_read_folio(struct file *file, struct folio *folio)
3231 {
3232         struct page *page = &folio->page;
3233         int ret = -EAGAIN;
3234         struct inode *inode = page->mapping->host;
3235
3236         trace_ext4_readpage(page);
3237
3238         if (ext4_has_inline_data(inode))
3239                 ret = ext4_readpage_inline(inode, page);
3240
3241         if (ret == -EAGAIN)
3242                 return ext4_mpage_readpages(inode, NULL, page);
3243
3244         return ret;
3245 }
3246
3247 static void ext4_readahead(struct readahead_control *rac)
3248 {
3249         struct inode *inode = rac->mapping->host;
3250
3251         /* If the file has inline data, no need to do readahead. */
3252         if (ext4_has_inline_data(inode))
3253                 return;
3254
3255         ext4_mpage_readpages(inode, rac, NULL);
3256 }
3257
3258 static void ext4_invalidate_folio(struct folio *folio, size_t offset,
3259                                 size_t length)
3260 {
3261         trace_ext4_invalidate_folio(folio, offset, length);
3262
3263         /* No journalling happens on data buffers when this function is used */
3264         WARN_ON(folio_buffers(folio) && buffer_jbd(folio_buffers(folio)));
3265
3266         block_invalidate_folio(folio, offset, length);
3267 }
3268
3269 static int __ext4_journalled_invalidate_folio(struct folio *folio,
3270                                             size_t offset, size_t length)
3271 {
3272         journal_t *journal = EXT4_JOURNAL(folio->mapping->host);
3273
3274         trace_ext4_journalled_invalidate_folio(folio, offset, length);
3275
3276         /*
3277          * If it's a full truncate we just forget about the pending dirtying
3278          */
3279         if (offset == 0 && length == folio_size(folio))
3280                 folio_clear_checked(folio);
3281
3282         return jbd2_journal_invalidate_folio(journal, folio, offset, length);
3283 }
3284
3285 /* Wrapper for aops... */
3286 static void ext4_journalled_invalidate_folio(struct folio *folio,
3287                                            size_t offset,
3288                                            size_t length)
3289 {
3290         WARN_ON(__ext4_journalled_invalidate_folio(folio, offset, length) < 0);
3291 }
3292
3293 static bool ext4_release_folio(struct folio *folio, gfp_t wait)
3294 {
3295         journal_t *journal = EXT4_JOURNAL(folio->mapping->host);
3296
3297         trace_ext4_releasepage(&folio->page);
3298
3299         /* Page has dirty journalled data -> cannot release */
3300         if (folio_test_checked(folio))
3301                 return false;
3302         if (journal)
3303                 return jbd2_journal_try_to_free_buffers(journal, folio);
3304         else
3305                 return try_to_free_buffers(folio);
3306 }
3307
3308 static bool ext4_inode_datasync_dirty(struct inode *inode)
3309 {
3310         journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
3311
3312         if (journal) {
3313                 if (jbd2_transaction_committed(journal,
3314                         EXT4_I(inode)->i_datasync_tid))
3315                         return false;
3316                 if (test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT))
3317                         return !list_empty(&EXT4_I(inode)->i_fc_list);
3318                 return true;
3319         }
3320
3321         /* Any metadata buffers to write? */
3322         if (!list_empty(&inode->i_mapping->private_list))
3323                 return true;
3324         return inode->i_state & I_DIRTY_DATASYNC;
3325 }
3326
3327 static void ext4_set_iomap(struct inode *inode, struct iomap *iomap,
3328                            struct ext4_map_blocks *map, loff_t offset,
3329                            loff_t length, unsigned int flags)
3330 {
3331         u8 blkbits = inode->i_blkbits;
3332
3333         /*
3334          * Writes that span EOF might trigger an I/O size update on completion,
3335          * so consider them to be dirty for the purpose of O_DSYNC, even if
3336          * there is no other metadata changes being made or are pending.
3337          */
3338         iomap->flags = 0;
3339         if (ext4_inode_datasync_dirty(inode) ||
3340             offset + length > i_size_read(inode))
3341                 iomap->flags |= IOMAP_F_DIRTY;
3342
3343         if (map->m_flags & EXT4_MAP_NEW)
3344                 iomap->flags |= IOMAP_F_NEW;
3345
3346         if (flags & IOMAP_DAX)
3347                 iomap->dax_dev = EXT4_SB(inode->i_sb)->s_daxdev;
3348         else
3349                 iomap->bdev = inode->i_sb->s_bdev;
3350         iomap->offset = (u64) map->m_lblk << blkbits;
3351         iomap->length = (u64) map->m_len << blkbits;
3352
3353         if ((map->m_flags & EXT4_MAP_MAPPED) &&
3354             !ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
3355                 iomap->flags |= IOMAP_F_MERGED;
3356
3357         /*
3358          * Flags passed to ext4_map_blocks() for direct I/O writes can result
3359          * in m_flags having both EXT4_MAP_MAPPED and EXT4_MAP_UNWRITTEN bits
3360          * set. In order for any allocated unwritten extents to be converted
3361          * into written extents correctly within the ->end_io() handler, we
3362          * need to ensure that the iomap->type is set appropriately. Hence, the
3363          * reason why we need to check whether the EXT4_MAP_UNWRITTEN bit has
3364          * been set first.
3365          */
3366         if (map->m_flags & EXT4_MAP_UNWRITTEN) {
3367                 iomap->type = IOMAP_UNWRITTEN;
3368                 iomap->addr = (u64) map->m_pblk << blkbits;
3369                 if (flags & IOMAP_DAX)
3370                         iomap->addr += EXT4_SB(inode->i_sb)->s_dax_part_off;
3371         } else if (map->m_flags & EXT4_MAP_MAPPED) {
3372                 iomap->type = IOMAP_MAPPED;
3373                 iomap->addr = (u64) map->m_pblk << blkbits;
3374                 if (flags & IOMAP_DAX)
3375                         iomap->addr += EXT4_SB(inode->i_sb)->s_dax_part_off;
3376         } else {
3377                 iomap->type = IOMAP_HOLE;
3378                 iomap->addr = IOMAP_NULL_ADDR;
3379         }
3380 }
3381
3382 static int ext4_iomap_alloc(struct inode *inode, struct ext4_map_blocks *map,
3383                             unsigned int flags)
3384 {
3385         handle_t *handle;
3386         u8 blkbits = inode->i_blkbits;
3387         int ret, dio_credits, m_flags = 0, retries = 0;
3388
3389         /*
3390          * Trim the mapping request to the maximum value that we can map at
3391          * once for direct I/O.
3392          */
3393         if (map->m_len > DIO_MAX_BLOCKS)
3394                 map->m_len = DIO_MAX_BLOCKS;
3395         dio_credits = ext4_chunk_trans_blocks(inode, map->m_len);
3396
3397 retry:
3398         /*
3399          * Either we allocate blocks and then don't get an unwritten extent, so
3400          * in that case we have reserved enough credits. Or, the blocks are
3401          * already allocated and unwritten. In that case, the extent conversion
3402          * fits into the credits as well.
3403          */
3404         handle = ext4_journal_start(inode, EXT4_HT_MAP_BLOCKS, dio_credits);
3405         if (IS_ERR(handle))
3406                 return PTR_ERR(handle);
3407
3408         /*
3409          * DAX and direct I/O are the only two operations that are currently
3410          * supported with IOMAP_WRITE.
3411          */
3412         WARN_ON(!(flags & (IOMAP_DAX | IOMAP_DIRECT)));
3413         if (flags & IOMAP_DAX)
3414                 m_flags = EXT4_GET_BLOCKS_CREATE_ZERO;
3415         /*
3416          * We use i_size instead of i_disksize here because delalloc writeback
3417          * can complete at any point during the I/O and subsequently push the
3418          * i_disksize out to i_size. This could be beyond where direct I/O is
3419          * happening and thus expose allocated blocks to direct I/O reads.
3420          */
3421         else if (((loff_t)map->m_lblk << blkbits) >= i_size_read(inode))
3422                 m_flags = EXT4_GET_BLOCKS_CREATE;
3423         else if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
3424                 m_flags = EXT4_GET_BLOCKS_IO_CREATE_EXT;
3425
3426         ret = ext4_map_blocks(handle, inode, map, m_flags);
3427
3428         /*
3429          * We cannot fill holes in indirect tree based inodes as that could
3430          * expose stale data in the case of a crash. Use the magic error code
3431          * to fallback to buffered I/O.
3432          */
3433         if (!m_flags && !ret)
3434                 ret = -ENOTBLK;
3435
3436         ext4_journal_stop(handle);
3437         if (ret == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
3438                 goto retry;
3439
3440         return ret;
3441 }
3442
3443
3444 static int ext4_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
3445                 unsigned flags, struct iomap *iomap, struct iomap *srcmap)
3446 {
3447         int ret;
3448         struct ext4_map_blocks map;
3449         u8 blkbits = inode->i_blkbits;
3450
3451         if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)
3452                 return -EINVAL;
3453
3454         if (WARN_ON_ONCE(ext4_has_inline_data(inode)))
3455                 return -ERANGE;
3456
3457         /*
3458          * Calculate the first and last logical blocks respectively.
3459          */
3460         map.m_lblk = offset >> blkbits;
3461         map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits,
3462                           EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1;
3463
3464         if (flags & IOMAP_WRITE) {
3465                 /*
3466                  * We check here if the blocks are already allocated, then we
3467                  * don't need to start a journal txn and we can directly return
3468                  * the mapping information. This could boost performance
3469                  * especially in multi-threaded overwrite requests.
3470                  */
3471                 if (offset + length <= i_size_read(inode)) {
3472                         ret = ext4_map_blocks(NULL, inode, &map, 0);
3473                         if (ret > 0 && (map.m_flags & EXT4_MAP_MAPPED))
3474                                 goto out;
3475                 }
3476                 ret = ext4_iomap_alloc(inode, &map, flags);
3477         } else {
3478                 ret = ext4_map_blocks(NULL, inode, &map, 0);
3479         }
3480
3481         if (ret < 0)
3482                 return ret;
3483 out:
3484         /*
3485          * When inline encryption is enabled, sometimes I/O to an encrypted file
3486          * has to be broken up to guarantee DUN contiguity.  Handle this by
3487          * limiting the length of the mapping returned.
3488          */
3489         map.m_len = fscrypt_limit_io_blocks(inode, map.m_lblk, map.m_len);
3490
3491         ext4_set_iomap(inode, iomap, &map, offset, length, flags);
3492
3493         return 0;
3494 }
3495
3496 static int ext4_iomap_overwrite_begin(struct inode *inode, loff_t offset,
3497                 loff_t length, unsigned flags, struct iomap *iomap,
3498                 struct iomap *srcmap)
3499 {
3500         int ret;
3501
3502         /*
3503          * Even for writes we don't need to allocate blocks, so just pretend
3504          * we are reading to save overhead of starting a transaction.
3505          */
3506         flags &= ~IOMAP_WRITE;
3507         ret = ext4_iomap_begin(inode, offset, length, flags, iomap, srcmap);
3508         WARN_ON_ONCE(!ret && iomap->type != IOMAP_MAPPED);
3509         return ret;
3510 }
3511
3512 static int ext4_iomap_end(struct inode *inode, loff_t offset, loff_t length,
3513                           ssize_t written, unsigned flags, struct iomap *iomap)
3514 {
3515         /*
3516          * Check to see whether an error occurred while writing out the data to
3517          * the allocated blocks. If so, return the magic error code so that we
3518          * fallback to buffered I/O and attempt to complete the remainder of
3519          * the I/O. Any blocks that may have been allocated in preparation for
3520          * the direct I/O will be reused during buffered I/O.
3521          */
3522         if (flags & (IOMAP_WRITE | IOMAP_DIRECT) && written == 0)
3523                 return -ENOTBLK;
3524
3525         return 0;
3526 }
3527
3528 const struct iomap_ops ext4_iomap_ops = {
3529         .iomap_begin            = ext4_iomap_begin,
3530         .iomap_end              = ext4_iomap_end,
3531 };
3532
3533 const struct iomap_ops ext4_iomap_overwrite_ops = {
3534         .iomap_begin            = ext4_iomap_overwrite_begin,
3535         .iomap_end              = ext4_iomap_end,
3536 };
3537
3538 static bool ext4_iomap_is_delalloc(struct inode *inode,
3539                                    struct ext4_map_blocks *map)
3540 {
3541         struct extent_status es;
3542         ext4_lblk_t offset = 0, end = map->m_lblk + map->m_len - 1;
3543
3544         ext4_es_find_extent_range(inode, &ext4_es_is_delayed,
3545                                   map->m_lblk, end, &es);
3546
3547         if (!es.es_len || es.es_lblk > end)
3548                 return false;
3549
3550         if (es.es_lblk > map->m_lblk) {
3551                 map->m_len = es.es_lblk - map->m_lblk;
3552                 return false;
3553         }
3554
3555         offset = map->m_lblk - es.es_lblk;
3556         map->m_len = es.es_len - offset;
3557
3558         return true;
3559 }
3560
3561 static int ext4_iomap_begin_report(struct inode *inode, loff_t offset,
3562                                    loff_t length, unsigned int flags,
3563                                    struct iomap *iomap, struct iomap *srcmap)
3564 {
3565         int ret;
3566         bool delalloc = false;
3567         struct ext4_map_blocks map;
3568         u8 blkbits = inode->i_blkbits;
3569
3570         if ((offset >> blkbits) > EXT4_MAX_LOGICAL_BLOCK)
3571                 return -EINVAL;
3572
3573         if (ext4_has_inline_data(inode)) {
3574                 ret = ext4_inline_data_iomap(inode, iomap);
3575                 if (ret != -EAGAIN) {
3576                         if (ret == 0 && offset >= iomap->length)
3577                                 ret = -ENOENT;
3578                         return ret;
3579                 }
3580         }
3581
3582         /*
3583          * Calculate the first and last logical block respectively.
3584          */
3585         map.m_lblk = offset >> blkbits;
3586         map.m_len = min_t(loff_t, (offset + length - 1) >> blkbits,
3587                           EXT4_MAX_LOGICAL_BLOCK) - map.m_lblk + 1;
3588
3589         /*
3590          * Fiemap callers may call for offset beyond s_bitmap_maxbytes.
3591          * So handle it here itself instead of querying ext4_map_blocks().
3592          * Since ext4_map_blocks() will warn about it and will return
3593          * -EIO error.
3594          */
3595         if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
3596                 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
3597
3598                 if (offset >= sbi->s_bitmap_maxbytes) {
3599                         map.m_flags = 0;
3600                         goto set_iomap;
3601                 }
3602         }
3603
3604         ret = ext4_map_blocks(NULL, inode, &map, 0);
3605         if (ret < 0)
3606                 return ret;
3607         if (ret == 0)
3608                 delalloc = ext4_iomap_is_delalloc(inode, &map);
3609
3610 set_iomap:
3611         ext4_set_iomap(inode, iomap, &map, offset, length, flags);
3612         if (delalloc && iomap->type == IOMAP_HOLE)
3613                 iomap->type = IOMAP_DELALLOC;
3614
3615         return 0;
3616 }
3617
3618 const struct iomap_ops ext4_iomap_report_ops = {
3619         .iomap_begin = ext4_iomap_begin_report,
3620 };
3621
3622 /*
3623  * Whenever the folio is being dirtied, corresponding buffers should already
3624  * be attached to the transaction (we take care of this in ext4_page_mkwrite()
3625  * and ext4_write_begin()). However we cannot move buffers to dirty transaction
3626  * lists here because ->dirty_folio is called under VFS locks and the folio
3627  * is not necessarily locked.
3628  *
3629  * We cannot just dirty the folio and leave attached buffers clean, because the
3630  * buffers' dirty state is "definitive".  We cannot just set the buffers dirty
3631  * or jbddirty because all the journalling code will explode.
3632  *
3633  * So what we do is to mark the folio "pending dirty" and next time writepage
3634  * is called, propagate that into the buffers appropriately.
3635  */
3636 static bool ext4_journalled_dirty_folio(struct address_space *mapping,
3637                 struct folio *folio)
3638 {
3639         WARN_ON_ONCE(!folio_buffers(folio));
3640         folio_set_checked(folio);
3641         return filemap_dirty_folio(mapping, folio);
3642 }
3643
3644 static bool ext4_dirty_folio(struct address_space *mapping, struct folio *folio)
3645 {
3646         WARN_ON_ONCE(!folio_test_locked(folio) && !folio_test_dirty(folio));
3647         WARN_ON_ONCE(!folio_buffers(folio));
3648         return block_dirty_folio(mapping, folio);
3649 }
3650
3651 static int ext4_iomap_swap_activate(struct swap_info_struct *sis,
3652                                     struct file *file, sector_t *span)
3653 {
3654         return iomap_swapfile_activate(sis, file, span,
3655                                        &ext4_iomap_report_ops);
3656 }
3657
3658 static const struct address_space_operations ext4_aops = {
3659         .read_folio             = ext4_read_folio,
3660         .readahead              = ext4_readahead,
3661         .writepage              = ext4_writepage,
3662         .writepages             = ext4_writepages,
3663         .write_begin            = ext4_write_begin,
3664         .write_end              = ext4_write_end,
3665         .dirty_folio            = ext4_dirty_folio,
3666         .bmap                   = ext4_bmap,
3667         .invalidate_folio       = ext4_invalidate_folio,
3668         .release_folio          = ext4_release_folio,
3669         .direct_IO              = noop_direct_IO,
3670         .migrate_folio          = buffer_migrate_folio,
3671         .is_partially_uptodate  = block_is_partially_uptodate,
3672         .error_remove_page      = generic_error_remove_page,
3673         .swap_activate          = ext4_iomap_swap_activate,
3674 };
3675
3676 static const struct address_space_operations ext4_journalled_aops = {
3677         .read_folio             = ext4_read_folio,
3678         .readahead              = ext4_readahead,
3679         .writepage              = ext4_writepage,
3680         .writepages             = ext4_writepages,
3681         .write_begin            = ext4_write_begin,
3682         .write_end              = ext4_journalled_write_end,
3683         .dirty_folio            = ext4_journalled_dirty_folio,
3684         .bmap                   = ext4_bmap,
3685         .invalidate_folio       = ext4_journalled_invalidate_folio,
3686         .release_folio          = ext4_release_folio,
3687         .direct_IO              = noop_direct_IO,
3688         .is_partially_uptodate  = block_is_partially_uptodate,
3689         .error_remove_page      = generic_error_remove_page,
3690         .swap_activate          = ext4_iomap_swap_activate,
3691 };
3692
3693 static const struct address_space_operations ext4_da_aops = {
3694         .read_folio             = ext4_read_folio,
3695         .readahead              = ext4_readahead,
3696         .writepage              = ext4_writepage,
3697         .writepages             = ext4_writepages,
3698         .write_begin            = ext4_da_write_begin,
3699         .write_end              = ext4_da_write_end,
3700         .dirty_folio            = ext4_dirty_folio,
3701         .bmap                   = ext4_bmap,
3702         .invalidate_folio       = ext4_invalidate_folio,
3703         .release_folio          = ext4_release_folio,
3704         .direct_IO              = noop_direct_IO,
3705         .migrate_folio          = buffer_migrate_folio,
3706         .is_partially_uptodate  = block_is_partially_uptodate,
3707         .error_remove_page      = generic_error_remove_page,
3708         .swap_activate          = ext4_iomap_swap_activate,
3709 };
3710
3711 static const struct address_space_operations ext4_dax_aops = {
3712         .writepages             = ext4_dax_writepages,
3713         .direct_IO              = noop_direct_IO,
3714         .dirty_folio            = noop_dirty_folio,
3715         .bmap                   = ext4_bmap,
3716         .swap_activate          = ext4_iomap_swap_activate,
3717 };
3718
3719 void ext4_set_aops(struct inode *inode)
3720 {
3721         switch (ext4_inode_journal_mode(inode)) {
3722         case EXT4_INODE_ORDERED_DATA_MODE:
3723         case EXT4_INODE_WRITEBACK_DATA_MODE:
3724                 break;
3725         case EXT4_INODE_JOURNAL_DATA_MODE:
3726                 inode->i_mapping->a_ops = &ext4_journalled_aops;
3727                 return;
3728         default:
3729                 BUG();
3730         }
3731         if (IS_DAX(inode))
3732                 inode->i_mapping->a_ops = &ext4_dax_aops;
3733         else if (test_opt(inode->i_sb, DELALLOC))
3734                 inode->i_mapping->a_ops = &ext4_da_aops;
3735         else
3736                 inode->i_mapping->a_ops = &ext4_aops;
3737 }
3738
3739 static int __ext4_block_zero_page_range(handle_t *handle,
3740                 struct address_space *mapping, loff_t from, loff_t length)
3741 {
3742         ext4_fsblk_t index = from >> PAGE_SHIFT;
3743         unsigned offset = from & (PAGE_SIZE-1);
3744         unsigned blocksize, pos;
3745         ext4_lblk_t iblock;
3746         struct inode *inode = mapping->host;
3747         struct buffer_head *bh;
3748         struct page *page;
3749         int err = 0;
3750
3751         page = find_or_create_page(mapping, from >> PAGE_SHIFT,
3752                                    mapping_gfp_constraint(mapping, ~__GFP_FS));
3753         if (!page)
3754                 return -ENOMEM;
3755
3756         blocksize = inode->i_sb->s_blocksize;
3757
3758         iblock = index << (PAGE_SHIFT - inode->i_sb->s_blocksize_bits);
3759
3760         if (!page_has_buffers(page))
3761                 create_empty_buffers(page, blocksize, 0);
3762
3763         /* Find the buffer that contains "offset" */
3764         bh = page_buffers(page);
3765         pos = blocksize;
3766         while (offset >= pos) {
3767                 bh = bh->b_this_page;
3768                 iblock++;
3769                 pos += blocksize;
3770         }
3771         if (buffer_freed(bh)) {
3772                 BUFFER_TRACE(bh, "freed: skip");
3773                 goto unlock;
3774         }
3775         if (!buffer_mapped(bh)) {
3776                 BUFFER_TRACE(bh, "unmapped");
3777                 ext4_get_block(inode, iblock, bh, 0);
3778                 /* unmapped? It's a hole - nothing to do */
3779                 if (!buffer_mapped(bh)) {
3780                         BUFFER_TRACE(bh, "still unmapped");
3781                         goto unlock;
3782                 }
3783         }
3784
3785         /* Ok, it's mapped. Make sure it's up-to-date */
3786         if (PageUptodate(page))
3787                 set_buffer_uptodate(bh);
3788
3789         if (!buffer_uptodate(bh)) {
3790                 err = ext4_read_bh_lock(bh, 0, true);
3791                 if (err)
3792                         goto unlock;
3793                 if (fscrypt_inode_uses_fs_layer_crypto(inode)) {
3794                         /* We expect the key to be set. */
3795                         BUG_ON(!fscrypt_has_encryption_key(inode));
3796                         err = fscrypt_decrypt_pagecache_blocks(page, blocksize,
3797                                                                bh_offset(bh));
3798                         if (err) {
3799                                 clear_buffer_uptodate(bh);
3800                                 goto unlock;
3801                         }
3802                 }
3803         }
3804         if (ext4_should_journal_data(inode)) {
3805                 BUFFER_TRACE(bh, "get write access");
3806                 err = ext4_journal_get_write_access(handle, inode->i_sb, bh,
3807                                                     EXT4_JTR_NONE);
3808                 if (err)
3809                         goto unlock;
3810         }
3811         zero_user(page, offset, length);
3812         BUFFER_TRACE(bh, "zeroed end of block");
3813
3814         if (ext4_should_journal_data(inode)) {
3815                 err = ext4_handle_dirty_metadata(handle, inode, bh);
3816         } else {
3817                 err = 0;
3818                 mark_buffer_dirty(bh);
3819                 if (ext4_should_order_data(inode))
3820                         err = ext4_jbd2_inode_add_write(handle, inode, from,
3821                                         length);
3822         }
3823
3824 unlock:
3825         unlock_page(page);
3826         put_page(page);
3827         return err;
3828 }
3829
3830 /*
3831  * ext4_block_zero_page_range() zeros out a mapping of length 'length'
3832  * starting from file offset 'from'.  The range to be zero'd must
3833  * be contained with in one block.  If the specified range exceeds
3834  * the end of the block it will be shortened to end of the block
3835  * that corresponds to 'from'
3836  */
3837 static int ext4_block_zero_page_range(handle_t *handle,
3838                 struct address_space *mapping, loff_t from, loff_t length)
3839 {
3840         struct inode *inode = mapping->host;
3841         unsigned offset = from & (PAGE_SIZE-1);
3842         unsigned blocksize = inode->i_sb->s_blocksize;
3843         unsigned max = blocksize - (offset & (blocksize - 1));
3844
3845         /*
3846          * correct length if it does not fall between
3847          * 'from' and the end of the block
3848          */
3849         if (length > max || length < 0)
3850                 length = max;
3851
3852         if (IS_DAX(inode)) {
3853                 return dax_zero_range(inode, from, length, NULL,
3854                                       &ext4_iomap_ops);
3855         }
3856         return __ext4_block_zero_page_range(handle, mapping, from, length);
3857 }
3858
3859 /*
3860  * ext4_block_truncate_page() zeroes out a mapping from file offset `from'
3861  * up to the end of the block which corresponds to `from'.
3862  * This required during truncate. We need to physically zero the tail end
3863  * of that block so it doesn't yield old data if the file is later grown.
3864  */
3865 static int ext4_block_truncate_page(handle_t *handle,
3866                 struct address_space *mapping, loff_t from)
3867 {
3868         unsigned offset = from & (PAGE_SIZE-1);
3869         unsigned length;
3870         unsigned blocksize;
3871         struct inode *inode = mapping->host;
3872
3873         /* If we are processing an encrypted inode during orphan list handling */
3874         if (IS_ENCRYPTED(inode) && !fscrypt_has_encryption_key(inode))
3875                 return 0;
3876
3877         blocksize = inode->i_sb->s_blocksize;
3878         length = blocksize - (offset & (blocksize - 1));
3879
3880         return ext4_block_zero_page_range(handle, mapping, from, length);
3881 }
3882
3883 int ext4_zero_partial_blocks(handle_t *handle, struct inode *inode,
3884                              loff_t lstart, loff_t length)
3885 {
3886         struct super_block *sb = inode->i_sb;
3887         struct address_space *mapping = inode->i_mapping;
3888         unsigned partial_start, partial_end;
3889         ext4_fsblk_t start, end;
3890         loff_t byte_end = (lstart + length - 1);
3891         int err = 0;
3892
3893         partial_start = lstart & (sb->s_blocksize - 1);
3894         partial_end = byte_end & (sb->s_blocksize - 1);
3895
3896         start = lstart >> sb->s_blocksize_bits;
3897         end = byte_end >> sb->s_blocksize_bits;
3898
3899         /* Handle partial zero within the single block */
3900         if (start == end &&
3901             (partial_start || (partial_end != sb->s_blocksize - 1))) {
3902                 err = ext4_block_zero_page_range(handle, mapping,
3903                                                  lstart, length);
3904                 return err;
3905         }
3906         /* Handle partial zero out on the start of the range */
3907         if (partial_start) {
3908                 err = ext4_block_zero_page_range(handle, mapping,
3909                                                  lstart, sb->s_blocksize);
3910                 if (err)
3911                         return err;
3912         }
3913         /* Handle partial zero out on the end of the range */
3914         if (partial_end != sb->s_blocksize - 1)
3915                 err = ext4_block_zero_page_range(handle, mapping,
3916                                                  byte_end - partial_end,
3917                                                  partial_end + 1);
3918         return err;
3919 }
3920
3921 int ext4_can_truncate(struct inode *inode)
3922 {
3923         if (S_ISREG(inode->i_mode))
3924                 return 1;
3925         if (S_ISDIR(inode->i_mode))
3926                 return 1;
3927         if (S_ISLNK(inode->i_mode))
3928                 return !ext4_inode_is_fast_symlink(inode);
3929         return 0;
3930 }
3931
3932 /*
3933  * We have to make sure i_disksize gets properly updated before we truncate
3934  * page cache due to hole punching or zero range. Otherwise i_disksize update
3935  * can get lost as it may have been postponed to submission of writeback but
3936  * that will never happen after we truncate page cache.
3937  */
3938 int ext4_update_disksize_before_punch(struct inode *inode, loff_t offset,
3939                                       loff_t len)
3940 {
3941         handle_t *handle;
3942         int ret;
3943
3944         loff_t size = i_size_read(inode);
3945
3946         WARN_ON(!inode_is_locked(inode));
3947         if (offset > size || offset + len < size)
3948                 return 0;
3949
3950         if (EXT4_I(inode)->i_disksize >= size)
3951                 return 0;
3952
3953         handle = ext4_journal_start(inode, EXT4_HT_MISC, 1);
3954         if (IS_ERR(handle))
3955                 return PTR_ERR(handle);
3956         ext4_update_i_disksize(inode, size);
3957         ret = ext4_mark_inode_dirty(handle, inode);
3958         ext4_journal_stop(handle);
3959
3960         return ret;
3961 }
3962
3963 static void ext4_wait_dax_page(struct inode *inode)
3964 {
3965         filemap_invalidate_unlock(inode->i_mapping);
3966         schedule();
3967         filemap_invalidate_lock(inode->i_mapping);
3968 }
3969
3970 int ext4_break_layouts(struct inode *inode)
3971 {
3972         struct page *page;
3973         int error;
3974
3975         if (WARN_ON_ONCE(!rwsem_is_locked(&inode->i_mapping->invalidate_lock)))
3976                 return -EINVAL;
3977
3978         do {
3979                 page = dax_layout_busy_page(inode->i_mapping);
3980                 if (!page)
3981                         return 0;
3982
3983                 error = ___wait_var_event(&page->_refcount,
3984                                 atomic_read(&page->_refcount) == 1,
3985                                 TASK_INTERRUPTIBLE, 0, 0,
3986                                 ext4_wait_dax_page(inode));
3987         } while (error == 0);
3988
3989         return error;
3990 }
3991
3992 /*
3993  * ext4_punch_hole: punches a hole in a file by releasing the blocks
3994  * associated with the given offset and length
3995  *
3996  * @inode:  File inode
3997  * @offset: The offset where the hole will begin
3998  * @len:    The length of the hole
3999  *
4000  * Returns: 0 on success or negative on failure
4001  */
4002
4003 int ext4_punch_hole(struct file *file, loff_t offset, loff_t length)
4004 {
4005         struct inode *inode = file_inode(file);
4006         struct super_block *sb = inode->i_sb;
4007         ext4_lblk_t first_block, stop_block;
4008         struct address_space *mapping = inode->i_mapping;
4009         loff_t first_block_offset, last_block_offset, max_length;
4010         struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
4011         handle_t *handle;
4012         unsigned int credits;
4013         int ret = 0, ret2 = 0;
4014
4015         trace_ext4_punch_hole(inode, offset, length, 0);
4016
4017         /*
4018          * Write out all dirty pages to avoid race conditions
4019          * Then release them.
4020          */
4021         if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) {
4022                 ret = filemap_write_and_wait_range(mapping, offset,
4023                                                    offset + length - 1);
4024                 if (ret)
4025                         return ret;
4026         }
4027
4028         inode_lock(inode);
4029
4030         /* No need to punch hole beyond i_size */
4031         if (offset >= inode->i_size)
4032                 goto out_mutex;
4033
4034         /*
4035          * If the hole extends beyond i_size, set the hole
4036          * to end after the page that contains i_size
4037          */
4038         if (offset + length > inode->i_size) {
4039                 length = inode->i_size +
4040                    PAGE_SIZE - (inode->i_size & (PAGE_SIZE - 1)) -
4041                    offset;
4042         }
4043
4044         /*
4045          * For punch hole the length + offset needs to be within one block
4046          * before last range. Adjust the length if it goes beyond that limit.
4047          */
4048         max_length = sbi->s_bitmap_maxbytes - inode->i_sb->s_blocksize;
4049         if (offset + length > max_length)
4050                 length = max_length - offset;
4051
4052         if (offset & (sb->s_blocksize - 1) ||
4053             (offset + length) & (sb->s_blocksize - 1)) {
4054                 /*
4055                  * Attach jinode to inode for jbd2 if we do any zeroing of
4056                  * partial block
4057                  */
4058                 ret = ext4_inode_attach_jinode(inode);
4059                 if (ret < 0)
4060                         goto out_mutex;
4061
4062         }
4063
4064         /* Wait all existing dio workers, newcomers will block on i_rwsem */
4065         inode_dio_wait(inode);
4066
4067         ret = file_modified(file);
4068         if (ret)
4069                 goto out_mutex;
4070
4071         /*
4072          * Prevent page faults from reinstantiating pages we have released from
4073          * page cache.
4074          */
4075         filemap_invalidate_lock(mapping);
4076
4077         ret = ext4_break_layouts(inode);
4078         if (ret)
4079                 goto out_dio;
4080
4081         first_block_offset = round_up(offset, sb->s_blocksize);
4082         last_block_offset = round_down((offset + length), sb->s_blocksize) - 1;
4083
4084         /* Now release the pages and zero block aligned part of pages*/
4085         if (last_block_offset > first_block_offset) {
4086                 ret = ext4_update_disksize_before_punch(inode, offset, length);
4087                 if (ret)
4088                         goto out_dio;
4089                 truncate_pagecache_range(inode, first_block_offset,
4090                                          last_block_offset);
4091         }
4092
4093         if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4094                 credits = ext4_writepage_trans_blocks(inode);
4095         else
4096                 credits = ext4_blocks_for_truncate(inode);
4097         handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
4098         if (IS_ERR(handle)) {
4099                 ret = PTR_ERR(handle);
4100                 ext4_std_error(sb, ret);
4101                 goto out_dio;
4102         }
4103
4104         ret = ext4_zero_partial_blocks(handle, inode, offset,
4105                                        length);
4106         if (ret)
4107                 goto out_stop;
4108
4109         first_block = (offset + sb->s_blocksize - 1) >>
4110                 EXT4_BLOCK_SIZE_BITS(sb);
4111         stop_block = (offset + length) >> EXT4_BLOCK_SIZE_BITS(sb);
4112
4113         /* If there are blocks to remove, do it */
4114         if (stop_block > first_block) {
4115
4116                 down_write(&EXT4_I(inode)->i_data_sem);
4117                 ext4_discard_preallocations(inode, 0);
4118
4119                 ret = ext4_es_remove_extent(inode, first_block,
4120                                             stop_block - first_block);
4121                 if (ret) {
4122                         up_write(&EXT4_I(inode)->i_data_sem);
4123                         goto out_stop;
4124                 }
4125
4126                 if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4127                         ret = ext4_ext_remove_space(inode, first_block,
4128                                                     stop_block - 1);
4129                 else
4130                         ret = ext4_ind_remove_space(handle, inode, first_block,
4131                                                     stop_block);
4132
4133                 up_write(&EXT4_I(inode)->i_data_sem);
4134         }
4135         ext4_fc_track_range(handle, inode, first_block, stop_block);
4136         if (IS_SYNC(inode))
4137                 ext4_handle_sync(handle);
4138
4139         inode->i_mtime = inode->i_ctime = current_time(inode);
4140         ret2 = ext4_mark_inode_dirty(handle, inode);
4141         if (unlikely(ret2))
4142                 ret = ret2;
4143         if (ret >= 0)
4144                 ext4_update_inode_fsync_trans(handle, inode, 1);
4145 out_stop:
4146         ext4_journal_stop(handle);
4147 out_dio:
4148         filemap_invalidate_unlock(mapping);
4149 out_mutex:
4150         inode_unlock(inode);
4151         return ret;
4152 }
4153
4154 int ext4_inode_attach_jinode(struct inode *inode)
4155 {
4156         struct ext4_inode_info *ei = EXT4_I(inode);
4157         struct jbd2_inode *jinode;
4158
4159         if (ei->jinode || !EXT4_SB(inode->i_sb)->s_journal)
4160                 return 0;
4161
4162         jinode = jbd2_alloc_inode(GFP_KERNEL);
4163         spin_lock(&inode->i_lock);
4164         if (!ei->jinode) {
4165                 if (!jinode) {
4166                         spin_unlock(&inode->i_lock);
4167                         return -ENOMEM;
4168                 }
4169                 ei->jinode = jinode;
4170                 jbd2_journal_init_jbd_inode(ei->jinode, inode);
4171                 jinode = NULL;
4172         }
4173         spin_unlock(&inode->i_lock);
4174         if (unlikely(jinode != NULL))
4175                 jbd2_free_inode(jinode);
4176         return 0;
4177 }
4178
4179 /*
4180  * ext4_truncate()
4181  *
4182  * We block out ext4_get_block() block instantiations across the entire
4183  * transaction, and VFS/VM ensures that ext4_truncate() cannot run
4184  * simultaneously on behalf of the same inode.
4185  *
4186  * As we work through the truncate and commit bits of it to the journal there
4187  * is one core, guiding principle: the file's tree must always be consistent on
4188  * disk.  We must be able to restart the truncate after a crash.
4189  *
4190  * The file's tree may be transiently inconsistent in memory (although it
4191  * probably isn't), but whenever we close off and commit a journal transaction,
4192  * the contents of (the filesystem + the journal) must be consistent and
4193  * restartable.  It's pretty simple, really: bottom up, right to left (although
4194  * left-to-right works OK too).
4195  *
4196  * Note that at recovery time, journal replay occurs *before* the restart of
4197  * truncate against the orphan inode list.
4198  *
4199  * The committed inode has the new, desired i_size (which is the same as
4200  * i_disksize in this case).  After a crash, ext4_orphan_cleanup() will see
4201  * that this inode's truncate did not complete and it will again call
4202  * ext4_truncate() to have another go.  So there will be instantiated blocks
4203  * to the right of the truncation point in a crashed ext4 filesystem.  But
4204  * that's fine - as long as they are linked from the inode, the post-crash
4205  * ext4_truncate() run will find them and release them.
4206  */
4207 int ext4_truncate(struct inode *inode)
4208 {
4209         struct ext4_inode_info *ei = EXT4_I(inode);
4210         unsigned int credits;
4211         int err = 0, err2;
4212         handle_t *handle;
4213         struct address_space *mapping = inode->i_mapping;
4214
4215         /*
4216          * There is a possibility that we're either freeing the inode
4217          * or it's a completely new inode. In those cases we might not
4218          * have i_rwsem locked because it's not necessary.
4219          */
4220         if (!(inode->i_state & (I_NEW|I_FREEING)))
4221                 WARN_ON(!inode_is_locked(inode));
4222         trace_ext4_truncate_enter(inode);
4223
4224         if (!ext4_can_truncate(inode))
4225                 goto out_trace;
4226
4227         if (inode->i_size == 0 && !test_opt(inode->i_sb, NO_AUTO_DA_ALLOC))
4228                 ext4_set_inode_state(inode, EXT4_STATE_DA_ALLOC_CLOSE);
4229
4230         if (ext4_has_inline_data(inode)) {
4231                 int has_inline = 1;
4232
4233                 err = ext4_inline_data_truncate(inode, &has_inline);
4234                 if (err || has_inline)
4235                         goto out_trace;
4236         }
4237
4238         /* If we zero-out tail of the page, we have to create jinode for jbd2 */
4239         if (inode->i_size & (inode->i_sb->s_blocksize - 1)) {
4240                 err = ext4_inode_attach_jinode(inode);
4241                 if (err)
4242                         goto out_trace;
4243         }
4244
4245         if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4246                 credits = ext4_writepage_trans_blocks(inode);
4247         else
4248                 credits = ext4_blocks_for_truncate(inode);
4249
4250         handle = ext4_journal_start(inode, EXT4_HT_TRUNCATE, credits);
4251         if (IS_ERR(handle)) {
4252                 err = PTR_ERR(handle);
4253                 goto out_trace;
4254         }
4255
4256         if (inode->i_size & (inode->i_sb->s_blocksize - 1))
4257                 ext4_block_truncate_page(handle, mapping, inode->i_size);
4258
4259         /*
4260          * We add the inode to the orphan list, so that if this
4261          * truncate spans multiple transactions, and we crash, we will
4262          * resume the truncate when the filesystem recovers.  It also
4263          * marks the inode dirty, to catch the new size.
4264          *
4265          * Implication: the file must always be in a sane, consistent
4266          * truncatable state while each transaction commits.
4267          */
4268         err = ext4_orphan_add(handle, inode);
4269         if (err)
4270                 goto out_stop;
4271
4272         down_write(&EXT4_I(inode)->i_data_sem);
4273
4274         ext4_discard_preallocations(inode, 0);
4275
4276         if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
4277                 err = ext4_ext_truncate(handle, inode);
4278         else
4279                 ext4_ind_truncate(handle, inode);
4280
4281         up_write(&ei->i_data_sem);
4282         if (err)
4283                 goto out_stop;
4284
4285         if (IS_SYNC(inode))
4286                 ext4_handle_sync(handle);
4287
4288 out_stop:
4289         /*
4290          * If this was a simple ftruncate() and the file will remain alive,
4291          * then we need to clear up the orphan record which we created above.
4292          * However, if this was a real unlink then we were called by
4293          * ext4_evict_inode(), and we allow that function to clean up the
4294          * orphan info for us.
4295          */
4296         if (inode->i_nlink)
4297                 ext4_orphan_del(handle, inode);
4298
4299         inode->i_mtime = inode->i_ctime = current_time(inode);
4300         err2 = ext4_mark_inode_dirty(handle, inode);
4301         if (unlikely(err2 && !err))
4302                 err = err2;
4303         ext4_journal_stop(handle);
4304
4305 out_trace:
4306         trace_ext4_truncate_exit(inode);
4307         return err;
4308 }
4309
4310 static inline u64 ext4_inode_peek_iversion(const struct inode *inode)
4311 {
4312         if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
4313                 return inode_peek_iversion_raw(inode);
4314         else
4315                 return inode_peek_iversion(inode);
4316 }
4317
4318 static int ext4_inode_blocks_set(struct ext4_inode *raw_inode,
4319                                  struct ext4_inode_info *ei)
4320 {
4321         struct inode *inode = &(ei->vfs_inode);
4322         u64 i_blocks = READ_ONCE(inode->i_blocks);
4323         struct super_block *sb = inode->i_sb;
4324
4325         if (i_blocks <= ~0U) {
4326                 /*
4327                  * i_blocks can be represented in a 32 bit variable
4328                  * as multiple of 512 bytes
4329                  */
4330                 raw_inode->i_blocks_lo   = cpu_to_le32(i_blocks);
4331                 raw_inode->i_blocks_high = 0;
4332                 ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
4333                 return 0;
4334         }
4335
4336         /*
4337          * This should never happen since sb->s_maxbytes should not have
4338          * allowed this, sb->s_maxbytes was set according to the huge_file
4339          * feature in ext4_fill_super().
4340          */
4341         if (!ext4_has_feature_huge_file(sb))
4342                 return -EFSCORRUPTED;
4343
4344         if (i_blocks <= 0xffffffffffffULL) {
4345                 /*
4346                  * i_blocks can be represented in a 48 bit variable
4347                  * as multiple of 512 bytes
4348                  */
4349                 raw_inode->i_blocks_lo   = cpu_to_le32(i_blocks);
4350                 raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
4351                 ext4_clear_inode_flag(inode, EXT4_INODE_HUGE_FILE);
4352         } else {
4353                 ext4_set_inode_flag(inode, EXT4_INODE_HUGE_FILE);
4354                 /* i_block is stored in file system block size */
4355                 i_blocks = i_blocks >> (inode->i_blkbits - 9);
4356                 raw_inode->i_blocks_lo   = cpu_to_le32(i_blocks);
4357                 raw_inode->i_blocks_high = cpu_to_le16(i_blocks >> 32);
4358         }
4359         return 0;
4360 }
4361
4362 static int ext4_fill_raw_inode(struct inode *inode, struct ext4_inode *raw_inode)
4363 {
4364         struct ext4_inode_info *ei = EXT4_I(inode);
4365         uid_t i_uid;
4366         gid_t i_gid;
4367         projid_t i_projid;
4368         int block;
4369         int err;
4370
4371         err = ext4_inode_blocks_set(raw_inode, ei);
4372
4373         raw_inode->i_mode = cpu_to_le16(inode->i_mode);
4374         i_uid = i_uid_read(inode);
4375         i_gid = i_gid_read(inode);
4376         i_projid = from_kprojid(&init_user_ns, ei->i_projid);
4377         if (!(test_opt(inode->i_sb, NO_UID32))) {
4378                 raw_inode->i_uid_low = cpu_to_le16(low_16_bits(i_uid));
4379                 raw_inode->i_gid_low = cpu_to_le16(low_16_bits(i_gid));
4380                 /*
4381                  * Fix up interoperability with old kernels. Otherwise,
4382                  * old inodes get re-used with the upper 16 bits of the
4383                  * uid/gid intact.
4384                  */
4385                 if (ei->i_dtime && list_empty(&ei->i_orphan)) {
4386                         raw_inode->i_uid_high = 0;
4387                         raw_inode->i_gid_high = 0;
4388                 } else {
4389                         raw_inode->i_uid_high =
4390                                 cpu_to_le16(high_16_bits(i_uid));
4391                         raw_inode->i_gid_high =
4392                                 cpu_to_le16(high_16_bits(i_gid));
4393                 }
4394         } else {
4395                 raw_inode->i_uid_low = cpu_to_le16(fs_high2lowuid(i_uid));
4396                 raw_inode->i_gid_low = cpu_to_le16(fs_high2lowgid(i_gid));
4397                 raw_inode->i_uid_high = 0;
4398                 raw_inode->i_gid_high = 0;
4399         }
4400         raw_inode->i_links_count = cpu_to_le16(inode->i_nlink);
4401
4402         EXT4_INODE_SET_XTIME(i_ctime, inode, raw_inode);
4403         EXT4_INODE_SET_XTIME(i_mtime, inode, raw_inode);
4404         EXT4_INODE_SET_XTIME(i_atime, inode, raw_inode);
4405         EXT4_EINODE_SET_XTIME(i_crtime, ei, raw_inode);
4406
4407         raw_inode->i_dtime = cpu_to_le32(ei->i_dtime);
4408         raw_inode->i_flags = cpu_to_le32(ei->i_flags & 0xFFFFFFFF);
4409         if (likely(!test_opt2(inode->i_sb, HURD_COMPAT)))
4410                 raw_inode->i_file_acl_high =
4411                         cpu_to_le16(ei->i_file_acl >> 32);
4412         raw_inode->i_file_acl_lo = cpu_to_le32(ei->i_file_acl);
4413         ext4_isize_set(raw_inode, ei->i_disksize);
4414
4415         raw_inode->i_generation = cpu_to_le32(inode->i_generation);
4416         if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode)) {
4417                 if (old_valid_dev(inode->i_rdev)) {
4418                         raw_inode->i_block[0] =
4419                                 cpu_to_le32(old_encode_dev(inode->i_rdev));
4420                         raw_inode->i_block[1] = 0;
4421                 } else {
4422                         raw_inode->i_block[0] = 0;
4423                         raw_inode->i_block[1] =
4424                                 cpu_to_le32(new_encode_dev(inode->i_rdev));
4425                         raw_inode->i_block[2] = 0;
4426                 }
4427         } else if (!ext4_has_inline_data(inode)) {
4428                 for (block = 0; block < EXT4_N_BLOCKS; block++)
4429                         raw_inode->i_block[block] = ei->i_data[block];
4430         }
4431
4432         if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
4433                 u64 ivers = ext4_inode_peek_iversion(inode);
4434
4435                 raw_inode->i_disk_version = cpu_to_le32(ivers);
4436                 if (ei->i_extra_isize) {
4437                         if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
4438                                 raw_inode->i_version_hi =
4439                                         cpu_to_le32(ivers >> 32);
4440                         raw_inode->i_extra_isize =
4441                                 cpu_to_le16(ei->i_extra_isize);
4442                 }
4443         }
4444
4445         if (i_projid != EXT4_DEF_PROJID &&
4446             !ext4_has_feature_project(inode->i_sb))
4447                 err = err ?: -EFSCORRUPTED;
4448
4449         if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE &&
4450             EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
4451                 raw_inode->i_projid = cpu_to_le32(i_projid);
4452
4453         ext4_inode_csum_set(inode, raw_inode, ei);
4454         return err;
4455 }
4456
4457 /*
4458  * ext4_get_inode_loc returns with an extra refcount against the inode's
4459  * underlying buffer_head on success. If we pass 'inode' and it does not
4460  * have in-inode xattr, we have all inode data in memory that is needed
4461  * to recreate the on-disk version of this inode.
4462  */
4463 static int __ext4_get_inode_loc(struct super_block *sb, unsigned long ino,
4464                                 struct inode *inode, struct ext4_iloc *iloc,
4465                                 ext4_fsblk_t *ret_block)
4466 {
4467         struct ext4_group_desc  *gdp;
4468         struct buffer_head      *bh;
4469         ext4_fsblk_t            block;
4470         struct blk_plug         plug;
4471         int                     inodes_per_block, inode_offset;
4472
4473         iloc->bh = NULL;
4474         if (ino < EXT4_ROOT_INO ||
4475             ino > le32_to_cpu(EXT4_SB(sb)->s_es->s_inodes_count))
4476                 return -EFSCORRUPTED;
4477
4478         iloc->block_group = (ino - 1) / EXT4_INODES_PER_GROUP(sb);
4479         gdp = ext4_get_group_desc(sb, iloc->block_group, NULL);
4480         if (!gdp)
4481                 return -EIO;
4482
4483         /*
4484          * Figure out the offset within the block group inode table
4485          */
4486         inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
4487         inode_offset = ((ino - 1) %
4488                         EXT4_INODES_PER_GROUP(sb));
4489         iloc->offset = (inode_offset % inodes_per_block) * EXT4_INODE_SIZE(sb);
4490
4491         block = ext4_inode_table(sb, gdp);
4492         if ((block <= le32_to_cpu(EXT4_SB(sb)->s_es->s_first_data_block)) ||
4493             (block >= ext4_blocks_count(EXT4_SB(sb)->s_es))) {
4494                 ext4_error(sb, "Invalid inode table block %llu in "
4495                            "block_group %u", block, iloc->block_group);
4496                 return -EFSCORRUPTED;
4497         }
4498         block += (inode_offset / inodes_per_block);
4499
4500         bh = sb_getblk(sb, block);
4501         if (unlikely(!bh))
4502                 return -ENOMEM;
4503         if (ext4_buffer_uptodate(bh))
4504                 goto has_buffer;
4505
4506         lock_buffer(bh);
4507         if (ext4_buffer_uptodate(bh)) {
4508                 /* Someone brought it uptodate while we waited */
4509                 unlock_buffer(bh);
4510                 goto has_buffer;
4511         }
4512
4513         /*
4514          * If we have all information of the inode in memory and this
4515          * is the only valid inode in the block, we need not read the
4516          * block.
4517          */
4518         if (inode && !ext4_test_inode_state(inode, EXT4_STATE_XATTR)) {
4519                 struct buffer_head *bitmap_bh;
4520                 int i, start;
4521
4522                 start = inode_offset & ~(inodes_per_block - 1);
4523
4524                 /* Is the inode bitmap in cache? */
4525                 bitmap_bh = sb_getblk(sb, ext4_inode_bitmap(sb, gdp));
4526                 if (unlikely(!bitmap_bh))
4527                         goto make_io;
4528
4529                 /*
4530                  * If the inode bitmap isn't in cache then the
4531                  * optimisation may end up performing two reads instead
4532                  * of one, so skip it.
4533                  */
4534                 if (!buffer_uptodate(bitmap_bh)) {
4535                         brelse(bitmap_bh);
4536                         goto make_io;
4537                 }
4538                 for (i = start; i < start + inodes_per_block; i++) {
4539                         if (i == inode_offset)
4540                                 continue;
4541                         if (ext4_test_bit(i, bitmap_bh->b_data))
4542                                 break;
4543                 }
4544                 brelse(bitmap_bh);
4545                 if (i == start + inodes_per_block) {
4546                         struct ext4_inode *raw_inode =
4547                                 (struct ext4_inode *) (bh->b_data + iloc->offset);
4548
4549                         /* all other inodes are free, so skip I/O */
4550                         memset(bh->b_data, 0, bh->b_size);
4551                         if (!ext4_test_inode_state(inode, EXT4_STATE_NEW))
4552                                 ext4_fill_raw_inode(inode, raw_inode);
4553                         set_buffer_uptodate(bh);
4554                         unlock_buffer(bh);
4555                         goto has_buffer;
4556                 }
4557         }
4558
4559 make_io:
4560         /*
4561          * If we need to do any I/O, try to pre-readahead extra
4562          * blocks from the inode table.
4563          */
4564         blk_start_plug(&plug);
4565         if (EXT4_SB(sb)->s_inode_readahead_blks) {
4566                 ext4_fsblk_t b, end, table;
4567                 unsigned num;
4568                 __u32 ra_blks = EXT4_SB(sb)->s_inode_readahead_blks;
4569
4570                 table = ext4_inode_table(sb, gdp);
4571                 /* s_inode_readahead_blks is always a power of 2 */
4572                 b = block & ~((ext4_fsblk_t) ra_blks - 1);
4573                 if (table > b)
4574                         b = table;
4575                 end = b + ra_blks;
4576                 num = EXT4_INODES_PER_GROUP(sb);
4577                 if (ext4_has_group_desc_csum(sb))
4578                         num -= ext4_itable_unused_count(sb, gdp);
4579                 table += num / inodes_per_block;
4580                 if (end > table)
4581                         end = table;
4582                 while (b <= end)
4583                         ext4_sb_breadahead_unmovable(sb, b++);
4584         }
4585
4586         /*
4587          * There are other valid inodes in the buffer, this inode
4588          * has in-inode xattrs, or we don't have this inode in memory.
4589          * Read the block from disk.
4590          */
4591         trace_ext4_load_inode(sb, ino);
4592         ext4_read_bh_nowait(bh, REQ_META | REQ_PRIO, NULL);
4593         blk_finish_plug(&plug);
4594         wait_on_buffer(bh);
4595         ext4_simulate_fail_bh(sb, bh, EXT4_SIM_INODE_EIO);
4596         if (!buffer_uptodate(bh)) {
4597                 if (ret_block)
4598                         *ret_block = block;
4599                 brelse(bh);
4600                 return -EIO;
4601         }
4602 has_buffer:
4603         iloc->bh = bh;
4604         return 0;
4605 }
4606
4607 static int __ext4_get_inode_loc_noinmem(struct inode *inode,
4608                                         struct ext4_iloc *iloc)
4609 {
4610         ext4_fsblk_t err_blk = 0;
4611         int ret;
4612
4613         ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, NULL, iloc,
4614                                         &err_blk);
4615
4616         if (ret == -EIO)
4617                 ext4_error_inode_block(inode, err_blk, EIO,
4618                                         "unable to read itable block");
4619
4620         return ret;
4621 }
4622
4623 int ext4_get_inode_loc(struct inode *inode, struct ext4_iloc *iloc)
4624 {
4625         ext4_fsblk_t err_blk = 0;
4626         int ret;
4627
4628         ret = __ext4_get_inode_loc(inode->i_sb, inode->i_ino, inode, iloc,
4629                                         &err_blk);
4630
4631         if (ret == -EIO)
4632                 ext4_error_inode_block(inode, err_blk, EIO,
4633                                         "unable to read itable block");
4634
4635         return ret;
4636 }
4637
4638
4639 int ext4_get_fc_inode_loc(struct super_block *sb, unsigned long ino,
4640                           struct ext4_iloc *iloc)
4641 {
4642         return __ext4_get_inode_loc(sb, ino, NULL, iloc, NULL);
4643 }
4644
4645 static bool ext4_should_enable_dax(struct inode *inode)
4646 {
4647         struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
4648
4649         if (test_opt2(inode->i_sb, DAX_NEVER))
4650                 return false;
4651         if (!S_ISREG(inode->i_mode))
4652                 return false;
4653         if (ext4_should_journal_data(inode))
4654                 return false;
4655         if (ext4_has_inline_data(inode))
4656                 return false;
4657         if (ext4_test_inode_flag(inode, EXT4_INODE_ENCRYPT))
4658                 return false;
4659         if (ext4_test_inode_flag(inode, EXT4_INODE_VERITY))
4660                 return false;
4661         if (!test_bit(EXT4_FLAGS_BDEV_IS_DAX, &sbi->s_ext4_flags))
4662                 return false;
4663         if (test_opt(inode->i_sb, DAX_ALWAYS))
4664                 return true;
4665
4666         return ext4_test_inode_flag(inode, EXT4_INODE_DAX);
4667 }
4668
4669 void ext4_set_inode_flags(struct inode *inode, bool init)
4670 {
4671         unsigned int flags = EXT4_I(inode)->i_flags;
4672         unsigned int new_fl = 0;
4673
4674         WARN_ON_ONCE(IS_DAX(inode) && init);
4675
4676         if (flags & EXT4_SYNC_FL)
4677                 new_fl |= S_SYNC;
4678         if (flags & EXT4_APPEND_FL)
4679                 new_fl |= S_APPEND;
4680         if (flags & EXT4_IMMUTABLE_FL)
4681                 new_fl |= S_IMMUTABLE;
4682         if (flags & EXT4_NOATIME_FL)
4683                 new_fl |= S_NOATIME;
4684         if (flags & EXT4_DIRSYNC_FL)
4685                 new_fl |= S_DIRSYNC;
4686
4687         /* Because of the way inode_set_flags() works we must preserve S_DAX
4688          * here if already set. */
4689         new_fl |= (inode->i_flags & S_DAX);
4690         if (init && ext4_should_enable_dax(inode))
4691                 new_fl |= S_DAX;
4692
4693         if (flags & EXT4_ENCRYPT_FL)
4694                 new_fl |= S_ENCRYPTED;
4695         if (flags & EXT4_CASEFOLD_FL)
4696                 new_fl |= S_CASEFOLD;
4697         if (flags & EXT4_VERITY_FL)
4698                 new_fl |= S_VERITY;
4699         inode_set_flags(inode, new_fl,
4700                         S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_DAX|
4701                         S_ENCRYPTED|S_CASEFOLD|S_VERITY);
4702 }
4703
4704 static blkcnt_t ext4_inode_blocks(struct ext4_inode *raw_inode,
4705                                   struct ext4_inode_info *ei)
4706 {
4707         blkcnt_t i_blocks ;
4708         struct inode *inode = &(ei->vfs_inode);
4709         struct super_block *sb = inode->i_sb;
4710
4711         if (ext4_has_feature_huge_file(sb)) {
4712                 /* we are using combined 48 bit field */
4713                 i_blocks = ((u64)le16_to_cpu(raw_inode->i_blocks_high)) << 32 |
4714                                         le32_to_cpu(raw_inode->i_blocks_lo);
4715                 if (ext4_test_inode_flag(inode, EXT4_INODE_HUGE_FILE)) {
4716                         /* i_blocks represent file system block size */
4717                         return i_blocks  << (inode->i_blkbits - 9);
4718                 } else {
4719                         return i_blocks;
4720                 }
4721         } else {
4722                 return le32_to_cpu(raw_inode->i_blocks_lo);
4723         }
4724 }
4725
4726 static inline int ext4_iget_extra_inode(struct inode *inode,
4727                                          struct ext4_inode *raw_inode,
4728                                          struct ext4_inode_info *ei)
4729 {
4730         __le32 *magic = (void *)raw_inode +
4731                         EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize;
4732
4733         if (EXT4_INODE_HAS_XATTR_SPACE(inode)  &&
4734             *magic == cpu_to_le32(EXT4_XATTR_MAGIC)) {
4735                 int err;
4736
4737                 ext4_set_inode_state(inode, EXT4_STATE_XATTR);
4738                 err = ext4_find_inline_data_nolock(inode);
4739                 if (!err && ext4_has_inline_data(inode))
4740                         ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
4741                 return err;
4742         } else
4743                 EXT4_I(inode)->i_inline_off = 0;
4744         return 0;
4745 }
4746
4747 int ext4_get_projid(struct inode *inode, kprojid_t *projid)
4748 {
4749         if (!ext4_has_feature_project(inode->i_sb))
4750                 return -EOPNOTSUPP;
4751         *projid = EXT4_I(inode)->i_projid;
4752         return 0;
4753 }
4754
4755 /*
4756  * ext4 has self-managed i_version for ea inodes, it stores the lower 32bit of
4757  * refcount in i_version, so use raw values if inode has EXT4_EA_INODE_FL flag
4758  * set.
4759  */
4760 static inline void ext4_inode_set_iversion_queried(struct inode *inode, u64 val)
4761 {
4762         if (unlikely(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
4763                 inode_set_iversion_raw(inode, val);
4764         else
4765                 inode_set_iversion_queried(inode, val);
4766 }
4767
4768 static const char *check_igot_inode(struct inode *inode, ext4_iget_flags flags)
4769
4770 {
4771         if (flags & EXT4_IGET_EA_INODE) {
4772                 if (!(EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
4773                         return "missing EA_INODE flag";
4774                 if (ext4_test_inode_state(inode, EXT4_STATE_XATTR) ||
4775                     EXT4_I(inode)->i_file_acl)
4776                         return "ea_inode with extended attributes";
4777         } else {
4778                 if ((EXT4_I(inode)->i_flags & EXT4_EA_INODE_FL))
4779                         return "unexpected EA_INODE flag";
4780         }
4781         if (is_bad_inode(inode) && !(flags & EXT4_IGET_BAD))
4782                 return "unexpected bad inode w/o EXT4_IGET_BAD";
4783         return NULL;
4784 }
4785
4786 struct inode *__ext4_iget(struct super_block *sb, unsigned long ino,
4787                           ext4_iget_flags flags, const char *function,
4788                           unsigned int line)
4789 {
4790         struct ext4_iloc iloc;
4791         struct ext4_inode *raw_inode;
4792         struct ext4_inode_info *ei;
4793         struct ext4_super_block *es = EXT4_SB(sb)->s_es;
4794         struct inode *inode;
4795         const char *err_str;
4796         journal_t *journal = EXT4_SB(sb)->s_journal;
4797         long ret;
4798         loff_t size;
4799         int block;
4800         uid_t i_uid;
4801         gid_t i_gid;
4802         projid_t i_projid;
4803
4804         if ((!(flags & EXT4_IGET_SPECIAL) &&
4805              ((ino < EXT4_FIRST_INO(sb) && ino != EXT4_ROOT_INO) ||
4806               ino == le32_to_cpu(es->s_usr_quota_inum) ||
4807               ino == le32_to_cpu(es->s_grp_quota_inum) ||
4808               ino == le32_to_cpu(es->s_prj_quota_inum) ||
4809               ino == le32_to_cpu(es->s_orphan_file_inum))) ||
4810             (ino < EXT4_ROOT_INO) ||
4811             (ino > le32_to_cpu(es->s_inodes_count))) {
4812                 if (flags & EXT4_IGET_HANDLE)
4813                         return ERR_PTR(-ESTALE);
4814                 __ext4_error(sb, function, line, false, EFSCORRUPTED, 0,
4815                              "inode #%lu: comm %s: iget: illegal inode #",
4816                              ino, current->comm);
4817                 return ERR_PTR(-EFSCORRUPTED);
4818         }
4819
4820         inode = iget_locked(sb, ino);
4821         if (!inode)
4822                 return ERR_PTR(-ENOMEM);
4823         if (!(inode->i_state & I_NEW)) {
4824                 if ((err_str = check_igot_inode(inode, flags)) != NULL) {
4825                         ext4_error_inode(inode, function, line, 0, err_str);
4826                         iput(inode);
4827                         return ERR_PTR(-EFSCORRUPTED);
4828                 }
4829                 return inode;
4830         }
4831
4832         ei = EXT4_I(inode);
4833         iloc.bh = NULL;
4834
4835         ret = __ext4_get_inode_loc_noinmem(inode, &iloc);
4836         if (ret < 0)
4837                 goto bad_inode;
4838         raw_inode = ext4_raw_inode(&iloc);
4839
4840         if ((flags & EXT4_IGET_HANDLE) &&
4841             (raw_inode->i_links_count == 0) && (raw_inode->i_mode == 0)) {
4842                 ret = -ESTALE;
4843                 goto bad_inode;
4844         }
4845
4846         if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
4847                 ei->i_extra_isize = le16_to_cpu(raw_inode->i_extra_isize);
4848                 if (EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize >
4849                         EXT4_INODE_SIZE(inode->i_sb) ||
4850                     (ei->i_extra_isize & 3)) {
4851                         ext4_error_inode(inode, function, line, 0,
4852                                          "iget: bad extra_isize %u "
4853                                          "(inode size %u)",
4854                                          ei->i_extra_isize,
4855                                          EXT4_INODE_SIZE(inode->i_sb));
4856                         ret = -EFSCORRUPTED;
4857                         goto bad_inode;
4858                 }
4859         } else
4860                 ei->i_extra_isize = 0;
4861
4862         /* Precompute checksum seed for inode metadata */
4863         if (ext4_has_metadata_csum(sb)) {
4864                 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
4865                 __u32 csum;
4866                 __le32 inum = cpu_to_le32(inode->i_ino);
4867                 __le32 gen = raw_inode->i_generation;
4868                 csum = ext4_chksum(sbi, sbi->s_csum_seed, (__u8 *)&inum,
4869                                    sizeof(inum));
4870                 ei->i_csum_seed = ext4_chksum(sbi, csum, (__u8 *)&gen,
4871                                               sizeof(gen));
4872         }
4873
4874         if ((!ext4_inode_csum_verify(inode, raw_inode, ei) ||
4875             ext4_simulate_fail(sb, EXT4_SIM_INODE_CRC)) &&
4876              (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY))) {
4877                 ext4_error_inode_err(inode, function, line, 0,
4878                                 EFSBADCRC, "iget: checksum invalid");
4879                 ret = -EFSBADCRC;
4880                 goto bad_inode;
4881         }
4882
4883         inode->i_mode = le16_to_cpu(raw_inode->i_mode);
4884         i_uid = (uid_t)le16_to_cpu(raw_inode->i_uid_low);
4885         i_gid = (gid_t)le16_to_cpu(raw_inode->i_gid_low);
4886         if (ext4_has_feature_project(sb) &&
4887             EXT4_INODE_SIZE(sb) > EXT4_GOOD_OLD_INODE_SIZE &&
4888             EXT4_FITS_IN_INODE(raw_inode, ei, i_projid))
4889                 i_projid = (projid_t)le32_to_cpu(raw_inode->i_projid);
4890         else
4891                 i_projid = EXT4_DEF_PROJID;
4892
4893         if (!(test_opt(inode->i_sb, NO_UID32))) {
4894                 i_uid |= le16_to_cpu(raw_inode->i_uid_high) << 16;
4895                 i_gid |= le16_to_cpu(raw_inode->i_gid_high) << 16;
4896         }
4897         i_uid_write(inode, i_uid);
4898         i_gid_write(inode, i_gid);
4899         ei->i_projid = make_kprojid(&init_user_ns, i_projid);
4900         set_nlink(inode, le16_to_cpu(raw_inode->i_links_count));
4901
4902         ext4_clear_state_flags(ei);     /* Only relevant on 32-bit archs */
4903         ei->i_inline_off = 0;
4904         ei->i_dir_start_lookup = 0;
4905         ei->i_dtime = le32_to_cpu(raw_inode->i_dtime);
4906         /* We now have enough fields to check if the inode was active or not.
4907          * This is needed because nfsd might try to access dead inodes
4908          * the test is that same one that e2fsck uses
4909          * NeilBrown 1999oct15
4910          */
4911         if (inode->i_nlink == 0) {
4912                 if ((inode->i_mode == 0 || flags & EXT4_IGET_SPECIAL ||
4913                      !(EXT4_SB(inode->i_sb)->s_mount_state & EXT4_ORPHAN_FS)) &&
4914                     ino != EXT4_BOOT_LOADER_INO) {
4915                         /* this inode is deleted or unallocated */
4916                         if (flags & EXT4_IGET_SPECIAL) {
4917                                 ext4_error_inode(inode, function, line, 0,
4918                                                  "iget: special inode unallocated");
4919                                 ret = -EFSCORRUPTED;
4920                         } else
4921                                 ret = -ESTALE;
4922                         goto bad_inode;
4923                 }
4924                 /* The only unlinked inodes we let through here have
4925                  * valid i_mode and are being read by the orphan
4926                  * recovery code: that's fine, we're about to complete
4927                  * the process of deleting those.
4928                  * OR it is the EXT4_BOOT_LOADER_INO which is
4929                  * not initialized on a new filesystem. */
4930         }
4931         ei->i_flags = le32_to_cpu(raw_inode->i_flags);
4932         ext4_set_inode_flags(inode, true);
4933         inode->i_blocks = ext4_inode_blocks(raw_inode, ei);
4934         ei->i_file_acl = le32_to_cpu(raw_inode->i_file_acl_lo);
4935         if (ext4_has_feature_64bit(sb))
4936                 ei->i_file_acl |=
4937                         ((__u64)le16_to_cpu(raw_inode->i_file_acl_high)) << 32;
4938         inode->i_size = ext4_isize(sb, raw_inode);
4939         if ((size = i_size_read(inode)) < 0) {
4940                 ext4_error_inode(inode, function, line, 0,
4941                                  "iget: bad i_size value: %lld", size);
4942                 ret = -EFSCORRUPTED;
4943                 goto bad_inode;
4944         }
4945         /*
4946          * If dir_index is not enabled but there's dir with INDEX flag set,
4947          * we'd normally treat htree data as empty space. But with metadata
4948          * checksumming that corrupts checksums so forbid that.
4949          */
4950         if (!ext4_has_feature_dir_index(sb) && ext4_has_metadata_csum(sb) &&
4951             ext4_test_inode_flag(inode, EXT4_INODE_INDEX)) {
4952                 ext4_error_inode(inode, function, line, 0,
4953                          "iget: Dir with htree data on filesystem without dir_index feature.");
4954                 ret = -EFSCORRUPTED;
4955                 goto bad_inode;
4956         }
4957         ei->i_disksize = inode->i_size;
4958 #ifdef CONFIG_QUOTA
4959         ei->i_reserved_quota = 0;
4960 #endif
4961         inode->i_generation = le32_to_cpu(raw_inode->i_generation);
4962         ei->i_block_group = iloc.block_group;
4963         ei->i_last_alloc_group = ~0;
4964         /*
4965          * NOTE! The in-memory inode i_data array is in little-endian order
4966          * even on big-endian machines: we do NOT byteswap the block numbers!
4967          */
4968         for (block = 0; block < EXT4_N_BLOCKS; block++)
4969                 ei->i_data[block] = raw_inode->i_block[block];
4970         INIT_LIST_HEAD(&ei->i_orphan);
4971         ext4_fc_init_inode(&ei->vfs_inode);
4972
4973         /*
4974          * Set transaction id's of transactions that have to be committed
4975          * to finish f[data]sync. We set them to currently running transaction
4976          * as we cannot be sure that the inode or some of its metadata isn't
4977          * part of the transaction - the inode could have been reclaimed and
4978          * now it is reread from disk.
4979          */
4980         if (journal) {
4981                 transaction_t *transaction;
4982                 tid_t tid;
4983
4984                 read_lock(&journal->j_state_lock);
4985                 if (journal->j_running_transaction)
4986                         transaction = journal->j_running_transaction;
4987                 else
4988                         transaction = journal->j_committing_transaction;
4989                 if (transaction)
4990                         tid = transaction->t_tid;
4991                 else
4992                         tid = journal->j_commit_sequence;
4993                 read_unlock(&journal->j_state_lock);
4994                 ei->i_sync_tid = tid;
4995                 ei->i_datasync_tid = tid;
4996         }
4997
4998         if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
4999                 if (ei->i_extra_isize == 0) {
5000                         /* The extra space is currently unused. Use it. */
5001                         BUILD_BUG_ON(sizeof(struct ext4_inode) & 3);
5002                         ei->i_extra_isize = sizeof(struct ext4_inode) -
5003                                             EXT4_GOOD_OLD_INODE_SIZE;
5004                 } else {
5005                         ret = ext4_iget_extra_inode(inode, raw_inode, ei);
5006                         if (ret)
5007                                 goto bad_inode;
5008                 }
5009         }
5010
5011         EXT4_INODE_GET_XTIME(i_ctime, inode, raw_inode);
5012         EXT4_INODE_GET_XTIME(i_mtime, inode, raw_inode);
5013         EXT4_INODE_GET_XTIME(i_atime, inode, raw_inode);
5014         EXT4_EINODE_GET_XTIME(i_crtime, ei, raw_inode);
5015
5016         if (likely(!test_opt2(inode->i_sb, HURD_COMPAT))) {
5017                 u64 ivers = le32_to_cpu(raw_inode->i_disk_version);
5018
5019                 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) {
5020                         if (EXT4_FITS_IN_INODE(raw_inode, ei, i_version_hi))
5021                                 ivers |=
5022                     (__u64)(le32_to_cpu(raw_inode->i_version_hi)) << 32;
5023                 }
5024                 ext4_inode_set_iversion_queried(inode, ivers);
5025         }
5026
5027         ret = 0;
5028         if (ei->i_file_acl &&
5029             !ext4_inode_block_valid(inode, ei->i_file_acl, 1)) {
5030                 ext4_error_inode(inode, function, line, 0,
5031                                  "iget: bad extended attribute block %llu",
5032                                  ei->i_file_acl);
5033                 ret = -EFSCORRUPTED;
5034                 goto bad_inode;
5035         } else if (!ext4_has_inline_data(inode)) {
5036                 /* validate the block references in the inode */
5037                 if (!(EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY) &&
5038                         (S_ISREG(inode->i_mode) || S_ISDIR(inode->i_mode) ||
5039                         (S_ISLNK(inode->i_mode) &&
5040                         !ext4_inode_is_fast_symlink(inode)))) {
5041                         if (ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))
5042                                 ret = ext4_ext_check_inode(inode);
5043                         else
5044                                 ret = ext4_ind_check_inode(inode);
5045                 }
5046         }
5047         if (ret)
5048                 goto bad_inode;
5049
5050         if (S_ISREG(inode->i_mode)) {
5051                 inode->i_op = &ext4_file_inode_operations;
5052                 inode->i_fop = &ext4_file_operations;
5053                 ext4_set_aops(inode);
5054         } else if (S_ISDIR(inode->i_mode)) {
5055                 inode->i_op = &ext4_dir_inode_operations;
5056                 inode->i_fop = &ext4_dir_operations;
5057         } else if (S_ISLNK(inode->i_mode)) {
5058                 /* VFS does not allow setting these so must be corruption */
5059                 if (IS_APPEND(inode) || IS_IMMUTABLE(inode)) {
5060                         ext4_error_inode(inode, function, line, 0,
5061                                          "iget: immutable or append flags "
5062                                          "not allowed on symlinks");
5063                         ret = -EFSCORRUPTED;
5064                         goto bad_inode;
5065                 }
5066                 if (IS_ENCRYPTED(inode)) {
5067                         inode->i_op = &ext4_encrypted_symlink_inode_operations;
5068                 } else if (ext4_inode_is_fast_symlink(inode)) {
5069                         inode->i_link = (char *)ei->i_data;
5070                         inode->i_op = &ext4_fast_symlink_inode_operations;
5071                         nd_terminate_link(ei->i_data, inode->i_size,
5072                                 sizeof(ei->i_data) - 1);
5073                 } else {
5074                         inode->i_op = &ext4_symlink_inode_operations;
5075                 }
5076         } else if (S_ISCHR(inode->i_mode) || S_ISBLK(inode->i_mode) ||
5077               S_ISFIFO(inode->i_mode) || S_ISSOCK(inode->i_mode)) {
5078                 inode->i_op = &ext4_special_inode_operations;
5079                 if (raw_inode->i_block[0])
5080                         init_special_inode(inode, inode->i_mode,
5081                            old_decode_dev(le32_to_cpu(raw_inode->i_block[0])));
5082                 else
5083                         init_special_inode(inode, inode->i_mode,
5084                            new_decode_dev(le32_to_cpu(raw_inode->i_block[1])));
5085         } else if (ino == EXT4_BOOT_LOADER_INO) {
5086                 make_bad_inode(inode);
5087         } else {
5088                 ret = -EFSCORRUPTED;
5089                 ext4_error_inode(inode, function, line, 0,
5090                                  "iget: bogus i_mode (%o)", inode->i_mode);
5091                 goto bad_inode;
5092         }
5093         if (IS_CASEFOLDED(inode) && !ext4_has_feature_casefold(inode->i_sb))
5094                 ext4_error_inode(inode, function, line, 0,
5095                                  "casefold flag without casefold feature");
5096         if ((err_str = check_igot_inode(inode, flags)) != NULL) {
5097                 ext4_error_inode(inode, function, line, 0, err_str);
5098                 ret = -EFSCORRUPTED;
5099                 goto bad_inode;
5100         }
5101
5102         brelse(iloc.bh);
5103         unlock_new_inode(inode);
5104         return inode;
5105
5106 bad_inode:
5107         brelse(iloc.bh);
5108         iget_failed(inode);
5109         return ERR_PTR(ret);
5110 }
5111
5112 static void __ext4_update_other_inode_time(struct super_block *sb,
5113                                            unsigned long orig_ino,
5114                                            unsigned long ino,
5115                                            struct ext4_inode *raw_inode)
5116 {
5117         struct inode *inode;
5118
5119         inode = find_inode_by_ino_rcu(sb, ino);
5120         if (!inode)
5121                 return;
5122
5123         if (!inode_is_dirtytime_only(inode))
5124                 return;
5125
5126         spin_lock(&inode->i_lock);
5127         if (inode_is_dirtytime_only(inode)) {
5128                 struct ext4_inode_info  *ei = EXT4_I(inode);
5129
5130                 inode->i_state &= ~I_DIRTY_TIME;
5131                 spin_unlock(&inode->i_lock);
5132
5133                 spin_lock(&ei->i_raw_lock);
5134                 EXT4_INODE_SET_XTIME(i_ctime, inode, raw_inode);
5135                 EXT4_INODE_SET_XTIME(i_mtime, inode, raw_inode);
5136                 EXT4_INODE_SET_XTIME(i_atime, inode, raw_inode);
5137                 ext4_inode_csum_set(inode, raw_inode, ei);
5138                 spin_unlock(&ei->i_raw_lock);
5139                 trace_ext4_other_inode_update_time(inode, orig_ino);
5140                 return;
5141         }
5142         spin_unlock(&inode->i_lock);
5143 }
5144
5145 /*
5146  * Opportunistically update the other time fields for other inodes in
5147  * the same inode table block.
5148  */
5149 static void ext4_update_other_inodes_time(struct super_block *sb,
5150                                           unsigned long orig_ino, char *buf)
5151 {
5152         unsigned long ino;
5153         int i, inodes_per_block = EXT4_SB(sb)->s_inodes_per_block;
5154         int inode_size = EXT4_INODE_SIZE(sb);
5155
5156         /*
5157          * Calculate the first inode in the inode table block.  Inode
5158          * numbers are one-based.  That is, the first inode in a block
5159          * (assuming 4k blocks and 256 byte inodes) is (n*16 + 1).
5160          */
5161         ino = ((orig_ino - 1) & ~(inodes_per_block - 1)) + 1;
5162         rcu_read_lock();
5163         for (i = 0; i < inodes_per_block; i++, ino++, buf += inode_size) {
5164                 if (ino == orig_ino)
5165                         continue;
5166                 __ext4_update_other_inode_time(sb, orig_ino, ino,
5167                                                (struct ext4_inode *)buf);
5168         }
5169         rcu_read_unlock();
5170 }
5171
5172 /*
5173  * Post the struct inode info into an on-disk inode location in the
5174  * buffer-cache.  This gobbles the caller's reference to the
5175  * buffer_head in the inode location struct.
5176  *
5177  * The caller must have write access to iloc->bh.
5178  */
5179 static int ext4_do_update_inode(handle_t *handle,
5180                                 struct inode *inode,
5181                                 struct ext4_iloc *iloc)
5182 {
5183         struct ext4_inode *raw_inode = ext4_raw_inode(iloc);
5184         struct ext4_inode_info *ei = EXT4_I(inode);
5185         struct buffer_head *bh = iloc->bh;
5186         struct super_block *sb = inode->i_sb;
5187         int err;
5188         int need_datasync = 0, set_large_file = 0;
5189
5190         spin_lock(&ei->i_raw_lock);
5191
5192         /*
5193          * For fields not tracked in the in-memory inode, initialise them
5194          * to zero for new inodes.
5195          */
5196         if (ext4_test_inode_state(inode, EXT4_STATE_NEW))
5197                 memset(raw_inode, 0, EXT4_SB(inode->i_sb)->s_inode_size);
5198
5199         if (READ_ONCE(ei->i_disksize) != ext4_isize(inode->i_sb, raw_inode))
5200                 need_datasync = 1;
5201         if (ei->i_disksize > 0x7fffffffULL) {
5202                 if (!ext4_has_feature_large_file(sb) ||
5203                     EXT4_SB(sb)->s_es->s_rev_level == cpu_to_le32(EXT4_GOOD_OLD_REV))
5204                         set_large_file = 1;
5205         }
5206
5207         err = ext4_fill_raw_inode(inode, raw_inode);
5208         spin_unlock(&ei->i_raw_lock);
5209         if (err) {
5210                 EXT4_ERROR_INODE(inode, "corrupted inode contents");
5211                 goto out_brelse;
5212         }
5213
5214         if (inode->i_sb->s_flags & SB_LAZYTIME)
5215                 ext4_update_other_inodes_time(inode->i_sb, inode->i_ino,
5216                                               bh->b_data);
5217
5218         BUFFER_TRACE(bh, "call ext4_handle_dirty_metadata");
5219         err = ext4_handle_dirty_metadata(handle, NULL, bh);
5220         if (err)
5221                 goto out_error;
5222         ext4_clear_inode_state(inode, EXT4_STATE_NEW);
5223         if (set_large_file) {
5224                 BUFFER_TRACE(EXT4_SB(sb)->s_sbh, "get write access");
5225                 err = ext4_journal_get_write_access(handle, sb,
5226                                                     EXT4_SB(sb)->s_sbh,
5227                                                     EXT4_JTR_NONE);
5228                 if (err)
5229                         goto out_error;
5230                 lock_buffer(EXT4_SB(sb)->s_sbh);
5231                 ext4_set_feature_large_file(sb);
5232                 ext4_superblock_csum_set(sb);
5233                 unlock_buffer(EXT4_SB(sb)->s_sbh);
5234                 ext4_handle_sync(handle);
5235                 err = ext4_handle_dirty_metadata(handle, NULL,
5236                                                  EXT4_SB(sb)->s_sbh);
5237         }
5238         ext4_update_inode_fsync_trans(handle, inode, need_datasync);
5239 out_error:
5240         ext4_std_error(inode->i_sb, err);
5241 out_brelse:
5242         brelse(bh);
5243         return err;
5244 }
5245
5246 /*
5247  * ext4_write_inode()
5248  *
5249  * We are called from a few places:
5250  *
5251  * - Within generic_file_aio_write() -> generic_write_sync() for O_SYNC files.
5252  *   Here, there will be no transaction running. We wait for any running
5253  *   transaction to commit.
5254  *
5255  * - Within flush work (sys_sync(), kupdate and such).
5256  *   We wait on commit, if told to.
5257  *
5258  * - Within iput_final() -> write_inode_now()
5259  *   We wait on commit, if told to.
5260  *
5261  * In all cases it is actually safe for us to return without doing anything,
5262  * because the inode has been copied into a raw inode buffer in
5263  * ext4_mark_inode_dirty().  This is a correctness thing for WB_SYNC_ALL
5264  * writeback.
5265  *
5266  * Note that we are absolutely dependent upon all inode dirtiers doing the
5267  * right thing: they *must* call mark_inode_dirty() after dirtying info in
5268  * which we are interested.
5269  *
5270  * It would be a bug for them to not do this.  The code:
5271  *
5272  *      mark_inode_dirty(inode)
5273  *      stuff();
5274  *      inode->i_size = expr;
5275  *
5276  * is in error because write_inode() could occur while `stuff()' is running,
5277  * and the new i_size will be lost.  Plus the inode will no longer be on the
5278  * superblock's dirty inode list.
5279  */
5280 int ext4_write_inode(struct inode *inode, struct writeback_control *wbc)
5281 {
5282         int err;
5283
5284         if (WARN_ON_ONCE(current->flags & PF_MEMALLOC) ||
5285             sb_rdonly(inode->i_sb))
5286                 return 0;
5287
5288         if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
5289                 return -EIO;
5290
5291         if (EXT4_SB(inode->i_sb)->s_journal) {
5292                 if (ext4_journal_current_handle()) {
5293                         ext4_debug("called recursively, non-PF_MEMALLOC!\n");
5294                         dump_stack();
5295                         return -EIO;
5296                 }
5297
5298                 /*
5299                  * No need to force transaction in WB_SYNC_NONE mode. Also
5300                  * ext4_sync_fs() will force the commit after everything is
5301                  * written.
5302                  */
5303                 if (wbc->sync_mode != WB_SYNC_ALL || wbc->for_sync)
5304                         return 0;
5305
5306                 err = ext4_fc_commit(EXT4_SB(inode->i_sb)->s_journal,
5307                                                 EXT4_I(inode)->i_sync_tid);
5308         } else {
5309                 struct ext4_iloc iloc;
5310
5311                 err = __ext4_get_inode_loc_noinmem(inode, &iloc);
5312                 if (err)
5313                         return err;
5314                 /*
5315                  * sync(2) will flush the whole buffer cache. No need to do
5316                  * it here separately for each inode.
5317                  */
5318                 if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync)
5319                         sync_dirty_buffer(iloc.bh);
5320                 if (buffer_req(iloc.bh) && !buffer_uptodate(iloc.bh)) {
5321                         ext4_error_inode_block(inode, iloc.bh->b_blocknr, EIO,
5322                                                "IO error syncing inode");
5323                         err = -EIO;
5324                 }
5325                 brelse(iloc.bh);
5326         }
5327         return err;
5328 }
5329
5330 /*
5331  * In data=journal mode ext4_journalled_invalidate_folio() may fail to invalidate
5332  * buffers that are attached to a folio straddling i_size and are undergoing
5333  * commit. In that case we have to wait for commit to finish and try again.
5334  */
5335 static void ext4_wait_for_tail_page_commit(struct inode *inode)
5336 {
5337         unsigned offset;
5338         journal_t *journal = EXT4_SB(inode->i_sb)->s_journal;
5339         tid_t commit_tid = 0;
5340         int ret;
5341
5342         offset = inode->i_size & (PAGE_SIZE - 1);
5343         /*
5344          * If the folio is fully truncated, we don't need to wait for any commit
5345          * (and we even should not as __ext4_journalled_invalidate_folio() may
5346          * strip all buffers from the folio but keep the folio dirty which can then
5347          * confuse e.g. concurrent ext4_writepage() seeing dirty folio without
5348          * buffers). Also we don't need to wait for any commit if all buffers in
5349          * the folio remain valid. This is most beneficial for the common case of
5350          * blocksize == PAGESIZE.
5351          */
5352         if (!offset || offset > (PAGE_SIZE - i_blocksize(inode)))
5353                 return;
5354         while (1) {
5355                 struct folio *folio = filemap_lock_folio(inode->i_mapping,
5356                                       inode->i_size >> PAGE_SHIFT);
5357                 if (!folio)
5358                         return;
5359                 ret = __ext4_journalled_invalidate_folio(folio, offset,
5360                                                 folio_size(folio) - offset);
5361                 folio_unlock(folio);
5362                 folio_put(folio);
5363                 if (ret != -EBUSY)
5364                         return;
5365                 commit_tid = 0;
5366                 read_lock(&journal->j_state_lock);
5367                 if (journal->j_committing_transaction)
5368                         commit_tid = journal->j_committing_transaction->t_tid;
5369                 read_unlock(&journal->j_state_lock);
5370                 if (commit_tid)
5371                         jbd2_log_wait_commit(journal, commit_tid);
5372         }
5373 }
5374
5375 /*
5376  * ext4_setattr()
5377  *
5378  * Called from notify_change.
5379  *
5380  * We want to trap VFS attempts to truncate the file as soon as
5381  * possible.  In particular, we want to make sure that when the VFS
5382  * shrinks i_size, we put the inode on the orphan list and modify
5383  * i_disksize immediately, so that during the subsequent flushing of
5384  * dirty pages and freeing of disk blocks, we can guarantee that any
5385  * commit will leave the blocks being flushed in an unused state on
5386  * disk.  (On recovery, the inode will get truncated and the blocks will
5387  * be freed, so we have a strong guarantee that no future commit will
5388  * leave these blocks visible to the user.)
5389  *
5390  * Another thing we have to assure is that if we are in ordered mode
5391  * and inode is still attached to the committing transaction, we must
5392  * we start writeout of all the dirty pages which are being truncated.
5393  * This way we are sure that all the data written in the previous
5394  * transaction are already on disk (truncate waits for pages under
5395  * writeback).
5396  *
5397  * Called with inode->i_rwsem down.
5398  */
5399 int ext4_setattr(struct user_namespace *mnt_userns, struct dentry *dentry,
5400                  struct iattr *attr)
5401 {
5402         struct inode *inode = d_inode(dentry);
5403         int error, rc = 0;
5404         int orphan = 0;
5405         const unsigned int ia_valid = attr->ia_valid;
5406         bool inc_ivers = true;
5407
5408         if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
5409                 return -EIO;
5410
5411         if (unlikely(IS_IMMUTABLE(inode)))
5412                 return -EPERM;
5413
5414         if (unlikely(IS_APPEND(inode) &&
5415                      (ia_valid & (ATTR_MODE | ATTR_UID |
5416                                   ATTR_GID | ATTR_TIMES_SET))))
5417                 return -EPERM;
5418
5419         error = setattr_prepare(mnt_userns, dentry, attr);
5420         if (error)
5421                 return error;
5422
5423         error = fscrypt_prepare_setattr(dentry, attr);
5424         if (error)
5425                 return error;
5426
5427         error = fsverity_prepare_setattr(dentry, attr);
5428         if (error)
5429                 return error;
5430
5431         if (is_quota_modification(mnt_userns, inode, attr)) {
5432                 error = dquot_initialize(inode);
5433                 if (error)
5434                         return error;
5435         }
5436
5437         if (i_uid_needs_update(mnt_userns, attr, inode) ||
5438             i_gid_needs_update(mnt_userns, attr, inode)) {
5439                 handle_t *handle;
5440
5441                 /* (user+group)*(old+new) structure, inode write (sb,
5442                  * inode block, ? - but truncate inode update has it) */
5443                 handle = ext4_journal_start(inode, EXT4_HT_QUOTA,
5444                         (EXT4_MAXQUOTAS_INIT_BLOCKS(inode->i_sb) +
5445                          EXT4_MAXQUOTAS_DEL_BLOCKS(inode->i_sb)) + 3);
5446                 if (IS_ERR(handle)) {
5447                         error = PTR_ERR(handle);
5448                         goto err_out;
5449                 }
5450
5451                 /* dquot_transfer() calls back ext4_get_inode_usage() which
5452                  * counts xattr inode references.
5453                  */
5454                 down_read(&EXT4_I(inode)->xattr_sem);
5455                 error = dquot_transfer(mnt_userns, inode, attr);
5456                 up_read(&EXT4_I(inode)->xattr_sem);
5457
5458                 if (error) {
5459                         ext4_journal_stop(handle);
5460                         return error;
5461                 }
5462                 /* Update corresponding info in inode so that everything is in
5463                  * one transaction */
5464                 i_uid_update(mnt_userns, attr, inode);
5465                 i_gid_update(mnt_userns, attr, inode);
5466                 error = ext4_mark_inode_dirty(handle, inode);
5467                 ext4_journal_stop(handle);
5468                 if (unlikely(error)) {
5469                         return error;
5470                 }
5471         }
5472
5473         if (attr->ia_valid & ATTR_SIZE) {
5474                 handle_t *handle;
5475                 loff_t oldsize = inode->i_size;
5476                 loff_t old_disksize;
5477                 int shrink = (attr->ia_size < inode->i_size);
5478
5479                 if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS))) {
5480                         struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
5481
5482                         if (attr->ia_size > sbi->s_bitmap_maxbytes) {
5483                                 return -EFBIG;
5484                         }
5485                 }
5486                 if (!S_ISREG(inode->i_mode)) {
5487                         return -EINVAL;
5488                 }
5489
5490                 if (attr->ia_size == inode->i_size)
5491                         inc_ivers = false;
5492
5493                 if (shrink) {
5494                         if (ext4_should_order_data(inode)) {
5495                                 error = ext4_begin_ordered_truncate(inode,
5496                                                             attr->ia_size);
5497                                 if (error)
5498                                         goto err_out;
5499                         }
5500                         /*
5501                          * Blocks are going to be removed from the inode. Wait
5502                          * for dio in flight.
5503                          */
5504                         inode_dio_wait(inode);
5505                 }
5506
5507                 filemap_invalidate_lock(inode->i_mapping);
5508
5509                 rc = ext4_break_layouts(inode);
5510                 if (rc) {
5511                         filemap_invalidate_unlock(inode->i_mapping);
5512                         goto err_out;
5513                 }
5514
5515                 if (attr->ia_size != inode->i_size) {
5516                         handle = ext4_journal_start(inode, EXT4_HT_INODE, 3);
5517                         if (IS_ERR(handle)) {
5518                                 error = PTR_ERR(handle);
5519                                 goto out_mmap_sem;
5520                         }
5521                         if (ext4_handle_valid(handle) && shrink) {
5522                                 error = ext4_orphan_add(handle, inode);
5523                                 orphan = 1;
5524                         }
5525                         /*
5526                          * Update c/mtime on truncate up, ext4_truncate() will
5527                          * update c/mtime in shrink case below
5528                          */
5529                         if (!shrink) {
5530                                 inode->i_mtime = current_time(inode);
5531                                 inode->i_ctime = inode->i_mtime;
5532                         }
5533
5534                         if (shrink)
5535                                 ext4_fc_track_range(handle, inode,
5536                                         (attr->ia_size > 0 ? attr->ia_size - 1 : 0) >>
5537                                         inode->i_sb->s_blocksize_bits,
5538                                         EXT_MAX_BLOCKS - 1);
5539                         else
5540                                 ext4_fc_track_range(
5541                                         handle, inode,
5542                                         (oldsize > 0 ? oldsize - 1 : oldsize) >>
5543                                         inode->i_sb->s_blocksize_bits,
5544                                         (attr->ia_size > 0 ? attr->ia_size - 1 : 0) >>
5545                                         inode->i_sb->s_blocksize_bits);
5546
5547                         down_write(&EXT4_I(inode)->i_data_sem);
5548                         old_disksize = EXT4_I(inode)->i_disksize;
5549                         EXT4_I(inode)->i_disksize = attr->ia_size;
5550                         rc = ext4_mark_inode_dirty(handle, inode);
5551                         if (!error)
5552                                 error = rc;
5553                         /*
5554                          * We have to update i_size under i_data_sem together
5555                          * with i_disksize to avoid races with writeback code
5556                          * running ext4_wb_update_i_disksize().
5557                          */
5558                         if (!error)
5559                                 i_size_write(inode, attr->ia_size);
5560                         else
5561                                 EXT4_I(inode)->i_disksize = old_disksize;
5562                         up_write(&EXT4_I(inode)->i_data_sem);
5563                         ext4_journal_stop(handle);
5564                         if (error)
5565                                 goto out_mmap_sem;
5566                         if (!shrink) {
5567                                 pagecache_isize_extended(inode, oldsize,
5568                                                          inode->i_size);
5569                         } else if (ext4_should_journal_data(inode)) {
5570                                 ext4_wait_for_tail_page_commit(inode);
5571                         }
5572                 }
5573
5574                 /*
5575                  * Truncate pagecache after we've waited for commit
5576                  * in data=journal mode to make pages freeable.
5577                  */
5578                 truncate_pagecache(inode, inode->i_size);
5579                 /*
5580                  * Call ext4_truncate() even if i_size didn't change to
5581                  * truncate possible preallocated blocks.
5582                  */
5583                 if (attr->ia_size <= oldsize) {
5584                         rc = ext4_truncate(inode);
5585                         if (rc)
5586                                 error = rc;
5587                 }
5588 out_mmap_sem:
5589                 filemap_invalidate_unlock(inode->i_mapping);
5590         }
5591
5592         if (!error) {
5593                 if (inc_ivers)
5594                         inode_inc_iversion(inode);
5595                 setattr_copy(mnt_userns, inode, attr);
5596                 mark_inode_dirty(inode);
5597         }
5598
5599         /*
5600          * If the call to ext4_truncate failed to get a transaction handle at
5601          * all, we need to clean up the in-core orphan list manually.
5602          */
5603         if (orphan && inode->i_nlink)
5604                 ext4_orphan_del(NULL, inode);
5605
5606         if (!error && (ia_valid & ATTR_MODE))
5607                 rc = posix_acl_chmod(mnt_userns, inode, inode->i_mode);
5608
5609 err_out:
5610         if  (error)
5611                 ext4_std_error(inode->i_sb, error);
5612         if (!error)
5613                 error = rc;
5614         return error;
5615 }
5616
5617 u32 ext4_dio_alignment(struct inode *inode)
5618 {
5619         if (fsverity_active(inode))
5620                 return 0;
5621         if (ext4_should_journal_data(inode))
5622                 return 0;
5623         if (ext4_has_inline_data(inode))
5624                 return 0;
5625         if (IS_ENCRYPTED(inode)) {
5626                 if (!fscrypt_dio_supported(inode))
5627                         return 0;
5628                 return i_blocksize(inode);
5629         }
5630         return 1; /* use the iomap defaults */
5631 }
5632
5633 int ext4_getattr(struct user_namespace *mnt_userns, const struct path *path,
5634                  struct kstat *stat, u32 request_mask, unsigned int query_flags)
5635 {
5636         struct inode *inode = d_inode(path->dentry);
5637         struct ext4_inode *raw_inode;
5638         struct ext4_inode_info *ei = EXT4_I(inode);
5639         unsigned int flags;
5640
5641         if ((request_mask & STATX_BTIME) &&
5642             EXT4_FITS_IN_INODE(raw_inode, ei, i_crtime)) {
5643                 stat->result_mask |= STATX_BTIME;
5644                 stat->btime.tv_sec = ei->i_crtime.tv_sec;
5645                 stat->btime.tv_nsec = ei->i_crtime.tv_nsec;
5646         }
5647
5648         /*
5649          * Return the DIO alignment restrictions if requested.  We only return
5650          * this information when requested, since on encrypted files it might
5651          * take a fair bit of work to get if the file wasn't opened recently.
5652          */
5653         if ((request_mask & STATX_DIOALIGN) && S_ISREG(inode->i_mode)) {
5654                 u32 dio_align = ext4_dio_alignment(inode);
5655
5656                 stat->result_mask |= STATX_DIOALIGN;
5657                 if (dio_align == 1) {
5658                         struct block_device *bdev = inode->i_sb->s_bdev;
5659
5660                         /* iomap defaults */
5661                         stat->dio_mem_align = bdev_dma_alignment(bdev) + 1;
5662                         stat->dio_offset_align = bdev_logical_block_size(bdev);
5663                 } else {
5664                         stat->dio_mem_align = dio_align;
5665                         stat->dio_offset_align = dio_align;
5666                 }
5667         }
5668
5669         flags = ei->i_flags & EXT4_FL_USER_VISIBLE;
5670         if (flags & EXT4_APPEND_FL)
5671                 stat->attributes |= STATX_ATTR_APPEND;
5672         if (flags & EXT4_COMPR_FL)
5673                 stat->attributes |= STATX_ATTR_COMPRESSED;
5674         if (flags & EXT4_ENCRYPT_FL)
5675                 stat->attributes |= STATX_ATTR_ENCRYPTED;
5676         if (flags & EXT4_IMMUTABLE_FL)
5677                 stat->attributes |= STATX_ATTR_IMMUTABLE;
5678         if (flags & EXT4_NODUMP_FL)
5679                 stat->attributes |= STATX_ATTR_NODUMP;
5680         if (flags & EXT4_VERITY_FL)
5681                 stat->attributes |= STATX_ATTR_VERITY;
5682
5683         stat->attributes_mask |= (STATX_ATTR_APPEND |
5684                                   STATX_ATTR_COMPRESSED |
5685                                   STATX_ATTR_ENCRYPTED |
5686                                   STATX_ATTR_IMMUTABLE |
5687                                   STATX_ATTR_NODUMP |
5688                                   STATX_ATTR_VERITY);
5689
5690         generic_fillattr(mnt_userns, inode, stat);
5691         return 0;
5692 }
5693
5694 int ext4_file_getattr(struct user_namespace *mnt_userns,
5695                       const struct path *path, struct kstat *stat,
5696                       u32 request_mask, unsigned int query_flags)
5697 {
5698         struct inode *inode = d_inode(path->dentry);
5699         u64 delalloc_blocks;
5700
5701         ext4_getattr(mnt_userns, path, stat, request_mask, query_flags);
5702
5703         /*
5704          * If there is inline data in the inode, the inode will normally not
5705          * have data blocks allocated (it may have an external xattr block).
5706          * Report at least one sector for such files, so tools like tar, rsync,
5707          * others don't incorrectly think the file is completely sparse.
5708          */
5709         if (unlikely(ext4_has_inline_data(inode)))
5710                 stat->blocks += (stat->size + 511) >> 9;
5711
5712         /*
5713          * We can't update i_blocks if the block allocation is delayed
5714          * otherwise in the case of system crash before the real block
5715          * allocation is done, we will have i_blocks inconsistent with
5716          * on-disk file blocks.
5717          * We always keep i_blocks updated together with real
5718          * allocation. But to not confuse with user, stat
5719          * will return the blocks that include the delayed allocation
5720          * blocks for this file.
5721          */
5722         delalloc_blocks = EXT4_C2B(EXT4_SB(inode->i_sb),
5723                                    EXT4_I(inode)->i_reserved_data_blocks);
5724         stat->blocks += delalloc_blocks << (inode->i_sb->s_blocksize_bits - 9);
5725         return 0;
5726 }
5727
5728 static int ext4_index_trans_blocks(struct inode *inode, int lblocks,
5729                                    int pextents)
5730 {
5731         if (!(ext4_test_inode_flag(inode, EXT4_INODE_EXTENTS)))
5732                 return ext4_ind_trans_blocks(inode, lblocks);
5733         return ext4_ext_index_trans_blocks(inode, pextents);
5734 }
5735
5736 /*
5737  * Account for index blocks, block groups bitmaps and block group
5738  * descriptor blocks if modify datablocks and index blocks
5739  * worse case, the indexs blocks spread over different block groups
5740  *
5741  * If datablocks are discontiguous, they are possible to spread over
5742  * different block groups too. If they are contiguous, with flexbg,
5743  * they could still across block group boundary.
5744  *
5745  * Also account for superblock, inode, quota and xattr blocks
5746  */
5747 static int ext4_meta_trans_blocks(struct inode *inode, int lblocks,
5748                                   int pextents)
5749 {
5750         ext4_group_t groups, ngroups = ext4_get_groups_count(inode->i_sb);
5751         int gdpblocks;
5752         int idxblocks;
5753         int ret = 0;
5754
5755         /*
5756          * How many index blocks need to touch to map @lblocks logical blocks
5757          * to @pextents physical extents?
5758          */
5759         idxblocks = ext4_index_trans_blocks(inode, lblocks, pextents);
5760
5761         ret = idxblocks;
5762
5763         /*
5764          * Now let's see how many group bitmaps and group descriptors need
5765          * to account
5766          */
5767         groups = idxblocks + pextents;
5768         gdpblocks = groups;
5769         if (groups > ngroups)
5770                 groups = ngroups;
5771         if (groups > EXT4_SB(inode->i_sb)->s_gdb_count)
5772                 gdpblocks = EXT4_SB(inode->i_sb)->s_gdb_count;
5773
5774         /* bitmaps and block group descriptor blocks */
5775         ret += groups + gdpblocks;
5776
5777         /* Blocks for super block, inode, quota and xattr blocks */
5778         ret += EXT4_META_TRANS_BLOCKS(inode->i_sb);
5779
5780         return ret;
5781 }
5782
5783 /*
5784  * Calculate the total number of credits to reserve to fit
5785  * the modification of a single pages into a single transaction,
5786  * which may include multiple chunks of block allocations.
5787  *
5788  * This could be called via ext4_write_begin()
5789  *
5790  * We need to consider the worse case, when
5791  * one new block per extent.
5792  */
5793 int ext4_writepage_trans_blocks(struct inode *inode)
5794 {
5795         int bpp = ext4_journal_blocks_per_page(inode);
5796         int ret;
5797
5798         ret = ext4_meta_trans_blocks(inode, bpp, bpp);
5799
5800         /* Account for data blocks for journalled mode */
5801         if (ext4_should_journal_data(inode))
5802                 ret += bpp;
5803         return ret;
5804 }
5805
5806 /*
5807  * Calculate the journal credits for a chunk of data modification.
5808  *
5809  * This is called from DIO, fallocate or whoever calling
5810  * ext4_map_blocks() to map/allocate a chunk of contiguous disk blocks.
5811  *
5812  * journal buffers for data blocks are not included here, as DIO
5813  * and fallocate do no need to journal data buffers.
5814  */
5815 int ext4_chunk_trans_blocks(struct inode *inode, int nrblocks)
5816 {
5817         return ext4_meta_trans_blocks(inode, nrblocks, 1);
5818 }
5819
5820 /*
5821  * The caller must have previously called ext4_reserve_inode_write().
5822  * Give this, we know that the caller already has write access to iloc->bh.
5823  */
5824 int ext4_mark_iloc_dirty(handle_t *handle,
5825                          struct inode *inode, struct ext4_iloc *iloc)
5826 {
5827         int err = 0;
5828
5829         if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb)))) {
5830                 put_bh(iloc->bh);
5831                 return -EIO;
5832         }
5833         ext4_fc_track_inode(handle, inode);
5834
5835         /* the do_update_inode consumes one bh->b_count */
5836         get_bh(iloc->bh);
5837
5838         /* ext4_do_update_inode() does jbd2_journal_dirty_metadata */
5839         err = ext4_do_update_inode(handle, inode, iloc);
5840         put_bh(iloc->bh);
5841         return err;
5842 }
5843
5844 /*
5845  * On success, We end up with an outstanding reference count against
5846  * iloc->bh.  This _must_ be cleaned up later.
5847  */
5848
5849 int
5850 ext4_reserve_inode_write(handle_t *handle, struct inode *inode,
5851                          struct ext4_iloc *iloc)
5852 {
5853         int err;
5854
5855         if (unlikely(ext4_forced_shutdown(EXT4_SB(inode->i_sb))))
5856                 return -EIO;
5857
5858         err = ext4_get_inode_loc(inode, iloc);
5859         if (!err) {
5860                 BUFFER_TRACE(iloc->bh, "get_write_access");
5861                 err = ext4_journal_get_write_access(handle, inode->i_sb,
5862                                                     iloc->bh, EXT4_JTR_NONE);
5863                 if (err) {
5864                         brelse(iloc->bh);
5865                         iloc->bh = NULL;
5866                 }
5867         }
5868         ext4_std_error(inode->i_sb, err);
5869         return err;
5870 }
5871
5872 static int __ext4_expand_extra_isize(struct inode *inode,
5873                                      unsigned int new_extra_isize,
5874                                      struct ext4_iloc *iloc,
5875                                      handle_t *handle, int *no_expand)
5876 {
5877         struct ext4_inode *raw_inode;
5878         struct ext4_xattr_ibody_header *header;
5879         unsigned int inode_size = EXT4_INODE_SIZE(inode->i_sb);
5880         struct ext4_inode_info *ei = EXT4_I(inode);
5881         int error;
5882
5883         /* this was checked at iget time, but double check for good measure */
5884         if ((EXT4_GOOD_OLD_INODE_SIZE + ei->i_extra_isize > inode_size) ||
5885             (ei->i_extra_isize & 3)) {
5886                 EXT4_ERROR_INODE(inode, "bad extra_isize %u (inode size %u)",
5887                                  ei->i_extra_isize,
5888                                  EXT4_INODE_SIZE(inode->i_sb));
5889                 return -EFSCORRUPTED;
5890         }
5891         if ((new_extra_isize < ei->i_extra_isize) ||
5892             (new_extra_isize < 4) ||
5893             (new_extra_isize > inode_size - EXT4_GOOD_OLD_INODE_SIZE))
5894                 return -EINVAL; /* Should never happen */
5895
5896         raw_inode = ext4_raw_inode(iloc);
5897
5898         header = IHDR(inode, raw_inode);
5899
5900         /* No extended attributes present */
5901         if (!ext4_test_inode_state(inode, EXT4_STATE_XATTR) ||
5902             header->h_magic != cpu_to_le32(EXT4_XATTR_MAGIC)) {
5903                 memset((void *)raw_inode + EXT4_GOOD_OLD_INODE_SIZE +
5904                        EXT4_I(inode)->i_extra_isize, 0,
5905                        new_extra_isize - EXT4_I(inode)->i_extra_isize);
5906                 EXT4_I(inode)->i_extra_isize = new_extra_isize;
5907                 return 0;
5908         }
5909
5910         /*
5911          * We may need to allocate external xattr block so we need quotas
5912          * initialized. Here we can be called with various locks held so we
5913          * cannot affort to initialize quotas ourselves. So just bail.
5914          */
5915         if (dquot_initialize_needed(inode))
5916                 return -EAGAIN;
5917
5918         /* try to expand with EAs present */
5919         error = ext4_expand_extra_isize_ea(inode, new_extra_isize,
5920                                            raw_inode, handle);
5921         if (error) {
5922                 /*
5923                  * Inode size expansion failed; don't try again
5924                  */
5925                 *no_expand = 1;
5926         }
5927
5928         return error;
5929 }
5930
5931 /*
5932  * Expand an inode by new_extra_isize bytes.
5933  * Returns 0 on success or negative error number on failure.
5934  */
5935 static int ext4_try_to_expand_extra_isize(struct inode *inode,
5936                                           unsigned int new_extra_isize,
5937                                           struct ext4_iloc iloc,
5938                                           handle_t *handle)
5939 {
5940         int no_expand;
5941         int error;
5942
5943         if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND))
5944                 return -EOVERFLOW;
5945
5946         /*
5947          * In nojournal mode, we can immediately attempt to expand
5948          * the inode.  When journaled, we first need to obtain extra
5949          * buffer credits since we may write into the EA block
5950          * with this same handle. If journal_extend fails, then it will
5951          * only result in a minor loss of functionality for that inode.
5952          * If this is felt to be critical, then e2fsck should be run to
5953          * force a large enough s_min_extra_isize.
5954          */
5955         if (ext4_journal_extend(handle,
5956                                 EXT4_DATA_TRANS_BLOCKS(inode->i_sb), 0) != 0)
5957                 return -ENOSPC;
5958
5959         if (ext4_write_trylock_xattr(inode, &no_expand) == 0)
5960                 return -EBUSY;
5961
5962         error = __ext4_expand_extra_isize(inode, new_extra_isize, &iloc,
5963                                           handle, &no_expand);
5964         ext4_write_unlock_xattr(inode, &no_expand);
5965
5966         return error;
5967 }
5968
5969 int ext4_expand_extra_isize(struct inode *inode,
5970                             unsigned int new_extra_isize,
5971                             struct ext4_iloc *iloc)
5972 {
5973         handle_t *handle;
5974         int no_expand;
5975         int error, rc;
5976
5977         if (ext4_test_inode_state(inode, EXT4_STATE_NO_EXPAND)) {
5978                 brelse(iloc->bh);
5979                 return -EOVERFLOW;
5980         }
5981
5982         handle = ext4_journal_start(inode, EXT4_HT_INODE,
5983                                     EXT4_DATA_TRANS_BLOCKS(inode->i_sb));
5984         if (IS_ERR(handle)) {
5985                 error = PTR_ERR(handle);
5986                 brelse(iloc->bh);
5987                 return error;
5988         }
5989
5990         ext4_write_lock_xattr(inode, &no_expand);
5991
5992         BUFFER_TRACE(iloc->bh, "get_write_access");
5993         error = ext4_journal_get_write_access(handle, inode->i_sb, iloc->bh,
5994                                               EXT4_JTR_NONE);
5995         if (error) {
5996                 brelse(iloc->bh);
5997                 goto out_unlock;
5998         }
5999
6000         error = __ext4_expand_extra_isize(inode, new_extra_isize, iloc,
6001                                           handle, &no_expand);
6002
6003         rc = ext4_mark_iloc_dirty(handle, inode, iloc);
6004         if (!error)
6005                 error = rc;
6006
6007 out_unlock:
6008         ext4_write_unlock_xattr(inode, &no_expand);
6009         ext4_journal_stop(handle);
6010         return error;
6011 }
6012
6013 /*
6014  * What we do here is to mark the in-core inode as clean with respect to inode
6015  * dirtiness (it may still be data-dirty).
6016  * This means that the in-core inode may be reaped by prune_icache
6017  * without having to perform any I/O.  This is a very good thing,
6018  * because *any* task may call prune_icache - even ones which
6019  * have a transaction open against a different journal.
6020  *
6021  * Is this cheating?  Not really.  Sure, we haven't written the
6022  * inode out, but prune_icache isn't a user-visible syncing function.
6023  * Whenever the user wants stuff synced (sys_sync, sys_msync, sys_fsync)
6024  * we start and wait on commits.
6025  */
6026 int __ext4_mark_inode_dirty(handle_t *handle, struct inode *inode,
6027                                 const char *func, unsigned int line)
6028 {
6029         struct ext4_iloc iloc;
6030         struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
6031         int err;
6032
6033         might_sleep();
6034         trace_ext4_mark_inode_dirty(inode, _RET_IP_);
6035         err = ext4_reserve_inode_write(handle, inode, &iloc);
6036         if (err)
6037                 goto out;
6038
6039         if (EXT4_I(inode)->i_extra_isize < sbi->s_want_extra_isize)
6040                 ext4_try_to_expand_extra_isize(inode, sbi->s_want_extra_isize,
6041                                                iloc, handle);
6042
6043         err = ext4_mark_iloc_dirty(handle, inode, &iloc);
6044 out:
6045         if (unlikely(err))
6046                 ext4_error_inode_err(inode, func, line, 0, err,
6047                                         "mark_inode_dirty error");
6048         return err;
6049 }
6050
6051 /*
6052  * ext4_dirty_inode() is called from __mark_inode_dirty()
6053  *
6054  * We're really interested in the case where a file is being extended.
6055  * i_size has been changed by generic_commit_write() and we thus need
6056  * to include the updated inode in the current transaction.
6057  *
6058  * Also, dquot_alloc_block() will always dirty the inode when blocks
6059  * are allocated to the file.
6060  *
6061  * If the inode is marked synchronous, we don't honour that here - doing
6062  * so would cause a commit on atime updates, which we don't bother doing.
6063  * We handle synchronous inodes at the highest possible level.
6064  */
6065 void ext4_dirty_inode(struct inode *inode, int flags)
6066 {
6067         handle_t *handle;
6068
6069         handle = ext4_journal_start(inode, EXT4_HT_INODE, 2);
6070         if (IS_ERR(handle))
6071                 return;
6072         ext4_mark_inode_dirty(handle, inode);
6073         ext4_journal_stop(handle);
6074 }
6075
6076 int ext4_change_inode_journal_flag(struct inode *inode, int val)
6077 {
6078         journal_t *journal;
6079         handle_t *handle;
6080         int err;
6081         struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb);
6082
6083         /*
6084          * We have to be very careful here: changing a data block's
6085          * journaling status dynamically is dangerous.  If we write a
6086          * data block to the journal, change the status and then delete
6087          * that block, we risk forgetting to revoke the old log record
6088          * from the journal and so a subsequent replay can corrupt data.
6089          * So, first we make sure that the journal is empty and that
6090          * nobody is changing anything.
6091          */
6092
6093         journal = EXT4_JOURNAL(inode);
6094         if (!journal)
6095                 return 0;
6096         if (is_journal_aborted(journal))
6097                 return -EROFS;
6098
6099         /* Wait for all existing dio workers */
6100         inode_dio_wait(inode);
6101
6102         /*
6103          * Before flushing the journal and switching inode's aops, we have
6104          * to flush all dirty data the inode has. There can be outstanding
6105          * delayed allocations, there can be unwritten extents created by
6106          * fallocate or buffered writes in dioread_nolock mode covered by
6107          * dirty data which can be converted only after flushing the dirty
6108          * data (and journalled aops don't know how to handle these cases).
6109          */
6110         if (val) {
6111                 filemap_invalidate_lock(inode->i_mapping);
6112                 err = filemap_write_and_wait(inode->i_mapping);
6113                 if (err < 0) {
6114                         filemap_invalidate_unlock(inode->i_mapping);
6115                         return err;
6116                 }
6117         }
6118
6119         percpu_down_write(&sbi->s_writepages_rwsem);
6120         jbd2_journal_lock_updates(journal);
6121
6122         /*
6123          * OK, there are no updates running now, and all cached data is
6124          * synced to disk.  We are now in a completely consistent state
6125          * which doesn't have anything in the journal, and we know that
6126          * no filesystem updates are running, so it is safe to modify
6127          * the inode's in-core data-journaling state flag now.
6128          */
6129
6130         if (val)
6131                 ext4_set_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
6132         else {
6133                 err = jbd2_journal_flush(journal, 0);
6134                 if (err < 0) {
6135                         jbd2_journal_unlock_updates(journal);
6136                         percpu_up_write(&sbi->s_writepages_rwsem);
6137                         return err;
6138                 }
6139                 ext4_clear_inode_flag(inode, EXT4_INODE_JOURNAL_DATA);
6140         }
6141         ext4_set_aops(inode);
6142
6143         jbd2_journal_unlock_updates(journal);
6144         percpu_up_write(&sbi->s_writepages_rwsem);
6145
6146         if (val)
6147                 filemap_invalidate_unlock(inode->i_mapping);
6148
6149         /* Finally we can mark the inode as dirty. */
6150
6151         handle = ext4_journal_start(inode, EXT4_HT_INODE, 1);
6152         if (IS_ERR(handle))
6153                 return PTR_ERR(handle);
6154
6155         ext4_fc_mark_ineligible(inode->i_sb,
6156                 EXT4_FC_REASON_JOURNAL_FLAG_CHANGE, handle);
6157         err = ext4_mark_inode_dirty(handle, inode);
6158         ext4_handle_sync(handle);
6159         ext4_journal_stop(handle);
6160         ext4_std_error(inode->i_sb, err);
6161
6162         return err;
6163 }
6164
6165 static int ext4_bh_unmapped(handle_t *handle, struct inode *inode,
6166                             struct buffer_head *bh)
6167 {
6168         return !buffer_mapped(bh);
6169 }
6170
6171 vm_fault_t ext4_page_mkwrite(struct vm_fault *vmf)
6172 {
6173         struct vm_area_struct *vma = vmf->vma;
6174         struct page *page = vmf->page;
6175         loff_t size;
6176         unsigned long len;
6177         int err;
6178         vm_fault_t ret;
6179         struct file *file = vma->vm_file;
6180         struct inode *inode = file_inode(file);
6181         struct address_space *mapping = inode->i_mapping;
6182         handle_t *handle;
6183         get_block_t *get_block;
6184         int retries = 0;
6185
6186         if (unlikely(IS_IMMUTABLE(inode)))
6187                 return VM_FAULT_SIGBUS;
6188
6189         sb_start_pagefault(inode->i_sb);
6190         file_update_time(vma->vm_file);
6191
6192         filemap_invalidate_lock_shared(mapping);
6193
6194         err = ext4_convert_inline_data(inode);
6195         if (err)
6196                 goto out_ret;
6197
6198         /*
6199          * On data journalling we skip straight to the transaction handle:
6200          * there's no delalloc; page truncated will be checked later; the
6201          * early return w/ all buffers mapped (calculates size/len) can't
6202          * be used; and there's no dioread_nolock, so only ext4_get_block.
6203          */
6204         if (ext4_should_journal_data(inode))
6205                 goto retry_alloc;
6206
6207         /* Delalloc case is easy... */
6208         if (test_opt(inode->i_sb, DELALLOC) &&
6209             !ext4_nonda_switch(inode->i_sb)) {
6210                 do {
6211                         err = block_page_mkwrite(vma, vmf,
6212                                                    ext4_da_get_block_prep);
6213                 } while (err == -ENOSPC &&
6214                        ext4_should_retry_alloc(inode->i_sb, &retries));
6215                 goto out_ret;
6216         }
6217
6218         lock_page(page);
6219         size = i_size_read(inode);
6220         /* Page got truncated from under us? */
6221         if (page->mapping != mapping || page_offset(page) > size) {
6222                 unlock_page(page);
6223                 ret = VM_FAULT_NOPAGE;
6224                 goto out;
6225         }
6226
6227         if (page->index == size >> PAGE_SHIFT)
6228                 len = size & ~PAGE_MASK;
6229         else
6230                 len = PAGE_SIZE;
6231         /*
6232          * Return if we have all the buffers mapped. This avoids the need to do
6233          * journal_start/journal_stop which can block and take a long time
6234          *
6235          * This cannot be done for data journalling, as we have to add the
6236          * inode to the transaction's list to writeprotect pages on commit.
6237          */
6238         if (page_has_buffers(page)) {
6239                 if (!ext4_walk_page_buffers(NULL, inode, page_buffers(page),
6240                                             0, len, NULL,
6241                                             ext4_bh_unmapped)) {
6242                         /* Wait so that we don't change page under IO */
6243                         wait_for_stable_page(page);
6244                         ret = VM_FAULT_LOCKED;
6245                         goto out;
6246                 }
6247         }
6248         unlock_page(page);
6249         /* OK, we need to fill the hole... */
6250         if (ext4_should_dioread_nolock(inode))
6251                 get_block = ext4_get_block_unwritten;
6252         else
6253                 get_block = ext4_get_block;
6254 retry_alloc:
6255         handle = ext4_journal_start(inode, EXT4_HT_WRITE_PAGE,
6256                                     ext4_writepage_trans_blocks(inode));
6257         if (IS_ERR(handle)) {
6258                 ret = VM_FAULT_SIGBUS;
6259                 goto out;
6260         }
6261         /*
6262          * Data journalling can't use block_page_mkwrite() because it
6263          * will set_buffer_dirty() before do_journal_get_write_access()
6264          * thus might hit warning messages for dirty metadata buffers.
6265          */
6266         if (!ext4_should_journal_data(inode)) {
6267                 err = block_page_mkwrite(vma, vmf, get_block);
6268         } else {
6269                 lock_page(page);
6270                 size = i_size_read(inode);
6271                 /* Page got truncated from under us? */
6272                 if (page->mapping != mapping || page_offset(page) > size) {
6273                         ret = VM_FAULT_NOPAGE;
6274                         goto out_error;
6275                 }
6276
6277                 if (page->index == size >> PAGE_SHIFT)
6278                         len = size & ~PAGE_MASK;
6279                 else
6280                         len = PAGE_SIZE;
6281
6282                 err = __block_write_begin(page, 0, len, ext4_get_block);
6283                 if (!err) {
6284                         ret = VM_FAULT_SIGBUS;
6285                         if (ext4_walk_page_buffers(handle, inode,
6286                                         page_buffers(page), 0, len, NULL,
6287                                         do_journal_get_write_access))
6288                                 goto out_error;
6289                         if (ext4_walk_page_buffers(handle, inode,
6290                                         page_buffers(page), 0, len, NULL,
6291                                         write_end_fn))
6292                                 goto out_error;
6293                         if (ext4_jbd2_inode_add_write(handle, inode,
6294                                                       page_offset(page), len))
6295                                 goto out_error;
6296                         ext4_set_inode_state(inode, EXT4_STATE_JDATA);
6297                 } else {
6298                         unlock_page(page);
6299                 }
6300         }
6301         ext4_journal_stop(handle);
6302         if (err == -ENOSPC && ext4_should_retry_alloc(inode->i_sb, &retries))
6303                 goto retry_alloc;
6304 out_ret:
6305         ret = block_page_mkwrite_return(err);
6306 out:
6307         filemap_invalidate_unlock_shared(mapping);
6308         sb_end_pagefault(inode->i_sb);
6309         return ret;
6310 out_error:
6311         unlock_page(page);
6312         ext4_journal_stop(handle);
6313         goto out;
6314 }