1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (c) 2000-2005 Silicon Graphics, Inc.
4 * Copyright (c) 2016-2018 Christoph Hellwig.
8 #include "xfs_shared.h"
9 #include "xfs_format.h"
10 #include "xfs_log_format.h"
11 #include "xfs_trans_resv.h"
12 #include "xfs_mount.h"
13 #include "xfs_inode.h"
14 #include "xfs_trans.h"
15 #include "xfs_iomap.h"
16 #include "xfs_trace.h"
18 #include "xfs_bmap_util.h"
19 #include "xfs_reflink.h"
20 #include "xfs_errortag.h"
21 #include "xfs_error.h"
23 struct xfs_writepage_ctx {
24 struct iomap_writepage_ctx ctx;
25 unsigned int data_seq;
29 static inline struct xfs_writepage_ctx *
30 XFS_WPC(struct iomap_writepage_ctx *ctx)
32 return container_of(ctx, struct xfs_writepage_ctx, ctx);
36 * Fast and loose check if this write could update the on-disk inode size.
38 static inline bool xfs_ioend_is_append(struct iomap_ioend *ioend)
40 return ioend->io_offset + ioend->io_size >
41 XFS_I(ioend->io_inode)->i_disk_size;
45 * Update on-disk file size now that data has been written to disk.
53 struct xfs_mount *mp = ip->i_mount;
58 error = xfs_trans_alloc(mp, &M_RES(mp)->tr_fsyncts, 0, 0, 0, &tp);
62 xfs_ilock(ip, XFS_ILOCK_EXCL);
63 isize = xfs_new_eof(ip, offset + size);
65 xfs_iunlock(ip, XFS_ILOCK_EXCL);
70 trace_xfs_setfilesize(ip, offset, size);
72 ip->i_disk_size = isize;
73 xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL);
74 xfs_trans_log_inode(tp, ip, XFS_ILOG_CORE);
76 return xfs_trans_commit(tp);
80 * IO write completion.
84 struct iomap_ioend *ioend)
86 struct xfs_inode *ip = XFS_I(ioend->io_inode);
87 struct xfs_mount *mp = ip->i_mount;
88 xfs_off_t offset = ioend->io_offset;
89 size_t size = ioend->io_size;
90 unsigned int nofs_flag;
94 * We can allocate memory here while doing writeback on behalf of
95 * memory reclaim. To avoid memory allocation deadlocks set the
96 * task-wide nofs context for the following operations.
98 nofs_flag = memalloc_nofs_save();
101 * Just clean up the in-memory structures if the fs has been shut down.
103 if (xfs_is_shutdown(mp)) {
109 * Clean up all COW blocks and underlying data fork delalloc blocks on
110 * I/O error. The delalloc punch is required because this ioend was
111 * mapped to blocks in the COW fork and the associated pages are no
112 * longer dirty. If we don't remove delalloc blocks here, they become
113 * stale and can corrupt free space accounting on unmount.
115 error = blk_status_to_errno(ioend->io_bio.bi_status);
116 if (unlikely(error)) {
117 if (ioend->io_flags & IOMAP_F_SHARED) {
118 xfs_reflink_cancel_cow_range(ip, offset, size, true);
119 xfs_bmap_punch_delalloc_range(ip, offset,
126 * Success: commit the COW or unwritten blocks if needed.
128 if (ioend->io_flags & IOMAP_F_SHARED)
129 error = xfs_reflink_end_cow(ip, offset, size);
130 else if (ioend->io_type == IOMAP_UNWRITTEN)
131 error = xfs_iomap_write_unwritten(ip, offset, size, false);
133 if (!error && xfs_ioend_is_append(ioend))
134 error = xfs_setfilesize(ip, ioend->io_offset, ioend->io_size);
136 iomap_finish_ioends(ioend, error);
137 memalloc_nofs_restore(nofs_flag);
141 * Finish all pending IO completions that require transactional modifications.
143 * We try to merge physical and logically contiguous ioends before completion to
144 * minimise the number of transactions we need to perform during IO completion.
145 * Both unwritten extent conversion and COW remapping need to iterate and modify
146 * one physical extent at a time, so we gain nothing by merging physically
147 * discontiguous extents here.
149 * The ioend chain length that we can be processing here is largely unbound in
150 * length and we may have to perform significant amounts of work on each ioend
151 * to complete it. Hence we have to be careful about holding the CPU for too
156 struct work_struct *work)
158 struct xfs_inode *ip =
159 container_of(work, struct xfs_inode, i_ioend_work);
160 struct iomap_ioend *ioend;
161 struct list_head tmp;
164 spin_lock_irqsave(&ip->i_ioend_lock, flags);
165 list_replace_init(&ip->i_ioend_list, &tmp);
166 spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
168 iomap_sort_ioends(&tmp);
169 while ((ioend = list_first_entry_or_null(&tmp, struct iomap_ioend,
171 list_del_init(&ioend->io_list);
172 iomap_ioend_try_merge(ioend, &tmp);
173 xfs_end_ioend(ioend);
182 struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);
183 struct xfs_inode *ip = XFS_I(ioend->io_inode);
186 spin_lock_irqsave(&ip->i_ioend_lock, flags);
187 if (list_empty(&ip->i_ioend_list))
188 WARN_ON_ONCE(!queue_work(ip->i_mount->m_unwritten_workqueue,
190 list_add_tail(&ioend->io_list, &ip->i_ioend_list);
191 spin_unlock_irqrestore(&ip->i_ioend_lock, flags);
195 * Fast revalidation of the cached writeback mapping. Return true if the current
196 * mapping is valid, false otherwise.
200 struct iomap_writepage_ctx *wpc,
201 struct xfs_inode *ip,
204 if (offset < wpc->iomap.offset ||
205 offset >= wpc->iomap.offset + wpc->iomap.length)
208 * If this is a COW mapping, it is sufficient to check that the mapping
209 * covers the offset. Be careful to check this first because the caller
210 * can revalidate a COW mapping without updating the data seqno.
212 if (wpc->iomap.flags & IOMAP_F_SHARED)
216 * This is not a COW mapping. Check the sequence number of the data fork
217 * because concurrent changes could have invalidated the extent. Check
218 * the COW fork because concurrent changes since the last time we
219 * checked (and found nothing at this offset) could have added
220 * overlapping blocks.
222 if (XFS_WPC(wpc)->data_seq != READ_ONCE(ip->i_df.if_seq)) {
223 trace_xfs_wb_data_iomap_invalid(ip, &wpc->iomap,
224 XFS_WPC(wpc)->data_seq, XFS_DATA_FORK);
227 if (xfs_inode_has_cow_data(ip) &&
228 XFS_WPC(wpc)->cow_seq != READ_ONCE(ip->i_cowfp->if_seq)) {
229 trace_xfs_wb_cow_iomap_invalid(ip, &wpc->iomap,
230 XFS_WPC(wpc)->cow_seq, XFS_COW_FORK);
237 * Pass in a dellalloc extent and convert it to real extents, return the real
238 * extent that maps offset_fsb in wpc->iomap.
240 * The current page is held locked so nothing could have removed the block
241 * backing offset_fsb, although it could have moved from the COW to the data
242 * fork by another thread.
246 struct iomap_writepage_ctx *wpc,
247 struct xfs_inode *ip,
254 if (whichfork == XFS_COW_FORK)
255 seq = &XFS_WPC(wpc)->cow_seq;
257 seq = &XFS_WPC(wpc)->data_seq;
260 * Attempt to allocate whatever delalloc extent currently backs offset
261 * and put the result into wpc->iomap. Allocate in a loop because it
262 * may take several attempts to allocate real blocks for a contiguous
263 * delalloc extent if free space is sufficiently fragmented.
266 error = xfs_bmapi_convert_delalloc(ip, whichfork, offset,
270 } while (wpc->iomap.offset + wpc->iomap.length <= offset);
277 struct iomap_writepage_ctx *wpc,
282 struct xfs_inode *ip = XFS_I(inode);
283 struct xfs_mount *mp = ip->i_mount;
284 ssize_t count = i_blocksize(inode);
285 xfs_fileoff_t offset_fsb = XFS_B_TO_FSBT(mp, offset);
286 xfs_fileoff_t end_fsb = XFS_B_TO_FSB(mp, offset + count);
287 xfs_fileoff_t cow_fsb;
289 struct xfs_bmbt_irec imap;
290 struct xfs_iext_cursor icur;
294 if (xfs_is_shutdown(mp))
297 XFS_ERRORTAG_DELAY(mp, XFS_ERRTAG_WB_DELAY_MS);
300 * COW fork blocks can overlap data fork blocks even if the blocks
301 * aren't shared. COW I/O always takes precedent, so we must always
302 * check for overlap on reflink inodes unless the mapping is already a
303 * COW one, or the COW fork hasn't changed from the last time we looked
306 * It's safe to check the COW fork if_seq here without the ILOCK because
307 * we've indirectly protected against concurrent updates: writeback has
308 * the page locked, which prevents concurrent invalidations by reflink
309 * and directio and prevents concurrent buffered writes to the same
310 * page. Changes to if_seq always happen under i_lock, which protects
311 * against concurrent updates and provides a memory barrier on the way
312 * out that ensures that we always see the current value.
314 if (xfs_imap_valid(wpc, ip, offset))
318 * If we don't have a valid map, now it's time to get a new one for this
319 * offset. This will convert delayed allocations (including COW ones)
320 * into real extents. If we return without a valid map, it means we
321 * landed in a hole and we skip the block.
324 cow_fsb = NULLFILEOFF;
325 whichfork = XFS_DATA_FORK;
326 xfs_ilock(ip, XFS_ILOCK_SHARED);
327 ASSERT(!xfs_need_iread_extents(&ip->i_df));
330 * Check if this is offset is covered by a COW extents, and if yes use
331 * it directly instead of looking up anything in the data fork.
333 if (xfs_inode_has_cow_data(ip) &&
334 xfs_iext_lookup_extent(ip, ip->i_cowfp, offset_fsb, &icur, &imap))
335 cow_fsb = imap.br_startoff;
336 if (cow_fsb != NULLFILEOFF && cow_fsb <= offset_fsb) {
337 XFS_WPC(wpc)->cow_seq = READ_ONCE(ip->i_cowfp->if_seq);
338 xfs_iunlock(ip, XFS_ILOCK_SHARED);
340 whichfork = XFS_COW_FORK;
341 goto allocate_blocks;
345 * No COW extent overlap. Revalidate now that we may have updated
346 * ->cow_seq. If the data mapping is still valid, we're done.
348 if (xfs_imap_valid(wpc, ip, offset)) {
349 xfs_iunlock(ip, XFS_ILOCK_SHARED);
354 * If we don't have a valid map, now it's time to get a new one for this
355 * offset. This will convert delayed allocations (including COW ones)
358 if (!xfs_iext_lookup_extent(ip, &ip->i_df, offset_fsb, &icur, &imap))
359 imap.br_startoff = end_fsb; /* fake a hole past EOF */
360 XFS_WPC(wpc)->data_seq = READ_ONCE(ip->i_df.if_seq);
361 xfs_iunlock(ip, XFS_ILOCK_SHARED);
363 /* landed in a hole or beyond EOF? */
364 if (imap.br_startoff > offset_fsb) {
365 imap.br_blockcount = imap.br_startoff - offset_fsb;
366 imap.br_startoff = offset_fsb;
367 imap.br_startblock = HOLESTARTBLOCK;
368 imap.br_state = XFS_EXT_NORM;
372 * Truncate to the next COW extent if there is one. This is the only
373 * opportunity to do this because we can skip COW fork lookups for the
374 * subsequent blocks in the mapping; however, the requirement to treat
375 * the COW range separately remains.
377 if (cow_fsb != NULLFILEOFF &&
378 cow_fsb < imap.br_startoff + imap.br_blockcount)
379 imap.br_blockcount = cow_fsb - imap.br_startoff;
381 /* got a delalloc extent? */
382 if (imap.br_startblock != HOLESTARTBLOCK &&
383 isnullstartblock(imap.br_startblock))
384 goto allocate_blocks;
386 xfs_bmbt_to_iomap(ip, &wpc->iomap, &imap, 0, 0, XFS_WPC(wpc)->data_seq);
387 trace_xfs_map_blocks_found(ip, offset, count, whichfork, &imap);
390 error = xfs_convert_blocks(wpc, ip, whichfork, offset);
393 * If we failed to find the extent in the COW fork we might have
394 * raced with a COW to data fork conversion or truncate.
395 * Restart the lookup to catch the extent in the data fork for
396 * the former case, but prevent additional retries to avoid
397 * looping forever for the latter case.
399 if (error == -EAGAIN && whichfork == XFS_COW_FORK && !retries++)
401 ASSERT(error != -EAGAIN);
406 * Due to merging the return real extent might be larger than the
407 * original delalloc one. Trim the return extent to the next COW
408 * boundary again to force a re-lookup.
410 if (whichfork != XFS_COW_FORK && cow_fsb != NULLFILEOFF) {
411 loff_t cow_offset = XFS_FSB_TO_B(mp, cow_fsb);
413 if (cow_offset < wpc->iomap.offset + wpc->iomap.length)
414 wpc->iomap.length = cow_offset - wpc->iomap.offset;
417 ASSERT(wpc->iomap.offset <= offset);
418 ASSERT(wpc->iomap.offset + wpc->iomap.length > offset);
419 trace_xfs_map_blocks_alloc(ip, offset, count, whichfork, &imap);
425 struct iomap_ioend *ioend,
428 unsigned int nofs_flag;
431 * We can allocate memory here while doing writeback on behalf of
432 * memory reclaim. To avoid memory allocation deadlocks set the
433 * task-wide nofs context for the following operations.
435 nofs_flag = memalloc_nofs_save();
437 /* Convert CoW extents to regular */
438 if (!status && (ioend->io_flags & IOMAP_F_SHARED)) {
439 status = xfs_reflink_convert_cow(XFS_I(ioend->io_inode),
440 ioend->io_offset, ioend->io_size);
443 memalloc_nofs_restore(nofs_flag);
445 /* send ioends that might require a transaction to the completion wq */
446 if (xfs_ioend_is_append(ioend) || ioend->io_type == IOMAP_UNWRITTEN ||
447 (ioend->io_flags & IOMAP_F_SHARED))
448 ioend->io_bio.bi_end_io = xfs_end_bio;
453 * If the folio has delalloc blocks on it, the caller is asking us to punch them
454 * out. If we don't, we can leave a stale delalloc mapping covered by a clean
455 * page that needs to be dirtied again before the delalloc mapping can be
456 * converted. This stale delalloc mapping can trip up a later direct I/O read
457 * operation on the same region.
459 * We prevent this by truncating away the delalloc regions on the folio. Because
460 * they are delalloc, we can do this without needing a transaction. Indeed - if
461 * we get ENOSPC errors, we have to be able to do this truncation without a
462 * transaction as there is no space left for block reservation (typically why
463 * we see a ENOSPC in writeback).
470 struct xfs_inode *ip = XFS_I(folio->mapping->host);
471 struct xfs_mount *mp = ip->i_mount;
474 if (xfs_is_shutdown(mp))
477 xfs_alert_ratelimited(mp,
478 "page discard on page "PTR_FMT", inode 0x%llx, pos %llu.",
479 folio, ip->i_ino, pos);
482 * The end of the punch range is always the offset of the first
483 * byte of the next folio. Hence the end offset is only dependent on the
484 * folio itself and not the start offset that is passed in.
486 error = xfs_bmap_punch_delalloc_range(ip, pos,
487 folio_pos(folio) + folio_size(folio));
489 if (error && !xfs_is_shutdown(mp))
490 xfs_alert(mp, "page discard unable to remove delalloc mapping.");
493 static const struct iomap_writeback_ops xfs_writeback_ops = {
494 .map_blocks = xfs_map_blocks,
495 .prepare_ioend = xfs_prepare_ioend,
496 .discard_folio = xfs_discard_folio,
501 struct address_space *mapping,
502 struct writeback_control *wbc)
504 struct xfs_writepage_ctx wpc = { };
506 xfs_iflags_clear(XFS_I(mapping->host), XFS_ITRUNCATED);
507 return iomap_writepages(mapping, wbc, &wpc.ctx, &xfs_writeback_ops);
512 struct address_space *mapping,
513 struct writeback_control *wbc)
515 struct xfs_inode *ip = XFS_I(mapping->host);
517 xfs_iflags_clear(ip, XFS_ITRUNCATED);
518 return dax_writeback_mapping_range(mapping,
519 xfs_inode_buftarg(ip)->bt_daxdev, wbc);
524 struct address_space *mapping,
527 struct xfs_inode *ip = XFS_I(mapping->host);
529 trace_xfs_vm_bmap(ip);
532 * The swap code (ab-)uses ->bmap to get a block mapping and then
533 * bypasses the file system for actual I/O. We really can't allow
534 * that on reflinks inodes, so we have to skip out here. And yes,
535 * 0 is the magic code for a bmap error.
537 * Since we don't pass back blockdev info, we can't return bmap
538 * information for rt files either.
540 if (xfs_is_cow_inode(ip) || XFS_IS_REALTIME_INODE(ip))
542 return iomap_bmap(mapping, block, &xfs_read_iomap_ops);
550 return iomap_read_folio(folio, &xfs_read_iomap_ops);
555 struct readahead_control *rac)
557 iomap_readahead(rac, &xfs_read_iomap_ops);
561 xfs_iomap_swapfile_activate(
562 struct swap_info_struct *sis,
563 struct file *swap_file,
566 sis->bdev = xfs_inode_buftarg(XFS_I(file_inode(swap_file)))->bt_bdev;
567 return iomap_swapfile_activate(sis, swap_file, span,
568 &xfs_read_iomap_ops);
571 const struct address_space_operations xfs_address_space_operations = {
572 .read_folio = xfs_vm_read_folio,
573 .readahead = xfs_vm_readahead,
574 .writepages = xfs_vm_writepages,
575 .dirty_folio = iomap_dirty_folio,
576 .release_folio = iomap_release_folio,
577 .invalidate_folio = iomap_invalidate_folio,
579 .migrate_folio = filemap_migrate_folio,
580 .is_partially_uptodate = iomap_is_partially_uptodate,
581 .error_remove_folio = generic_error_remove_folio,
582 .swap_activate = xfs_iomap_swapfile_activate,
585 const struct address_space_operations xfs_dax_aops = {
586 .writepages = xfs_dax_writepages,
587 .dirty_folio = noop_dirty_folio,
588 .swap_activate = xfs_iomap_swapfile_activate,