1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved.
4 * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved.
7 #include <linux/spinlock.h>
8 #include <linux/completion.h>
9 #include <linux/buffer_head.h>
10 #include <linux/blkdev.h>
11 #include <linux/gfs2_ondisk.h>
12 #include <linux/crc32.h>
13 #include <linux/iomap.h>
14 #include <linux/ktime.h>
30 #include "trace_gfs2.h"
32 /* This doesn't need to be that large as max 64 bit pointers in a 4k
33 * block is 512, so __u16 is fine for that. It saves stack space to
37 struct buffer_head *mp_bh[GFS2_MAX_META_HEIGHT];
38 __u16 mp_list[GFS2_MAX_META_HEIGHT];
39 int mp_fheight; /* find_metapath height */
40 int mp_aheight; /* actual height (lookup height) */
43 static int punch_hole(struct gfs2_inode *ip, u64 offset, u64 length);
46 * gfs2_unstuffer_page - unstuff a stuffed inode into a block cached by a page
48 * @dibh: the dinode buffer
49 * @block: the block number that was allocated
50 * @page: The (optional) page. This is looked up if @page is NULL
55 static int gfs2_unstuffer_page(struct gfs2_inode *ip, struct buffer_head *dibh,
56 u64 block, struct page *page)
58 struct inode *inode = &ip->i_inode;
60 if (!PageUptodate(page)) {
61 void *kaddr = kmap(page);
62 u64 dsize = i_size_read(inode);
64 if (dsize > gfs2_max_stuffed_size(ip))
65 dsize = gfs2_max_stuffed_size(ip);
67 memcpy(kaddr, dibh->b_data + sizeof(struct gfs2_dinode), dsize);
68 memset(kaddr + dsize, 0, PAGE_SIZE - dsize);
71 SetPageUptodate(page);
74 if (gfs2_is_jdata(ip)) {
75 struct buffer_head *bh;
77 if (!page_has_buffers(page))
78 create_empty_buffers(page, BIT(inode->i_blkbits),
81 bh = page_buffers(page);
82 if (!buffer_mapped(bh))
83 map_bh(bh, inode->i_sb, block);
85 set_buffer_uptodate(bh);
86 gfs2_trans_add_data(ip->i_gl, bh);
89 gfs2_ordered_add_inode(ip);
95 static int __gfs2_unstuff_inode(struct gfs2_inode *ip, struct page *page)
97 struct buffer_head *bh, *dibh;
98 struct gfs2_dinode *di;
100 int isdir = gfs2_is_dir(ip);
103 error = gfs2_meta_inode_buffer(ip, &dibh);
107 if (i_size_read(&ip->i_inode)) {
108 /* Get a free block, fill it with the stuffed data,
109 and write it out to disk */
112 error = gfs2_alloc_blocks(ip, &block, &n, 0, NULL);
116 gfs2_trans_remove_revoke(GFS2_SB(&ip->i_inode), block, 1);
117 error = gfs2_dir_get_new_buffer(ip, block, &bh);
120 gfs2_buffer_copy_tail(bh, sizeof(struct gfs2_meta_header),
121 dibh, sizeof(struct gfs2_dinode));
124 error = gfs2_unstuffer_page(ip, dibh, block, page);
130 /* Set up the pointer to the new block */
132 gfs2_trans_add_meta(ip->i_gl, dibh);
133 di = (struct gfs2_dinode *)dibh->b_data;
134 gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode));
136 if (i_size_read(&ip->i_inode)) {
137 *(__be64 *)(di + 1) = cpu_to_be64(block);
138 gfs2_add_inode_blocks(&ip->i_inode, 1);
139 di->di_blocks = cpu_to_be64(gfs2_get_inode_blocks(&ip->i_inode));
143 di->di_height = cpu_to_be16(1);
151 * gfs2_unstuff_dinode - Unstuff a dinode when the data has grown too big
152 * @ip: The GFS2 inode to unstuff
154 * This routine unstuffs a dinode and returns it to a "normal" state such
155 * that the height can be grown in the traditional way.
160 int gfs2_unstuff_dinode(struct gfs2_inode *ip)
162 struct inode *inode = &ip->i_inode;
166 down_write(&ip->i_rw_mutex);
167 page = find_or_create_page(inode->i_mapping, 0, GFP_NOFS);
171 error = __gfs2_unstuff_inode(ip, page);
175 up_write(&ip->i_rw_mutex);
180 * find_metapath - Find path through the metadata tree
181 * @sdp: The superblock
182 * @block: The disk block to look up
183 * @mp: The metapath to return the result in
184 * @height: The pre-calculated height of the metadata tree
186 * This routine returns a struct metapath structure that defines a path
187 * through the metadata of inode "ip" to get to block "block".
190 * Given: "ip" is a height 3 file, "offset" is 101342453, and this is a
191 * filesystem with a blocksize of 4096.
193 * find_metapath() would return a struct metapath structure set to:
194 * mp_fheight = 3, mp_list[0] = 0, mp_list[1] = 48, and mp_list[2] = 165.
196 * That means that in order to get to the block containing the byte at
197 * offset 101342453, we would load the indirect block pointed to by pointer
198 * 0 in the dinode. We would then load the indirect block pointed to by
199 * pointer 48 in that indirect block. We would then load the data block
200 * pointed to by pointer 165 in that indirect block.
202 * ----------------------------------------
207 * ----------------------------------------
211 * ----------------------------------------
215 * |0 5 6 7 8 9 0 1 2|
216 * ----------------------------------------
220 * ----------------------------------------
225 * ----------------------------------------
229 * ----------------------------------------
230 * | Data block containing offset |
234 * ----------------------------------------
238 static void find_metapath(const struct gfs2_sbd *sdp, u64 block,
239 struct metapath *mp, unsigned int height)
243 mp->mp_fheight = height;
244 for (i = height; i--;)
245 mp->mp_list[i] = do_div(block, sdp->sd_inptrs);
248 static inline unsigned int metapath_branch_start(const struct metapath *mp)
250 if (mp->mp_list[0] == 0)
256 * metaptr1 - Return the first possible metadata pointer in a metapath buffer
257 * @height: The metadata height (0 = dinode)
260 static inline __be64 *metaptr1(unsigned int height, const struct metapath *mp)
262 struct buffer_head *bh = mp->mp_bh[height];
264 return ((__be64 *)(bh->b_data + sizeof(struct gfs2_dinode)));
265 return ((__be64 *)(bh->b_data + sizeof(struct gfs2_meta_header)));
269 * metapointer - Return pointer to start of metadata in a buffer
270 * @height: The metadata height (0 = dinode)
273 * Return a pointer to the block number of the next height of the metadata
274 * tree given a buffer containing the pointer to the current height of the
278 static inline __be64 *metapointer(unsigned int height, const struct metapath *mp)
280 __be64 *p = metaptr1(height, mp);
281 return p + mp->mp_list[height];
284 static inline const __be64 *metaend(unsigned int height, const struct metapath *mp)
286 const struct buffer_head *bh = mp->mp_bh[height];
287 return (const __be64 *)(bh->b_data + bh->b_size);
290 static void clone_metapath(struct metapath *clone, struct metapath *mp)
295 for (hgt = 0; hgt < mp->mp_aheight; hgt++)
296 get_bh(clone->mp_bh[hgt]);
299 static void gfs2_metapath_ra(struct gfs2_glock *gl, __be64 *start, __be64 *end)
303 for (t = start; t < end; t++) {
304 struct buffer_head *rabh;
309 rabh = gfs2_getbuf(gl, be64_to_cpu(*t), CREATE);
310 if (trylock_buffer(rabh)) {
311 if (!buffer_uptodate(rabh)) {
312 rabh->b_end_io = end_buffer_read_sync;
313 submit_bh(REQ_OP_READ,
314 REQ_RAHEAD | REQ_META | REQ_PRIO,
324 static int __fillup_metapath(struct gfs2_inode *ip, struct metapath *mp,
325 unsigned int x, unsigned int h)
328 __be64 *ptr = metapointer(x, mp);
329 u64 dblock = be64_to_cpu(*ptr);
334 ret = gfs2_meta_buffer(ip, GFS2_METATYPE_IN, dblock, &mp->mp_bh[x + 1]);
338 mp->mp_aheight = x + 1;
343 * lookup_metapath - Walk the metadata tree to a specific point
347 * Assumes that the inode's buffer has already been looked up and
348 * hooked onto mp->mp_bh[0] and that the metapath has been initialised
349 * by find_metapath().
351 * If this function encounters part of the tree which has not been
352 * allocated, it returns the current height of the tree at the point
353 * at which it found the unallocated block. Blocks which are found are
354 * added to the mp->mp_bh[] list.
359 static int lookup_metapath(struct gfs2_inode *ip, struct metapath *mp)
361 return __fillup_metapath(ip, mp, 0, ip->i_height - 1);
365 * fillup_metapath - fill up buffers for the metadata path to a specific height
368 * @h: The height to which it should be mapped
370 * Similar to lookup_metapath, but does lookups for a range of heights
372 * Returns: error or the number of buffers filled
375 static int fillup_metapath(struct gfs2_inode *ip, struct metapath *mp, int h)
381 /* find the first buffer we need to look up. */
382 for (x = h - 1; x > 0; x--) {
387 ret = __fillup_metapath(ip, mp, x, h);
390 return mp->mp_aheight - x - 1;
393 static sector_t metapath_to_block(struct gfs2_sbd *sdp, struct metapath *mp)
395 sector_t factor = 1, block = 0;
398 for (hgt = mp->mp_fheight - 1; hgt >= 0; hgt--) {
399 if (hgt < mp->mp_aheight)
400 block += mp->mp_list[hgt] * factor;
401 factor *= sdp->sd_inptrs;
406 static void release_metapath(struct metapath *mp)
410 for (i = 0; i < GFS2_MAX_META_HEIGHT; i++) {
411 if (mp->mp_bh[i] == NULL)
413 brelse(mp->mp_bh[i]);
419 * gfs2_extent_length - Returns length of an extent of blocks
420 * @bh: The metadata block
421 * @ptr: Current position in @bh
422 * @limit: Max extent length to return
423 * @eob: Set to 1 if we hit "end of block"
425 * Returns: The length of the extent (minimum of one block)
428 static inline unsigned int gfs2_extent_length(struct buffer_head *bh, __be64 *ptr, size_t limit, int *eob)
430 const __be64 *end = (__be64 *)(bh->b_data + bh->b_size);
431 const __be64 *first = ptr;
432 u64 d = be64_to_cpu(*ptr);
440 } while(be64_to_cpu(*ptr) == d);
446 enum walker_status { WALK_STOP, WALK_FOLLOW, WALK_CONTINUE };
449 * gfs2_metadata_walker - walk an indirect block
450 * @mp: Metapath to indirect block
451 * @ptrs: Number of pointers to look at
453 * When returning WALK_FOLLOW, the walker must update @mp to point at the right
454 * indirect block to follow.
456 typedef enum walker_status (*gfs2_metadata_walker)(struct metapath *mp,
460 * gfs2_walk_metadata - walk a tree of indirect blocks
462 * @mp: Starting point of walk
463 * @max_len: Maximum number of blocks to walk
464 * @walker: Called during the walk
466 * Returns 1 if the walk was stopped by @walker, 0 if we went past @max_len or
467 * past the end of metadata, and a negative error code otherwise.
470 static int gfs2_walk_metadata(struct inode *inode, struct metapath *mp,
471 u64 max_len, gfs2_metadata_walker walker)
473 struct gfs2_inode *ip = GFS2_I(inode);
474 struct gfs2_sbd *sdp = GFS2_SB(inode);
480 * The walk starts in the lowest allocated indirect block, which may be
481 * before the position indicated by @mp. Adjust @max_len accordingly
482 * to avoid a short walk.
484 for (hgt = mp->mp_fheight - 1; hgt >= mp->mp_aheight; hgt--) {
485 max_len += mp->mp_list[hgt] * factor;
486 mp->mp_list[hgt] = 0;
487 factor *= sdp->sd_inptrs;
491 u16 start = mp->mp_list[hgt];
492 enum walker_status status;
496 /* Walk indirect block. */
497 ptrs = (hgt >= 1 ? sdp->sd_inptrs : sdp->sd_diptrs) - start;
500 ptrs = DIV_ROUND_UP_ULL(max_len, factor);
501 status = walker(mp, ptrs);
506 BUG_ON(mp->mp_aheight == mp->mp_fheight);
507 ptrs = mp->mp_list[hgt] - start;
516 if (status == WALK_FOLLOW)
517 goto fill_up_metapath;
520 /* Decrease height of metapath. */
521 brelse(mp->mp_bh[hgt]);
522 mp->mp_bh[hgt] = NULL;
523 mp->mp_list[hgt] = 0;
527 factor *= sdp->sd_inptrs;
529 /* Advance in metadata tree. */
530 (mp->mp_list[hgt])++;
532 if (mp->mp_list[hgt] >= sdp->sd_inptrs)
535 if (mp->mp_list[hgt] >= sdp->sd_diptrs)
540 /* Increase height of metapath. */
541 ret = fillup_metapath(ip, mp, ip->i_height - 1);
546 do_div(factor, sdp->sd_inptrs);
547 mp->mp_aheight = hgt + 1;
552 static enum walker_status gfs2_hole_walker(struct metapath *mp,
555 const __be64 *start, *ptr, *end;
558 hgt = mp->mp_aheight - 1;
559 start = metapointer(hgt, mp);
562 for (ptr = start; ptr < end; ptr++) {
564 mp->mp_list[hgt] += ptr - start;
565 if (mp->mp_aheight == mp->mp_fheight)
570 return WALK_CONTINUE;
574 * gfs2_hole_size - figure out the size of a hole
576 * @lblock: The logical starting block number
577 * @len: How far to look (in blocks)
578 * @mp: The metapath at lblock
579 * @iomap: The iomap to store the hole size in
581 * This function modifies @mp.
583 * Returns: errno on error
585 static int gfs2_hole_size(struct inode *inode, sector_t lblock, u64 len,
586 struct metapath *mp, struct iomap *iomap)
588 struct metapath clone;
592 clone_metapath(&clone, mp);
593 ret = gfs2_walk_metadata(inode, &clone, len, gfs2_hole_walker);
598 hole_size = metapath_to_block(GFS2_SB(inode), &clone) - lblock;
601 iomap->length = hole_size << inode->i_blkbits;
605 release_metapath(&clone);
609 static inline void gfs2_indirect_init(struct metapath *mp,
610 struct gfs2_glock *gl, unsigned int i,
611 unsigned offset, u64 bn)
613 __be64 *ptr = (__be64 *)(mp->mp_bh[i - 1]->b_data +
614 ((i > 1) ? sizeof(struct gfs2_meta_header) :
615 sizeof(struct gfs2_dinode)));
617 BUG_ON(mp->mp_bh[i] != NULL);
618 mp->mp_bh[i] = gfs2_meta_new(gl, bn);
619 gfs2_trans_add_meta(gl, mp->mp_bh[i]);
620 gfs2_metatype_set(mp->mp_bh[i], GFS2_METATYPE_IN, GFS2_FORMAT_IN);
621 gfs2_buffer_clear_tail(mp->mp_bh[i], sizeof(struct gfs2_meta_header));
623 *ptr = cpu_to_be64(bn);
628 ALLOC_GROW_DEPTH = 1,
629 ALLOC_GROW_HEIGHT = 2,
630 /* ALLOC_UNSTUFF = 3, TBD and rather complicated */
634 * __gfs2_iomap_alloc - Build a metadata tree of the requested height
635 * @inode: The GFS2 inode
636 * @iomap: The iomap structure
637 * @mp: The metapath, with proper height information calculated
639 * In this routine we may have to alloc:
640 * i) Indirect blocks to grow the metadata tree height
641 * ii) Indirect blocks to fill in lower part of the metadata tree
644 * This function is called after __gfs2_iomap_get, which works out the
645 * total number of blocks which we need via gfs2_alloc_size.
647 * We then do the actual allocation asking for an extent at a time (if
648 * enough contiguous free blocks are available, there will only be one
649 * allocation request per call) and uses the state machine to initialise
650 * the blocks in order.
652 * Right now, this function will allocate at most one indirect block
653 * worth of data -- with a default block size of 4K, that's slightly
654 * less than 2M. If this limitation is ever removed to allow huge
655 * allocations, we would probably still want to limit the iomap size we
656 * return to avoid stalling other tasks during huge writes; the next
657 * iomap iteration would then find the blocks already allocated.
659 * Returns: errno on error
662 static int __gfs2_iomap_alloc(struct inode *inode, struct iomap *iomap,
665 struct gfs2_inode *ip = GFS2_I(inode);
666 struct gfs2_sbd *sdp = GFS2_SB(inode);
667 struct buffer_head *dibh = mp->mp_bh[0];
669 unsigned n, i, blks, alloced = 0, iblks = 0, branch_start = 0;
670 size_t dblks = iomap->length >> inode->i_blkbits;
671 const unsigned end_of_metadata = mp->mp_fheight - 1;
673 enum alloc_state state;
677 BUG_ON(mp->mp_aheight < 1);
678 BUG_ON(dibh == NULL);
681 gfs2_trans_add_meta(ip->i_gl, dibh);
683 down_write(&ip->i_rw_mutex);
685 if (mp->mp_fheight == mp->mp_aheight) {
686 /* Bottom indirect block exists */
689 /* Need to allocate indirect blocks */
690 if (mp->mp_fheight == ip->i_height) {
691 /* Writing into existing tree, extend tree down */
692 iblks = mp->mp_fheight - mp->mp_aheight;
693 state = ALLOC_GROW_DEPTH;
695 /* Building up tree height */
696 state = ALLOC_GROW_HEIGHT;
697 iblks = mp->mp_fheight - ip->i_height;
698 branch_start = metapath_branch_start(mp);
699 iblks += (mp->mp_fheight - branch_start);
703 /* start of the second part of the function (state machine) */
705 blks = dblks + iblks;
709 ret = gfs2_alloc_blocks(ip, &bn, &n, 0, NULL);
713 if (state != ALLOC_DATA || gfs2_is_jdata(ip))
714 gfs2_trans_remove_revoke(sdp, bn, n);
716 /* Growing height of tree */
717 case ALLOC_GROW_HEIGHT:
719 ptr = (__be64 *)(dibh->b_data +
720 sizeof(struct gfs2_dinode));
723 for (; i - 1 < mp->mp_fheight - ip->i_height && n > 0;
725 gfs2_indirect_init(mp, ip->i_gl, i, 0, bn++);
726 if (i - 1 == mp->mp_fheight - ip->i_height) {
728 gfs2_buffer_copy_tail(mp->mp_bh[i],
729 sizeof(struct gfs2_meta_header),
730 dibh, sizeof(struct gfs2_dinode));
731 gfs2_buffer_clear_tail(dibh,
732 sizeof(struct gfs2_dinode) +
734 ptr = (__be64 *)(mp->mp_bh[i]->b_data +
735 sizeof(struct gfs2_meta_header));
737 state = ALLOC_GROW_DEPTH;
738 for(i = branch_start; i < mp->mp_fheight; i++) {
739 if (mp->mp_bh[i] == NULL)
741 brelse(mp->mp_bh[i]);
748 fallthrough; /* To branching from existing tree */
749 case ALLOC_GROW_DEPTH:
750 if (i > 1 && i < mp->mp_fheight)
751 gfs2_trans_add_meta(ip->i_gl, mp->mp_bh[i-1]);
752 for (; i < mp->mp_fheight && n > 0; i++, n--)
753 gfs2_indirect_init(mp, ip->i_gl, i,
754 mp->mp_list[i-1], bn++);
755 if (i == mp->mp_fheight)
759 fallthrough; /* To tree complete, adding data blocks */
762 BUG_ON(mp->mp_bh[end_of_metadata] == NULL);
763 gfs2_trans_add_meta(ip->i_gl, mp->mp_bh[end_of_metadata]);
765 ptr = metapointer(end_of_metadata, mp);
766 iomap->addr = bn << inode->i_blkbits;
767 iomap->flags |= IOMAP_F_MERGED | IOMAP_F_NEW;
769 *ptr++ = cpu_to_be64(bn++);
772 } while (iomap->addr == IOMAP_NULL_ADDR);
774 iomap->type = IOMAP_MAPPED;
775 iomap->length = (u64)dblks << inode->i_blkbits;
776 ip->i_height = mp->mp_fheight;
777 gfs2_add_inode_blocks(&ip->i_inode, alloced);
778 gfs2_dinode_out(ip, dibh->b_data);
780 up_write(&ip->i_rw_mutex);
784 #define IOMAP_F_GFS2_BOUNDARY IOMAP_F_PRIVATE
787 * gfs2_alloc_size - Compute the maximum allocation size
790 * @size: Requested size in blocks
792 * Compute the maximum size of the next allocation at @mp.
794 * Returns: size in blocks
796 static u64 gfs2_alloc_size(struct inode *inode, struct metapath *mp, u64 size)
798 struct gfs2_inode *ip = GFS2_I(inode);
799 struct gfs2_sbd *sdp = GFS2_SB(inode);
800 const __be64 *first, *ptr, *end;
803 * For writes to stuffed files, this function is called twice via
804 * __gfs2_iomap_get, before and after unstuffing. The size we return the
805 * first time needs to be large enough to get the reservation and
806 * allocation sizes right. The size we return the second time must
807 * be exact or else __gfs2_iomap_alloc won't do the right thing.
810 if (gfs2_is_stuffed(ip) || mp->mp_fheight != mp->mp_aheight) {
811 unsigned int maxsize = mp->mp_fheight > 1 ?
812 sdp->sd_inptrs : sdp->sd_diptrs;
813 maxsize -= mp->mp_list[mp->mp_fheight - 1];
819 first = metapointer(ip->i_height - 1, mp);
820 end = metaend(ip->i_height - 1, mp);
821 if (end - first > size)
823 for (ptr = first; ptr < end; ptr++) {
831 * __gfs2_iomap_get - Map blocks from an inode to disk blocks
833 * @pos: Starting position in bytes
834 * @length: Length to map, in bytes
835 * @flags: iomap flags
836 * @iomap: The iomap structure
841 static int __gfs2_iomap_get(struct inode *inode, loff_t pos, loff_t length,
842 unsigned flags, struct iomap *iomap,
845 struct gfs2_inode *ip = GFS2_I(inode);
846 struct gfs2_sbd *sdp = GFS2_SB(inode);
847 loff_t size = i_size_read(inode);
850 sector_t lblock_stop;
854 struct buffer_head *dibh = NULL, *bh;
860 down_read(&ip->i_rw_mutex);
862 ret = gfs2_meta_inode_buffer(ip, &dibh);
867 if (gfs2_is_stuffed(ip)) {
868 if (flags & IOMAP_WRITE) {
869 loff_t max_size = gfs2_max_stuffed_size(ip);
871 if (pos + length > max_size)
873 iomap->length = max_size;
876 if (flags & IOMAP_REPORT) {
881 iomap->length = length;
885 iomap->length = size;
887 iomap->addr = (ip->i_no_addr << inode->i_blkbits) +
888 sizeof(struct gfs2_dinode);
889 iomap->type = IOMAP_INLINE;
890 iomap->inline_data = dibh->b_data + sizeof(struct gfs2_dinode);
895 lblock = pos >> inode->i_blkbits;
896 iomap->offset = lblock << inode->i_blkbits;
897 lblock_stop = (pos + length - 1) >> inode->i_blkbits;
898 len = lblock_stop - lblock + 1;
899 iomap->length = len << inode->i_blkbits;
901 height = ip->i_height;
902 while ((lblock + 1) * sdp->sd_sb.sb_bsize > sdp->sd_heightsize[height])
904 find_metapath(sdp, lblock, mp, height);
905 if (height > ip->i_height || gfs2_is_stuffed(ip))
908 ret = lookup_metapath(ip, mp);
912 if (mp->mp_aheight != ip->i_height)
915 ptr = metapointer(ip->i_height - 1, mp);
919 bh = mp->mp_bh[ip->i_height - 1];
920 len = gfs2_extent_length(bh, ptr, len, &eob);
922 iomap->addr = be64_to_cpu(*ptr) << inode->i_blkbits;
923 iomap->length = len << inode->i_blkbits;
924 iomap->type = IOMAP_MAPPED;
925 iomap->flags |= IOMAP_F_MERGED;
927 iomap->flags |= IOMAP_F_GFS2_BOUNDARY;
930 iomap->bdev = inode->i_sb->s_bdev;
932 up_read(&ip->i_rw_mutex);
936 if (flags & IOMAP_REPORT) {
939 else if (height == ip->i_height)
940 ret = gfs2_hole_size(inode, lblock, len, mp, iomap);
942 iomap->length = size - iomap->offset;
943 } else if (flags & IOMAP_WRITE) {
946 if (flags & IOMAP_DIRECT)
947 goto out; /* (see gfs2_file_direct_write) */
949 len = gfs2_alloc_size(inode, mp, len);
950 alloc_size = len << inode->i_blkbits;
951 if (alloc_size < iomap->length)
952 iomap->length = alloc_size;
954 if (pos < size && height == ip->i_height)
955 ret = gfs2_hole_size(inode, lblock, len, mp, iomap);
958 iomap->addr = IOMAP_NULL_ADDR;
959 iomap->type = IOMAP_HOLE;
963 static int gfs2_iomap_page_prepare(struct inode *inode, loff_t pos,
966 unsigned int blockmask = i_blocksize(inode) - 1;
967 struct gfs2_sbd *sdp = GFS2_SB(inode);
970 blocks = ((pos & blockmask) + len + blockmask) >> inode->i_blkbits;
971 return gfs2_trans_begin(sdp, RES_DINODE + blocks, 0);
974 static void gfs2_iomap_page_done(struct inode *inode, loff_t pos,
975 unsigned copied, struct page *page)
977 struct gfs2_trans *tr = current->journal_info;
978 struct gfs2_inode *ip = GFS2_I(inode);
979 struct gfs2_sbd *sdp = GFS2_SB(inode);
981 if (page && !gfs2_is_stuffed(ip))
982 gfs2_page_add_databufs(ip, page, offset_in_page(pos), copied);
984 if (tr->tr_num_buf_new)
985 __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
990 static const struct iomap_page_ops gfs2_iomap_page_ops = {
991 .page_prepare = gfs2_iomap_page_prepare,
992 .page_done = gfs2_iomap_page_done,
995 static int gfs2_iomap_begin_write(struct inode *inode, loff_t pos,
996 loff_t length, unsigned flags,
1000 struct gfs2_inode *ip = GFS2_I(inode);
1001 struct gfs2_sbd *sdp = GFS2_SB(inode);
1005 unstuff = gfs2_is_stuffed(ip) &&
1006 pos + length > gfs2_max_stuffed_size(ip);
1008 if (unstuff || iomap->type == IOMAP_HOLE) {
1009 unsigned int data_blocks, ind_blocks;
1010 struct gfs2_alloc_parms ap = {};
1011 unsigned int rblocks;
1012 struct gfs2_trans *tr;
1014 gfs2_write_calc_reserv(ip, iomap->length, &data_blocks,
1016 ap.target = data_blocks + ind_blocks;
1017 ret = gfs2_quota_lock_check(ip, &ap);
1021 ret = gfs2_inplace_reserve(ip, &ap);
1025 rblocks = RES_DINODE + ind_blocks;
1026 if (gfs2_is_jdata(ip))
1027 rblocks += data_blocks;
1028 if (ind_blocks || data_blocks)
1029 rblocks += RES_STATFS + RES_QUOTA;
1030 if (inode == sdp->sd_rindex)
1031 rblocks += 2 * RES_STATFS;
1032 rblocks += gfs2_rg_blocks(ip, data_blocks + ind_blocks);
1034 ret = gfs2_trans_begin(sdp, rblocks,
1035 iomap->length >> inode->i_blkbits);
1037 goto out_trans_fail;
1040 ret = gfs2_unstuff_dinode(ip);
1043 release_metapath(mp);
1044 ret = __gfs2_iomap_get(inode, iomap->offset,
1045 iomap->length, flags, iomap, mp);
1050 if (iomap->type == IOMAP_HOLE) {
1051 ret = __gfs2_iomap_alloc(inode, iomap, mp);
1053 gfs2_trans_end(sdp);
1054 gfs2_inplace_release(ip);
1055 punch_hole(ip, iomap->offset, iomap->length);
1060 tr = current->journal_info;
1061 if (tr->tr_num_buf_new)
1062 __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
1064 gfs2_trans_end(sdp);
1067 if (gfs2_is_stuffed(ip) || gfs2_is_jdata(ip))
1068 iomap->page_ops = &gfs2_iomap_page_ops;
1072 gfs2_trans_end(sdp);
1074 gfs2_inplace_release(ip);
1076 gfs2_quota_unlock(ip);
1080 static int gfs2_iomap_begin(struct inode *inode, loff_t pos, loff_t length,
1081 unsigned flags, struct iomap *iomap,
1082 struct iomap *srcmap)
1084 struct gfs2_inode *ip = GFS2_I(inode);
1085 struct metapath mp = { .mp_aheight = 1, };
1088 if (gfs2_is_jdata(ip))
1089 iomap->flags |= IOMAP_F_BUFFER_HEAD;
1091 trace_gfs2_iomap_start(ip, pos, length, flags);
1092 ret = __gfs2_iomap_get(inode, pos, length, flags, iomap, &mp);
1096 switch(flags & (IOMAP_WRITE | IOMAP_ZERO)) {
1098 if (flags & IOMAP_DIRECT) {
1100 * Silently fall back to buffered I/O for stuffed files
1101 * or if we've got a hole (see gfs2_file_direct_write).
1103 if (iomap->type != IOMAP_MAPPED)
1109 if (iomap->type == IOMAP_HOLE)
1116 ret = gfs2_iomap_begin_write(inode, pos, length, flags, iomap, &mp);
1119 release_metapath(&mp);
1120 trace_gfs2_iomap_end(ip, iomap, ret);
1124 static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length,
1125 ssize_t written, unsigned flags, struct iomap *iomap)
1127 struct gfs2_inode *ip = GFS2_I(inode);
1128 struct gfs2_sbd *sdp = GFS2_SB(inode);
1130 switch (flags & (IOMAP_WRITE | IOMAP_ZERO)) {
1132 if (flags & IOMAP_DIRECT)
1136 if (iomap->type == IOMAP_HOLE)
1143 if (!gfs2_is_stuffed(ip))
1144 gfs2_ordered_add_inode(ip);
1146 if (inode == sdp->sd_rindex)
1147 adjust_fs_space(inode);
1149 gfs2_inplace_release(ip);
1151 if (ip->i_qadata && ip->i_qadata->qa_qd_num)
1152 gfs2_quota_unlock(ip);
1154 if (length != written && (iomap->flags & IOMAP_F_NEW)) {
1155 /* Deallocate blocks that were just allocated. */
1156 loff_t hstart = round_up(pos + written, i_blocksize(inode));
1157 loff_t hend = iomap->offset + iomap->length;
1159 if (hstart < hend) {
1160 truncate_pagecache_range(inode, hstart, hend - 1);
1161 punch_hole(ip, hstart, hend - hstart);
1165 if (unlikely(!written))
1168 if (iomap->flags & IOMAP_F_SIZE_CHANGED)
1169 mark_inode_dirty(inode);
1170 set_bit(GLF_DIRTY, &ip->i_gl->gl_flags);
1174 const struct iomap_ops gfs2_iomap_ops = {
1175 .iomap_begin = gfs2_iomap_begin,
1176 .iomap_end = gfs2_iomap_end,
1180 * gfs2_block_map - Map one or more blocks of an inode to a disk block
1182 * @lblock: The logical block number
1183 * @bh_map: The bh to be mapped
1184 * @create: True if its ok to alloc blocks to satify the request
1186 * The size of the requested mapping is defined in bh_map->b_size.
1188 * Clears buffer_mapped(bh_map) and leaves bh_map->b_size unchanged
1189 * when @lblock is not mapped. Sets buffer_mapped(bh_map) and
1190 * bh_map->b_size to indicate the size of the mapping when @lblock and
1191 * successive blocks are mapped, up to the requested size.
1193 * Sets buffer_boundary() if a read of metadata will be required
1194 * before the next block can be mapped. Sets buffer_new() if new
1195 * blocks were allocated.
1200 int gfs2_block_map(struct inode *inode, sector_t lblock,
1201 struct buffer_head *bh_map, int create)
1203 struct gfs2_inode *ip = GFS2_I(inode);
1204 loff_t pos = (loff_t)lblock << inode->i_blkbits;
1205 loff_t length = bh_map->b_size;
1206 struct iomap iomap = { };
1209 clear_buffer_mapped(bh_map);
1210 clear_buffer_new(bh_map);
1211 clear_buffer_boundary(bh_map);
1212 trace_gfs2_bmap(ip, bh_map, lblock, create, 1);
1215 ret = gfs2_iomap_get(inode, pos, length, &iomap);
1217 ret = gfs2_iomap_alloc(inode, pos, length, &iomap);
1221 if (iomap.length > bh_map->b_size) {
1222 iomap.length = bh_map->b_size;
1223 iomap.flags &= ~IOMAP_F_GFS2_BOUNDARY;
1225 if (iomap.addr != IOMAP_NULL_ADDR)
1226 map_bh(bh_map, inode->i_sb, iomap.addr >> inode->i_blkbits);
1227 bh_map->b_size = iomap.length;
1228 if (iomap.flags & IOMAP_F_GFS2_BOUNDARY)
1229 set_buffer_boundary(bh_map);
1230 if (iomap.flags & IOMAP_F_NEW)
1231 set_buffer_new(bh_map);
1234 trace_gfs2_bmap(ip, bh_map, lblock, create, ret);
1238 int gfs2_get_extent(struct inode *inode, u64 lblock, u64 *dblock,
1239 unsigned int *extlen)
1241 unsigned int blkbits = inode->i_blkbits;
1242 struct iomap iomap = { };
1246 ret = gfs2_iomap_get(inode, lblock << blkbits, *extlen << blkbits,
1250 if (iomap.type != IOMAP_MAPPED)
1252 *dblock = iomap.addr >> blkbits;
1253 len = iomap.length >> blkbits;
1259 int gfs2_alloc_extent(struct inode *inode, u64 lblock, u64 *dblock,
1260 unsigned int *extlen, bool *new)
1262 unsigned int blkbits = inode->i_blkbits;
1263 struct iomap iomap = { };
1267 ret = gfs2_iomap_alloc(inode, lblock << blkbits, *extlen << blkbits,
1271 if (iomap.type != IOMAP_MAPPED)
1273 *dblock = iomap.addr >> blkbits;
1274 len = iomap.length >> blkbits;
1277 *new = iomap.flags & IOMAP_F_NEW;
1282 * NOTE: Never call gfs2_block_zero_range with an open transaction because it
1283 * uses iomap write to perform its actions, which begin their own transactions
1284 * (iomap_begin, page_prepare, etc.)
1286 static int gfs2_block_zero_range(struct inode *inode, loff_t from,
1287 unsigned int length)
1289 BUG_ON(current->journal_info);
1290 return iomap_zero_range(inode, from, length, NULL, &gfs2_iomap_ops);
1293 #define GFS2_JTRUNC_REVOKES 8192
1296 * gfs2_journaled_truncate - Wrapper for truncate_pagecache for jdata files
1297 * @inode: The inode being truncated
1298 * @oldsize: The original (larger) size
1299 * @newsize: The new smaller size
1301 * With jdata files, we have to journal a revoke for each block which is
1302 * truncated. As a result, we need to split this into separate transactions
1303 * if the number of pages being truncated gets too large.
1306 static int gfs2_journaled_truncate(struct inode *inode, u64 oldsize, u64 newsize)
1308 struct gfs2_sbd *sdp = GFS2_SB(inode);
1309 u64 max_chunk = GFS2_JTRUNC_REVOKES * sdp->sd_vfs->s_blocksize;
1313 while (oldsize != newsize) {
1314 struct gfs2_trans *tr;
1317 chunk = oldsize - newsize;
1318 if (chunk > max_chunk)
1321 offs = oldsize & ~PAGE_MASK;
1322 if (offs && chunk > PAGE_SIZE)
1323 chunk = offs + ((chunk - offs) & PAGE_MASK);
1325 truncate_pagecache(inode, oldsize - chunk);
1328 tr = current->journal_info;
1329 if (!test_bit(TR_TOUCHED, &tr->tr_flags))
1332 gfs2_trans_end(sdp);
1333 error = gfs2_trans_begin(sdp, RES_DINODE, GFS2_JTRUNC_REVOKES);
1341 static int trunc_start(struct inode *inode, u64 newsize)
1343 struct gfs2_inode *ip = GFS2_I(inode);
1344 struct gfs2_sbd *sdp = GFS2_SB(inode);
1345 struct buffer_head *dibh = NULL;
1346 int journaled = gfs2_is_jdata(ip);
1347 u64 oldsize = inode->i_size;
1350 if (!gfs2_is_stuffed(ip)) {
1351 unsigned int blocksize = i_blocksize(inode);
1352 unsigned int offs = newsize & (blocksize - 1);
1354 error = gfs2_block_zero_range(inode, newsize,
1361 error = gfs2_trans_begin(sdp, RES_DINODE + RES_JDATA, GFS2_JTRUNC_REVOKES);
1363 error = gfs2_trans_begin(sdp, RES_DINODE, 0);
1367 error = gfs2_meta_inode_buffer(ip, &dibh);
1371 gfs2_trans_add_meta(ip->i_gl, dibh);
1373 if (gfs2_is_stuffed(ip))
1374 gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode) + newsize);
1376 ip->i_diskflags |= GFS2_DIF_TRUNC_IN_PROG;
1378 i_size_write(inode, newsize);
1379 ip->i_inode.i_mtime = ip->i_inode.i_ctime = current_time(&ip->i_inode);
1380 gfs2_dinode_out(ip, dibh->b_data);
1383 error = gfs2_journaled_truncate(inode, oldsize, newsize);
1385 truncate_pagecache(inode, newsize);
1389 if (current->journal_info)
1390 gfs2_trans_end(sdp);
1394 int gfs2_iomap_get(struct inode *inode, loff_t pos, loff_t length,
1395 struct iomap *iomap)
1397 struct metapath mp = { .mp_aheight = 1, };
1400 ret = __gfs2_iomap_get(inode, pos, length, 0, iomap, &mp);
1401 release_metapath(&mp);
1405 int gfs2_iomap_alloc(struct inode *inode, loff_t pos, loff_t length,
1406 struct iomap *iomap)
1408 struct metapath mp = { .mp_aheight = 1, };
1411 ret = __gfs2_iomap_get(inode, pos, length, IOMAP_WRITE, iomap, &mp);
1412 if (!ret && iomap->type == IOMAP_HOLE)
1413 ret = __gfs2_iomap_alloc(inode, iomap, &mp);
1414 release_metapath(&mp);
1419 * sweep_bh_for_rgrps - find an rgrp in a meta buffer and free blocks therein
1421 * @rd_gh: holder of resource group glock
1422 * @bh: buffer head to sweep
1423 * @start: starting point in bh
1424 * @end: end point in bh
1425 * @meta: true if bh points to metadata (rather than data)
1426 * @btotal: place to keep count of total blocks freed
1428 * We sweep a metadata buffer (provided by the metapath) for blocks we need to
1429 * free, and free them all. However, we do it one rgrp at a time. If this
1430 * block has references to multiple rgrps, we break it into individual
1431 * transactions. This allows other processes to use the rgrps while we're
1432 * focused on a single one, for better concurrency / performance.
1433 * At every transaction boundary, we rewrite the inode into the journal.
1434 * That way the bitmaps are kept consistent with the inode and we can recover
1435 * if we're interrupted by power-outages.
1437 * Returns: 0, or return code if an error occurred.
1438 * *btotal has the total number of blocks freed
1440 static int sweep_bh_for_rgrps(struct gfs2_inode *ip, struct gfs2_holder *rd_gh,
1441 struct buffer_head *bh, __be64 *start, __be64 *end,
1442 bool meta, u32 *btotal)
1444 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
1445 struct gfs2_rgrpd *rgd;
1446 struct gfs2_trans *tr;
1448 int blks_outside_rgrp;
1449 u64 bn, bstart, isize_blks;
1450 s64 blen; /* needs to be s64 or gfs2_add_inode_blocks breaks */
1452 bool buf_in_tr = false; /* buffer was added to transaction */
1456 if (gfs2_holder_initialized(rd_gh)) {
1457 rgd = gfs2_glock2rgrp(rd_gh->gh_gl);
1458 gfs2_assert_withdraw(sdp,
1459 gfs2_glock_is_locked_by_me(rd_gh->gh_gl));
1461 blks_outside_rgrp = 0;
1465 for (p = start; p < end; p++) {
1468 bn = be64_to_cpu(*p);
1471 if (!rgrp_contains_block(rgd, bn)) {
1472 blks_outside_rgrp++;
1476 rgd = gfs2_blk2rgrpd(sdp, bn, true);
1477 if (unlikely(!rgd)) {
1481 ret = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE,
1482 LM_FLAG_NODE_SCOPE, rd_gh);
1486 /* Must be done with the rgrp glock held: */
1487 if (gfs2_rs_active(&ip->i_res) &&
1488 rgd == ip->i_res.rs_rgd)
1489 gfs2_rs_deltree(&ip->i_res);
1492 /* The size of our transactions will be unknown until we
1493 actually process all the metadata blocks that relate to
1494 the rgrp. So we estimate. We know it can't be more than
1495 the dinode's i_blocks and we don't want to exceed the
1496 journal flush threshold, sd_log_thresh2. */
1497 if (current->journal_info == NULL) {
1498 unsigned int jblocks_rqsted, revokes;
1500 jblocks_rqsted = rgd->rd_length + RES_DINODE +
1502 isize_blks = gfs2_get_inode_blocks(&ip->i_inode);
1503 if (isize_blks > atomic_read(&sdp->sd_log_thresh2))
1505 atomic_read(&sdp->sd_log_thresh2);
1507 jblocks_rqsted += isize_blks;
1508 revokes = jblocks_rqsted;
1510 revokes += end - start;
1511 else if (ip->i_depth)
1512 revokes += sdp->sd_inptrs;
1513 ret = gfs2_trans_begin(sdp, jblocks_rqsted, revokes);
1516 down_write(&ip->i_rw_mutex);
1518 /* check if we will exceed the transaction blocks requested */
1519 tr = current->journal_info;
1520 if (tr->tr_num_buf_new + RES_STATFS +
1521 RES_QUOTA >= atomic_read(&sdp->sd_log_thresh2)) {
1522 /* We set blks_outside_rgrp to ensure the loop will
1523 be repeated for the same rgrp, but with a new
1525 blks_outside_rgrp++;
1526 /* This next part is tricky. If the buffer was added
1527 to the transaction, we've already set some block
1528 pointers to 0, so we better follow through and free
1529 them, or we will introduce corruption (so break).
1530 This may be impossible, or at least rare, but I
1531 decided to cover the case regardless.
1533 If the buffer was not added to the transaction
1534 (this call), doing so would exceed our transaction
1535 size, so we need to end the transaction and start a
1536 new one (so goto). */
1543 gfs2_trans_add_meta(ip->i_gl, bh);
1546 if (bstart + blen == bn) {
1551 __gfs2_free_blocks(ip, rgd, bstart, (u32)blen, meta);
1553 gfs2_add_inode_blocks(&ip->i_inode, -blen);
1559 __gfs2_free_blocks(ip, rgd, bstart, (u32)blen, meta);
1561 gfs2_add_inode_blocks(&ip->i_inode, -blen);
1564 if (!ret && blks_outside_rgrp) { /* If buffer still has non-zero blocks
1565 outside the rgrp we just processed,
1566 do it all over again. */
1567 if (current->journal_info) {
1568 struct buffer_head *dibh;
1570 ret = gfs2_meta_inode_buffer(ip, &dibh);
1574 /* Every transaction boundary, we rewrite the dinode
1575 to keep its di_blocks current in case of failure. */
1576 ip->i_inode.i_mtime = ip->i_inode.i_ctime =
1577 current_time(&ip->i_inode);
1578 gfs2_trans_add_meta(ip->i_gl, dibh);
1579 gfs2_dinode_out(ip, dibh->b_data);
1581 up_write(&ip->i_rw_mutex);
1582 gfs2_trans_end(sdp);
1585 gfs2_glock_dq_uninit(rd_gh);
1593 static bool mp_eq_to_hgt(struct metapath *mp, __u16 *list, unsigned int h)
1595 if (memcmp(mp->mp_list, list, h * sizeof(mp->mp_list[0])))
1601 * find_nonnull_ptr - find a non-null pointer given a metapath and height
1602 * @sdp: The superblock
1603 * @mp: starting metapath
1604 * @h: desired height to search
1605 * @end_list: See punch_hole().
1606 * @end_aligned: See punch_hole().
1608 * Assumes the metapath is valid (with buffers) out to height h.
1609 * Returns: true if a non-null pointer was found in the metapath buffer
1610 * false if all remaining pointers are NULL in the buffer
1612 static bool find_nonnull_ptr(struct gfs2_sbd *sdp, struct metapath *mp,
1614 __u16 *end_list, unsigned int end_aligned)
1616 struct buffer_head *bh = mp->mp_bh[h];
1617 __be64 *first, *ptr, *end;
1619 first = metaptr1(h, mp);
1620 ptr = first + mp->mp_list[h];
1621 end = (__be64 *)(bh->b_data + bh->b_size);
1622 if (end_list && mp_eq_to_hgt(mp, end_list, h)) {
1623 bool keep_end = h < end_aligned;
1624 end = first + end_list[h] + keep_end;
1628 if (*ptr) { /* if we have a non-null pointer */
1629 mp->mp_list[h] = ptr - first;
1631 if (h < GFS2_MAX_META_HEIGHT)
1640 enum dealloc_states {
1641 DEALLOC_MP_FULL = 0, /* Strip a metapath with all buffers read in */
1642 DEALLOC_MP_LOWER = 1, /* lower the metapath strip height */
1643 DEALLOC_FILL_MP = 2, /* Fill in the metapath to the given height. */
1644 DEALLOC_DONE = 3, /* process complete */
1648 metapointer_range(struct metapath *mp, int height,
1649 __u16 *start_list, unsigned int start_aligned,
1650 __u16 *end_list, unsigned int end_aligned,
1651 __be64 **start, __be64 **end)
1653 struct buffer_head *bh = mp->mp_bh[height];
1656 first = metaptr1(height, mp);
1658 if (mp_eq_to_hgt(mp, start_list, height)) {
1659 bool keep_start = height < start_aligned;
1660 *start = first + start_list[height] + keep_start;
1662 *end = (__be64 *)(bh->b_data + bh->b_size);
1663 if (end_list && mp_eq_to_hgt(mp, end_list, height)) {
1664 bool keep_end = height < end_aligned;
1665 *end = first + end_list[height] + keep_end;
1669 static inline bool walk_done(struct gfs2_sbd *sdp,
1670 struct metapath *mp, int height,
1671 __u16 *end_list, unsigned int end_aligned)
1676 bool keep_end = height < end_aligned;
1677 if (!mp_eq_to_hgt(mp, end_list, height))
1679 end = end_list[height] + keep_end;
1681 end = (height > 0) ? sdp->sd_inptrs : sdp->sd_diptrs;
1682 return mp->mp_list[height] >= end;
1686 * punch_hole - deallocate blocks in a file
1687 * @ip: inode to truncate
1688 * @offset: the start of the hole
1689 * @length: the size of the hole (or 0 for truncate)
1691 * Punch a hole into a file or truncate a file at a given position. This
1692 * function operates in whole blocks (@offset and @length are rounded
1693 * accordingly); partially filled blocks must be cleared otherwise.
1695 * This function works from the bottom up, and from the right to the left. In
1696 * other words, it strips off the highest layer (data) before stripping any of
1697 * the metadata. Doing it this way is best in case the operation is interrupted
1698 * by power failure, etc. The dinode is rewritten in every transaction to
1699 * guarantee integrity.
1701 static int punch_hole(struct gfs2_inode *ip, u64 offset, u64 length)
1703 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
1704 u64 maxsize = sdp->sd_heightsize[ip->i_height];
1705 struct metapath mp = {};
1706 struct buffer_head *dibh, *bh;
1707 struct gfs2_holder rd_gh;
1708 unsigned int bsize_shift = sdp->sd_sb.sb_bsize_shift;
1709 u64 lblock = (offset + (1 << bsize_shift) - 1) >> bsize_shift;
1710 __u16 start_list[GFS2_MAX_META_HEIGHT];
1711 __u16 __end_list[GFS2_MAX_META_HEIGHT], *end_list = NULL;
1712 unsigned int start_aligned, end_aligned;
1713 unsigned int strip_h = ip->i_height - 1;
1716 int mp_h; /* metapath buffers are read in to this height */
1718 __be64 *start, *end;
1720 if (offset >= maxsize) {
1722 * The starting point lies beyond the allocated meta-data;
1723 * there are no blocks do deallocate.
1729 * The start position of the hole is defined by lblock, start_list, and
1730 * start_aligned. The end position of the hole is defined by lend,
1731 * end_list, and end_aligned.
1733 * start_aligned and end_aligned define down to which height the start
1734 * and end positions are aligned to the metadata tree (i.e., the
1735 * position is a multiple of the metadata granularity at the height
1736 * above). This determines at which heights additional meta pointers
1737 * needs to be preserved for the remaining data.
1741 u64 end_offset = offset + length;
1745 * Clip the end at the maximum file size for the given height:
1746 * that's how far the metadata goes; files bigger than that
1747 * will have additional layers of indirection.
1749 if (end_offset > maxsize)
1750 end_offset = maxsize;
1751 lend = end_offset >> bsize_shift;
1756 find_metapath(sdp, lend, &mp, ip->i_height);
1757 end_list = __end_list;
1758 memcpy(end_list, mp.mp_list, sizeof(mp.mp_list));
1760 for (mp_h = ip->i_height - 1; mp_h > 0; mp_h--) {
1767 find_metapath(sdp, lblock, &mp, ip->i_height);
1768 memcpy(start_list, mp.mp_list, sizeof(start_list));
1770 for (mp_h = ip->i_height - 1; mp_h > 0; mp_h--) {
1771 if (start_list[mp_h])
1774 start_aligned = mp_h;
1776 ret = gfs2_meta_inode_buffer(ip, &dibh);
1781 ret = lookup_metapath(ip, &mp);
1785 /* issue read-ahead on metadata */
1786 for (mp_h = 0; mp_h < mp.mp_aheight - 1; mp_h++) {
1787 metapointer_range(&mp, mp_h, start_list, start_aligned,
1788 end_list, end_aligned, &start, &end);
1789 gfs2_metapath_ra(ip->i_gl, start, end);
1792 if (mp.mp_aheight == ip->i_height)
1793 state = DEALLOC_MP_FULL; /* We have a complete metapath */
1795 state = DEALLOC_FILL_MP; /* deal with partial metapath */
1797 ret = gfs2_rindex_update(sdp);
1801 ret = gfs2_quota_hold(ip, NO_UID_QUOTA_CHANGE, NO_GID_QUOTA_CHANGE);
1804 gfs2_holder_mark_uninitialized(&rd_gh);
1808 while (state != DEALLOC_DONE) {
1810 /* Truncate a full metapath at the given strip height.
1811 * Note that strip_h == mp_h in order to be in this state. */
1812 case DEALLOC_MP_FULL:
1813 bh = mp.mp_bh[mp_h];
1814 gfs2_assert_withdraw(sdp, bh);
1815 if (gfs2_assert_withdraw(sdp,
1816 prev_bnr != bh->b_blocknr)) {
1817 fs_emerg(sdp, "inode %llu, block:%llu, i_h:%u,"
1818 "s_h:%u, mp_h:%u\n",
1819 (unsigned long long)ip->i_no_addr,
1820 prev_bnr, ip->i_height, strip_h, mp_h);
1822 prev_bnr = bh->b_blocknr;
1824 if (gfs2_metatype_check(sdp, bh,
1825 (mp_h ? GFS2_METATYPE_IN :
1826 GFS2_METATYPE_DI))) {
1832 * Below, passing end_aligned as 0 gives us the
1833 * metapointer range excluding the end point: the end
1834 * point is the first metapath we must not deallocate!
1837 metapointer_range(&mp, mp_h, start_list, start_aligned,
1838 end_list, 0 /* end_aligned */,
1840 ret = sweep_bh_for_rgrps(ip, &rd_gh, mp.mp_bh[mp_h],
1842 mp_h != ip->i_height - 1,
1845 /* If we hit an error or just swept dinode buffer,
1848 state = DEALLOC_DONE;
1851 state = DEALLOC_MP_LOWER;
1854 /* lower the metapath strip height */
1855 case DEALLOC_MP_LOWER:
1856 /* We're done with the current buffer, so release it,
1857 unless it's the dinode buffer. Then back up to the
1858 previous pointer. */
1860 brelse(mp.mp_bh[mp_h]);
1861 mp.mp_bh[mp_h] = NULL;
1863 /* If we can't get any lower in height, we've stripped
1864 off all we can. Next step is to back up and start
1865 stripping the previous level of metadata. */
1868 memcpy(mp.mp_list, start_list, sizeof(start_list));
1870 state = DEALLOC_FILL_MP;
1873 mp.mp_list[mp_h] = 0;
1874 mp_h--; /* search one metadata height down */
1876 if (walk_done(sdp, &mp, mp_h, end_list, end_aligned))
1878 /* Here we've found a part of the metapath that is not
1879 * allocated. We need to search at that height for the
1880 * next non-null pointer. */
1881 if (find_nonnull_ptr(sdp, &mp, mp_h, end_list, end_aligned)) {
1882 state = DEALLOC_FILL_MP;
1885 /* No more non-null pointers at this height. Back up
1886 to the previous height and try again. */
1887 break; /* loop around in the same state */
1889 /* Fill the metapath with buffers to the given height. */
1890 case DEALLOC_FILL_MP:
1891 /* Fill the buffers out to the current height. */
1892 ret = fillup_metapath(ip, &mp, mp_h);
1896 /* On the first pass, issue read-ahead on metadata. */
1897 if (mp.mp_aheight > 1 && strip_h == ip->i_height - 1) {
1898 unsigned int height = mp.mp_aheight - 1;
1900 /* No read-ahead for data blocks. */
1901 if (mp.mp_aheight - 1 == strip_h)
1904 for (; height >= mp.mp_aheight - ret; height--) {
1905 metapointer_range(&mp, height,
1906 start_list, start_aligned,
1907 end_list, end_aligned,
1909 gfs2_metapath_ra(ip->i_gl, start, end);
1913 /* If buffers found for the entire strip height */
1914 if (mp.mp_aheight - 1 == strip_h) {
1915 state = DEALLOC_MP_FULL;
1918 if (mp.mp_aheight < ip->i_height) /* We have a partial height */
1919 mp_h = mp.mp_aheight - 1;
1921 /* If we find a non-null block pointer, crawl a bit
1922 higher up in the metapath and try again, otherwise
1923 we need to look lower for a new starting point. */
1924 if (find_nonnull_ptr(sdp, &mp, mp_h, end_list, end_aligned))
1927 state = DEALLOC_MP_LOWER;
1933 if (current->journal_info == NULL) {
1934 ret = gfs2_trans_begin(sdp, RES_DINODE + RES_STATFS +
1938 down_write(&ip->i_rw_mutex);
1940 gfs2_statfs_change(sdp, 0, +btotal, 0);
1941 gfs2_quota_change(ip, -(s64)btotal, ip->i_inode.i_uid,
1943 ip->i_inode.i_mtime = ip->i_inode.i_ctime = current_time(&ip->i_inode);
1944 gfs2_trans_add_meta(ip->i_gl, dibh);
1945 gfs2_dinode_out(ip, dibh->b_data);
1946 up_write(&ip->i_rw_mutex);
1947 gfs2_trans_end(sdp);
1951 if (gfs2_holder_initialized(&rd_gh))
1952 gfs2_glock_dq_uninit(&rd_gh);
1953 if (current->journal_info) {
1954 up_write(&ip->i_rw_mutex);
1955 gfs2_trans_end(sdp);
1958 gfs2_quota_unhold(ip);
1960 release_metapath(&mp);
1964 static int trunc_end(struct gfs2_inode *ip)
1966 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
1967 struct buffer_head *dibh;
1970 error = gfs2_trans_begin(sdp, RES_DINODE, 0);
1974 down_write(&ip->i_rw_mutex);
1976 error = gfs2_meta_inode_buffer(ip, &dibh);
1980 if (!i_size_read(&ip->i_inode)) {
1982 ip->i_goal = ip->i_no_addr;
1983 gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode));
1984 gfs2_ordered_del_inode(ip);
1986 ip->i_inode.i_mtime = ip->i_inode.i_ctime = current_time(&ip->i_inode);
1987 ip->i_diskflags &= ~GFS2_DIF_TRUNC_IN_PROG;
1989 gfs2_trans_add_meta(ip->i_gl, dibh);
1990 gfs2_dinode_out(ip, dibh->b_data);
1994 up_write(&ip->i_rw_mutex);
1995 gfs2_trans_end(sdp);
2000 * do_shrink - make a file smaller
2002 * @newsize: the size to make the file
2004 * Called with an exclusive lock on @inode. The @size must
2005 * be equal to or smaller than the current inode size.
2010 static int do_shrink(struct inode *inode, u64 newsize)
2012 struct gfs2_inode *ip = GFS2_I(inode);
2015 error = trunc_start(inode, newsize);
2018 if (gfs2_is_stuffed(ip))
2021 error = punch_hole(ip, newsize, 0);
2023 error = trunc_end(ip);
2028 void gfs2_trim_blocks(struct inode *inode)
2032 ret = do_shrink(inode, inode->i_size);
2037 * do_grow - Touch and update inode size
2039 * @size: The new size
2041 * This function updates the timestamps on the inode and
2042 * may also increase the size of the inode. This function
2043 * must not be called with @size any smaller than the current
2046 * Although it is not strictly required to unstuff files here,
2047 * earlier versions of GFS2 have a bug in the stuffed file reading
2048 * code which will result in a buffer overrun if the size is larger
2049 * than the max stuffed file size. In order to prevent this from
2050 * occurring, such files are unstuffed, but in other cases we can
2051 * just update the inode size directly.
2053 * Returns: 0 on success, or -ve on error
2056 static int do_grow(struct inode *inode, u64 size)
2058 struct gfs2_inode *ip = GFS2_I(inode);
2059 struct gfs2_sbd *sdp = GFS2_SB(inode);
2060 struct gfs2_alloc_parms ap = { .target = 1, };
2061 struct buffer_head *dibh;
2065 if (gfs2_is_stuffed(ip) && size > gfs2_max_stuffed_size(ip)) {
2066 error = gfs2_quota_lock_check(ip, &ap);
2070 error = gfs2_inplace_reserve(ip, &ap);
2072 goto do_grow_qunlock;
2076 error = gfs2_trans_begin(sdp, RES_DINODE + RES_STATFS + RES_RG_BIT +
2078 gfs2_is_jdata(ip) ? RES_JDATA : 0) +
2079 (sdp->sd_args.ar_quota == GFS2_QUOTA_OFF ?
2082 goto do_grow_release;
2085 error = gfs2_unstuff_dinode(ip);
2090 error = gfs2_meta_inode_buffer(ip, &dibh);
2094 truncate_setsize(inode, size);
2095 ip->i_inode.i_mtime = ip->i_inode.i_ctime = current_time(&ip->i_inode);
2096 gfs2_trans_add_meta(ip->i_gl, dibh);
2097 gfs2_dinode_out(ip, dibh->b_data);
2101 gfs2_trans_end(sdp);
2104 gfs2_inplace_release(ip);
2106 gfs2_quota_unlock(ip);
2112 * gfs2_setattr_size - make a file a given size
2114 * @newsize: the size to make the file
2116 * The file size can grow, shrink, or stay the same size. This
2117 * is called holding i_rwsem and an exclusive glock on the inode
2123 int gfs2_setattr_size(struct inode *inode, u64 newsize)
2125 struct gfs2_inode *ip = GFS2_I(inode);
2128 BUG_ON(!S_ISREG(inode->i_mode));
2130 ret = inode_newsize_ok(inode, newsize);
2134 inode_dio_wait(inode);
2136 ret = gfs2_qa_get(ip);
2140 if (newsize >= inode->i_size) {
2141 ret = do_grow(inode, newsize);
2145 ret = do_shrink(inode, newsize);
2152 int gfs2_truncatei_resume(struct gfs2_inode *ip)
2155 error = punch_hole(ip, i_size_read(&ip->i_inode), 0);
2157 error = trunc_end(ip);
2161 int gfs2_file_dealloc(struct gfs2_inode *ip)
2163 return punch_hole(ip, 0, 0);
2167 * gfs2_free_journal_extents - Free cached journal bmap info
2172 void gfs2_free_journal_extents(struct gfs2_jdesc *jd)
2174 struct gfs2_journal_extent *jext;
2176 while(!list_empty(&jd->extent_list)) {
2177 jext = list_first_entry(&jd->extent_list, struct gfs2_journal_extent, list);
2178 list_del(&jext->list);
2184 * gfs2_add_jextent - Add or merge a new extent to extent cache
2185 * @jd: The journal descriptor
2186 * @lblock: The logical block at start of new extent
2187 * @dblock: The physical block at start of new extent
2188 * @blocks: Size of extent in fs blocks
2190 * Returns: 0 on success or -ENOMEM
2193 static int gfs2_add_jextent(struct gfs2_jdesc *jd, u64 lblock, u64 dblock, u64 blocks)
2195 struct gfs2_journal_extent *jext;
2197 if (!list_empty(&jd->extent_list)) {
2198 jext = list_last_entry(&jd->extent_list, struct gfs2_journal_extent, list);
2199 if ((jext->dblock + jext->blocks) == dblock) {
2200 jext->blocks += blocks;
2205 jext = kzalloc(sizeof(struct gfs2_journal_extent), GFP_NOFS);
2208 jext->dblock = dblock;
2209 jext->lblock = lblock;
2210 jext->blocks = blocks;
2211 list_add_tail(&jext->list, &jd->extent_list);
2217 * gfs2_map_journal_extents - Cache journal bmap info
2218 * @sdp: The super block
2219 * @jd: The journal to map
2221 * Create a reusable "extent" mapping from all logical
2222 * blocks to all physical blocks for the given journal. This will save
2223 * us time when writing journal blocks. Most journals will have only one
2224 * extent that maps all their logical blocks. That's because gfs2.mkfs
2225 * arranges the journal blocks sequentially to maximize performance.
2226 * So the extent would map the first block for the entire file length.
2227 * However, gfs2_jadd can happen while file activity is happening, so
2228 * those journals may not be sequential. Less likely is the case where
2229 * the users created their own journals by mounting the metafs and
2230 * laying it out. But it's still possible. These journals might have
2233 * Returns: 0 on success, or error on failure
2236 int gfs2_map_journal_extents(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd)
2240 struct gfs2_inode *ip = GFS2_I(jd->jd_inode);
2241 struct buffer_head bh;
2242 unsigned int shift = sdp->sd_sb.sb_bsize_shift;
2247 start = ktime_get();
2248 lblock_stop = i_size_read(jd->jd_inode) >> shift;
2249 size = (lblock_stop - lblock) << shift;
2251 WARN_ON(!list_empty(&jd->extent_list));
2257 rc = gfs2_block_map(jd->jd_inode, lblock, &bh, 0);
2258 if (rc || !buffer_mapped(&bh))
2260 rc = gfs2_add_jextent(jd, lblock, bh.b_blocknr, bh.b_size >> shift);
2264 lblock += (bh.b_size >> ip->i_inode.i_blkbits);
2268 fs_info(sdp, "journal %d mapped with %u extents in %lldms\n", jd->jd_jid,
2269 jd->nr_extents, ktime_ms_delta(end, start));
2273 fs_warn(sdp, "error %d mapping journal %u at offset %llu (extent %u)\n",
2275 (unsigned long long)(i_size_read(jd->jd_inode) - size),
2277 fs_warn(sdp, "bmap=%d lblock=%llu block=%llu, state=0x%08lx, size=%llu\n",
2278 rc, (unsigned long long)lblock, (unsigned long long)bh.b_blocknr,
2279 bh.b_state, (unsigned long long)bh.b_size);
2280 gfs2_free_journal_extents(jd);
2285 * gfs2_write_alloc_required - figure out if a write will require an allocation
2286 * @ip: the file being written to
2287 * @offset: the offset to write to
2288 * @len: the number of bytes being written
2290 * Returns: 1 if an alloc is required, 0 otherwise
2293 int gfs2_write_alloc_required(struct gfs2_inode *ip, u64 offset,
2296 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
2297 struct buffer_head bh;
2299 u64 lblock, lblock_stop, size;
2305 if (gfs2_is_stuffed(ip)) {
2306 if (offset + len > gfs2_max_stuffed_size(ip))
2311 shift = sdp->sd_sb.sb_bsize_shift;
2312 BUG_ON(gfs2_is_dir(ip));
2313 end_of_file = (i_size_read(&ip->i_inode) + sdp->sd_sb.sb_bsize - 1) >> shift;
2314 lblock = offset >> shift;
2315 lblock_stop = (offset + len + sdp->sd_sb.sb_bsize - 1) >> shift;
2316 if (lblock_stop > end_of_file && ip != GFS2_I(sdp->sd_rindex))
2319 size = (lblock_stop - lblock) << shift;
2323 gfs2_block_map(&ip->i_inode, lblock, &bh, 0);
2324 if (!buffer_mapped(&bh))
2327 lblock += (bh.b_size >> ip->i_inode.i_blkbits);
2333 static int stuffed_zero_range(struct inode *inode, loff_t offset, loff_t length)
2335 struct gfs2_inode *ip = GFS2_I(inode);
2336 struct buffer_head *dibh;
2339 if (offset >= inode->i_size)
2341 if (offset + length > inode->i_size)
2342 length = inode->i_size - offset;
2344 error = gfs2_meta_inode_buffer(ip, &dibh);
2347 gfs2_trans_add_meta(ip->i_gl, dibh);
2348 memset(dibh->b_data + sizeof(struct gfs2_dinode) + offset, 0,
2354 static int gfs2_journaled_truncate_range(struct inode *inode, loff_t offset,
2357 struct gfs2_sbd *sdp = GFS2_SB(inode);
2358 loff_t max_chunk = GFS2_JTRUNC_REVOKES * sdp->sd_vfs->s_blocksize;
2362 struct gfs2_trans *tr;
2367 if (chunk > max_chunk)
2370 offs = offset & ~PAGE_MASK;
2371 if (offs && chunk > PAGE_SIZE)
2372 chunk = offs + ((chunk - offs) & PAGE_MASK);
2374 truncate_pagecache_range(inode, offset, chunk);
2378 tr = current->journal_info;
2379 if (!test_bit(TR_TOUCHED, &tr->tr_flags))
2382 gfs2_trans_end(sdp);
2383 error = gfs2_trans_begin(sdp, RES_DINODE, GFS2_JTRUNC_REVOKES);
2390 int __gfs2_punch_hole(struct file *file, loff_t offset, loff_t length)
2392 struct inode *inode = file_inode(file);
2393 struct gfs2_inode *ip = GFS2_I(inode);
2394 struct gfs2_sbd *sdp = GFS2_SB(inode);
2395 unsigned int blocksize = i_blocksize(inode);
2399 if (!gfs2_is_stuffed(ip)) {
2400 unsigned int start_off, end_len;
2402 start_off = offset & (blocksize - 1);
2403 end_len = (offset + length) & (blocksize - 1);
2405 unsigned int len = length;
2406 if (length > blocksize - start_off)
2407 len = blocksize - start_off;
2408 error = gfs2_block_zero_range(inode, offset, len);
2411 if (start_off + length < blocksize)
2415 error = gfs2_block_zero_range(inode,
2416 offset + length - end_len, end_len);
2422 start = round_down(offset, blocksize);
2423 end = round_up(offset + length, blocksize) - 1;
2424 error = filemap_write_and_wait_range(inode->i_mapping, start, end);
2428 if (gfs2_is_jdata(ip))
2429 error = gfs2_trans_begin(sdp, RES_DINODE + 2 * RES_JDATA,
2430 GFS2_JTRUNC_REVOKES);
2432 error = gfs2_trans_begin(sdp, RES_DINODE, 0);
2436 if (gfs2_is_stuffed(ip)) {
2437 error = stuffed_zero_range(inode, offset, length);
2442 if (gfs2_is_jdata(ip)) {
2443 BUG_ON(!current->journal_info);
2444 gfs2_journaled_truncate_range(inode, offset, length);
2446 truncate_pagecache_range(inode, offset, offset + length - 1);
2448 file_update_time(file);
2449 mark_inode_dirty(inode);
2451 if (current->journal_info)
2452 gfs2_trans_end(sdp);
2454 if (!gfs2_is_stuffed(ip))
2455 error = punch_hole(ip, offset, length);
2458 if (current->journal_info)
2459 gfs2_trans_end(sdp);
2463 static int gfs2_map_blocks(struct iomap_writepage_ctx *wpc, struct inode *inode,
2468 if (WARN_ON_ONCE(gfs2_is_stuffed(GFS2_I(inode))))
2471 if (offset >= wpc->iomap.offset &&
2472 offset < wpc->iomap.offset + wpc->iomap.length)
2475 memset(&wpc->iomap, 0, sizeof(wpc->iomap));
2476 ret = gfs2_iomap_get(inode, offset, INT_MAX, &wpc->iomap);
2480 const struct iomap_writeback_ops gfs2_writeback_ops = {
2481 .map_blocks = gfs2_map_blocks,