1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved.
4 * Copyright (C) 2004-2006 Red Hat, Inc. All rights reserved.
7 #include <linux/spinlock.h>
8 #include <linux/completion.h>
9 #include <linux/buffer_head.h>
10 #include <linux/blkdev.h>
11 #include <linux/gfs2_ondisk.h>
12 #include <linux/crc32.h>
13 #include <linux/iomap.h>
14 #include <linux/ktime.h>
30 #include "trace_gfs2.h"
32 /* This doesn't need to be that large as max 64 bit pointers in a 4k
33 * block is 512, so __u16 is fine for that. It saves stack space to
37 struct buffer_head *mp_bh[GFS2_MAX_META_HEIGHT];
38 __u16 mp_list[GFS2_MAX_META_HEIGHT];
39 int mp_fheight; /* find_metapath height */
40 int mp_aheight; /* actual height (lookup height) */
43 static int punch_hole(struct gfs2_inode *ip, u64 offset, u64 length);
46 * gfs2_unstuffer_page - unstuff a stuffed inode into a block cached by a page
48 * @dibh: the dinode buffer
49 * @block: the block number that was allocated
50 * @page: The (optional) page. This is looked up if @page is NULL
55 static int gfs2_unstuffer_page(struct gfs2_inode *ip, struct buffer_head *dibh,
56 u64 block, struct page *page)
58 struct inode *inode = &ip->i_inode;
60 if (!PageUptodate(page)) {
61 void *kaddr = kmap(page);
62 u64 dsize = i_size_read(inode);
64 if (dsize > gfs2_max_stuffed_size(ip))
65 dsize = gfs2_max_stuffed_size(ip);
67 memcpy(kaddr, dibh->b_data + sizeof(struct gfs2_dinode), dsize);
68 memset(kaddr + dsize, 0, PAGE_SIZE - dsize);
71 SetPageUptodate(page);
74 if (gfs2_is_jdata(ip)) {
75 struct buffer_head *bh;
77 if (!page_has_buffers(page))
78 create_empty_buffers(page, BIT(inode->i_blkbits),
81 bh = page_buffers(page);
82 if (!buffer_mapped(bh))
83 map_bh(bh, inode->i_sb, block);
85 set_buffer_uptodate(bh);
86 gfs2_trans_add_data(ip->i_gl, bh);
89 gfs2_ordered_add_inode(ip);
95 static int __gfs2_unstuff_inode(struct gfs2_inode *ip, struct page *page)
97 struct buffer_head *bh, *dibh;
98 struct gfs2_dinode *di;
100 int isdir = gfs2_is_dir(ip);
103 error = gfs2_meta_inode_buffer(ip, &dibh);
107 if (i_size_read(&ip->i_inode)) {
108 /* Get a free block, fill it with the stuffed data,
109 and write it out to disk */
112 error = gfs2_alloc_blocks(ip, &block, &n, 0, NULL);
116 gfs2_trans_remove_revoke(GFS2_SB(&ip->i_inode), block, 1);
117 error = gfs2_dir_get_new_buffer(ip, block, &bh);
120 gfs2_buffer_copy_tail(bh, sizeof(struct gfs2_meta_header),
121 dibh, sizeof(struct gfs2_dinode));
124 error = gfs2_unstuffer_page(ip, dibh, block, page);
130 /* Set up the pointer to the new block */
132 gfs2_trans_add_meta(ip->i_gl, dibh);
133 di = (struct gfs2_dinode *)dibh->b_data;
134 gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode));
136 if (i_size_read(&ip->i_inode)) {
137 *(__be64 *)(di + 1) = cpu_to_be64(block);
138 gfs2_add_inode_blocks(&ip->i_inode, 1);
139 di->di_blocks = cpu_to_be64(gfs2_get_inode_blocks(&ip->i_inode));
143 di->di_height = cpu_to_be16(1);
151 * gfs2_unstuff_dinode - Unstuff a dinode when the data has grown too big
152 * @ip: The GFS2 inode to unstuff
154 * This routine unstuffs a dinode and returns it to a "normal" state such
155 * that the height can be grown in the traditional way.
160 int gfs2_unstuff_dinode(struct gfs2_inode *ip)
162 struct inode *inode = &ip->i_inode;
166 down_write(&ip->i_rw_mutex);
167 page = find_or_create_page(inode->i_mapping, 0, GFP_NOFS);
171 error = __gfs2_unstuff_inode(ip, page);
175 up_write(&ip->i_rw_mutex);
180 * find_metapath - Find path through the metadata tree
181 * @sdp: The superblock
182 * @block: The disk block to look up
183 * @mp: The metapath to return the result in
184 * @height: The pre-calculated height of the metadata tree
186 * This routine returns a struct metapath structure that defines a path
187 * through the metadata of inode "ip" to get to block "block".
190 * Given: "ip" is a height 3 file, "offset" is 101342453, and this is a
191 * filesystem with a blocksize of 4096.
193 * find_metapath() would return a struct metapath structure set to:
194 * mp_fheight = 3, mp_list[0] = 0, mp_list[1] = 48, and mp_list[2] = 165.
196 * That means that in order to get to the block containing the byte at
197 * offset 101342453, we would load the indirect block pointed to by pointer
198 * 0 in the dinode. We would then load the indirect block pointed to by
199 * pointer 48 in that indirect block. We would then load the data block
200 * pointed to by pointer 165 in that indirect block.
202 * ----------------------------------------
207 * ----------------------------------------
211 * ----------------------------------------
215 * |0 5 6 7 8 9 0 1 2|
216 * ----------------------------------------
220 * ----------------------------------------
225 * ----------------------------------------
229 * ----------------------------------------
230 * | Data block containing offset |
234 * ----------------------------------------
238 static void find_metapath(const struct gfs2_sbd *sdp, u64 block,
239 struct metapath *mp, unsigned int height)
243 mp->mp_fheight = height;
244 for (i = height; i--;)
245 mp->mp_list[i] = do_div(block, sdp->sd_inptrs);
248 static inline unsigned int metapath_branch_start(const struct metapath *mp)
250 if (mp->mp_list[0] == 0)
256 * metaptr1 - Return the first possible metadata pointer in a metapath buffer
257 * @height: The metadata height (0 = dinode)
260 static inline __be64 *metaptr1(unsigned int height, const struct metapath *mp)
262 struct buffer_head *bh = mp->mp_bh[height];
264 return ((__be64 *)(bh->b_data + sizeof(struct gfs2_dinode)));
265 return ((__be64 *)(bh->b_data + sizeof(struct gfs2_meta_header)));
269 * metapointer - Return pointer to start of metadata in a buffer
270 * @height: The metadata height (0 = dinode)
273 * Return a pointer to the block number of the next height of the metadata
274 * tree given a buffer containing the pointer to the current height of the
278 static inline __be64 *metapointer(unsigned int height, const struct metapath *mp)
280 __be64 *p = metaptr1(height, mp);
281 return p + mp->mp_list[height];
284 static inline const __be64 *metaend(unsigned int height, const struct metapath *mp)
286 const struct buffer_head *bh = mp->mp_bh[height];
287 return (const __be64 *)(bh->b_data + bh->b_size);
290 static void clone_metapath(struct metapath *clone, struct metapath *mp)
295 for (hgt = 0; hgt < mp->mp_aheight; hgt++)
296 get_bh(clone->mp_bh[hgt]);
299 static void gfs2_metapath_ra(struct gfs2_glock *gl, __be64 *start, __be64 *end)
303 for (t = start; t < end; t++) {
304 struct buffer_head *rabh;
309 rabh = gfs2_getbuf(gl, be64_to_cpu(*t), CREATE);
310 if (trylock_buffer(rabh)) {
311 if (!buffer_uptodate(rabh)) {
312 rabh->b_end_io = end_buffer_read_sync;
313 submit_bh(REQ_OP_READ | REQ_RAHEAD | REQ_META |
323 static int __fillup_metapath(struct gfs2_inode *ip, struct metapath *mp,
324 unsigned int x, unsigned int h)
327 __be64 *ptr = metapointer(x, mp);
328 u64 dblock = be64_to_cpu(*ptr);
333 ret = gfs2_meta_buffer(ip, GFS2_METATYPE_IN, dblock, &mp->mp_bh[x + 1]);
337 mp->mp_aheight = x + 1;
342 * lookup_metapath - Walk the metadata tree to a specific point
346 * Assumes that the inode's buffer has already been looked up and
347 * hooked onto mp->mp_bh[0] and that the metapath has been initialised
348 * by find_metapath().
350 * If this function encounters part of the tree which has not been
351 * allocated, it returns the current height of the tree at the point
352 * at which it found the unallocated block. Blocks which are found are
353 * added to the mp->mp_bh[] list.
358 static int lookup_metapath(struct gfs2_inode *ip, struct metapath *mp)
360 return __fillup_metapath(ip, mp, 0, ip->i_height - 1);
364 * fillup_metapath - fill up buffers for the metadata path to a specific height
367 * @h: The height to which it should be mapped
369 * Similar to lookup_metapath, but does lookups for a range of heights
371 * Returns: error or the number of buffers filled
374 static int fillup_metapath(struct gfs2_inode *ip, struct metapath *mp, int h)
380 /* find the first buffer we need to look up. */
381 for (x = h - 1; x > 0; x--) {
386 ret = __fillup_metapath(ip, mp, x, h);
389 return mp->mp_aheight - x - 1;
392 static sector_t metapath_to_block(struct gfs2_sbd *sdp, struct metapath *mp)
394 sector_t factor = 1, block = 0;
397 for (hgt = mp->mp_fheight - 1; hgt >= 0; hgt--) {
398 if (hgt < mp->mp_aheight)
399 block += mp->mp_list[hgt] * factor;
400 factor *= sdp->sd_inptrs;
405 static void release_metapath(struct metapath *mp)
409 for (i = 0; i < GFS2_MAX_META_HEIGHT; i++) {
410 if (mp->mp_bh[i] == NULL)
412 brelse(mp->mp_bh[i]);
418 * gfs2_extent_length - Returns length of an extent of blocks
419 * @bh: The metadata block
420 * @ptr: Current position in @bh
421 * @limit: Max extent length to return
422 * @eob: Set to 1 if we hit "end of block"
424 * Returns: The length of the extent (minimum of one block)
427 static inline unsigned int gfs2_extent_length(struct buffer_head *bh, __be64 *ptr, size_t limit, int *eob)
429 const __be64 *end = (__be64 *)(bh->b_data + bh->b_size);
430 const __be64 *first = ptr;
431 u64 d = be64_to_cpu(*ptr);
439 } while(be64_to_cpu(*ptr) == d);
445 enum walker_status { WALK_STOP, WALK_FOLLOW, WALK_CONTINUE };
448 * gfs2_metadata_walker - walk an indirect block
449 * @mp: Metapath to indirect block
450 * @ptrs: Number of pointers to look at
452 * When returning WALK_FOLLOW, the walker must update @mp to point at the right
453 * indirect block to follow.
455 typedef enum walker_status (*gfs2_metadata_walker)(struct metapath *mp,
459 * gfs2_walk_metadata - walk a tree of indirect blocks
461 * @mp: Starting point of walk
462 * @max_len: Maximum number of blocks to walk
463 * @walker: Called during the walk
465 * Returns 1 if the walk was stopped by @walker, 0 if we went past @max_len or
466 * past the end of metadata, and a negative error code otherwise.
469 static int gfs2_walk_metadata(struct inode *inode, struct metapath *mp,
470 u64 max_len, gfs2_metadata_walker walker)
472 struct gfs2_inode *ip = GFS2_I(inode);
473 struct gfs2_sbd *sdp = GFS2_SB(inode);
479 * The walk starts in the lowest allocated indirect block, which may be
480 * before the position indicated by @mp. Adjust @max_len accordingly
481 * to avoid a short walk.
483 for (hgt = mp->mp_fheight - 1; hgt >= mp->mp_aheight; hgt--) {
484 max_len += mp->mp_list[hgt] * factor;
485 mp->mp_list[hgt] = 0;
486 factor *= sdp->sd_inptrs;
490 u16 start = mp->mp_list[hgt];
491 enum walker_status status;
495 /* Walk indirect block. */
496 ptrs = (hgt >= 1 ? sdp->sd_inptrs : sdp->sd_diptrs) - start;
499 ptrs = DIV_ROUND_UP_ULL(max_len, factor);
500 status = walker(mp, ptrs);
505 BUG_ON(mp->mp_aheight == mp->mp_fheight);
506 ptrs = mp->mp_list[hgt] - start;
515 if (status == WALK_FOLLOW)
516 goto fill_up_metapath;
519 /* Decrease height of metapath. */
520 brelse(mp->mp_bh[hgt]);
521 mp->mp_bh[hgt] = NULL;
522 mp->mp_list[hgt] = 0;
526 factor *= sdp->sd_inptrs;
528 /* Advance in metadata tree. */
529 (mp->mp_list[hgt])++;
531 if (mp->mp_list[hgt] >= sdp->sd_inptrs)
534 if (mp->mp_list[hgt] >= sdp->sd_diptrs)
539 /* Increase height of metapath. */
540 ret = fillup_metapath(ip, mp, ip->i_height - 1);
545 do_div(factor, sdp->sd_inptrs);
546 mp->mp_aheight = hgt + 1;
551 static enum walker_status gfs2_hole_walker(struct metapath *mp,
554 const __be64 *start, *ptr, *end;
557 hgt = mp->mp_aheight - 1;
558 start = metapointer(hgt, mp);
561 for (ptr = start; ptr < end; ptr++) {
563 mp->mp_list[hgt] += ptr - start;
564 if (mp->mp_aheight == mp->mp_fheight)
569 return WALK_CONTINUE;
573 * gfs2_hole_size - figure out the size of a hole
575 * @lblock: The logical starting block number
576 * @len: How far to look (in blocks)
577 * @mp: The metapath at lblock
578 * @iomap: The iomap to store the hole size in
580 * This function modifies @mp.
582 * Returns: errno on error
584 static int gfs2_hole_size(struct inode *inode, sector_t lblock, u64 len,
585 struct metapath *mp, struct iomap *iomap)
587 struct metapath clone;
591 clone_metapath(&clone, mp);
592 ret = gfs2_walk_metadata(inode, &clone, len, gfs2_hole_walker);
597 hole_size = metapath_to_block(GFS2_SB(inode), &clone) - lblock;
600 iomap->length = hole_size << inode->i_blkbits;
604 release_metapath(&clone);
608 static inline void gfs2_indirect_init(struct metapath *mp,
609 struct gfs2_glock *gl, unsigned int i,
610 unsigned offset, u64 bn)
612 __be64 *ptr = (__be64 *)(mp->mp_bh[i - 1]->b_data +
613 ((i > 1) ? sizeof(struct gfs2_meta_header) :
614 sizeof(struct gfs2_dinode)));
616 BUG_ON(mp->mp_bh[i] != NULL);
617 mp->mp_bh[i] = gfs2_meta_new(gl, bn);
618 gfs2_trans_add_meta(gl, mp->mp_bh[i]);
619 gfs2_metatype_set(mp->mp_bh[i], GFS2_METATYPE_IN, GFS2_FORMAT_IN);
620 gfs2_buffer_clear_tail(mp->mp_bh[i], sizeof(struct gfs2_meta_header));
622 *ptr = cpu_to_be64(bn);
627 ALLOC_GROW_DEPTH = 1,
628 ALLOC_GROW_HEIGHT = 2,
629 /* ALLOC_UNSTUFF = 3, TBD and rather complicated */
633 * __gfs2_iomap_alloc - Build a metadata tree of the requested height
634 * @inode: The GFS2 inode
635 * @iomap: The iomap structure
636 * @mp: The metapath, with proper height information calculated
638 * In this routine we may have to alloc:
639 * i) Indirect blocks to grow the metadata tree height
640 * ii) Indirect blocks to fill in lower part of the metadata tree
643 * This function is called after __gfs2_iomap_get, which works out the
644 * total number of blocks which we need via gfs2_alloc_size.
646 * We then do the actual allocation asking for an extent at a time (if
647 * enough contiguous free blocks are available, there will only be one
648 * allocation request per call) and uses the state machine to initialise
649 * the blocks in order.
651 * Right now, this function will allocate at most one indirect block
652 * worth of data -- with a default block size of 4K, that's slightly
653 * less than 2M. If this limitation is ever removed to allow huge
654 * allocations, we would probably still want to limit the iomap size we
655 * return to avoid stalling other tasks during huge writes; the next
656 * iomap iteration would then find the blocks already allocated.
658 * Returns: errno on error
661 static int __gfs2_iomap_alloc(struct inode *inode, struct iomap *iomap,
664 struct gfs2_inode *ip = GFS2_I(inode);
665 struct gfs2_sbd *sdp = GFS2_SB(inode);
666 struct buffer_head *dibh = mp->mp_bh[0];
668 unsigned n, i, blks, alloced = 0, iblks = 0, branch_start = 0;
669 size_t dblks = iomap->length >> inode->i_blkbits;
670 const unsigned end_of_metadata = mp->mp_fheight - 1;
672 enum alloc_state state;
676 BUG_ON(mp->mp_aheight < 1);
677 BUG_ON(dibh == NULL);
680 gfs2_trans_add_meta(ip->i_gl, dibh);
682 down_write(&ip->i_rw_mutex);
684 if (mp->mp_fheight == mp->mp_aheight) {
685 /* Bottom indirect block exists */
688 /* Need to allocate indirect blocks */
689 if (mp->mp_fheight == ip->i_height) {
690 /* Writing into existing tree, extend tree down */
691 iblks = mp->mp_fheight - mp->mp_aheight;
692 state = ALLOC_GROW_DEPTH;
694 /* Building up tree height */
695 state = ALLOC_GROW_HEIGHT;
696 iblks = mp->mp_fheight - ip->i_height;
697 branch_start = metapath_branch_start(mp);
698 iblks += (mp->mp_fheight - branch_start);
702 /* start of the second part of the function (state machine) */
704 blks = dblks + iblks;
708 ret = gfs2_alloc_blocks(ip, &bn, &n, 0, NULL);
712 if (state != ALLOC_DATA || gfs2_is_jdata(ip))
713 gfs2_trans_remove_revoke(sdp, bn, n);
715 /* Growing height of tree */
716 case ALLOC_GROW_HEIGHT:
718 ptr = (__be64 *)(dibh->b_data +
719 sizeof(struct gfs2_dinode));
722 for (; i - 1 < mp->mp_fheight - ip->i_height && n > 0;
724 gfs2_indirect_init(mp, ip->i_gl, i, 0, bn++);
725 if (i - 1 == mp->mp_fheight - ip->i_height) {
727 gfs2_buffer_copy_tail(mp->mp_bh[i],
728 sizeof(struct gfs2_meta_header),
729 dibh, sizeof(struct gfs2_dinode));
730 gfs2_buffer_clear_tail(dibh,
731 sizeof(struct gfs2_dinode) +
733 ptr = (__be64 *)(mp->mp_bh[i]->b_data +
734 sizeof(struct gfs2_meta_header));
736 state = ALLOC_GROW_DEPTH;
737 for(i = branch_start; i < mp->mp_fheight; i++) {
738 if (mp->mp_bh[i] == NULL)
740 brelse(mp->mp_bh[i]);
747 fallthrough; /* To branching from existing tree */
748 case ALLOC_GROW_DEPTH:
749 if (i > 1 && i < mp->mp_fheight)
750 gfs2_trans_add_meta(ip->i_gl, mp->mp_bh[i-1]);
751 for (; i < mp->mp_fheight && n > 0; i++, n--)
752 gfs2_indirect_init(mp, ip->i_gl, i,
753 mp->mp_list[i-1], bn++);
754 if (i == mp->mp_fheight)
758 fallthrough; /* To tree complete, adding data blocks */
761 BUG_ON(mp->mp_bh[end_of_metadata] == NULL);
762 gfs2_trans_add_meta(ip->i_gl, mp->mp_bh[end_of_metadata]);
764 ptr = metapointer(end_of_metadata, mp);
765 iomap->addr = bn << inode->i_blkbits;
766 iomap->flags |= IOMAP_F_MERGED | IOMAP_F_NEW;
768 *ptr++ = cpu_to_be64(bn++);
771 } while (iomap->addr == IOMAP_NULL_ADDR);
773 iomap->type = IOMAP_MAPPED;
774 iomap->length = (u64)dblks << inode->i_blkbits;
775 ip->i_height = mp->mp_fheight;
776 gfs2_add_inode_blocks(&ip->i_inode, alloced);
777 gfs2_dinode_out(ip, dibh->b_data);
779 up_write(&ip->i_rw_mutex);
783 #define IOMAP_F_GFS2_BOUNDARY IOMAP_F_PRIVATE
786 * gfs2_alloc_size - Compute the maximum allocation size
789 * @size: Requested size in blocks
791 * Compute the maximum size of the next allocation at @mp.
793 * Returns: size in blocks
795 static u64 gfs2_alloc_size(struct inode *inode, struct metapath *mp, u64 size)
797 struct gfs2_inode *ip = GFS2_I(inode);
798 struct gfs2_sbd *sdp = GFS2_SB(inode);
799 const __be64 *first, *ptr, *end;
802 * For writes to stuffed files, this function is called twice via
803 * __gfs2_iomap_get, before and after unstuffing. The size we return the
804 * first time needs to be large enough to get the reservation and
805 * allocation sizes right. The size we return the second time must
806 * be exact or else __gfs2_iomap_alloc won't do the right thing.
809 if (gfs2_is_stuffed(ip) || mp->mp_fheight != mp->mp_aheight) {
810 unsigned int maxsize = mp->mp_fheight > 1 ?
811 sdp->sd_inptrs : sdp->sd_diptrs;
812 maxsize -= mp->mp_list[mp->mp_fheight - 1];
818 first = metapointer(ip->i_height - 1, mp);
819 end = metaend(ip->i_height - 1, mp);
820 if (end - first > size)
822 for (ptr = first; ptr < end; ptr++) {
830 * __gfs2_iomap_get - Map blocks from an inode to disk blocks
832 * @pos: Starting position in bytes
833 * @length: Length to map, in bytes
834 * @flags: iomap flags
835 * @iomap: The iomap structure
840 static int __gfs2_iomap_get(struct inode *inode, loff_t pos, loff_t length,
841 unsigned flags, struct iomap *iomap,
844 struct gfs2_inode *ip = GFS2_I(inode);
845 struct gfs2_sbd *sdp = GFS2_SB(inode);
846 loff_t size = i_size_read(inode);
849 sector_t lblock_stop;
853 struct buffer_head *dibh = NULL, *bh;
859 down_read(&ip->i_rw_mutex);
861 ret = gfs2_meta_inode_buffer(ip, &dibh);
866 if (gfs2_is_stuffed(ip)) {
867 if (flags & IOMAP_WRITE) {
868 loff_t max_size = gfs2_max_stuffed_size(ip);
870 if (pos + length > max_size)
872 iomap->length = max_size;
875 if (flags & IOMAP_REPORT) {
880 iomap->length = length;
884 iomap->length = size;
886 iomap->addr = (ip->i_no_addr << inode->i_blkbits) +
887 sizeof(struct gfs2_dinode);
888 iomap->type = IOMAP_INLINE;
889 iomap->inline_data = dibh->b_data + sizeof(struct gfs2_dinode);
894 lblock = pos >> inode->i_blkbits;
895 iomap->offset = lblock << inode->i_blkbits;
896 lblock_stop = (pos + length - 1) >> inode->i_blkbits;
897 len = lblock_stop - lblock + 1;
898 iomap->length = len << inode->i_blkbits;
900 height = ip->i_height;
901 while ((lblock + 1) * sdp->sd_sb.sb_bsize > sdp->sd_heightsize[height])
903 find_metapath(sdp, lblock, mp, height);
904 if (height > ip->i_height || gfs2_is_stuffed(ip))
907 ret = lookup_metapath(ip, mp);
911 if (mp->mp_aheight != ip->i_height)
914 ptr = metapointer(ip->i_height - 1, mp);
918 bh = mp->mp_bh[ip->i_height - 1];
919 len = gfs2_extent_length(bh, ptr, len, &eob);
921 iomap->addr = be64_to_cpu(*ptr) << inode->i_blkbits;
922 iomap->length = len << inode->i_blkbits;
923 iomap->type = IOMAP_MAPPED;
924 iomap->flags |= IOMAP_F_MERGED;
926 iomap->flags |= IOMAP_F_GFS2_BOUNDARY;
929 iomap->bdev = inode->i_sb->s_bdev;
931 up_read(&ip->i_rw_mutex);
935 if (flags & IOMAP_REPORT) {
938 else if (height == ip->i_height)
939 ret = gfs2_hole_size(inode, lblock, len, mp, iomap);
941 iomap->length = size - iomap->offset;
942 } else if (flags & IOMAP_WRITE) {
945 if (flags & IOMAP_DIRECT)
946 goto out; /* (see gfs2_file_direct_write) */
948 len = gfs2_alloc_size(inode, mp, len);
949 alloc_size = len << inode->i_blkbits;
950 if (alloc_size < iomap->length)
951 iomap->length = alloc_size;
953 if (pos < size && height == ip->i_height)
954 ret = gfs2_hole_size(inode, lblock, len, mp, iomap);
957 iomap->addr = IOMAP_NULL_ADDR;
958 iomap->type = IOMAP_HOLE;
962 static int gfs2_iomap_page_prepare(struct inode *inode, loff_t pos,
965 unsigned int blockmask = i_blocksize(inode) - 1;
966 struct gfs2_sbd *sdp = GFS2_SB(inode);
969 blocks = ((pos & blockmask) + len + blockmask) >> inode->i_blkbits;
970 return gfs2_trans_begin(sdp, RES_DINODE + blocks, 0);
973 static void gfs2_iomap_page_done(struct inode *inode, loff_t pos,
974 unsigned copied, struct page *page)
976 struct gfs2_trans *tr = current->journal_info;
977 struct gfs2_inode *ip = GFS2_I(inode);
978 struct gfs2_sbd *sdp = GFS2_SB(inode);
980 if (page && !gfs2_is_stuffed(ip))
981 gfs2_page_add_databufs(ip, page, offset_in_page(pos), copied);
983 if (tr->tr_num_buf_new)
984 __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
989 static const struct iomap_page_ops gfs2_iomap_page_ops = {
990 .page_prepare = gfs2_iomap_page_prepare,
991 .page_done = gfs2_iomap_page_done,
994 static int gfs2_iomap_begin_write(struct inode *inode, loff_t pos,
995 loff_t length, unsigned flags,
999 struct gfs2_inode *ip = GFS2_I(inode);
1000 struct gfs2_sbd *sdp = GFS2_SB(inode);
1004 unstuff = gfs2_is_stuffed(ip) &&
1005 pos + length > gfs2_max_stuffed_size(ip);
1007 if (unstuff || iomap->type == IOMAP_HOLE) {
1008 unsigned int data_blocks, ind_blocks;
1009 struct gfs2_alloc_parms ap = {};
1010 unsigned int rblocks;
1011 struct gfs2_trans *tr;
1013 gfs2_write_calc_reserv(ip, iomap->length, &data_blocks,
1015 ap.target = data_blocks + ind_blocks;
1016 ret = gfs2_quota_lock_check(ip, &ap);
1020 ret = gfs2_inplace_reserve(ip, &ap);
1024 rblocks = RES_DINODE + ind_blocks;
1025 if (gfs2_is_jdata(ip))
1026 rblocks += data_blocks;
1027 if (ind_blocks || data_blocks)
1028 rblocks += RES_STATFS + RES_QUOTA;
1029 if (inode == sdp->sd_rindex)
1030 rblocks += 2 * RES_STATFS;
1031 rblocks += gfs2_rg_blocks(ip, data_blocks + ind_blocks);
1033 ret = gfs2_trans_begin(sdp, rblocks,
1034 iomap->length >> inode->i_blkbits);
1036 goto out_trans_fail;
1039 ret = gfs2_unstuff_dinode(ip);
1042 release_metapath(mp);
1043 ret = __gfs2_iomap_get(inode, iomap->offset,
1044 iomap->length, flags, iomap, mp);
1049 if (iomap->type == IOMAP_HOLE) {
1050 ret = __gfs2_iomap_alloc(inode, iomap, mp);
1052 gfs2_trans_end(sdp);
1053 gfs2_inplace_release(ip);
1054 punch_hole(ip, iomap->offset, iomap->length);
1059 tr = current->journal_info;
1060 if (tr->tr_num_buf_new)
1061 __mark_inode_dirty(inode, I_DIRTY_DATASYNC);
1063 gfs2_trans_end(sdp);
1066 if (gfs2_is_stuffed(ip) || gfs2_is_jdata(ip))
1067 iomap->page_ops = &gfs2_iomap_page_ops;
1071 gfs2_trans_end(sdp);
1073 gfs2_inplace_release(ip);
1075 gfs2_quota_unlock(ip);
1079 static int gfs2_iomap_begin(struct inode *inode, loff_t pos, loff_t length,
1080 unsigned flags, struct iomap *iomap,
1081 struct iomap *srcmap)
1083 struct gfs2_inode *ip = GFS2_I(inode);
1084 struct metapath mp = { .mp_aheight = 1, };
1087 if (gfs2_is_jdata(ip))
1088 iomap->flags |= IOMAP_F_BUFFER_HEAD;
1090 trace_gfs2_iomap_start(ip, pos, length, flags);
1091 ret = __gfs2_iomap_get(inode, pos, length, flags, iomap, &mp);
1095 switch(flags & (IOMAP_WRITE | IOMAP_ZERO)) {
1097 if (flags & IOMAP_DIRECT) {
1099 * Silently fall back to buffered I/O for stuffed files
1100 * or if we've got a hole (see gfs2_file_direct_write).
1102 if (iomap->type != IOMAP_MAPPED)
1108 if (iomap->type == IOMAP_HOLE)
1115 ret = gfs2_iomap_begin_write(inode, pos, length, flags, iomap, &mp);
1118 release_metapath(&mp);
1119 trace_gfs2_iomap_end(ip, iomap, ret);
1123 static int gfs2_iomap_end(struct inode *inode, loff_t pos, loff_t length,
1124 ssize_t written, unsigned flags, struct iomap *iomap)
1126 struct gfs2_inode *ip = GFS2_I(inode);
1127 struct gfs2_sbd *sdp = GFS2_SB(inode);
1129 switch (flags & (IOMAP_WRITE | IOMAP_ZERO)) {
1131 if (flags & IOMAP_DIRECT)
1135 if (iomap->type == IOMAP_HOLE)
1142 if (!gfs2_is_stuffed(ip))
1143 gfs2_ordered_add_inode(ip);
1145 if (inode == sdp->sd_rindex)
1146 adjust_fs_space(inode);
1148 gfs2_inplace_release(ip);
1150 if (ip->i_qadata && ip->i_qadata->qa_qd_num)
1151 gfs2_quota_unlock(ip);
1153 if (length != written && (iomap->flags & IOMAP_F_NEW)) {
1154 /* Deallocate blocks that were just allocated. */
1155 loff_t hstart = round_up(pos + written, i_blocksize(inode));
1156 loff_t hend = iomap->offset + iomap->length;
1158 if (hstart < hend) {
1159 truncate_pagecache_range(inode, hstart, hend - 1);
1160 punch_hole(ip, hstart, hend - hstart);
1164 if (unlikely(!written))
1167 if (iomap->flags & IOMAP_F_SIZE_CHANGED)
1168 mark_inode_dirty(inode);
1169 set_bit(GLF_DIRTY, &ip->i_gl->gl_flags);
1173 const struct iomap_ops gfs2_iomap_ops = {
1174 .iomap_begin = gfs2_iomap_begin,
1175 .iomap_end = gfs2_iomap_end,
1179 * gfs2_block_map - Map one or more blocks of an inode to a disk block
1181 * @lblock: The logical block number
1182 * @bh_map: The bh to be mapped
1183 * @create: True if its ok to alloc blocks to satify the request
1185 * The size of the requested mapping is defined in bh_map->b_size.
1187 * Clears buffer_mapped(bh_map) and leaves bh_map->b_size unchanged
1188 * when @lblock is not mapped. Sets buffer_mapped(bh_map) and
1189 * bh_map->b_size to indicate the size of the mapping when @lblock and
1190 * successive blocks are mapped, up to the requested size.
1192 * Sets buffer_boundary() if a read of metadata will be required
1193 * before the next block can be mapped. Sets buffer_new() if new
1194 * blocks were allocated.
1199 int gfs2_block_map(struct inode *inode, sector_t lblock,
1200 struct buffer_head *bh_map, int create)
1202 struct gfs2_inode *ip = GFS2_I(inode);
1203 loff_t pos = (loff_t)lblock << inode->i_blkbits;
1204 loff_t length = bh_map->b_size;
1205 struct iomap iomap = { };
1208 clear_buffer_mapped(bh_map);
1209 clear_buffer_new(bh_map);
1210 clear_buffer_boundary(bh_map);
1211 trace_gfs2_bmap(ip, bh_map, lblock, create, 1);
1214 ret = gfs2_iomap_get(inode, pos, length, &iomap);
1216 ret = gfs2_iomap_alloc(inode, pos, length, &iomap);
1220 if (iomap.length > bh_map->b_size) {
1221 iomap.length = bh_map->b_size;
1222 iomap.flags &= ~IOMAP_F_GFS2_BOUNDARY;
1224 if (iomap.addr != IOMAP_NULL_ADDR)
1225 map_bh(bh_map, inode->i_sb, iomap.addr >> inode->i_blkbits);
1226 bh_map->b_size = iomap.length;
1227 if (iomap.flags & IOMAP_F_GFS2_BOUNDARY)
1228 set_buffer_boundary(bh_map);
1229 if (iomap.flags & IOMAP_F_NEW)
1230 set_buffer_new(bh_map);
1233 trace_gfs2_bmap(ip, bh_map, lblock, create, ret);
1237 int gfs2_get_extent(struct inode *inode, u64 lblock, u64 *dblock,
1238 unsigned int *extlen)
1240 unsigned int blkbits = inode->i_blkbits;
1241 struct iomap iomap = { };
1245 ret = gfs2_iomap_get(inode, lblock << blkbits, *extlen << blkbits,
1249 if (iomap.type != IOMAP_MAPPED)
1251 *dblock = iomap.addr >> blkbits;
1252 len = iomap.length >> blkbits;
1258 int gfs2_alloc_extent(struct inode *inode, u64 lblock, u64 *dblock,
1259 unsigned int *extlen, bool *new)
1261 unsigned int blkbits = inode->i_blkbits;
1262 struct iomap iomap = { };
1266 ret = gfs2_iomap_alloc(inode, lblock << blkbits, *extlen << blkbits,
1270 if (iomap.type != IOMAP_MAPPED)
1272 *dblock = iomap.addr >> blkbits;
1273 len = iomap.length >> blkbits;
1276 *new = iomap.flags & IOMAP_F_NEW;
1281 * NOTE: Never call gfs2_block_zero_range with an open transaction because it
1282 * uses iomap write to perform its actions, which begin their own transactions
1283 * (iomap_begin, page_prepare, etc.)
1285 static int gfs2_block_zero_range(struct inode *inode, loff_t from,
1286 unsigned int length)
1288 BUG_ON(current->journal_info);
1289 return iomap_zero_range(inode, from, length, NULL, &gfs2_iomap_ops);
1292 #define GFS2_JTRUNC_REVOKES 8192
1295 * gfs2_journaled_truncate - Wrapper for truncate_pagecache for jdata files
1296 * @inode: The inode being truncated
1297 * @oldsize: The original (larger) size
1298 * @newsize: The new smaller size
1300 * With jdata files, we have to journal a revoke for each block which is
1301 * truncated. As a result, we need to split this into separate transactions
1302 * if the number of pages being truncated gets too large.
1305 static int gfs2_journaled_truncate(struct inode *inode, u64 oldsize, u64 newsize)
1307 struct gfs2_sbd *sdp = GFS2_SB(inode);
1308 u64 max_chunk = GFS2_JTRUNC_REVOKES * sdp->sd_vfs->s_blocksize;
1312 while (oldsize != newsize) {
1313 struct gfs2_trans *tr;
1316 chunk = oldsize - newsize;
1317 if (chunk > max_chunk)
1320 offs = oldsize & ~PAGE_MASK;
1321 if (offs && chunk > PAGE_SIZE)
1322 chunk = offs + ((chunk - offs) & PAGE_MASK);
1324 truncate_pagecache(inode, oldsize - chunk);
1327 tr = current->journal_info;
1328 if (!test_bit(TR_TOUCHED, &tr->tr_flags))
1331 gfs2_trans_end(sdp);
1332 error = gfs2_trans_begin(sdp, RES_DINODE, GFS2_JTRUNC_REVOKES);
1340 static int trunc_start(struct inode *inode, u64 newsize)
1342 struct gfs2_inode *ip = GFS2_I(inode);
1343 struct gfs2_sbd *sdp = GFS2_SB(inode);
1344 struct buffer_head *dibh = NULL;
1345 int journaled = gfs2_is_jdata(ip);
1346 u64 oldsize = inode->i_size;
1349 if (!gfs2_is_stuffed(ip)) {
1350 unsigned int blocksize = i_blocksize(inode);
1351 unsigned int offs = newsize & (blocksize - 1);
1353 error = gfs2_block_zero_range(inode, newsize,
1360 error = gfs2_trans_begin(sdp, RES_DINODE + RES_JDATA, GFS2_JTRUNC_REVOKES);
1362 error = gfs2_trans_begin(sdp, RES_DINODE, 0);
1366 error = gfs2_meta_inode_buffer(ip, &dibh);
1370 gfs2_trans_add_meta(ip->i_gl, dibh);
1372 if (gfs2_is_stuffed(ip))
1373 gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode) + newsize);
1375 ip->i_diskflags |= GFS2_DIF_TRUNC_IN_PROG;
1377 i_size_write(inode, newsize);
1378 ip->i_inode.i_mtime = ip->i_inode.i_ctime = current_time(&ip->i_inode);
1379 gfs2_dinode_out(ip, dibh->b_data);
1382 error = gfs2_journaled_truncate(inode, oldsize, newsize);
1384 truncate_pagecache(inode, newsize);
1388 if (current->journal_info)
1389 gfs2_trans_end(sdp);
1393 int gfs2_iomap_get(struct inode *inode, loff_t pos, loff_t length,
1394 struct iomap *iomap)
1396 struct metapath mp = { .mp_aheight = 1, };
1399 ret = __gfs2_iomap_get(inode, pos, length, 0, iomap, &mp);
1400 release_metapath(&mp);
1404 int gfs2_iomap_alloc(struct inode *inode, loff_t pos, loff_t length,
1405 struct iomap *iomap)
1407 struct metapath mp = { .mp_aheight = 1, };
1410 ret = __gfs2_iomap_get(inode, pos, length, IOMAP_WRITE, iomap, &mp);
1411 if (!ret && iomap->type == IOMAP_HOLE)
1412 ret = __gfs2_iomap_alloc(inode, iomap, &mp);
1413 release_metapath(&mp);
1418 * sweep_bh_for_rgrps - find an rgrp in a meta buffer and free blocks therein
1420 * @rd_gh: holder of resource group glock
1421 * @bh: buffer head to sweep
1422 * @start: starting point in bh
1423 * @end: end point in bh
1424 * @meta: true if bh points to metadata (rather than data)
1425 * @btotal: place to keep count of total blocks freed
1427 * We sweep a metadata buffer (provided by the metapath) for blocks we need to
1428 * free, and free them all. However, we do it one rgrp at a time. If this
1429 * block has references to multiple rgrps, we break it into individual
1430 * transactions. This allows other processes to use the rgrps while we're
1431 * focused on a single one, for better concurrency / performance.
1432 * At every transaction boundary, we rewrite the inode into the journal.
1433 * That way the bitmaps are kept consistent with the inode and we can recover
1434 * if we're interrupted by power-outages.
1436 * Returns: 0, or return code if an error occurred.
1437 * *btotal has the total number of blocks freed
1439 static int sweep_bh_for_rgrps(struct gfs2_inode *ip, struct gfs2_holder *rd_gh,
1440 struct buffer_head *bh, __be64 *start, __be64 *end,
1441 bool meta, u32 *btotal)
1443 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
1444 struct gfs2_rgrpd *rgd;
1445 struct gfs2_trans *tr;
1447 int blks_outside_rgrp;
1448 u64 bn, bstart, isize_blks;
1449 s64 blen; /* needs to be s64 or gfs2_add_inode_blocks breaks */
1451 bool buf_in_tr = false; /* buffer was added to transaction */
1455 if (gfs2_holder_initialized(rd_gh)) {
1456 rgd = gfs2_glock2rgrp(rd_gh->gh_gl);
1457 gfs2_assert_withdraw(sdp,
1458 gfs2_glock_is_locked_by_me(rd_gh->gh_gl));
1460 blks_outside_rgrp = 0;
1464 for (p = start; p < end; p++) {
1467 bn = be64_to_cpu(*p);
1470 if (!rgrp_contains_block(rgd, bn)) {
1471 blks_outside_rgrp++;
1475 rgd = gfs2_blk2rgrpd(sdp, bn, true);
1476 if (unlikely(!rgd)) {
1480 ret = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE,
1481 LM_FLAG_NODE_SCOPE, rd_gh);
1485 /* Must be done with the rgrp glock held: */
1486 if (gfs2_rs_active(&ip->i_res) &&
1487 rgd == ip->i_res.rs_rgd)
1488 gfs2_rs_deltree(&ip->i_res);
1491 /* The size of our transactions will be unknown until we
1492 actually process all the metadata blocks that relate to
1493 the rgrp. So we estimate. We know it can't be more than
1494 the dinode's i_blocks and we don't want to exceed the
1495 journal flush threshold, sd_log_thresh2. */
1496 if (current->journal_info == NULL) {
1497 unsigned int jblocks_rqsted, revokes;
1499 jblocks_rqsted = rgd->rd_length + RES_DINODE +
1501 isize_blks = gfs2_get_inode_blocks(&ip->i_inode);
1502 if (isize_blks > atomic_read(&sdp->sd_log_thresh2))
1504 atomic_read(&sdp->sd_log_thresh2);
1506 jblocks_rqsted += isize_blks;
1507 revokes = jblocks_rqsted;
1509 revokes += end - start;
1510 else if (ip->i_depth)
1511 revokes += sdp->sd_inptrs;
1512 ret = gfs2_trans_begin(sdp, jblocks_rqsted, revokes);
1515 down_write(&ip->i_rw_mutex);
1517 /* check if we will exceed the transaction blocks requested */
1518 tr = current->journal_info;
1519 if (tr->tr_num_buf_new + RES_STATFS +
1520 RES_QUOTA >= atomic_read(&sdp->sd_log_thresh2)) {
1521 /* We set blks_outside_rgrp to ensure the loop will
1522 be repeated for the same rgrp, but with a new
1524 blks_outside_rgrp++;
1525 /* This next part is tricky. If the buffer was added
1526 to the transaction, we've already set some block
1527 pointers to 0, so we better follow through and free
1528 them, or we will introduce corruption (so break).
1529 This may be impossible, or at least rare, but I
1530 decided to cover the case regardless.
1532 If the buffer was not added to the transaction
1533 (this call), doing so would exceed our transaction
1534 size, so we need to end the transaction and start a
1535 new one (so goto). */
1542 gfs2_trans_add_meta(ip->i_gl, bh);
1545 if (bstart + blen == bn) {
1550 __gfs2_free_blocks(ip, rgd, bstart, (u32)blen, meta);
1552 gfs2_add_inode_blocks(&ip->i_inode, -blen);
1558 __gfs2_free_blocks(ip, rgd, bstart, (u32)blen, meta);
1560 gfs2_add_inode_blocks(&ip->i_inode, -blen);
1563 if (!ret && blks_outside_rgrp) { /* If buffer still has non-zero blocks
1564 outside the rgrp we just processed,
1565 do it all over again. */
1566 if (current->journal_info) {
1567 struct buffer_head *dibh;
1569 ret = gfs2_meta_inode_buffer(ip, &dibh);
1573 /* Every transaction boundary, we rewrite the dinode
1574 to keep its di_blocks current in case of failure. */
1575 ip->i_inode.i_mtime = ip->i_inode.i_ctime =
1576 current_time(&ip->i_inode);
1577 gfs2_trans_add_meta(ip->i_gl, dibh);
1578 gfs2_dinode_out(ip, dibh->b_data);
1580 up_write(&ip->i_rw_mutex);
1581 gfs2_trans_end(sdp);
1584 gfs2_glock_dq_uninit(rd_gh);
1592 static bool mp_eq_to_hgt(struct metapath *mp, __u16 *list, unsigned int h)
1594 if (memcmp(mp->mp_list, list, h * sizeof(mp->mp_list[0])))
1600 * find_nonnull_ptr - find a non-null pointer given a metapath and height
1601 * @sdp: The superblock
1602 * @mp: starting metapath
1603 * @h: desired height to search
1604 * @end_list: See punch_hole().
1605 * @end_aligned: See punch_hole().
1607 * Assumes the metapath is valid (with buffers) out to height h.
1608 * Returns: true if a non-null pointer was found in the metapath buffer
1609 * false if all remaining pointers are NULL in the buffer
1611 static bool find_nonnull_ptr(struct gfs2_sbd *sdp, struct metapath *mp,
1613 __u16 *end_list, unsigned int end_aligned)
1615 struct buffer_head *bh = mp->mp_bh[h];
1616 __be64 *first, *ptr, *end;
1618 first = metaptr1(h, mp);
1619 ptr = first + mp->mp_list[h];
1620 end = (__be64 *)(bh->b_data + bh->b_size);
1621 if (end_list && mp_eq_to_hgt(mp, end_list, h)) {
1622 bool keep_end = h < end_aligned;
1623 end = first + end_list[h] + keep_end;
1627 if (*ptr) { /* if we have a non-null pointer */
1628 mp->mp_list[h] = ptr - first;
1630 if (h < GFS2_MAX_META_HEIGHT)
1639 enum dealloc_states {
1640 DEALLOC_MP_FULL = 0, /* Strip a metapath with all buffers read in */
1641 DEALLOC_MP_LOWER = 1, /* lower the metapath strip height */
1642 DEALLOC_FILL_MP = 2, /* Fill in the metapath to the given height. */
1643 DEALLOC_DONE = 3, /* process complete */
1647 metapointer_range(struct metapath *mp, int height,
1648 __u16 *start_list, unsigned int start_aligned,
1649 __u16 *end_list, unsigned int end_aligned,
1650 __be64 **start, __be64 **end)
1652 struct buffer_head *bh = mp->mp_bh[height];
1655 first = metaptr1(height, mp);
1657 if (mp_eq_to_hgt(mp, start_list, height)) {
1658 bool keep_start = height < start_aligned;
1659 *start = first + start_list[height] + keep_start;
1661 *end = (__be64 *)(bh->b_data + bh->b_size);
1662 if (end_list && mp_eq_to_hgt(mp, end_list, height)) {
1663 bool keep_end = height < end_aligned;
1664 *end = first + end_list[height] + keep_end;
1668 static inline bool walk_done(struct gfs2_sbd *sdp,
1669 struct metapath *mp, int height,
1670 __u16 *end_list, unsigned int end_aligned)
1675 bool keep_end = height < end_aligned;
1676 if (!mp_eq_to_hgt(mp, end_list, height))
1678 end = end_list[height] + keep_end;
1680 end = (height > 0) ? sdp->sd_inptrs : sdp->sd_diptrs;
1681 return mp->mp_list[height] >= end;
1685 * punch_hole - deallocate blocks in a file
1686 * @ip: inode to truncate
1687 * @offset: the start of the hole
1688 * @length: the size of the hole (or 0 for truncate)
1690 * Punch a hole into a file or truncate a file at a given position. This
1691 * function operates in whole blocks (@offset and @length are rounded
1692 * accordingly); partially filled blocks must be cleared otherwise.
1694 * This function works from the bottom up, and from the right to the left. In
1695 * other words, it strips off the highest layer (data) before stripping any of
1696 * the metadata. Doing it this way is best in case the operation is interrupted
1697 * by power failure, etc. The dinode is rewritten in every transaction to
1698 * guarantee integrity.
1700 static int punch_hole(struct gfs2_inode *ip, u64 offset, u64 length)
1702 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
1703 u64 maxsize = sdp->sd_heightsize[ip->i_height];
1704 struct metapath mp = {};
1705 struct buffer_head *dibh, *bh;
1706 struct gfs2_holder rd_gh;
1707 unsigned int bsize_shift = sdp->sd_sb.sb_bsize_shift;
1708 u64 lblock = (offset + (1 << bsize_shift) - 1) >> bsize_shift;
1709 __u16 start_list[GFS2_MAX_META_HEIGHT];
1710 __u16 __end_list[GFS2_MAX_META_HEIGHT], *end_list = NULL;
1711 unsigned int start_aligned, end_aligned;
1712 unsigned int strip_h = ip->i_height - 1;
1715 int mp_h; /* metapath buffers are read in to this height */
1717 __be64 *start, *end;
1719 if (offset >= maxsize) {
1721 * The starting point lies beyond the allocated meta-data;
1722 * there are no blocks do deallocate.
1728 * The start position of the hole is defined by lblock, start_list, and
1729 * start_aligned. The end position of the hole is defined by lend,
1730 * end_list, and end_aligned.
1732 * start_aligned and end_aligned define down to which height the start
1733 * and end positions are aligned to the metadata tree (i.e., the
1734 * position is a multiple of the metadata granularity at the height
1735 * above). This determines at which heights additional meta pointers
1736 * needs to be preserved for the remaining data.
1740 u64 end_offset = offset + length;
1744 * Clip the end at the maximum file size for the given height:
1745 * that's how far the metadata goes; files bigger than that
1746 * will have additional layers of indirection.
1748 if (end_offset > maxsize)
1749 end_offset = maxsize;
1750 lend = end_offset >> bsize_shift;
1755 find_metapath(sdp, lend, &mp, ip->i_height);
1756 end_list = __end_list;
1757 memcpy(end_list, mp.mp_list, sizeof(mp.mp_list));
1759 for (mp_h = ip->i_height - 1; mp_h > 0; mp_h--) {
1766 find_metapath(sdp, lblock, &mp, ip->i_height);
1767 memcpy(start_list, mp.mp_list, sizeof(start_list));
1769 for (mp_h = ip->i_height - 1; mp_h > 0; mp_h--) {
1770 if (start_list[mp_h])
1773 start_aligned = mp_h;
1775 ret = gfs2_meta_inode_buffer(ip, &dibh);
1780 ret = lookup_metapath(ip, &mp);
1784 /* issue read-ahead on metadata */
1785 for (mp_h = 0; mp_h < mp.mp_aheight - 1; mp_h++) {
1786 metapointer_range(&mp, mp_h, start_list, start_aligned,
1787 end_list, end_aligned, &start, &end);
1788 gfs2_metapath_ra(ip->i_gl, start, end);
1791 if (mp.mp_aheight == ip->i_height)
1792 state = DEALLOC_MP_FULL; /* We have a complete metapath */
1794 state = DEALLOC_FILL_MP; /* deal with partial metapath */
1796 ret = gfs2_rindex_update(sdp);
1800 ret = gfs2_quota_hold(ip, NO_UID_QUOTA_CHANGE, NO_GID_QUOTA_CHANGE);
1803 gfs2_holder_mark_uninitialized(&rd_gh);
1807 while (state != DEALLOC_DONE) {
1809 /* Truncate a full metapath at the given strip height.
1810 * Note that strip_h == mp_h in order to be in this state. */
1811 case DEALLOC_MP_FULL:
1812 bh = mp.mp_bh[mp_h];
1813 gfs2_assert_withdraw(sdp, bh);
1814 if (gfs2_assert_withdraw(sdp,
1815 prev_bnr != bh->b_blocknr)) {
1816 fs_emerg(sdp, "inode %llu, block:%llu, i_h:%u,"
1817 "s_h:%u, mp_h:%u\n",
1818 (unsigned long long)ip->i_no_addr,
1819 prev_bnr, ip->i_height, strip_h, mp_h);
1821 prev_bnr = bh->b_blocknr;
1823 if (gfs2_metatype_check(sdp, bh,
1824 (mp_h ? GFS2_METATYPE_IN :
1825 GFS2_METATYPE_DI))) {
1831 * Below, passing end_aligned as 0 gives us the
1832 * metapointer range excluding the end point: the end
1833 * point is the first metapath we must not deallocate!
1836 metapointer_range(&mp, mp_h, start_list, start_aligned,
1837 end_list, 0 /* end_aligned */,
1839 ret = sweep_bh_for_rgrps(ip, &rd_gh, mp.mp_bh[mp_h],
1841 mp_h != ip->i_height - 1,
1844 /* If we hit an error or just swept dinode buffer,
1847 state = DEALLOC_DONE;
1850 state = DEALLOC_MP_LOWER;
1853 /* lower the metapath strip height */
1854 case DEALLOC_MP_LOWER:
1855 /* We're done with the current buffer, so release it,
1856 unless it's the dinode buffer. Then back up to the
1857 previous pointer. */
1859 brelse(mp.mp_bh[mp_h]);
1860 mp.mp_bh[mp_h] = NULL;
1862 /* If we can't get any lower in height, we've stripped
1863 off all we can. Next step is to back up and start
1864 stripping the previous level of metadata. */
1867 memcpy(mp.mp_list, start_list, sizeof(start_list));
1869 state = DEALLOC_FILL_MP;
1872 mp.mp_list[mp_h] = 0;
1873 mp_h--; /* search one metadata height down */
1875 if (walk_done(sdp, &mp, mp_h, end_list, end_aligned))
1877 /* Here we've found a part of the metapath that is not
1878 * allocated. We need to search at that height for the
1879 * next non-null pointer. */
1880 if (find_nonnull_ptr(sdp, &mp, mp_h, end_list, end_aligned)) {
1881 state = DEALLOC_FILL_MP;
1884 /* No more non-null pointers at this height. Back up
1885 to the previous height and try again. */
1886 break; /* loop around in the same state */
1888 /* Fill the metapath with buffers to the given height. */
1889 case DEALLOC_FILL_MP:
1890 /* Fill the buffers out to the current height. */
1891 ret = fillup_metapath(ip, &mp, mp_h);
1895 /* On the first pass, issue read-ahead on metadata. */
1896 if (mp.mp_aheight > 1 && strip_h == ip->i_height - 1) {
1897 unsigned int height = mp.mp_aheight - 1;
1899 /* No read-ahead for data blocks. */
1900 if (mp.mp_aheight - 1 == strip_h)
1903 for (; height >= mp.mp_aheight - ret; height--) {
1904 metapointer_range(&mp, height,
1905 start_list, start_aligned,
1906 end_list, end_aligned,
1908 gfs2_metapath_ra(ip->i_gl, start, end);
1912 /* If buffers found for the entire strip height */
1913 if (mp.mp_aheight - 1 == strip_h) {
1914 state = DEALLOC_MP_FULL;
1917 if (mp.mp_aheight < ip->i_height) /* We have a partial height */
1918 mp_h = mp.mp_aheight - 1;
1920 /* If we find a non-null block pointer, crawl a bit
1921 higher up in the metapath and try again, otherwise
1922 we need to look lower for a new starting point. */
1923 if (find_nonnull_ptr(sdp, &mp, mp_h, end_list, end_aligned))
1926 state = DEALLOC_MP_LOWER;
1932 if (current->journal_info == NULL) {
1933 ret = gfs2_trans_begin(sdp, RES_DINODE + RES_STATFS +
1937 down_write(&ip->i_rw_mutex);
1939 gfs2_statfs_change(sdp, 0, +btotal, 0);
1940 gfs2_quota_change(ip, -(s64)btotal, ip->i_inode.i_uid,
1942 ip->i_inode.i_mtime = ip->i_inode.i_ctime = current_time(&ip->i_inode);
1943 gfs2_trans_add_meta(ip->i_gl, dibh);
1944 gfs2_dinode_out(ip, dibh->b_data);
1945 up_write(&ip->i_rw_mutex);
1946 gfs2_trans_end(sdp);
1950 if (gfs2_holder_initialized(&rd_gh))
1951 gfs2_glock_dq_uninit(&rd_gh);
1952 if (current->journal_info) {
1953 up_write(&ip->i_rw_mutex);
1954 gfs2_trans_end(sdp);
1957 gfs2_quota_unhold(ip);
1959 release_metapath(&mp);
1963 static int trunc_end(struct gfs2_inode *ip)
1965 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
1966 struct buffer_head *dibh;
1969 error = gfs2_trans_begin(sdp, RES_DINODE, 0);
1973 down_write(&ip->i_rw_mutex);
1975 error = gfs2_meta_inode_buffer(ip, &dibh);
1979 if (!i_size_read(&ip->i_inode)) {
1981 ip->i_goal = ip->i_no_addr;
1982 gfs2_buffer_clear_tail(dibh, sizeof(struct gfs2_dinode));
1983 gfs2_ordered_del_inode(ip);
1985 ip->i_inode.i_mtime = ip->i_inode.i_ctime = current_time(&ip->i_inode);
1986 ip->i_diskflags &= ~GFS2_DIF_TRUNC_IN_PROG;
1988 gfs2_trans_add_meta(ip->i_gl, dibh);
1989 gfs2_dinode_out(ip, dibh->b_data);
1993 up_write(&ip->i_rw_mutex);
1994 gfs2_trans_end(sdp);
1999 * do_shrink - make a file smaller
2001 * @newsize: the size to make the file
2003 * Called with an exclusive lock on @inode. The @size must
2004 * be equal to or smaller than the current inode size.
2009 static int do_shrink(struct inode *inode, u64 newsize)
2011 struct gfs2_inode *ip = GFS2_I(inode);
2014 error = trunc_start(inode, newsize);
2017 if (gfs2_is_stuffed(ip))
2020 error = punch_hole(ip, newsize, 0);
2022 error = trunc_end(ip);
2027 void gfs2_trim_blocks(struct inode *inode)
2031 ret = do_shrink(inode, inode->i_size);
2036 * do_grow - Touch and update inode size
2038 * @size: The new size
2040 * This function updates the timestamps on the inode and
2041 * may also increase the size of the inode. This function
2042 * must not be called with @size any smaller than the current
2045 * Although it is not strictly required to unstuff files here,
2046 * earlier versions of GFS2 have a bug in the stuffed file reading
2047 * code which will result in a buffer overrun if the size is larger
2048 * than the max stuffed file size. In order to prevent this from
2049 * occurring, such files are unstuffed, but in other cases we can
2050 * just update the inode size directly.
2052 * Returns: 0 on success, or -ve on error
2055 static int do_grow(struct inode *inode, u64 size)
2057 struct gfs2_inode *ip = GFS2_I(inode);
2058 struct gfs2_sbd *sdp = GFS2_SB(inode);
2059 struct gfs2_alloc_parms ap = { .target = 1, };
2060 struct buffer_head *dibh;
2064 if (gfs2_is_stuffed(ip) && size > gfs2_max_stuffed_size(ip)) {
2065 error = gfs2_quota_lock_check(ip, &ap);
2069 error = gfs2_inplace_reserve(ip, &ap);
2071 goto do_grow_qunlock;
2075 error = gfs2_trans_begin(sdp, RES_DINODE + RES_STATFS + RES_RG_BIT +
2077 gfs2_is_jdata(ip) ? RES_JDATA : 0) +
2078 (sdp->sd_args.ar_quota == GFS2_QUOTA_OFF ?
2081 goto do_grow_release;
2084 error = gfs2_unstuff_dinode(ip);
2089 error = gfs2_meta_inode_buffer(ip, &dibh);
2093 truncate_setsize(inode, size);
2094 ip->i_inode.i_mtime = ip->i_inode.i_ctime = current_time(&ip->i_inode);
2095 gfs2_trans_add_meta(ip->i_gl, dibh);
2096 gfs2_dinode_out(ip, dibh->b_data);
2100 gfs2_trans_end(sdp);
2103 gfs2_inplace_release(ip);
2105 gfs2_quota_unlock(ip);
2111 * gfs2_setattr_size - make a file a given size
2113 * @newsize: the size to make the file
2115 * The file size can grow, shrink, or stay the same size. This
2116 * is called holding i_rwsem and an exclusive glock on the inode
2122 int gfs2_setattr_size(struct inode *inode, u64 newsize)
2124 struct gfs2_inode *ip = GFS2_I(inode);
2127 BUG_ON(!S_ISREG(inode->i_mode));
2129 ret = inode_newsize_ok(inode, newsize);
2133 inode_dio_wait(inode);
2135 ret = gfs2_qa_get(ip);
2139 if (newsize >= inode->i_size) {
2140 ret = do_grow(inode, newsize);
2144 ret = do_shrink(inode, newsize);
2151 int gfs2_truncatei_resume(struct gfs2_inode *ip)
2154 error = punch_hole(ip, i_size_read(&ip->i_inode), 0);
2156 error = trunc_end(ip);
2160 int gfs2_file_dealloc(struct gfs2_inode *ip)
2162 return punch_hole(ip, 0, 0);
2166 * gfs2_free_journal_extents - Free cached journal bmap info
2171 void gfs2_free_journal_extents(struct gfs2_jdesc *jd)
2173 struct gfs2_journal_extent *jext;
2175 while(!list_empty(&jd->extent_list)) {
2176 jext = list_first_entry(&jd->extent_list, struct gfs2_journal_extent, list);
2177 list_del(&jext->list);
2183 * gfs2_add_jextent - Add or merge a new extent to extent cache
2184 * @jd: The journal descriptor
2185 * @lblock: The logical block at start of new extent
2186 * @dblock: The physical block at start of new extent
2187 * @blocks: Size of extent in fs blocks
2189 * Returns: 0 on success or -ENOMEM
2192 static int gfs2_add_jextent(struct gfs2_jdesc *jd, u64 lblock, u64 dblock, u64 blocks)
2194 struct gfs2_journal_extent *jext;
2196 if (!list_empty(&jd->extent_list)) {
2197 jext = list_last_entry(&jd->extent_list, struct gfs2_journal_extent, list);
2198 if ((jext->dblock + jext->blocks) == dblock) {
2199 jext->blocks += blocks;
2204 jext = kzalloc(sizeof(struct gfs2_journal_extent), GFP_NOFS);
2207 jext->dblock = dblock;
2208 jext->lblock = lblock;
2209 jext->blocks = blocks;
2210 list_add_tail(&jext->list, &jd->extent_list);
2216 * gfs2_map_journal_extents - Cache journal bmap info
2217 * @sdp: The super block
2218 * @jd: The journal to map
2220 * Create a reusable "extent" mapping from all logical
2221 * blocks to all physical blocks for the given journal. This will save
2222 * us time when writing journal blocks. Most journals will have only one
2223 * extent that maps all their logical blocks. That's because gfs2.mkfs
2224 * arranges the journal blocks sequentially to maximize performance.
2225 * So the extent would map the first block for the entire file length.
2226 * However, gfs2_jadd can happen while file activity is happening, so
2227 * those journals may not be sequential. Less likely is the case where
2228 * the users created their own journals by mounting the metafs and
2229 * laying it out. But it's still possible. These journals might have
2232 * Returns: 0 on success, or error on failure
2235 int gfs2_map_journal_extents(struct gfs2_sbd *sdp, struct gfs2_jdesc *jd)
2239 struct gfs2_inode *ip = GFS2_I(jd->jd_inode);
2240 struct buffer_head bh;
2241 unsigned int shift = sdp->sd_sb.sb_bsize_shift;
2246 start = ktime_get();
2247 lblock_stop = i_size_read(jd->jd_inode) >> shift;
2248 size = (lblock_stop - lblock) << shift;
2250 WARN_ON(!list_empty(&jd->extent_list));
2256 rc = gfs2_block_map(jd->jd_inode, lblock, &bh, 0);
2257 if (rc || !buffer_mapped(&bh))
2259 rc = gfs2_add_jextent(jd, lblock, bh.b_blocknr, bh.b_size >> shift);
2263 lblock += (bh.b_size >> ip->i_inode.i_blkbits);
2267 fs_info(sdp, "journal %d mapped with %u extents in %lldms\n", jd->jd_jid,
2268 jd->nr_extents, ktime_ms_delta(end, start));
2272 fs_warn(sdp, "error %d mapping journal %u at offset %llu (extent %u)\n",
2274 (unsigned long long)(i_size_read(jd->jd_inode) - size),
2276 fs_warn(sdp, "bmap=%d lblock=%llu block=%llu, state=0x%08lx, size=%llu\n",
2277 rc, (unsigned long long)lblock, (unsigned long long)bh.b_blocknr,
2278 bh.b_state, (unsigned long long)bh.b_size);
2279 gfs2_free_journal_extents(jd);
2284 * gfs2_write_alloc_required - figure out if a write will require an allocation
2285 * @ip: the file being written to
2286 * @offset: the offset to write to
2287 * @len: the number of bytes being written
2289 * Returns: 1 if an alloc is required, 0 otherwise
2292 int gfs2_write_alloc_required(struct gfs2_inode *ip, u64 offset,
2295 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
2296 struct buffer_head bh;
2298 u64 lblock, lblock_stop, size;
2304 if (gfs2_is_stuffed(ip)) {
2305 if (offset + len > gfs2_max_stuffed_size(ip))
2310 shift = sdp->sd_sb.sb_bsize_shift;
2311 BUG_ON(gfs2_is_dir(ip));
2312 end_of_file = (i_size_read(&ip->i_inode) + sdp->sd_sb.sb_bsize - 1) >> shift;
2313 lblock = offset >> shift;
2314 lblock_stop = (offset + len + sdp->sd_sb.sb_bsize - 1) >> shift;
2315 if (lblock_stop > end_of_file && ip != GFS2_I(sdp->sd_rindex))
2318 size = (lblock_stop - lblock) << shift;
2322 gfs2_block_map(&ip->i_inode, lblock, &bh, 0);
2323 if (!buffer_mapped(&bh))
2326 lblock += (bh.b_size >> ip->i_inode.i_blkbits);
2332 static int stuffed_zero_range(struct inode *inode, loff_t offset, loff_t length)
2334 struct gfs2_inode *ip = GFS2_I(inode);
2335 struct buffer_head *dibh;
2338 if (offset >= inode->i_size)
2340 if (offset + length > inode->i_size)
2341 length = inode->i_size - offset;
2343 error = gfs2_meta_inode_buffer(ip, &dibh);
2346 gfs2_trans_add_meta(ip->i_gl, dibh);
2347 memset(dibh->b_data + sizeof(struct gfs2_dinode) + offset, 0,
2353 static int gfs2_journaled_truncate_range(struct inode *inode, loff_t offset,
2356 struct gfs2_sbd *sdp = GFS2_SB(inode);
2357 loff_t max_chunk = GFS2_JTRUNC_REVOKES * sdp->sd_vfs->s_blocksize;
2361 struct gfs2_trans *tr;
2366 if (chunk > max_chunk)
2369 offs = offset & ~PAGE_MASK;
2370 if (offs && chunk > PAGE_SIZE)
2371 chunk = offs + ((chunk - offs) & PAGE_MASK);
2373 truncate_pagecache_range(inode, offset, chunk);
2377 tr = current->journal_info;
2378 if (!test_bit(TR_TOUCHED, &tr->tr_flags))
2381 gfs2_trans_end(sdp);
2382 error = gfs2_trans_begin(sdp, RES_DINODE, GFS2_JTRUNC_REVOKES);
2389 int __gfs2_punch_hole(struct file *file, loff_t offset, loff_t length)
2391 struct inode *inode = file_inode(file);
2392 struct gfs2_inode *ip = GFS2_I(inode);
2393 struct gfs2_sbd *sdp = GFS2_SB(inode);
2394 unsigned int blocksize = i_blocksize(inode);
2398 if (!gfs2_is_stuffed(ip)) {
2399 unsigned int start_off, end_len;
2401 start_off = offset & (blocksize - 1);
2402 end_len = (offset + length) & (blocksize - 1);
2404 unsigned int len = length;
2405 if (length > blocksize - start_off)
2406 len = blocksize - start_off;
2407 error = gfs2_block_zero_range(inode, offset, len);
2410 if (start_off + length < blocksize)
2414 error = gfs2_block_zero_range(inode,
2415 offset + length - end_len, end_len);
2421 start = round_down(offset, blocksize);
2422 end = round_up(offset + length, blocksize) - 1;
2423 error = filemap_write_and_wait_range(inode->i_mapping, start, end);
2427 if (gfs2_is_jdata(ip))
2428 error = gfs2_trans_begin(sdp, RES_DINODE + 2 * RES_JDATA,
2429 GFS2_JTRUNC_REVOKES);
2431 error = gfs2_trans_begin(sdp, RES_DINODE, 0);
2435 if (gfs2_is_stuffed(ip)) {
2436 error = stuffed_zero_range(inode, offset, length);
2441 if (gfs2_is_jdata(ip)) {
2442 BUG_ON(!current->journal_info);
2443 gfs2_journaled_truncate_range(inode, offset, length);
2445 truncate_pagecache_range(inode, offset, offset + length - 1);
2447 file_update_time(file);
2448 mark_inode_dirty(inode);
2450 if (current->journal_info)
2451 gfs2_trans_end(sdp);
2453 if (!gfs2_is_stuffed(ip))
2454 error = punch_hole(ip, offset, length);
2457 if (current->journal_info)
2458 gfs2_trans_end(sdp);
2462 static int gfs2_map_blocks(struct iomap_writepage_ctx *wpc, struct inode *inode,
2467 if (WARN_ON_ONCE(gfs2_is_stuffed(GFS2_I(inode))))
2470 if (offset >= wpc->iomap.offset &&
2471 offset < wpc->iomap.offset + wpc->iomap.length)
2474 memset(&wpc->iomap, 0, sizeof(wpc->iomap));
2475 ret = gfs2_iomap_get(inode, offset, INT_MAX, &wpc->iomap);
2479 const struct iomap_writeback_ops gfs2_writeback_ops = {
2480 .map_blocks = gfs2_map_blocks,