2 * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
5 * This copyrighted material is made available to anyone wishing to use,
6 * modify, copy, or redistribute it subject to the terms and conditions
7 * of the GNU General Public License version 2.
10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12 #include <linux/slab.h>
13 #include <linux/spinlock.h>
14 #include <linux/completion.h>
15 #include <linux/buffer_head.h>
17 #include <linux/gfs2_ondisk.h>
18 #include <linux/prefetch.h>
19 #include <linux/blkdev.h>
20 #include <linux/rbtree.h>
21 #include <linux/random.h>
36 #include "trace_gfs2.h"
38 #define BFITNOENT ((u32)~0)
39 #define NO_BLOCK ((u64)~0)
41 #if BITS_PER_LONG == 32
42 #define LBITMASK (0x55555555UL)
43 #define LBITSKIP55 (0x55555555UL)
44 #define LBITSKIP00 (0x00000000UL)
46 #define LBITMASK (0x5555555555555555UL)
47 #define LBITSKIP55 (0x5555555555555555UL)
48 #define LBITSKIP00 (0x0000000000000000UL)
52 * These routines are used by the resource group routines (rgrp.c)
53 * to keep track of block allocation. Each block is represented by two
54 * bits. So, each byte represents GFS2_NBBY (i.e. 4) blocks.
57 * 1 = Used (not metadata)
58 * 2 = Unlinked (still in use) inode
67 static const char valid_change[16] = {
75 static int gfs2_rbm_find(struct gfs2_rbm *rbm, u8 state, u32 *minext,
76 const struct gfs2_inode *ip, bool nowrap);
80 * gfs2_setbit - Set a bit in the bitmaps
81 * @rbm: The position of the bit to set
82 * @do_clone: Also set the clone bitmap, if it exists
83 * @new_state: the new state of the block
87 static inline void gfs2_setbit(const struct gfs2_rbm *rbm, bool do_clone,
88 unsigned char new_state)
90 unsigned char *byte1, *byte2, *end, cur_state;
91 struct gfs2_bitmap *bi = rbm_bi(rbm);
92 unsigned int buflen = bi->bi_len;
93 const unsigned int bit = (rbm->offset % GFS2_NBBY) * GFS2_BIT_SIZE;
95 byte1 = bi->bi_bh->b_data + bi->bi_offset + (rbm->offset / GFS2_NBBY);
96 end = bi->bi_bh->b_data + bi->bi_offset + buflen;
100 cur_state = (*byte1 >> bit) & GFS2_BIT_MASK;
102 if (unlikely(!valid_change[new_state * 4 + cur_state])) {
103 pr_warn("buf_blk = 0x%x old_state=%d, new_state=%d\n",
104 rbm->offset, cur_state, new_state);
105 pr_warn("rgrp=0x%llx bi_start=0x%x\n",
106 (unsigned long long)rbm->rgd->rd_addr, bi->bi_start);
107 pr_warn("bi_offset=0x%x bi_len=0x%x\n",
108 bi->bi_offset, bi->bi_len);
110 gfs2_consist_rgrpd(rbm->rgd);
113 *byte1 ^= (cur_state ^ new_state) << bit;
115 if (do_clone && bi->bi_clone) {
116 byte2 = bi->bi_clone + bi->bi_offset + (rbm->offset / GFS2_NBBY);
117 cur_state = (*byte2 >> bit) & GFS2_BIT_MASK;
118 *byte2 ^= (cur_state ^ new_state) << bit;
123 * gfs2_testbit - test a bit in the bitmaps
124 * @rbm: The bit to test
126 * Returns: The two bit block state of the requested bit
129 static inline u8 gfs2_testbit(const struct gfs2_rbm *rbm)
131 struct gfs2_bitmap *bi = rbm_bi(rbm);
132 const u8 *buffer = bi->bi_bh->b_data + bi->bi_offset;
136 byte = buffer + (rbm->offset / GFS2_NBBY);
137 bit = (rbm->offset % GFS2_NBBY) * GFS2_BIT_SIZE;
139 return (*byte >> bit) & GFS2_BIT_MASK;
144 * @ptr: Pointer to bitmap data
145 * @mask: Mask to use (normally 0x55555.... but adjusted for search start)
146 * @state: The state we are searching for
148 * We xor the bitmap data with a patter which is the bitwise opposite
149 * of what we are looking for, this gives rise to a pattern of ones
150 * wherever there is a match. Since we have two bits per entry, we
151 * take this pattern, shift it down by one place and then and it with
152 * the original. All the even bit positions (0,2,4, etc) then represent
153 * successful matches, so we mask with 0x55555..... to remove the unwanted
156 * This allows searching of a whole u64 at once (32 blocks) with a
157 * single test (on 64 bit arches).
160 static inline u64 gfs2_bit_search(const __le64 *ptr, u64 mask, u8 state)
163 static const u64 search[] = {
164 [0] = 0xffffffffffffffffULL,
165 [1] = 0xaaaaaaaaaaaaaaaaULL,
166 [2] = 0x5555555555555555ULL,
167 [3] = 0x0000000000000000ULL,
169 tmp = le64_to_cpu(*ptr) ^ search[state];
176 * rs_cmp - multi-block reservation range compare
177 * @blk: absolute file system block number of the new reservation
178 * @len: number of blocks in the new reservation
179 * @rs: existing reservation to compare against
181 * returns: 1 if the block range is beyond the reach of the reservation
182 * -1 if the block range is before the start of the reservation
183 * 0 if the block range overlaps with the reservation
185 static inline int rs_cmp(u64 blk, u32 len, struct gfs2_blkreserv *rs)
187 u64 startblk = gfs2_rbm_to_block(&rs->rs_rbm);
189 if (blk >= startblk + rs->rs_free)
191 if (blk + len - 1 < startblk)
197 * gfs2_bitfit - Search an rgrp's bitmap buffer to find a bit-pair representing
198 * a block in a given allocation state.
199 * @buf: the buffer that holds the bitmaps
200 * @len: the length (in bytes) of the buffer
201 * @goal: start search at this block's bit-pair (within @buffer)
202 * @state: GFS2_BLKST_XXX the state of the block we're looking for.
204 * Scope of @goal and returned block number is only within this bitmap buffer,
205 * not entire rgrp or filesystem. @buffer will be offset from the actual
206 * beginning of a bitmap block buffer, skipping any header structures, but
207 * headers are always a multiple of 64 bits long so that the buffer is
208 * always aligned to a 64 bit boundary.
210 * The size of the buffer is in bytes, but is it assumed that it is
211 * always ok to read a complete multiple of 64 bits at the end
212 * of the block in case the end is no aligned to a natural boundary.
214 * Return: the block number (bitmap buffer scope) that was found
217 static u32 gfs2_bitfit(const u8 *buf, const unsigned int len,
220 u32 spoint = (goal << 1) & ((8*sizeof(u64)) - 1);
221 const __le64 *ptr = ((__le64 *)buf) + (goal >> 5);
222 const __le64 *end = (__le64 *)(buf + ALIGN(len, sizeof(u64)));
224 u64 mask = 0x5555555555555555ULL;
227 /* Mask off bits we don't care about at the start of the search */
229 tmp = gfs2_bit_search(ptr, mask, state);
231 while(tmp == 0 && ptr < end) {
232 tmp = gfs2_bit_search(ptr, 0x5555555555555555ULL, state);
235 /* Mask off any bits which are more than len bytes from the start */
236 if (ptr == end && (len & (sizeof(u64) - 1)))
237 tmp &= (((u64)~0) >> (64 - 8*(len & (sizeof(u64) - 1))));
238 /* Didn't find anything, so return */
243 bit /= 2; /* two bits per entry in the bitmap */
244 return (((const unsigned char *)ptr - buf) * GFS2_NBBY) + bit;
248 * gfs2_rbm_from_block - Set the rbm based upon rgd and block number
249 * @rbm: The rbm with rgd already set correctly
250 * @block: The block number (filesystem relative)
252 * This sets the bi and offset members of an rbm based on a
253 * resource group and a filesystem relative block number. The
254 * resource group must be set in the rbm on entry, the bi and
255 * offset members will be set by this function.
257 * Returns: 0 on success, or an error code
260 static int gfs2_rbm_from_block(struct gfs2_rbm *rbm, u64 block)
262 u64 rblock = block - rbm->rgd->rd_data0;
264 if (WARN_ON_ONCE(rblock > UINT_MAX))
266 if (block >= rbm->rgd->rd_data0 + rbm->rgd->rd_data)
270 rbm->offset = (u32)(rblock);
271 /* Check if the block is within the first block */
272 if (rbm->offset < rbm_bi(rbm)->bi_blocks)
275 /* Adjust for the size diff between gfs2_meta_header and gfs2_rgrp */
276 rbm->offset += (sizeof(struct gfs2_rgrp) -
277 sizeof(struct gfs2_meta_header)) * GFS2_NBBY;
278 rbm->bii = rbm->offset / rbm->rgd->rd_sbd->sd_blocks_per_bitmap;
279 rbm->offset -= rbm->bii * rbm->rgd->rd_sbd->sd_blocks_per_bitmap;
284 * gfs2_rbm_incr - increment an rbm structure
285 * @rbm: The rbm with rgd already set correctly
287 * This function takes an existing rbm structure and increments it to the next
288 * viable block offset.
290 * Returns: If incrementing the offset would cause the rbm to go past the
291 * end of the rgrp, true is returned, otherwise false.
295 static bool gfs2_rbm_incr(struct gfs2_rbm *rbm)
297 if (rbm->offset + 1 < rbm_bi(rbm)->bi_blocks) { /* in the same bitmap */
301 if (rbm->bii == rbm->rgd->rd_length - 1) /* at the last bitmap */
310 * gfs2_unaligned_extlen - Look for free blocks which are not byte aligned
311 * @rbm: Position to search (value/result)
312 * @n_unaligned: Number of unaligned blocks to check
313 * @len: Decremented for each block found (terminate on zero)
315 * Returns: true if a non-free block is encountered
318 static bool gfs2_unaligned_extlen(struct gfs2_rbm *rbm, u32 n_unaligned, u32 *len)
323 for (n = 0; n < n_unaligned; n++) {
324 res = gfs2_testbit(rbm);
325 if (res != GFS2_BLKST_FREE)
330 if (gfs2_rbm_incr(rbm))
338 * gfs2_free_extlen - Return extent length of free blocks
339 * @rrbm: Starting position
340 * @len: Max length to check
342 * Starting at the block specified by the rbm, see how many free blocks
343 * there are, not reading more than len blocks ahead. This can be done
344 * using memchr_inv when the blocks are byte aligned, but has to be done
345 * on a block by block basis in case of unaligned blocks. Also this
346 * function can cope with bitmap boundaries (although it must stop on
347 * a resource group boundary)
349 * Returns: Number of free blocks in the extent
352 static u32 gfs2_free_extlen(const struct gfs2_rbm *rrbm, u32 len)
354 struct gfs2_rbm rbm = *rrbm;
355 u32 n_unaligned = rbm.offset & 3;
359 u8 *ptr, *start, *end;
361 struct gfs2_bitmap *bi;
364 gfs2_unaligned_extlen(&rbm, 4 - n_unaligned, &len))
367 n_unaligned = len & 3;
368 /* Start is now byte aligned */
371 start = bi->bi_bh->b_data;
373 start = bi->bi_clone;
374 end = start + bi->bi_bh->b_size;
375 start += bi->bi_offset;
376 BUG_ON(rbm.offset & 3);
377 start += (rbm.offset / GFS2_NBBY);
378 bytes = min_t(u32, len / GFS2_NBBY, (end - start));
379 ptr = memchr_inv(start, 0, bytes);
380 chunk_size = ((ptr == NULL) ? bytes : (ptr - start));
381 chunk_size *= GFS2_NBBY;
382 BUG_ON(len < chunk_size);
384 block = gfs2_rbm_to_block(&rbm);
385 if (gfs2_rbm_from_block(&rbm, block + chunk_size)) {
393 n_unaligned = len & 3;
396 /* Deal with any bits left over at the end */
398 gfs2_unaligned_extlen(&rbm, n_unaligned, &len);
404 * gfs2_bitcount - count the number of bits in a certain state
405 * @rgd: the resource group descriptor
406 * @buffer: the buffer that holds the bitmaps
407 * @buflen: the length (in bytes) of the buffer
408 * @state: the state of the block we're looking for
410 * Returns: The number of bits
413 static u32 gfs2_bitcount(struct gfs2_rgrpd *rgd, const u8 *buffer,
414 unsigned int buflen, u8 state)
416 const u8 *byte = buffer;
417 const u8 *end = buffer + buflen;
418 const u8 state1 = state << 2;
419 const u8 state2 = state << 4;
420 const u8 state3 = state << 6;
423 for (; byte < end; byte++) {
424 if (((*byte) & 0x03) == state)
426 if (((*byte) & 0x0C) == state1)
428 if (((*byte) & 0x30) == state2)
430 if (((*byte) & 0xC0) == state3)
438 * gfs2_rgrp_verify - Verify that a resource group is consistent
443 void gfs2_rgrp_verify(struct gfs2_rgrpd *rgd)
445 struct gfs2_sbd *sdp = rgd->rd_sbd;
446 struct gfs2_bitmap *bi = NULL;
447 u32 length = rgd->rd_length;
451 memset(count, 0, 4 * sizeof(u32));
453 /* Count # blocks in each of 4 possible allocation states */
454 for (buf = 0; buf < length; buf++) {
455 bi = rgd->rd_bits + buf;
456 for (x = 0; x < 4; x++)
457 count[x] += gfs2_bitcount(rgd,
463 if (count[0] != rgd->rd_free) {
464 if (gfs2_consist_rgrpd(rgd))
465 fs_err(sdp, "free data mismatch: %u != %u\n",
466 count[0], rgd->rd_free);
470 tmp = rgd->rd_data - rgd->rd_free - rgd->rd_dinodes;
471 if (count[1] != tmp) {
472 if (gfs2_consist_rgrpd(rgd))
473 fs_err(sdp, "used data mismatch: %u != %u\n",
478 if (count[2] + count[3] != rgd->rd_dinodes) {
479 if (gfs2_consist_rgrpd(rgd))
480 fs_err(sdp, "used metadata mismatch: %u != %u\n",
481 count[2] + count[3], rgd->rd_dinodes);
486 static inline int rgrp_contains_block(struct gfs2_rgrpd *rgd, u64 block)
488 u64 first = rgd->rd_data0;
489 u64 last = first + rgd->rd_data;
490 return first <= block && block < last;
494 * gfs2_blk2rgrpd - Find resource group for a given data/meta block number
495 * @sdp: The GFS2 superblock
496 * @blk: The data block number
497 * @exact: True if this needs to be an exact match
499 * Returns: The resource group, or NULL if not found
502 struct gfs2_rgrpd *gfs2_blk2rgrpd(struct gfs2_sbd *sdp, u64 blk, bool exact)
504 struct rb_node *n, *next;
505 struct gfs2_rgrpd *cur;
507 spin_lock(&sdp->sd_rindex_spin);
508 n = sdp->sd_rindex_tree.rb_node;
510 cur = rb_entry(n, struct gfs2_rgrpd, rd_node);
512 if (blk < cur->rd_addr)
514 else if (blk >= cur->rd_data0 + cur->rd_data)
517 spin_unlock(&sdp->sd_rindex_spin);
519 if (blk < cur->rd_addr)
521 if (blk >= cur->rd_data0 + cur->rd_data)
528 spin_unlock(&sdp->sd_rindex_spin);
534 * gfs2_rgrpd_get_first - get the first Resource Group in the filesystem
535 * @sdp: The GFS2 superblock
537 * Returns: The first rgrp in the filesystem
540 struct gfs2_rgrpd *gfs2_rgrpd_get_first(struct gfs2_sbd *sdp)
542 const struct rb_node *n;
543 struct gfs2_rgrpd *rgd;
545 spin_lock(&sdp->sd_rindex_spin);
546 n = rb_first(&sdp->sd_rindex_tree);
547 rgd = rb_entry(n, struct gfs2_rgrpd, rd_node);
548 spin_unlock(&sdp->sd_rindex_spin);
554 * gfs2_rgrpd_get_next - get the next RG
555 * @rgd: the resource group descriptor
557 * Returns: The next rgrp
560 struct gfs2_rgrpd *gfs2_rgrpd_get_next(struct gfs2_rgrpd *rgd)
562 struct gfs2_sbd *sdp = rgd->rd_sbd;
563 const struct rb_node *n;
565 spin_lock(&sdp->sd_rindex_spin);
566 n = rb_next(&rgd->rd_node);
568 n = rb_first(&sdp->sd_rindex_tree);
570 if (unlikely(&rgd->rd_node == n)) {
571 spin_unlock(&sdp->sd_rindex_spin);
574 rgd = rb_entry(n, struct gfs2_rgrpd, rd_node);
575 spin_unlock(&sdp->sd_rindex_spin);
579 void check_and_update_goal(struct gfs2_inode *ip)
581 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
582 if (!ip->i_goal || gfs2_blk2rgrpd(sdp, ip->i_goal, 1) == NULL)
583 ip->i_goal = ip->i_no_addr;
586 void gfs2_free_clones(struct gfs2_rgrpd *rgd)
590 for (x = 0; x < rgd->rd_length; x++) {
591 struct gfs2_bitmap *bi = rgd->rd_bits + x;
598 * gfs2_rsqa_alloc - make sure we have a reservation assigned to the inode
599 * plus a quota allocations data structure, if necessary
600 * @ip: the inode for this reservation
602 int gfs2_rsqa_alloc(struct gfs2_inode *ip)
604 return gfs2_qa_alloc(ip);
607 static void dump_rs(struct seq_file *seq, const struct gfs2_blkreserv *rs)
609 gfs2_print_dbg(seq, " B: n:%llu s:%llu b:%u f:%u\n",
610 (unsigned long long)rs->rs_inum,
611 (unsigned long long)gfs2_rbm_to_block(&rs->rs_rbm),
612 rs->rs_rbm.offset, rs->rs_free);
616 * __rs_deltree - remove a multi-block reservation from the rgd tree
617 * @rs: The reservation to remove
620 static void __rs_deltree(struct gfs2_blkreserv *rs)
622 struct gfs2_rgrpd *rgd;
624 if (!gfs2_rs_active(rs))
627 rgd = rs->rs_rbm.rgd;
628 trace_gfs2_rs(rs, TRACE_RS_TREEDEL);
629 rb_erase(&rs->rs_node, &rgd->rd_rstree);
630 RB_CLEAR_NODE(&rs->rs_node);
633 u64 last_block = gfs2_rbm_to_block(&rs->rs_rbm) +
635 struct gfs2_rbm last_rbm = { .rgd = rs->rs_rbm.rgd, };
636 struct gfs2_bitmap *start, *last;
638 /* return reserved blocks to the rgrp */
639 BUG_ON(rs->rs_rbm.rgd->rd_reserved < rs->rs_free);
640 rs->rs_rbm.rgd->rd_reserved -= rs->rs_free;
641 /* The rgrp extent failure point is likely not to increase;
642 it will only do so if the freed blocks are somehow
643 contiguous with a span of free blocks that follows. Still,
644 it will force the number to be recalculated later. */
645 rgd->rd_extfail_pt += rs->rs_free;
647 if (gfs2_rbm_from_block(&last_rbm, last_block))
649 start = rbm_bi(&rs->rs_rbm);
650 last = rbm_bi(&last_rbm);
652 clear_bit(GBF_FULL, &start->bi_flags);
653 while (start++ != last);
658 * gfs2_rs_deltree - remove a multi-block reservation from the rgd tree
659 * @rs: The reservation to remove
662 void gfs2_rs_deltree(struct gfs2_blkreserv *rs)
664 struct gfs2_rgrpd *rgd;
666 rgd = rs->rs_rbm.rgd;
668 spin_lock(&rgd->rd_rsspin);
671 spin_unlock(&rgd->rd_rsspin);
676 * gfs2_rsqa_delete - delete a multi-block reservation and quota allocation
677 * @ip: The inode for this reservation
678 * @wcount: The inode's write count, or NULL
681 void gfs2_rsqa_delete(struct gfs2_inode *ip, atomic_t *wcount)
683 down_write(&ip->i_rw_mutex);
684 if ((wcount == NULL) || (atomic_read(wcount) <= 1))
685 gfs2_rs_deltree(&ip->i_res);
686 up_write(&ip->i_rw_mutex);
687 gfs2_qa_delete(ip, wcount);
691 * return_all_reservations - return all reserved blocks back to the rgrp.
692 * @rgd: the rgrp that needs its space back
694 * We previously reserved a bunch of blocks for allocation. Now we need to
695 * give them back. This leave the reservation structures in tact, but removes
696 * all of their corresponding "no-fly zones".
698 static void return_all_reservations(struct gfs2_rgrpd *rgd)
701 struct gfs2_blkreserv *rs;
703 spin_lock(&rgd->rd_rsspin);
704 while ((n = rb_first(&rgd->rd_rstree))) {
705 rs = rb_entry(n, struct gfs2_blkreserv, rs_node);
708 spin_unlock(&rgd->rd_rsspin);
711 void gfs2_clear_rgrpd(struct gfs2_sbd *sdp)
714 struct gfs2_rgrpd *rgd;
715 struct gfs2_glock *gl;
717 while ((n = rb_first(&sdp->sd_rindex_tree))) {
718 rgd = rb_entry(n, struct gfs2_rgrpd, rd_node);
721 rb_erase(n, &sdp->sd_rindex_tree);
724 spin_lock(&gl->gl_lockref.lock);
725 gl->gl_object = NULL;
726 spin_unlock(&gl->gl_lockref.lock);
727 gfs2_rgrp_brelse(rgd);
728 gfs2_glock_add_to_lru(gl);
732 gfs2_free_clones(rgd);
733 return_all_reservations(rgd);
736 kmem_cache_free(gfs2_rgrpd_cachep, rgd);
740 static void gfs2_rindex_print(const struct gfs2_rgrpd *rgd)
742 pr_info("ri_addr = %llu\n", (unsigned long long)rgd->rd_addr);
743 pr_info("ri_length = %u\n", rgd->rd_length);
744 pr_info("ri_data0 = %llu\n", (unsigned long long)rgd->rd_data0);
745 pr_info("ri_data = %u\n", rgd->rd_data);
746 pr_info("ri_bitbytes = %u\n", rgd->rd_bitbytes);
750 * gfs2_compute_bitstructs - Compute the bitmap sizes
751 * @rgd: The resource group descriptor
753 * Calculates bitmap descriptors, one for each block that contains bitmap data
758 static int compute_bitstructs(struct gfs2_rgrpd *rgd)
760 struct gfs2_sbd *sdp = rgd->rd_sbd;
761 struct gfs2_bitmap *bi;
762 u32 length = rgd->rd_length; /* # blocks in hdr & bitmap */
763 u32 bytes_left, bytes;
769 rgd->rd_bits = kcalloc(length, sizeof(struct gfs2_bitmap), GFP_NOFS);
773 bytes_left = rgd->rd_bitbytes;
775 for (x = 0; x < length; x++) {
776 bi = rgd->rd_bits + x;
779 /* small rgrp; bitmap stored completely in header block */
782 bi->bi_offset = sizeof(struct gfs2_rgrp);
785 bi->bi_blocks = bytes * GFS2_NBBY;
788 bytes = sdp->sd_sb.sb_bsize - sizeof(struct gfs2_rgrp);
789 bi->bi_offset = sizeof(struct gfs2_rgrp);
792 bi->bi_blocks = bytes * GFS2_NBBY;
794 } else if (x + 1 == length) {
796 bi->bi_offset = sizeof(struct gfs2_meta_header);
797 bi->bi_start = rgd->rd_bitbytes - bytes_left;
799 bi->bi_blocks = bytes * GFS2_NBBY;
802 bytes = sdp->sd_sb.sb_bsize -
803 sizeof(struct gfs2_meta_header);
804 bi->bi_offset = sizeof(struct gfs2_meta_header);
805 bi->bi_start = rgd->rd_bitbytes - bytes_left;
807 bi->bi_blocks = bytes * GFS2_NBBY;
814 gfs2_consist_rgrpd(rgd);
817 bi = rgd->rd_bits + (length - 1);
818 if ((bi->bi_start + bi->bi_len) * GFS2_NBBY != rgd->rd_data) {
819 if (gfs2_consist_rgrpd(rgd)) {
820 gfs2_rindex_print(rgd);
821 fs_err(sdp, "start=%u len=%u offset=%u\n",
822 bi->bi_start, bi->bi_len, bi->bi_offset);
831 * gfs2_ri_total - Total up the file system space, according to the rindex.
832 * @sdp: the filesystem
835 u64 gfs2_ri_total(struct gfs2_sbd *sdp)
838 struct inode *inode = sdp->sd_rindex;
839 struct gfs2_inode *ip = GFS2_I(inode);
840 char buf[sizeof(struct gfs2_rindex)];
843 for (rgrps = 0;; rgrps++) {
844 loff_t pos = rgrps * sizeof(struct gfs2_rindex);
846 if (pos + sizeof(struct gfs2_rindex) > i_size_read(inode))
848 error = gfs2_internal_read(ip, buf, &pos,
849 sizeof(struct gfs2_rindex));
850 if (error != sizeof(struct gfs2_rindex))
852 total_data += be32_to_cpu(((struct gfs2_rindex *)buf)->ri_data);
857 static int rgd_insert(struct gfs2_rgrpd *rgd)
859 struct gfs2_sbd *sdp = rgd->rd_sbd;
860 struct rb_node **newn = &sdp->sd_rindex_tree.rb_node, *parent = NULL;
862 /* Figure out where to put new node */
864 struct gfs2_rgrpd *cur = rb_entry(*newn, struct gfs2_rgrpd,
868 if (rgd->rd_addr < cur->rd_addr)
869 newn = &((*newn)->rb_left);
870 else if (rgd->rd_addr > cur->rd_addr)
871 newn = &((*newn)->rb_right);
876 rb_link_node(&rgd->rd_node, parent, newn);
877 rb_insert_color(&rgd->rd_node, &sdp->sd_rindex_tree);
883 * read_rindex_entry - Pull in a new resource index entry from the disk
884 * @ip: Pointer to the rindex inode
886 * Returns: 0 on success, > 0 on EOF, error code otherwise
889 static int read_rindex_entry(struct gfs2_inode *ip)
891 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
892 const unsigned bsize = sdp->sd_sb.sb_bsize;
893 loff_t pos = sdp->sd_rgrps * sizeof(struct gfs2_rindex);
894 struct gfs2_rindex buf;
896 struct gfs2_rgrpd *rgd;
898 if (pos >= i_size_read(&ip->i_inode))
901 error = gfs2_internal_read(ip, (char *)&buf, &pos,
902 sizeof(struct gfs2_rindex));
904 if (error != sizeof(struct gfs2_rindex))
905 return (error == 0) ? 1 : error;
907 rgd = kmem_cache_zalloc(gfs2_rgrpd_cachep, GFP_NOFS);
913 rgd->rd_addr = be64_to_cpu(buf.ri_addr);
914 rgd->rd_length = be32_to_cpu(buf.ri_length);
915 rgd->rd_data0 = be64_to_cpu(buf.ri_data0);
916 rgd->rd_data = be32_to_cpu(buf.ri_data);
917 rgd->rd_bitbytes = be32_to_cpu(buf.ri_bitbytes);
918 spin_lock_init(&rgd->rd_rsspin);
920 error = gfs2_glock_get(sdp, rgd->rd_addr,
921 &gfs2_rgrp_glops, CREATE, &rgd->rd_gl);
925 error = compute_bitstructs(rgd);
929 rgd->rd_rgl = (struct gfs2_rgrp_lvb *)rgd->rd_gl->gl_lksb.sb_lvbptr;
930 rgd->rd_flags &= ~(GFS2_RDF_UPTODATE | GFS2_RDF_PREFERRED);
931 if (rgd->rd_data > sdp->sd_max_rg_data)
932 sdp->sd_max_rg_data = rgd->rd_data;
933 spin_lock(&sdp->sd_rindex_spin);
934 error = rgd_insert(rgd);
935 spin_unlock(&sdp->sd_rindex_spin);
937 rgd->rd_gl->gl_object = rgd;
938 rgd->rd_gl->gl_vm.start = (rgd->rd_addr * bsize) & PAGE_MASK;
939 rgd->rd_gl->gl_vm.end = PAGE_ALIGN((rgd->rd_addr +
940 rgd->rd_length) * bsize) - 1;
944 error = 0; /* someone else read in the rgrp; free it and ignore it */
946 gfs2_glock_put(rgd->rd_gl);
951 kmem_cache_free(gfs2_rgrpd_cachep, rgd);
956 * set_rgrp_preferences - Run all the rgrps, selecting some we prefer to use
957 * @sdp: the GFS2 superblock
959 * The purpose of this function is to select a subset of the resource groups
960 * and mark them as PREFERRED. We do it in such a way that each node prefers
961 * to use a unique set of rgrps to minimize glock contention.
963 static void set_rgrp_preferences(struct gfs2_sbd *sdp)
965 struct gfs2_rgrpd *rgd, *first;
968 /* Skip an initial number of rgrps, based on this node's journal ID.
969 That should start each node out on its own set. */
970 rgd = gfs2_rgrpd_get_first(sdp);
971 for (i = 0; i < sdp->sd_lockstruct.ls_jid; i++)
972 rgd = gfs2_rgrpd_get_next(rgd);
976 rgd->rd_flags |= GFS2_RDF_PREFERRED;
977 for (i = 0; i < sdp->sd_journals; i++) {
978 rgd = gfs2_rgrpd_get_next(rgd);
979 if (!rgd || rgd == first)
982 } while (rgd && rgd != first);
986 * gfs2_ri_update - Pull in a new resource index from the disk
987 * @ip: pointer to the rindex inode
989 * Returns: 0 on successful update, error code otherwise
992 static int gfs2_ri_update(struct gfs2_inode *ip)
994 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
998 error = read_rindex_entry(ip);
999 } while (error == 0);
1004 if (RB_EMPTY_ROOT(&sdp->sd_rindex_tree)) {
1005 fs_err(sdp, "no resource groups found in the file system.\n");
1008 set_rgrp_preferences(sdp);
1010 sdp->sd_rindex_uptodate = 1;
1015 * gfs2_rindex_update - Update the rindex if required
1016 * @sdp: The GFS2 superblock
1018 * We grab a lock on the rindex inode to make sure that it doesn't
1019 * change whilst we are performing an operation. We keep this lock
1020 * for quite long periods of time compared to other locks. This
1021 * doesn't matter, since it is shared and it is very, very rarely
1022 * accessed in the exclusive mode (i.e. only when expanding the filesystem).
1024 * This makes sure that we're using the latest copy of the resource index
1025 * special file, which might have been updated if someone expanded the
1026 * filesystem (via gfs2_grow utility), which adds new resource groups.
1028 * Returns: 0 on succeess, error code otherwise
1031 int gfs2_rindex_update(struct gfs2_sbd *sdp)
1033 struct gfs2_inode *ip = GFS2_I(sdp->sd_rindex);
1034 struct gfs2_glock *gl = ip->i_gl;
1035 struct gfs2_holder ri_gh;
1037 int unlock_required = 0;
1039 /* Read new copy from disk if we don't have the latest */
1040 if (!sdp->sd_rindex_uptodate) {
1041 if (!gfs2_glock_is_locked_by_me(gl)) {
1042 error = gfs2_glock_nq_init(gl, LM_ST_SHARED, 0, &ri_gh);
1045 unlock_required = 1;
1047 if (!sdp->sd_rindex_uptodate)
1048 error = gfs2_ri_update(ip);
1049 if (unlock_required)
1050 gfs2_glock_dq_uninit(&ri_gh);
1056 static void gfs2_rgrp_in(struct gfs2_rgrpd *rgd, const void *buf)
1058 const struct gfs2_rgrp *str = buf;
1061 rg_flags = be32_to_cpu(str->rg_flags);
1062 rg_flags &= ~GFS2_RDF_MASK;
1063 rgd->rd_flags &= GFS2_RDF_MASK;
1064 rgd->rd_flags |= rg_flags;
1065 rgd->rd_free = be32_to_cpu(str->rg_free);
1066 rgd->rd_dinodes = be32_to_cpu(str->rg_dinodes);
1067 rgd->rd_igeneration = be64_to_cpu(str->rg_igeneration);
1070 static void gfs2_rgrp_out(struct gfs2_rgrpd *rgd, void *buf)
1072 struct gfs2_rgrp *str = buf;
1074 str->rg_flags = cpu_to_be32(rgd->rd_flags & ~GFS2_RDF_MASK);
1075 str->rg_free = cpu_to_be32(rgd->rd_free);
1076 str->rg_dinodes = cpu_to_be32(rgd->rd_dinodes);
1077 str->__pad = cpu_to_be32(0);
1078 str->rg_igeneration = cpu_to_be64(rgd->rd_igeneration);
1079 memset(&str->rg_reserved, 0, sizeof(str->rg_reserved));
1082 static int gfs2_rgrp_lvb_valid(struct gfs2_rgrpd *rgd)
1084 struct gfs2_rgrp_lvb *rgl = rgd->rd_rgl;
1085 struct gfs2_rgrp *str = (struct gfs2_rgrp *)rgd->rd_bits[0].bi_bh->b_data;
1087 if (rgl->rl_flags != str->rg_flags || rgl->rl_free != str->rg_free ||
1088 rgl->rl_dinodes != str->rg_dinodes ||
1089 rgl->rl_igeneration != str->rg_igeneration)
1094 static void gfs2_rgrp_ondisk2lvb(struct gfs2_rgrp_lvb *rgl, const void *buf)
1096 const struct gfs2_rgrp *str = buf;
1098 rgl->rl_magic = cpu_to_be32(GFS2_MAGIC);
1099 rgl->rl_flags = str->rg_flags;
1100 rgl->rl_free = str->rg_free;
1101 rgl->rl_dinodes = str->rg_dinodes;
1102 rgl->rl_igeneration = str->rg_igeneration;
1106 static void update_rgrp_lvb_unlinked(struct gfs2_rgrpd *rgd, u32 change)
1108 struct gfs2_rgrp_lvb *rgl = rgd->rd_rgl;
1109 u32 unlinked = be32_to_cpu(rgl->rl_unlinked) + change;
1110 rgl->rl_unlinked = cpu_to_be32(unlinked);
1113 static u32 count_unlinked(struct gfs2_rgrpd *rgd)
1115 struct gfs2_bitmap *bi;
1116 const u32 length = rgd->rd_length;
1117 const u8 *buffer = NULL;
1118 u32 i, goal, count = 0;
1120 for (i = 0, bi = rgd->rd_bits; i < length; i++, bi++) {
1122 buffer = bi->bi_bh->b_data + bi->bi_offset;
1123 WARN_ON(!buffer_uptodate(bi->bi_bh));
1124 while (goal < bi->bi_len * GFS2_NBBY) {
1125 goal = gfs2_bitfit(buffer, bi->bi_len, goal,
1126 GFS2_BLKST_UNLINKED);
1127 if (goal == BFITNOENT)
1139 * gfs2_rgrp_bh_get - Read in a RG's header and bitmaps
1140 * @rgd: the struct gfs2_rgrpd describing the RG to read in
1142 * Read in all of a Resource Group's header and bitmap blocks.
1143 * Caller must eventually call gfs2_rgrp_brelse() to free the bitmaps.
1148 static int gfs2_rgrp_bh_get(struct gfs2_rgrpd *rgd)
1150 struct gfs2_sbd *sdp = rgd->rd_sbd;
1151 struct gfs2_glock *gl = rgd->rd_gl;
1152 unsigned int length = rgd->rd_length;
1153 struct gfs2_bitmap *bi;
1157 if (rgd->rd_bits[0].bi_bh != NULL)
1160 for (x = 0; x < length; x++) {
1161 bi = rgd->rd_bits + x;
1162 error = gfs2_meta_read(gl, rgd->rd_addr + x, 0, 0, &bi->bi_bh);
1167 for (y = length; y--;) {
1168 bi = rgd->rd_bits + y;
1169 error = gfs2_meta_wait(sdp, bi->bi_bh);
1172 if (gfs2_metatype_check(sdp, bi->bi_bh, y ? GFS2_METATYPE_RB :
1173 GFS2_METATYPE_RG)) {
1179 if (!(rgd->rd_flags & GFS2_RDF_UPTODATE)) {
1180 for (x = 0; x < length; x++)
1181 clear_bit(GBF_FULL, &rgd->rd_bits[x].bi_flags);
1182 gfs2_rgrp_in(rgd, (rgd->rd_bits[0].bi_bh)->b_data);
1183 rgd->rd_flags |= (GFS2_RDF_UPTODATE | GFS2_RDF_CHECK);
1184 rgd->rd_free_clone = rgd->rd_free;
1185 /* max out the rgrp allocation failure point */
1186 rgd->rd_extfail_pt = rgd->rd_free;
1188 if (cpu_to_be32(GFS2_MAGIC) != rgd->rd_rgl->rl_magic) {
1189 rgd->rd_rgl->rl_unlinked = cpu_to_be32(count_unlinked(rgd));
1190 gfs2_rgrp_ondisk2lvb(rgd->rd_rgl,
1191 rgd->rd_bits[0].bi_bh->b_data);
1193 else if (sdp->sd_args.ar_rgrplvb) {
1194 if (!gfs2_rgrp_lvb_valid(rgd)){
1195 gfs2_consist_rgrpd(rgd);
1199 if (rgd->rd_rgl->rl_unlinked == 0)
1200 rgd->rd_flags &= ~GFS2_RDF_CHECK;
1206 bi = rgd->rd_bits + x;
1209 gfs2_assert_warn(sdp, !bi->bi_clone);
1215 static int update_rgrp_lvb(struct gfs2_rgrpd *rgd)
1219 if (rgd->rd_flags & GFS2_RDF_UPTODATE)
1222 if (cpu_to_be32(GFS2_MAGIC) != rgd->rd_rgl->rl_magic)
1223 return gfs2_rgrp_bh_get(rgd);
1225 rl_flags = be32_to_cpu(rgd->rd_rgl->rl_flags);
1226 rl_flags &= ~GFS2_RDF_MASK;
1227 rgd->rd_flags &= GFS2_RDF_MASK;
1228 rgd->rd_flags |= (rl_flags | GFS2_RDF_CHECK);
1229 if (rgd->rd_rgl->rl_unlinked == 0)
1230 rgd->rd_flags &= ~GFS2_RDF_CHECK;
1231 rgd->rd_free = be32_to_cpu(rgd->rd_rgl->rl_free);
1232 rgd->rd_free_clone = rgd->rd_free;
1233 rgd->rd_dinodes = be32_to_cpu(rgd->rd_rgl->rl_dinodes);
1234 rgd->rd_igeneration = be64_to_cpu(rgd->rd_rgl->rl_igeneration);
1238 int gfs2_rgrp_go_lock(struct gfs2_holder *gh)
1240 struct gfs2_rgrpd *rgd = gh->gh_gl->gl_object;
1241 struct gfs2_sbd *sdp = rgd->rd_sbd;
1243 if (gh->gh_flags & GL_SKIP && sdp->sd_args.ar_rgrplvb)
1245 return gfs2_rgrp_bh_get(rgd);
1249 * gfs2_rgrp_brelse - Release RG bitmaps read in with gfs2_rgrp_bh_get()
1250 * @rgd: The resource group
1254 void gfs2_rgrp_brelse(struct gfs2_rgrpd *rgd)
1256 int x, length = rgd->rd_length;
1258 for (x = 0; x < length; x++) {
1259 struct gfs2_bitmap *bi = rgd->rd_bits + x;
1269 * gfs2_rgrp_go_unlock - Unlock a rgrp glock
1270 * @gh: The glock holder for the resource group
1274 void gfs2_rgrp_go_unlock(struct gfs2_holder *gh)
1276 struct gfs2_rgrpd *rgd = gh->gh_gl->gl_object;
1277 int demote_requested = test_bit(GLF_DEMOTE, &gh->gh_gl->gl_flags) |
1278 test_bit(GLF_PENDING_DEMOTE, &gh->gh_gl->gl_flags);
1280 if (rgd && demote_requested)
1281 gfs2_rgrp_brelse(rgd);
1284 int gfs2_rgrp_send_discards(struct gfs2_sbd *sdp, u64 offset,
1285 struct buffer_head *bh,
1286 const struct gfs2_bitmap *bi, unsigned minlen, u64 *ptrimmed)
1288 struct super_block *sb = sdp->sd_vfs;
1291 sector_t nr_blks = 0;
1297 for (x = 0; x < bi->bi_len; x++) {
1298 const u8 *clone = bi->bi_clone ? bi->bi_clone : bi->bi_bh->b_data;
1299 clone += bi->bi_offset;
1302 const u8 *orig = bh->b_data + bi->bi_offset + x;
1303 diff = ~(*orig | (*orig >> 1)) & (*clone | (*clone >> 1));
1305 diff = ~(*clone | (*clone >> 1));
1310 blk = offset + ((bi->bi_start + x) * GFS2_NBBY);
1314 goto start_new_extent;
1315 if ((start + nr_blks) != blk) {
1316 if (nr_blks >= minlen) {
1317 rv = sb_issue_discard(sb,
1334 if (nr_blks >= minlen) {
1335 rv = sb_issue_discard(sb, start, nr_blks, GFP_NOFS, 0);
1341 *ptrimmed = trimmed;
1345 if (sdp->sd_args.ar_discard)
1346 fs_warn(sdp, "error %d on discard request, turning discards off for this filesystem", rv);
1347 sdp->sd_args.ar_discard = 0;
1352 * gfs2_fitrim - Generate discard requests for unused bits of the filesystem
1353 * @filp: Any file on the filesystem
1354 * @argp: Pointer to the arguments (also used to pass result)
1356 * Returns: 0 on success, otherwise error code
1359 int gfs2_fitrim(struct file *filp, void __user *argp)
1361 struct inode *inode = file_inode(filp);
1362 struct gfs2_sbd *sdp = GFS2_SB(inode);
1363 struct request_queue *q = bdev_get_queue(sdp->sd_vfs->s_bdev);
1364 struct buffer_head *bh;
1365 struct gfs2_rgrpd *rgd;
1366 struct gfs2_rgrpd *rgd_end;
1367 struct gfs2_holder gh;
1368 struct fstrim_range r;
1372 u64 start, end, minlen;
1374 unsigned bs_shift = sdp->sd_sb.sb_bsize_shift;
1376 if (!capable(CAP_SYS_ADMIN))
1379 if (!test_bit(SDF_JOURNAL_LIVE, &sdp->sd_flags))
1382 if (!blk_queue_discard(q))
1385 if (copy_from_user(&r, argp, sizeof(r)))
1388 ret = gfs2_rindex_update(sdp);
1392 start = r.start >> bs_shift;
1393 end = start + (r.len >> bs_shift);
1394 minlen = max_t(u64, r.minlen, sdp->sd_sb.sb_bsize);
1395 minlen = max_t(u64, minlen,
1396 q->limits.discard_granularity) >> bs_shift;
1398 if (end <= start || minlen > sdp->sd_max_rg_data)
1401 rgd = gfs2_blk2rgrpd(sdp, start, 0);
1402 rgd_end = gfs2_blk2rgrpd(sdp, end, 0);
1404 if ((gfs2_rgrpd_get_first(sdp) == gfs2_rgrpd_get_next(rgd_end))
1405 && (start > rgd_end->rd_data0 + rgd_end->rd_data))
1406 return -EINVAL; /* start is beyond the end of the fs */
1410 ret = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE, 0, &gh);
1414 if (!(rgd->rd_flags & GFS2_RGF_TRIMMED)) {
1415 /* Trim each bitmap in the rgrp */
1416 for (x = 0; x < rgd->rd_length; x++) {
1417 struct gfs2_bitmap *bi = rgd->rd_bits + x;
1418 ret = gfs2_rgrp_send_discards(sdp,
1419 rgd->rd_data0, NULL, bi, minlen,
1422 gfs2_glock_dq_uninit(&gh);
1428 /* Mark rgrp as having been trimmed */
1429 ret = gfs2_trans_begin(sdp, RES_RG_HDR, 0);
1431 bh = rgd->rd_bits[0].bi_bh;
1432 rgd->rd_flags |= GFS2_RGF_TRIMMED;
1433 gfs2_trans_add_meta(rgd->rd_gl, bh);
1434 gfs2_rgrp_out(rgd, bh->b_data);
1435 gfs2_rgrp_ondisk2lvb(rgd->rd_rgl, bh->b_data);
1436 gfs2_trans_end(sdp);
1439 gfs2_glock_dq_uninit(&gh);
1444 rgd = gfs2_rgrpd_get_next(rgd);
1448 r.len = trimmed << bs_shift;
1449 if (copy_to_user(argp, &r, sizeof(r)))
1456 * rs_insert - insert a new multi-block reservation into the rgrp's rb_tree
1457 * @ip: the inode structure
1460 static void rs_insert(struct gfs2_inode *ip)
1462 struct rb_node **newn, *parent = NULL;
1464 struct gfs2_blkreserv *rs = &ip->i_res;
1465 struct gfs2_rgrpd *rgd = rs->rs_rbm.rgd;
1466 u64 fsblock = gfs2_rbm_to_block(&rs->rs_rbm);
1468 BUG_ON(gfs2_rs_active(rs));
1470 spin_lock(&rgd->rd_rsspin);
1471 newn = &rgd->rd_rstree.rb_node;
1473 struct gfs2_blkreserv *cur =
1474 rb_entry(*newn, struct gfs2_blkreserv, rs_node);
1477 rc = rs_cmp(fsblock, rs->rs_free, cur);
1479 newn = &((*newn)->rb_right);
1481 newn = &((*newn)->rb_left);
1483 spin_unlock(&rgd->rd_rsspin);
1489 rb_link_node(&rs->rs_node, parent, newn);
1490 rb_insert_color(&rs->rs_node, &rgd->rd_rstree);
1492 /* Do our rgrp accounting for the reservation */
1493 rgd->rd_reserved += rs->rs_free; /* blocks reserved */
1494 spin_unlock(&rgd->rd_rsspin);
1495 trace_gfs2_rs(rs, TRACE_RS_INSERT);
1499 * rg_mblk_search - find a group of multiple free blocks to form a reservation
1500 * @rgd: the resource group descriptor
1501 * @ip: pointer to the inode for which we're reserving blocks
1502 * @ap: the allocation parameters
1506 static void rg_mblk_search(struct gfs2_rgrpd *rgd, struct gfs2_inode *ip,
1507 const struct gfs2_alloc_parms *ap)
1509 struct gfs2_rbm rbm = { .rgd = rgd, };
1511 struct gfs2_blkreserv *rs = &ip->i_res;
1513 u32 free_blocks = rgd->rd_free_clone - rgd->rd_reserved;
1515 struct inode *inode = &ip->i_inode;
1517 if (S_ISDIR(inode->i_mode))
1520 extlen = max_t(u32, atomic_read(&rs->rs_sizehint), ap->target);
1521 extlen = clamp(extlen, RGRP_RSRV_MINBLKS, free_blocks);
1523 if ((rgd->rd_free_clone < rgd->rd_reserved) || (free_blocks < extlen))
1526 /* Find bitmap block that contains bits for goal block */
1527 if (rgrp_contains_block(rgd, ip->i_goal))
1530 goal = rgd->rd_last_alloc + rgd->rd_data0;
1532 if (WARN_ON(gfs2_rbm_from_block(&rbm, goal)))
1535 ret = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, &extlen, ip, true);
1538 rs->rs_free = extlen;
1539 rs->rs_inum = ip->i_no_addr;
1542 if (goal == rgd->rd_last_alloc + rgd->rd_data0)
1543 rgd->rd_last_alloc = 0;
1548 * gfs2_next_unreserved_block - Return next block that is not reserved
1549 * @rgd: The resource group
1550 * @block: The starting block
1551 * @length: The required length
1552 * @ip: Ignore any reservations for this inode
1554 * If the block does not appear in any reservation, then return the
1555 * block number unchanged. If it does appear in the reservation, then
1556 * keep looking through the tree of reservations in order to find the
1557 * first block number which is not reserved.
1560 static u64 gfs2_next_unreserved_block(struct gfs2_rgrpd *rgd, u64 block,
1562 const struct gfs2_inode *ip)
1564 struct gfs2_blkreserv *rs;
1568 spin_lock(&rgd->rd_rsspin);
1569 n = rgd->rd_rstree.rb_node;
1571 rs = rb_entry(n, struct gfs2_blkreserv, rs_node);
1572 rc = rs_cmp(block, length, rs);
1582 while ((rs_cmp(block, length, rs) == 0) && (&ip->i_res != rs)) {
1583 block = gfs2_rbm_to_block(&rs->rs_rbm) + rs->rs_free;
1587 rs = rb_entry(n, struct gfs2_blkreserv, rs_node);
1591 spin_unlock(&rgd->rd_rsspin);
1596 * gfs2_reservation_check_and_update - Check for reservations during block alloc
1597 * @rbm: The current position in the resource group
1598 * @ip: The inode for which we are searching for blocks
1599 * @minext: The minimum extent length
1600 * @maxext: A pointer to the maximum extent structure
1602 * This checks the current position in the rgrp to see whether there is
1603 * a reservation covering this block. If not then this function is a
1604 * no-op. If there is, then the position is moved to the end of the
1605 * contiguous reservation(s) so that we are pointing at the first
1606 * non-reserved block.
1608 * Returns: 0 if no reservation, 1 if @rbm has changed, otherwise an error
1611 static int gfs2_reservation_check_and_update(struct gfs2_rbm *rbm,
1612 const struct gfs2_inode *ip,
1614 struct gfs2_extent *maxext)
1616 u64 block = gfs2_rbm_to_block(rbm);
1622 * If we have a minimum extent length, then skip over any extent
1623 * which is less than the min extent length in size.
1626 extlen = gfs2_free_extlen(rbm, minext);
1627 if (extlen <= maxext->len)
1632 * Check the extent which has been found against the reservations
1633 * and skip if parts of it are already reserved
1635 nblock = gfs2_next_unreserved_block(rbm->rgd, block, extlen, ip);
1636 if (nblock == block) {
1637 if (!minext || extlen >= minext)
1640 if (extlen > maxext->len) {
1641 maxext->len = extlen;
1645 nblock = block + extlen;
1647 ret = gfs2_rbm_from_block(rbm, nblock);
1654 * gfs2_rbm_find - Look for blocks of a particular state
1655 * @rbm: Value/result starting position and final position
1656 * @state: The state which we want to find
1657 * @minext: Pointer to the requested extent length (NULL for a single block)
1658 * This is updated to be the actual reservation size.
1659 * @ip: If set, check for reservations
1660 * @nowrap: Stop looking at the end of the rgrp, rather than wrapping
1661 * around until we've reached the starting point.
1664 * - If looking for free blocks, we set GBF_FULL on each bitmap which
1665 * has no free blocks in it.
1666 * - If looking for free blocks, we set rd_extfail_pt on each rgrp which
1667 * has come up short on a free block search.
1669 * Returns: 0 on success, -ENOSPC if there is no block of the requested state
1672 static int gfs2_rbm_find(struct gfs2_rbm *rbm, u8 state, u32 *minext,
1673 const struct gfs2_inode *ip, bool nowrap)
1675 struct buffer_head *bh;
1678 int first_bii = rbm->bii;
1679 u32 first_offset = rbm->offset;
1683 int iters = rbm->rgd->rd_length;
1685 struct gfs2_bitmap *bi;
1686 struct gfs2_extent maxext = { .rbm.rgd = rbm->rgd, };
1688 /* If we are not starting at the beginning of a bitmap, then we
1689 * need to add one to the bitmap count to ensure that we search
1690 * the starting bitmap twice.
1692 if (rbm->offset != 0)
1697 if ((ip == NULL || !gfs2_rs_active(&ip->i_res)) &&
1698 test_bit(GBF_FULL, &bi->bi_flags) &&
1699 (state == GFS2_BLKST_FREE))
1703 buffer = bh->b_data + bi->bi_offset;
1704 WARN_ON(!buffer_uptodate(bh));
1705 if (state != GFS2_BLKST_UNLINKED && bi->bi_clone)
1706 buffer = bi->bi_clone + bi->bi_offset;
1707 initial_offset = rbm->offset;
1708 offset = gfs2_bitfit(buffer, bi->bi_len, rbm->offset, state);
1709 if (offset == BFITNOENT)
1711 rbm->offset = offset;
1715 initial_bii = rbm->bii;
1716 ret = gfs2_reservation_check_and_update(rbm, ip,
1717 minext ? *minext : 0,
1722 n += (rbm->bii - initial_bii);
1725 if (ret == -E2BIG) {
1728 n += (rbm->bii - initial_bii);
1729 goto res_covered_end_of_rgrp;
1733 bitmap_full: /* Mark bitmap as full and fall through */
1734 if ((state == GFS2_BLKST_FREE) && initial_offset == 0)
1735 set_bit(GBF_FULL, &bi->bi_flags);
1737 next_bitmap: /* Find next bitmap in the rgrp */
1740 if (rbm->bii == rbm->rgd->rd_length)
1742 res_covered_end_of_rgrp:
1743 if ((rbm->bii == 0) && nowrap)
1751 if (minext == NULL || state != GFS2_BLKST_FREE)
1754 /* If the extent was too small, and it's smaller than the smallest
1755 to have failed before, remember for future reference that it's
1756 useless to search this rgrp again for this amount or more. */
1757 if ((first_offset == 0) && (first_bii == 0) &&
1758 (*minext < rbm->rgd->rd_extfail_pt))
1759 rbm->rgd->rd_extfail_pt = *minext;
1761 /* If the maximum extent we found is big enough to fulfill the
1762 minimum requirements, use it anyway. */
1765 *minext = maxext.len;
1773 * try_rgrp_unlink - Look for any unlinked, allocated, but unused inodes
1775 * @last_unlinked: block address of the last dinode we unlinked
1776 * @skip: block address we should explicitly not unlink
1778 * Returns: 0 if no error
1779 * The inode, if one has been found, in inode.
1782 static void try_rgrp_unlink(struct gfs2_rgrpd *rgd, u64 *last_unlinked, u64 skip)
1785 struct gfs2_sbd *sdp = rgd->rd_sbd;
1786 struct gfs2_glock *gl;
1787 struct gfs2_inode *ip;
1790 struct gfs2_rbm rbm = { .rgd = rgd, .bii = 0, .offset = 0 };
1793 down_write(&sdp->sd_log_flush_lock);
1794 error = gfs2_rbm_find(&rbm, GFS2_BLKST_UNLINKED, NULL, NULL,
1796 up_write(&sdp->sd_log_flush_lock);
1797 if (error == -ENOSPC)
1799 if (WARN_ON_ONCE(error))
1802 block = gfs2_rbm_to_block(&rbm);
1803 if (gfs2_rbm_from_block(&rbm, block + 1))
1805 if (*last_unlinked != NO_BLOCK && block <= *last_unlinked)
1809 *last_unlinked = block;
1811 error = gfs2_glock_get(sdp, block, &gfs2_iopen_glops, CREATE, &gl);
1815 /* If the inode is already in cache, we can ignore it here
1816 * because the existing inode disposal code will deal with
1817 * it when all refs have gone away. Accessing gl_object like
1818 * this is not safe in general. Here it is ok because we do
1819 * not dereference the pointer, and we only need an approx
1820 * answer to whether it is NULL or not.
1824 if (ip || queue_work(gfs2_delete_workqueue, &gl->gl_delete) == 0)
1829 /* Limit reclaim to sensible number of tasks */
1830 if (found > NR_CPUS)
1834 rgd->rd_flags &= ~GFS2_RDF_CHECK;
1839 * gfs2_rgrp_congested - Use stats to figure out whether an rgrp is congested
1840 * @rgd: The rgrp in question
1841 * @loops: An indication of how picky we can be (0=very, 1=less so)
1843 * This function uses the recently added glock statistics in order to
1844 * figure out whether a parciular resource group is suffering from
1845 * contention from multiple nodes. This is done purely on the basis
1846 * of timings, since this is the only data we have to work with and
1847 * our aim here is to reject a resource group which is highly contended
1848 * but (very important) not to do this too often in order to ensure that
1849 * we do not land up introducing fragmentation by changing resource
1850 * groups when not actually required.
1852 * The calculation is fairly simple, we want to know whether the SRTTB
1853 * (i.e. smoothed round trip time for blocking operations) to acquire
1854 * the lock for this rgrp's glock is significantly greater than the
1855 * time taken for resource groups on average. We introduce a margin in
1856 * the form of the variable @var which is computed as the sum of the two
1857 * respective variences, and multiplied by a factor depending on @loops
1858 * and whether we have a lot of data to base the decision on. This is
1859 * then tested against the square difference of the means in order to
1860 * decide whether the result is statistically significant or not.
1862 * Returns: A boolean verdict on the congestion status
1865 static bool gfs2_rgrp_congested(const struct gfs2_rgrpd *rgd, int loops)
1867 const struct gfs2_glock *gl = rgd->rd_gl;
1868 const struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1869 struct gfs2_lkstats *st;
1870 u64 r_dcount, l_dcount;
1871 u64 l_srttb, a_srttb = 0;
1875 int cpu, nonzero = 0;
1878 for_each_present_cpu(cpu) {
1879 st = &per_cpu_ptr(sdp->sd_lkstats, cpu)->lkstats[LM_TYPE_RGRP];
1880 if (st->stats[GFS2_LKS_SRTTB]) {
1881 a_srttb += st->stats[GFS2_LKS_SRTTB];
1885 st = &this_cpu_ptr(sdp->sd_lkstats)->lkstats[LM_TYPE_RGRP];
1887 do_div(a_srttb, nonzero);
1888 r_dcount = st->stats[GFS2_LKS_DCOUNT];
1889 var = st->stats[GFS2_LKS_SRTTVARB] +
1890 gl->gl_stats.stats[GFS2_LKS_SRTTVARB];
1893 l_srttb = gl->gl_stats.stats[GFS2_LKS_SRTTB];
1894 l_dcount = gl->gl_stats.stats[GFS2_LKS_DCOUNT];
1896 if ((l_dcount < 1) || (r_dcount < 1) || (a_srttb == 0))
1899 srttb_diff = a_srttb - l_srttb;
1900 sqr_diff = srttb_diff * srttb_diff;
1903 if (l_dcount < 8 || r_dcount < 8)
1908 return ((srttb_diff < 0) && (sqr_diff > var));
1912 * gfs2_rgrp_used_recently
1913 * @rs: The block reservation with the rgrp to test
1914 * @msecs: The time limit in milliseconds
1916 * Returns: True if the rgrp glock has been used within the time limit
1918 static bool gfs2_rgrp_used_recently(const struct gfs2_blkreserv *rs,
1923 tdiff = ktime_to_ns(ktime_sub(ktime_get_real(),
1924 rs->rs_rbm.rgd->rd_gl->gl_dstamp));
1926 return tdiff > (msecs * 1000 * 1000);
1929 static u32 gfs2_orlov_skip(const struct gfs2_inode *ip)
1931 const struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
1934 get_random_bytes(&skip, sizeof(skip));
1935 return skip % sdp->sd_rgrps;
1938 static bool gfs2_select_rgrp(struct gfs2_rgrpd **pos, const struct gfs2_rgrpd *begin)
1940 struct gfs2_rgrpd *rgd = *pos;
1941 struct gfs2_sbd *sdp = rgd->rd_sbd;
1943 rgd = gfs2_rgrpd_get_next(rgd);
1945 rgd = gfs2_rgrpd_get_first(sdp);
1947 if (rgd != begin) /* If we didn't wrap */
1953 * fast_to_acquire - determine if a resource group will be fast to acquire
1955 * If this is one of our preferred rgrps, it should be quicker to acquire,
1956 * because we tried to set ourselves up as dlm lock master.
1958 static inline int fast_to_acquire(struct gfs2_rgrpd *rgd)
1960 struct gfs2_glock *gl = rgd->rd_gl;
1962 if (gl->gl_state != LM_ST_UNLOCKED && list_empty(&gl->gl_holders) &&
1963 !test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags) &&
1964 !test_bit(GLF_DEMOTE, &gl->gl_flags))
1966 if (rgd->rd_flags & GFS2_RDF_PREFERRED)
1972 * gfs2_inplace_reserve - Reserve space in the filesystem
1973 * @ip: the inode to reserve space for
1974 * @ap: the allocation parameters
1976 * We try our best to find an rgrp that has at least ap->target blocks
1977 * available. After a couple of passes (loops == 2), the prospects of finding
1978 * such an rgrp diminish. At this stage, we return the first rgrp that has
1979 * atleast ap->min_target blocks available. Either way, we set ap->allowed to
1980 * the number of blocks available in the chosen rgrp.
1982 * Returns: 0 on success,
1983 * -ENOMEM if a suitable rgrp can't be found
1987 int gfs2_inplace_reserve(struct gfs2_inode *ip, struct gfs2_alloc_parms *ap)
1989 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
1990 struct gfs2_rgrpd *begin = NULL;
1991 struct gfs2_blkreserv *rs = &ip->i_res;
1992 int error = 0, rg_locked, flags = 0;
1993 u64 last_unlinked = NO_BLOCK;
1997 if (sdp->sd_args.ar_rgrplvb)
1999 if (gfs2_assert_warn(sdp, ap->target))
2001 if (gfs2_rs_active(rs)) {
2002 begin = rs->rs_rbm.rgd;
2003 } else if (ip->i_rgd && rgrp_contains_block(ip->i_rgd, ip->i_goal)) {
2004 rs->rs_rbm.rgd = begin = ip->i_rgd;
2006 check_and_update_goal(ip);
2007 rs->rs_rbm.rgd = begin = gfs2_blk2rgrpd(sdp, ip->i_goal, 1);
2009 if (S_ISDIR(ip->i_inode.i_mode) && (ap->aflags & GFS2_AF_ORLOV))
2010 skip = gfs2_orlov_skip(ip);
2011 if (rs->rs_rbm.rgd == NULL)
2017 if (!gfs2_glock_is_locked_by_me(rs->rs_rbm.rgd->rd_gl)) {
2021 if (!gfs2_rs_active(rs)) {
2023 !fast_to_acquire(rs->rs_rbm.rgd))
2026 gfs2_rgrp_used_recently(rs, 1000) &&
2027 gfs2_rgrp_congested(rs->rs_rbm.rgd, loops))
2030 error = gfs2_glock_nq_init(rs->rs_rbm.rgd->rd_gl,
2031 LM_ST_EXCLUSIVE, flags,
2033 if (unlikely(error))
2035 if (!gfs2_rs_active(rs) && (loops < 2) &&
2036 gfs2_rgrp_congested(rs->rs_rbm.rgd, loops))
2038 if (sdp->sd_args.ar_rgrplvb) {
2039 error = update_rgrp_lvb(rs->rs_rbm.rgd);
2040 if (unlikely(error)) {
2041 gfs2_glock_dq_uninit(&rs->rs_rgd_gh);
2047 /* Skip unuseable resource groups */
2048 if ((rs->rs_rbm.rgd->rd_flags & (GFS2_RGF_NOALLOC |
2050 (loops == 0 && ap->target > rs->rs_rbm.rgd->rd_extfail_pt))
2053 if (sdp->sd_args.ar_rgrplvb)
2054 gfs2_rgrp_bh_get(rs->rs_rbm.rgd);
2056 /* Get a reservation if we don't already have one */
2057 if (!gfs2_rs_active(rs))
2058 rg_mblk_search(rs->rs_rbm.rgd, ip, ap);
2060 /* Skip rgrps when we can't get a reservation on first pass */
2061 if (!gfs2_rs_active(rs) && (loops < 1))
2064 /* If rgrp has enough free space, use it */
2065 if (rs->rs_rbm.rgd->rd_free_clone >= ap->target ||
2066 (loops == 2 && ap->min_target &&
2067 rs->rs_rbm.rgd->rd_free_clone >= ap->min_target)) {
2068 ip->i_rgd = rs->rs_rbm.rgd;
2069 ap->allowed = ip->i_rgd->rd_free_clone;
2073 /* Check for unlinked inodes which can be reclaimed */
2074 if (rs->rs_rbm.rgd->rd_flags & GFS2_RDF_CHECK)
2075 try_rgrp_unlink(rs->rs_rbm.rgd, &last_unlinked,
2078 /* Drop reservation, if we couldn't use reserved rgrp */
2079 if (gfs2_rs_active(rs))
2080 gfs2_rs_deltree(rs);
2082 /* Unlock rgrp if required */
2084 gfs2_glock_dq_uninit(&rs->rs_rgd_gh);
2086 /* Find the next rgrp, and continue looking */
2087 if (gfs2_select_rgrp(&rs->rs_rbm.rgd, begin))
2092 /* If we've scanned all the rgrps, but found no free blocks
2093 * then this checks for some less likely conditions before
2097 /* Check that fs hasn't grown if writing to rindex */
2098 if (ip == GFS2_I(sdp->sd_rindex) && !sdp->sd_rindex_uptodate) {
2099 error = gfs2_ri_update(ip);
2103 /* Flushing the log may release space */
2105 gfs2_log_flush(sdp, NULL, NORMAL_FLUSH);
2112 * gfs2_inplace_release - release an inplace reservation
2113 * @ip: the inode the reservation was taken out on
2115 * Release a reservation made by gfs2_inplace_reserve().
2118 void gfs2_inplace_release(struct gfs2_inode *ip)
2120 struct gfs2_blkreserv *rs = &ip->i_res;
2122 if (gfs2_holder_initialized(&rs->rs_rgd_gh))
2123 gfs2_glock_dq_uninit(&rs->rs_rgd_gh);
2127 * gfs2_get_block_type - Check a block in a RG is of given type
2128 * @rgd: the resource group holding the block
2129 * @block: the block number
2131 * Returns: The block type (GFS2_BLKST_*)
2134 static unsigned char gfs2_get_block_type(struct gfs2_rgrpd *rgd, u64 block)
2136 struct gfs2_rbm rbm = { .rgd = rgd, };
2139 ret = gfs2_rbm_from_block(&rbm, block);
2140 WARN_ON_ONCE(ret != 0);
2142 return gfs2_testbit(&rbm);
2147 * gfs2_alloc_extent - allocate an extent from a given bitmap
2148 * @rbm: the resource group information
2149 * @dinode: TRUE if the first block we allocate is for a dinode
2150 * @n: The extent length (value/result)
2152 * Add the bitmap buffer to the transaction.
2153 * Set the found bits to @new_state to change block's allocation state.
2155 static void gfs2_alloc_extent(const struct gfs2_rbm *rbm, bool dinode,
2158 struct gfs2_rbm pos = { .rgd = rbm->rgd, };
2159 const unsigned int elen = *n;
2164 block = gfs2_rbm_to_block(rbm);
2165 gfs2_trans_add_meta(rbm->rgd->rd_gl, rbm_bi(rbm)->bi_bh);
2166 gfs2_setbit(rbm, true, dinode ? GFS2_BLKST_DINODE : GFS2_BLKST_USED);
2169 ret = gfs2_rbm_from_block(&pos, block);
2170 if (ret || gfs2_testbit(&pos) != GFS2_BLKST_FREE)
2172 gfs2_trans_add_meta(pos.rgd->rd_gl, rbm_bi(&pos)->bi_bh);
2173 gfs2_setbit(&pos, true, GFS2_BLKST_USED);
2180 * rgblk_free - Change alloc state of given block(s)
2181 * @sdp: the filesystem
2182 * @bstart: the start of a run of blocks to free
2183 * @blen: the length of the block run (all must lie within ONE RG!)
2184 * @new_state: GFS2_BLKST_XXX the after-allocation block state
2186 * Returns: Resource group containing the block(s)
2189 static struct gfs2_rgrpd *rgblk_free(struct gfs2_sbd *sdp, u64 bstart,
2190 u32 blen, unsigned char new_state)
2192 struct gfs2_rbm rbm;
2193 struct gfs2_bitmap *bi, *bi_prev = NULL;
2195 rbm.rgd = gfs2_blk2rgrpd(sdp, bstart, 1);
2197 if (gfs2_consist(sdp))
2198 fs_err(sdp, "block = %llu\n", (unsigned long long)bstart);
2202 gfs2_rbm_from_block(&rbm, bstart);
2205 if (bi != bi_prev) {
2206 if (!bi->bi_clone) {
2207 bi->bi_clone = kmalloc(bi->bi_bh->b_size,
2208 GFP_NOFS | __GFP_NOFAIL);
2209 memcpy(bi->bi_clone + bi->bi_offset,
2210 bi->bi_bh->b_data + bi->bi_offset,
2213 gfs2_trans_add_meta(rbm.rgd->rd_gl, bi->bi_bh);
2216 gfs2_setbit(&rbm, false, new_state);
2217 gfs2_rbm_incr(&rbm);
2224 * gfs2_rgrp_dump - print out an rgrp
2225 * @seq: The iterator
2226 * @gl: The glock in question
2230 void gfs2_rgrp_dump(struct seq_file *seq, const struct gfs2_glock *gl)
2232 struct gfs2_rgrpd *rgd = gl->gl_object;
2233 struct gfs2_blkreserv *trs;
2234 const struct rb_node *n;
2238 gfs2_print_dbg(seq, " R: n:%llu f:%02x b:%u/%u i:%u r:%u e:%u\n",
2239 (unsigned long long)rgd->rd_addr, rgd->rd_flags,
2240 rgd->rd_free, rgd->rd_free_clone, rgd->rd_dinodes,
2241 rgd->rd_reserved, rgd->rd_extfail_pt);
2242 spin_lock(&rgd->rd_rsspin);
2243 for (n = rb_first(&rgd->rd_rstree); n; n = rb_next(&trs->rs_node)) {
2244 trs = rb_entry(n, struct gfs2_blkreserv, rs_node);
2247 spin_unlock(&rgd->rd_rsspin);
2250 static void gfs2_rgrp_error(struct gfs2_rgrpd *rgd)
2252 struct gfs2_sbd *sdp = rgd->rd_sbd;
2253 fs_warn(sdp, "rgrp %llu has an error, marking it readonly until umount\n",
2254 (unsigned long long)rgd->rd_addr);
2255 fs_warn(sdp, "umount on all nodes and run fsck.gfs2 to fix the error\n");
2256 gfs2_rgrp_dump(NULL, rgd->rd_gl);
2257 rgd->rd_flags |= GFS2_RDF_ERROR;
2261 * gfs2_adjust_reservation - Adjust (or remove) a reservation after allocation
2262 * @ip: The inode we have just allocated blocks for
2263 * @rbm: The start of the allocated blocks
2264 * @len: The extent length
2266 * Adjusts a reservation after an allocation has taken place. If the
2267 * reservation does not match the allocation, or if it is now empty
2268 * then it is removed.
2271 static void gfs2_adjust_reservation(struct gfs2_inode *ip,
2272 const struct gfs2_rbm *rbm, unsigned len)
2274 struct gfs2_blkreserv *rs = &ip->i_res;
2275 struct gfs2_rgrpd *rgd = rbm->rgd;
2280 spin_lock(&rgd->rd_rsspin);
2281 if (gfs2_rs_active(rs)) {
2282 if (gfs2_rbm_eq(&rs->rs_rbm, rbm)) {
2283 block = gfs2_rbm_to_block(rbm);
2284 ret = gfs2_rbm_from_block(&rs->rs_rbm, block + len);
2285 rlen = min(rs->rs_free, len);
2286 rs->rs_free -= rlen;
2287 rgd->rd_reserved -= rlen;
2288 trace_gfs2_rs(rs, TRACE_RS_CLAIM);
2289 if (rs->rs_free && !ret)
2291 /* We used up our block reservation, so we should
2292 reserve more blocks next time. */
2293 atomic_add(RGRP_RSRV_ADDBLKS, &rs->rs_sizehint);
2298 spin_unlock(&rgd->rd_rsspin);
2302 * gfs2_set_alloc_start - Set starting point for block allocation
2303 * @rbm: The rbm which will be set to the required location
2304 * @ip: The gfs2 inode
2305 * @dinode: Flag to say if allocation includes a new inode
2307 * This sets the starting point from the reservation if one is active
2308 * otherwise it falls back to guessing a start point based on the
2309 * inode's goal block or the last allocation point in the rgrp.
2312 static void gfs2_set_alloc_start(struct gfs2_rbm *rbm,
2313 const struct gfs2_inode *ip, bool dinode)
2317 if (gfs2_rs_active(&ip->i_res)) {
2318 *rbm = ip->i_res.rs_rbm;
2322 if (!dinode && rgrp_contains_block(rbm->rgd, ip->i_goal))
2325 goal = rbm->rgd->rd_last_alloc + rbm->rgd->rd_data0;
2327 gfs2_rbm_from_block(rbm, goal);
2331 * gfs2_alloc_blocks - Allocate one or more blocks of data and/or a dinode
2332 * @ip: the inode to allocate the block for
2333 * @bn: Used to return the starting block number
2334 * @nblocks: requested number of blocks/extent length (value/result)
2335 * @dinode: 1 if we're allocating a dinode block, else 0
2336 * @generation: the generation number of the inode
2338 * Returns: 0 or error
2341 int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *nblocks,
2342 bool dinode, u64 *generation)
2344 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
2345 struct buffer_head *dibh;
2346 struct gfs2_rbm rbm = { .rgd = ip->i_rgd, };
2348 u64 block; /* block, within the file system scope */
2351 gfs2_set_alloc_start(&rbm, ip, dinode);
2352 error = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, NULL, ip, false);
2354 if (error == -ENOSPC) {
2355 gfs2_set_alloc_start(&rbm, ip, dinode);
2356 error = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, NULL, NULL, false);
2359 /* Since all blocks are reserved in advance, this shouldn't happen */
2361 fs_warn(sdp, "inum=%llu error=%d, nblocks=%u, full=%d fail_pt=%d\n",
2362 (unsigned long long)ip->i_no_addr, error, *nblocks,
2363 test_bit(GBF_FULL, &rbm.rgd->rd_bits->bi_flags),
2364 rbm.rgd->rd_extfail_pt);
2368 gfs2_alloc_extent(&rbm, dinode, nblocks);
2369 block = gfs2_rbm_to_block(&rbm);
2370 rbm.rgd->rd_last_alloc = block - rbm.rgd->rd_data0;
2371 if (gfs2_rs_active(&ip->i_res))
2372 gfs2_adjust_reservation(ip, &rbm, *nblocks);
2378 ip->i_goal = block + ndata - 1;
2379 error = gfs2_meta_inode_buffer(ip, &dibh);
2381 struct gfs2_dinode *di =
2382 (struct gfs2_dinode *)dibh->b_data;
2383 gfs2_trans_add_meta(ip->i_gl, dibh);
2384 di->di_goal_meta = di->di_goal_data =
2385 cpu_to_be64(ip->i_goal);
2389 if (rbm.rgd->rd_free < *nblocks) {
2390 pr_warn("nblocks=%u\n", *nblocks);
2394 rbm.rgd->rd_free -= *nblocks;
2396 rbm.rgd->rd_dinodes++;
2397 *generation = rbm.rgd->rd_igeneration++;
2398 if (*generation == 0)
2399 *generation = rbm.rgd->rd_igeneration++;
2402 gfs2_trans_add_meta(rbm.rgd->rd_gl, rbm.rgd->rd_bits[0].bi_bh);
2403 gfs2_rgrp_out(rbm.rgd, rbm.rgd->rd_bits[0].bi_bh->b_data);
2404 gfs2_rgrp_ondisk2lvb(rbm.rgd->rd_rgl, rbm.rgd->rd_bits[0].bi_bh->b_data);
2406 gfs2_statfs_change(sdp, 0, -(s64)*nblocks, dinode ? 1 : 0);
2408 gfs2_trans_add_unrevoke(sdp, block, *nblocks);
2410 gfs2_quota_change(ip, *nblocks, ip->i_inode.i_uid, ip->i_inode.i_gid);
2412 rbm.rgd->rd_free_clone -= *nblocks;
2413 trace_gfs2_block_alloc(ip, rbm.rgd, block, *nblocks,
2414 dinode ? GFS2_BLKST_DINODE : GFS2_BLKST_USED);
2419 gfs2_rgrp_error(rbm.rgd);
2424 * __gfs2_free_blocks - free a contiguous run of block(s)
2425 * @ip: the inode these blocks are being freed from
2426 * @bstart: first block of a run of contiguous blocks
2427 * @blen: the length of the block run
2428 * @meta: 1 if the blocks represent metadata
2432 void __gfs2_free_blocks(struct gfs2_inode *ip, u64 bstart, u32 blen, int meta)
2434 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
2435 struct gfs2_rgrpd *rgd;
2437 rgd = rgblk_free(sdp, bstart, blen, GFS2_BLKST_FREE);
2440 trace_gfs2_block_alloc(ip, rgd, bstart, blen, GFS2_BLKST_FREE);
2441 rgd->rd_free += blen;
2442 rgd->rd_flags &= ~GFS2_RGF_TRIMMED;
2443 gfs2_trans_add_meta(rgd->rd_gl, rgd->rd_bits[0].bi_bh);
2444 gfs2_rgrp_out(rgd, rgd->rd_bits[0].bi_bh->b_data);
2445 gfs2_rgrp_ondisk2lvb(rgd->rd_rgl, rgd->rd_bits[0].bi_bh->b_data);
2447 /* Directories keep their data in the metadata address space */
2448 if (meta || ip->i_depth)
2449 gfs2_meta_wipe(ip, bstart, blen);
2453 * gfs2_free_meta - free a contiguous run of data block(s)
2454 * @ip: the inode these blocks are being freed from
2455 * @bstart: first block of a run of contiguous blocks
2456 * @blen: the length of the block run
2460 void gfs2_free_meta(struct gfs2_inode *ip, u64 bstart, u32 blen)
2462 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
2464 __gfs2_free_blocks(ip, bstart, blen, 1);
2465 gfs2_statfs_change(sdp, 0, +blen, 0);
2466 gfs2_quota_change(ip, -(s64)blen, ip->i_inode.i_uid, ip->i_inode.i_gid);
2469 void gfs2_unlink_di(struct inode *inode)
2471 struct gfs2_inode *ip = GFS2_I(inode);
2472 struct gfs2_sbd *sdp = GFS2_SB(inode);
2473 struct gfs2_rgrpd *rgd;
2474 u64 blkno = ip->i_no_addr;
2476 rgd = rgblk_free(sdp, blkno, 1, GFS2_BLKST_UNLINKED);
2479 trace_gfs2_block_alloc(ip, rgd, blkno, 1, GFS2_BLKST_UNLINKED);
2480 gfs2_trans_add_meta(rgd->rd_gl, rgd->rd_bits[0].bi_bh);
2481 gfs2_rgrp_out(rgd, rgd->rd_bits[0].bi_bh->b_data);
2482 gfs2_rgrp_ondisk2lvb(rgd->rd_rgl, rgd->rd_bits[0].bi_bh->b_data);
2483 update_rgrp_lvb_unlinked(rgd, 1);
2486 static void gfs2_free_uninit_di(struct gfs2_rgrpd *rgd, u64 blkno)
2488 struct gfs2_sbd *sdp = rgd->rd_sbd;
2489 struct gfs2_rgrpd *tmp_rgd;
2491 tmp_rgd = rgblk_free(sdp, blkno, 1, GFS2_BLKST_FREE);
2494 gfs2_assert_withdraw(sdp, rgd == tmp_rgd);
2496 if (!rgd->rd_dinodes)
2497 gfs2_consist_rgrpd(rgd);
2501 gfs2_trans_add_meta(rgd->rd_gl, rgd->rd_bits[0].bi_bh);
2502 gfs2_rgrp_out(rgd, rgd->rd_bits[0].bi_bh->b_data);
2503 gfs2_rgrp_ondisk2lvb(rgd->rd_rgl, rgd->rd_bits[0].bi_bh->b_data);
2504 update_rgrp_lvb_unlinked(rgd, -1);
2506 gfs2_statfs_change(sdp, 0, +1, -1);
2510 void gfs2_free_di(struct gfs2_rgrpd *rgd, struct gfs2_inode *ip)
2512 gfs2_free_uninit_di(rgd, ip->i_no_addr);
2513 trace_gfs2_block_alloc(ip, rgd, ip->i_no_addr, 1, GFS2_BLKST_FREE);
2514 gfs2_quota_change(ip, -1, ip->i_inode.i_uid, ip->i_inode.i_gid);
2515 gfs2_meta_wipe(ip, ip->i_no_addr, 1);
2519 * gfs2_check_blk_type - Check the type of a block
2520 * @sdp: The superblock
2521 * @no_addr: The block number to check
2522 * @type: The block type we are looking for
2524 * Returns: 0 if the block type matches the expected type
2525 * -ESTALE if it doesn't match
2526 * or -ve errno if something went wrong while checking
2529 int gfs2_check_blk_type(struct gfs2_sbd *sdp, u64 no_addr, unsigned int type)
2531 struct gfs2_rgrpd *rgd;
2532 struct gfs2_holder rgd_gh;
2533 int error = -EINVAL;
2535 rgd = gfs2_blk2rgrpd(sdp, no_addr, 1);
2539 error = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_SHARED, 0, &rgd_gh);
2543 if (gfs2_get_block_type(rgd, no_addr) != type)
2546 gfs2_glock_dq_uninit(&rgd_gh);
2552 * gfs2_rlist_add - add a RG to a list of RGs
2554 * @rlist: the list of resource groups
2557 * Figure out what RG a block belongs to and add that RG to the list
2559 * FIXME: Don't use NOFAIL
2563 void gfs2_rlist_add(struct gfs2_inode *ip, struct gfs2_rgrp_list *rlist,
2566 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
2567 struct gfs2_rgrpd *rgd;
2568 struct gfs2_rgrpd **tmp;
2569 unsigned int new_space;
2572 if (gfs2_assert_warn(sdp, !rlist->rl_ghs))
2575 if (ip->i_rgd && rgrp_contains_block(ip->i_rgd, block))
2578 rgd = gfs2_blk2rgrpd(sdp, block, 1);
2580 fs_err(sdp, "rlist_add: no rgrp for block %llu\n", (unsigned long long)block);
2585 for (x = 0; x < rlist->rl_rgrps; x++)
2586 if (rlist->rl_rgd[x] == rgd)
2589 if (rlist->rl_rgrps == rlist->rl_space) {
2590 new_space = rlist->rl_space + 10;
2592 tmp = kcalloc(new_space, sizeof(struct gfs2_rgrpd *),
2593 GFP_NOFS | __GFP_NOFAIL);
2595 if (rlist->rl_rgd) {
2596 memcpy(tmp, rlist->rl_rgd,
2597 rlist->rl_space * sizeof(struct gfs2_rgrpd *));
2598 kfree(rlist->rl_rgd);
2601 rlist->rl_space = new_space;
2602 rlist->rl_rgd = tmp;
2605 rlist->rl_rgd[rlist->rl_rgrps++] = rgd;
2609 * gfs2_rlist_alloc - all RGs have been added to the rlist, now allocate
2610 * and initialize an array of glock holders for them
2611 * @rlist: the list of resource groups
2612 * @state: the lock state to acquire the RG lock in
2614 * FIXME: Don't use NOFAIL
2618 void gfs2_rlist_alloc(struct gfs2_rgrp_list *rlist, unsigned int state)
2622 rlist->rl_ghs = kmalloc(rlist->rl_rgrps * sizeof(struct gfs2_holder),
2623 GFP_NOFS | __GFP_NOFAIL);
2624 for (x = 0; x < rlist->rl_rgrps; x++)
2625 gfs2_holder_init(rlist->rl_rgd[x]->rd_gl,
2631 * gfs2_rlist_free - free a resource group list
2632 * @rlist: the list of resource groups
2636 void gfs2_rlist_free(struct gfs2_rgrp_list *rlist)
2640 kfree(rlist->rl_rgd);
2642 if (rlist->rl_ghs) {
2643 for (x = 0; x < rlist->rl_rgrps; x++)
2644 gfs2_holder_uninit(&rlist->rl_ghs[x]);
2645 kfree(rlist->rl_ghs);
2646 rlist->rl_ghs = NULL;