1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2011 STRATO. All rights reserved.
6 #include <linux/sched.h>
7 #include <linux/pagemap.h>
8 #include <linux/writeback.h>
9 #include <linux/blkdev.h>
10 #include <linux/rbtree.h>
11 #include <linux/slab.h>
12 #include <linux/workqueue.h>
13 #include <linux/btrfs.h>
14 #include <linux/sched/mm.h>
17 #include "transaction.h"
22 #include "extent_io.h"
24 #include "block-group.h"
26 #include "tree-mod-log.h"
28 #include "accessors.h"
29 #include "extent-tree.h"
30 #include "root-tree.h"
31 #include "tree-checker.h"
33 enum btrfs_qgroup_mode btrfs_qgroup_mode(struct btrfs_fs_info *fs_info)
35 if (!test_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags))
36 return BTRFS_QGROUP_MODE_DISABLED;
37 if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE)
38 return BTRFS_QGROUP_MODE_SIMPLE;
39 return BTRFS_QGROUP_MODE_FULL;
42 bool btrfs_qgroup_enabled(struct btrfs_fs_info *fs_info)
44 return btrfs_qgroup_mode(fs_info) != BTRFS_QGROUP_MODE_DISABLED;
47 bool btrfs_qgroup_full_accounting(struct btrfs_fs_info *fs_info)
49 return btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_FULL;
53 * Helpers to access qgroup reservation
55 * Callers should ensure the lock context and type are valid
58 static u64 qgroup_rsv_total(const struct btrfs_qgroup *qgroup)
63 for (i = 0; i < BTRFS_QGROUP_RSV_LAST; i++)
64 ret += qgroup->rsv.values[i];
69 #ifdef CONFIG_BTRFS_DEBUG
70 static const char *qgroup_rsv_type_str(enum btrfs_qgroup_rsv_type type)
72 if (type == BTRFS_QGROUP_RSV_DATA)
74 if (type == BTRFS_QGROUP_RSV_META_PERTRANS)
75 return "meta_pertrans";
76 if (type == BTRFS_QGROUP_RSV_META_PREALLOC)
77 return "meta_prealloc";
82 static void qgroup_rsv_add(struct btrfs_fs_info *fs_info,
83 struct btrfs_qgroup *qgroup, u64 num_bytes,
84 enum btrfs_qgroup_rsv_type type)
86 trace_qgroup_update_reserve(fs_info, qgroup, num_bytes, type);
87 qgroup->rsv.values[type] += num_bytes;
90 static void qgroup_rsv_release(struct btrfs_fs_info *fs_info,
91 struct btrfs_qgroup *qgroup, u64 num_bytes,
92 enum btrfs_qgroup_rsv_type type)
94 trace_qgroup_update_reserve(fs_info, qgroup, -(s64)num_bytes, type);
95 if (qgroup->rsv.values[type] >= num_bytes) {
96 qgroup->rsv.values[type] -= num_bytes;
99 #ifdef CONFIG_BTRFS_DEBUG
101 "qgroup %llu %s reserved space underflow, have %llu to free %llu",
102 qgroup->qgroupid, qgroup_rsv_type_str(type),
103 qgroup->rsv.values[type], num_bytes);
105 qgroup->rsv.values[type] = 0;
108 static void qgroup_rsv_add_by_qgroup(struct btrfs_fs_info *fs_info,
109 struct btrfs_qgroup *dest,
110 struct btrfs_qgroup *src)
114 for (i = 0; i < BTRFS_QGROUP_RSV_LAST; i++)
115 qgroup_rsv_add(fs_info, dest, src->rsv.values[i], i);
118 static void qgroup_rsv_release_by_qgroup(struct btrfs_fs_info *fs_info,
119 struct btrfs_qgroup *dest,
120 struct btrfs_qgroup *src)
124 for (i = 0; i < BTRFS_QGROUP_RSV_LAST; i++)
125 qgroup_rsv_release(fs_info, dest, src->rsv.values[i], i);
128 static void btrfs_qgroup_update_old_refcnt(struct btrfs_qgroup *qg, u64 seq,
131 if (qg->old_refcnt < seq)
132 qg->old_refcnt = seq;
133 qg->old_refcnt += mod;
136 static void btrfs_qgroup_update_new_refcnt(struct btrfs_qgroup *qg, u64 seq,
139 if (qg->new_refcnt < seq)
140 qg->new_refcnt = seq;
141 qg->new_refcnt += mod;
144 static inline u64 btrfs_qgroup_get_old_refcnt(struct btrfs_qgroup *qg, u64 seq)
146 if (qg->old_refcnt < seq)
148 return qg->old_refcnt - seq;
151 static inline u64 btrfs_qgroup_get_new_refcnt(struct btrfs_qgroup *qg, u64 seq)
153 if (qg->new_refcnt < seq)
155 return qg->new_refcnt - seq;
159 * glue structure to represent the relations between qgroups.
161 struct btrfs_qgroup_list {
162 struct list_head next_group;
163 struct list_head next_member;
164 struct btrfs_qgroup *group;
165 struct btrfs_qgroup *member;
169 qgroup_rescan_init(struct btrfs_fs_info *fs_info, u64 progress_objectid,
171 static void qgroup_rescan_zero_tracking(struct btrfs_fs_info *fs_info);
173 /* must be called with qgroup_ioctl_lock held */
174 static struct btrfs_qgroup *find_qgroup_rb(struct btrfs_fs_info *fs_info,
177 struct rb_node *n = fs_info->qgroup_tree.rb_node;
178 struct btrfs_qgroup *qgroup;
181 qgroup = rb_entry(n, struct btrfs_qgroup, node);
182 if (qgroup->qgroupid < qgroupid)
184 else if (qgroup->qgroupid > qgroupid)
193 * Add qgroup to the filesystem's qgroup tree.
195 * Must be called with qgroup_lock held and @prealloc preallocated.
197 * The control on the lifespan of @prealloc would be transfered to this
198 * function, thus caller should no longer touch @prealloc.
200 static struct btrfs_qgroup *add_qgroup_rb(struct btrfs_fs_info *fs_info,
201 struct btrfs_qgroup *prealloc,
204 struct rb_node **p = &fs_info->qgroup_tree.rb_node;
205 struct rb_node *parent = NULL;
206 struct btrfs_qgroup *qgroup;
208 /* Caller must have pre-allocated @prealloc. */
213 qgroup = rb_entry(parent, struct btrfs_qgroup, node);
215 if (qgroup->qgroupid < qgroupid) {
217 } else if (qgroup->qgroupid > qgroupid) {
226 qgroup->qgroupid = qgroupid;
227 INIT_LIST_HEAD(&qgroup->groups);
228 INIT_LIST_HEAD(&qgroup->members);
229 INIT_LIST_HEAD(&qgroup->dirty);
230 INIT_LIST_HEAD(&qgroup->iterator);
231 INIT_LIST_HEAD(&qgroup->nested_iterator);
233 rb_link_node(&qgroup->node, parent, p);
234 rb_insert_color(&qgroup->node, &fs_info->qgroup_tree);
239 static void __del_qgroup_rb(struct btrfs_fs_info *fs_info,
240 struct btrfs_qgroup *qgroup)
242 struct btrfs_qgroup_list *list;
244 list_del(&qgroup->dirty);
245 while (!list_empty(&qgroup->groups)) {
246 list = list_first_entry(&qgroup->groups,
247 struct btrfs_qgroup_list, next_group);
248 list_del(&list->next_group);
249 list_del(&list->next_member);
253 while (!list_empty(&qgroup->members)) {
254 list = list_first_entry(&qgroup->members,
255 struct btrfs_qgroup_list, next_member);
256 list_del(&list->next_group);
257 list_del(&list->next_member);
262 /* must be called with qgroup_lock held */
263 static int del_qgroup_rb(struct btrfs_fs_info *fs_info, u64 qgroupid)
265 struct btrfs_qgroup *qgroup = find_qgroup_rb(fs_info, qgroupid);
270 rb_erase(&qgroup->node, &fs_info->qgroup_tree);
271 __del_qgroup_rb(fs_info, qgroup);
276 * Add relation specified by two qgroups.
278 * Must be called with qgroup_lock held, the ownership of @prealloc is
279 * transferred to this function and caller should not touch it anymore.
281 * Return: 0 on success
282 * -ENOENT if one of the qgroups is NULL
285 static int __add_relation_rb(struct btrfs_qgroup_list *prealloc,
286 struct btrfs_qgroup *member,
287 struct btrfs_qgroup *parent)
289 if (!member || !parent) {
294 prealloc->group = parent;
295 prealloc->member = member;
296 list_add_tail(&prealloc->next_group, &member->groups);
297 list_add_tail(&prealloc->next_member, &parent->members);
303 * Add relation specified by two qgroup ids.
305 * Must be called with qgroup_lock held.
307 * Return: 0 on success
308 * -ENOENT if one of the ids does not exist
311 static int add_relation_rb(struct btrfs_fs_info *fs_info,
312 struct btrfs_qgroup_list *prealloc,
313 u64 memberid, u64 parentid)
315 struct btrfs_qgroup *member;
316 struct btrfs_qgroup *parent;
318 member = find_qgroup_rb(fs_info, memberid);
319 parent = find_qgroup_rb(fs_info, parentid);
321 return __add_relation_rb(prealloc, member, parent);
324 /* Must be called with qgroup_lock held */
325 static int del_relation_rb(struct btrfs_fs_info *fs_info,
326 u64 memberid, u64 parentid)
328 struct btrfs_qgroup *member;
329 struct btrfs_qgroup *parent;
330 struct btrfs_qgroup_list *list;
332 member = find_qgroup_rb(fs_info, memberid);
333 parent = find_qgroup_rb(fs_info, parentid);
334 if (!member || !parent)
337 list_for_each_entry(list, &member->groups, next_group) {
338 if (list->group == parent) {
339 list_del(&list->next_group);
340 list_del(&list->next_member);
348 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
349 int btrfs_verify_qgroup_counts(struct btrfs_fs_info *fs_info, u64 qgroupid,
352 struct btrfs_qgroup *qgroup;
354 qgroup = find_qgroup_rb(fs_info, qgroupid);
357 if (qgroup->rfer != rfer || qgroup->excl != excl)
363 static void qgroup_mark_inconsistent(struct btrfs_fs_info *fs_info)
365 if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE)
367 fs_info->qgroup_flags |= (BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT |
368 BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN |
369 BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING);
372 static void qgroup_read_enable_gen(struct btrfs_fs_info *fs_info,
373 struct extent_buffer *leaf, int slot,
374 struct btrfs_qgroup_status_item *ptr)
376 ASSERT(btrfs_fs_incompat(fs_info, SIMPLE_QUOTA));
377 ASSERT(btrfs_item_size(leaf, slot) >= sizeof(*ptr));
378 fs_info->qgroup_enable_gen = btrfs_qgroup_status_enable_gen(leaf, ptr);
382 * The full config is read in one go, only called from open_ctree()
383 * It doesn't use any locking, as at this point we're still single-threaded
385 int btrfs_read_qgroup_config(struct btrfs_fs_info *fs_info)
387 struct btrfs_key key;
388 struct btrfs_key found_key;
389 struct btrfs_root *quota_root = fs_info->quota_root;
390 struct btrfs_path *path = NULL;
391 struct extent_buffer *l;
395 u64 rescan_progress = 0;
397 if (!fs_info->quota_root)
400 fs_info->qgroup_ulist = ulist_alloc(GFP_KERNEL);
401 if (!fs_info->qgroup_ulist) {
406 path = btrfs_alloc_path();
412 ret = btrfs_sysfs_add_qgroups(fs_info);
415 /* default this to quota off, in case no status key is found */
416 fs_info->qgroup_flags = 0;
419 * pass 1: read status, all qgroup infos and limits
424 ret = btrfs_search_slot_for_read(quota_root, &key, path, 1, 1);
429 struct btrfs_qgroup *qgroup;
431 slot = path->slots[0];
433 btrfs_item_key_to_cpu(l, &found_key, slot);
435 if (found_key.type == BTRFS_QGROUP_STATUS_KEY) {
436 struct btrfs_qgroup_status_item *ptr;
438 ptr = btrfs_item_ptr(l, slot,
439 struct btrfs_qgroup_status_item);
441 if (btrfs_qgroup_status_version(l, ptr) !=
442 BTRFS_QGROUP_STATUS_VERSION) {
444 "old qgroup version, quota disabled");
447 fs_info->qgroup_flags = btrfs_qgroup_status_flags(l, ptr);
448 if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE) {
449 qgroup_read_enable_gen(fs_info, l, slot, ptr);
450 } else if (btrfs_qgroup_status_generation(l, ptr) != fs_info->generation) {
451 qgroup_mark_inconsistent(fs_info);
453 "qgroup generation mismatch, marked as inconsistent");
455 rescan_progress = btrfs_qgroup_status_rescan(l, ptr);
459 if (found_key.type != BTRFS_QGROUP_INFO_KEY &&
460 found_key.type != BTRFS_QGROUP_LIMIT_KEY)
463 qgroup = find_qgroup_rb(fs_info, found_key.offset);
464 if ((qgroup && found_key.type == BTRFS_QGROUP_INFO_KEY) ||
465 (!qgroup && found_key.type == BTRFS_QGROUP_LIMIT_KEY)) {
466 btrfs_err(fs_info, "inconsistent qgroup config");
467 qgroup_mark_inconsistent(fs_info);
470 struct btrfs_qgroup *prealloc;
472 prealloc = kzalloc(sizeof(*prealloc), GFP_KERNEL);
477 qgroup = add_qgroup_rb(fs_info, prealloc, found_key.offset);
479 ret = btrfs_sysfs_add_one_qgroup(fs_info, qgroup);
483 switch (found_key.type) {
484 case BTRFS_QGROUP_INFO_KEY: {
485 struct btrfs_qgroup_info_item *ptr;
487 ptr = btrfs_item_ptr(l, slot,
488 struct btrfs_qgroup_info_item);
489 qgroup->rfer = btrfs_qgroup_info_rfer(l, ptr);
490 qgroup->rfer_cmpr = btrfs_qgroup_info_rfer_cmpr(l, ptr);
491 qgroup->excl = btrfs_qgroup_info_excl(l, ptr);
492 qgroup->excl_cmpr = btrfs_qgroup_info_excl_cmpr(l, ptr);
493 /* generation currently unused */
496 case BTRFS_QGROUP_LIMIT_KEY: {
497 struct btrfs_qgroup_limit_item *ptr;
499 ptr = btrfs_item_ptr(l, slot,
500 struct btrfs_qgroup_limit_item);
501 qgroup->lim_flags = btrfs_qgroup_limit_flags(l, ptr);
502 qgroup->max_rfer = btrfs_qgroup_limit_max_rfer(l, ptr);
503 qgroup->max_excl = btrfs_qgroup_limit_max_excl(l, ptr);
504 qgroup->rsv_rfer = btrfs_qgroup_limit_rsv_rfer(l, ptr);
505 qgroup->rsv_excl = btrfs_qgroup_limit_rsv_excl(l, ptr);
510 ret = btrfs_next_item(quota_root, path);
516 btrfs_release_path(path);
519 * pass 2: read all qgroup relations
522 key.type = BTRFS_QGROUP_RELATION_KEY;
524 ret = btrfs_search_slot_for_read(quota_root, &key, path, 1, 0);
528 struct btrfs_qgroup_list *list = NULL;
530 slot = path->slots[0];
532 btrfs_item_key_to_cpu(l, &found_key, slot);
534 if (found_key.type != BTRFS_QGROUP_RELATION_KEY)
537 if (found_key.objectid > found_key.offset) {
538 /* parent <- member, not needed to build config */
539 /* FIXME should we omit the key completely? */
543 list = kzalloc(sizeof(*list), GFP_KERNEL);
548 ret = add_relation_rb(fs_info, list, found_key.objectid,
551 if (ret == -ENOENT) {
553 "orphan qgroup relation 0x%llx->0x%llx",
554 found_key.objectid, found_key.offset);
555 ret = 0; /* ignore the error */
560 ret = btrfs_next_item(quota_root, path);
567 btrfs_free_path(path);
568 fs_info->qgroup_flags |= flags;
570 if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_ON)
571 set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags);
572 if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_RESCAN)
573 ret = qgroup_rescan_init(fs_info, rescan_progress, 0);
575 ulist_free(fs_info->qgroup_ulist);
576 fs_info->qgroup_ulist = NULL;
577 fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_RESCAN;
578 btrfs_sysfs_del_qgroups(fs_info);
581 return ret < 0 ? ret : 0;
585 * Called in close_ctree() when quota is still enabled. This verifies we don't
586 * leak some reserved space.
588 * Return false if no reserved space is left.
589 * Return true if some reserved space is leaked.
591 bool btrfs_check_quota_leak(struct btrfs_fs_info *fs_info)
593 struct rb_node *node;
596 if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_DISABLED)
599 * Since we're unmounting, there is no race and no need to grab qgroup
600 * lock. And here we don't go post-order to provide a more user
601 * friendly sorted result.
603 for (node = rb_first(&fs_info->qgroup_tree); node; node = rb_next(node)) {
604 struct btrfs_qgroup *qgroup;
607 qgroup = rb_entry(node, struct btrfs_qgroup, node);
608 for (i = 0; i < BTRFS_QGROUP_RSV_LAST; i++) {
609 if (qgroup->rsv.values[i]) {
612 "qgroup %hu/%llu has unreleased space, type %d rsv %llu",
613 btrfs_qgroup_level(qgroup->qgroupid),
614 btrfs_qgroup_subvolid(qgroup->qgroupid),
615 i, qgroup->rsv.values[i]);
623 * This is called from close_ctree() or open_ctree() or btrfs_quota_disable(),
624 * first two are in single-threaded paths.And for the third one, we have set
625 * quota_root to be null with qgroup_lock held before, so it is safe to clean
626 * up the in-memory structures without qgroup_lock held.
628 void btrfs_free_qgroup_config(struct btrfs_fs_info *fs_info)
631 struct btrfs_qgroup *qgroup;
633 while ((n = rb_first(&fs_info->qgroup_tree))) {
634 qgroup = rb_entry(n, struct btrfs_qgroup, node);
635 rb_erase(n, &fs_info->qgroup_tree);
636 __del_qgroup_rb(fs_info, qgroup);
637 btrfs_sysfs_del_one_qgroup(fs_info, qgroup);
641 * We call btrfs_free_qgroup_config() when unmounting
642 * filesystem and disabling quota, so we set qgroup_ulist
643 * to be null here to avoid double free.
645 ulist_free(fs_info->qgroup_ulist);
646 fs_info->qgroup_ulist = NULL;
647 btrfs_sysfs_del_qgroups(fs_info);
650 static int add_qgroup_relation_item(struct btrfs_trans_handle *trans, u64 src,
654 struct btrfs_root *quota_root = trans->fs_info->quota_root;
655 struct btrfs_path *path;
656 struct btrfs_key key;
658 path = btrfs_alloc_path();
663 key.type = BTRFS_QGROUP_RELATION_KEY;
666 ret = btrfs_insert_empty_item(trans, quota_root, path, &key, 0);
668 btrfs_mark_buffer_dirty(trans, path->nodes[0]);
670 btrfs_free_path(path);
674 static int del_qgroup_relation_item(struct btrfs_trans_handle *trans, u64 src,
678 struct btrfs_root *quota_root = trans->fs_info->quota_root;
679 struct btrfs_path *path;
680 struct btrfs_key key;
682 path = btrfs_alloc_path();
687 key.type = BTRFS_QGROUP_RELATION_KEY;
690 ret = btrfs_search_slot(trans, quota_root, &key, path, -1, 1);
699 ret = btrfs_del_item(trans, quota_root, path);
701 btrfs_free_path(path);
705 static int add_qgroup_item(struct btrfs_trans_handle *trans,
706 struct btrfs_root *quota_root, u64 qgroupid)
709 struct btrfs_path *path;
710 struct btrfs_qgroup_info_item *qgroup_info;
711 struct btrfs_qgroup_limit_item *qgroup_limit;
712 struct extent_buffer *leaf;
713 struct btrfs_key key;
715 if (btrfs_is_testing(quota_root->fs_info))
718 path = btrfs_alloc_path();
723 key.type = BTRFS_QGROUP_INFO_KEY;
724 key.offset = qgroupid;
727 * Avoid a transaction abort by catching -EEXIST here. In that
728 * case, we proceed by re-initializing the existing structure
732 ret = btrfs_insert_empty_item(trans, quota_root, path, &key,
733 sizeof(*qgroup_info));
734 if (ret && ret != -EEXIST)
737 leaf = path->nodes[0];
738 qgroup_info = btrfs_item_ptr(leaf, path->slots[0],
739 struct btrfs_qgroup_info_item);
740 btrfs_set_qgroup_info_generation(leaf, qgroup_info, trans->transid);
741 btrfs_set_qgroup_info_rfer(leaf, qgroup_info, 0);
742 btrfs_set_qgroup_info_rfer_cmpr(leaf, qgroup_info, 0);
743 btrfs_set_qgroup_info_excl(leaf, qgroup_info, 0);
744 btrfs_set_qgroup_info_excl_cmpr(leaf, qgroup_info, 0);
746 btrfs_mark_buffer_dirty(trans, leaf);
748 btrfs_release_path(path);
750 key.type = BTRFS_QGROUP_LIMIT_KEY;
751 ret = btrfs_insert_empty_item(trans, quota_root, path, &key,
752 sizeof(*qgroup_limit));
753 if (ret && ret != -EEXIST)
756 leaf = path->nodes[0];
757 qgroup_limit = btrfs_item_ptr(leaf, path->slots[0],
758 struct btrfs_qgroup_limit_item);
759 btrfs_set_qgroup_limit_flags(leaf, qgroup_limit, 0);
760 btrfs_set_qgroup_limit_max_rfer(leaf, qgroup_limit, 0);
761 btrfs_set_qgroup_limit_max_excl(leaf, qgroup_limit, 0);
762 btrfs_set_qgroup_limit_rsv_rfer(leaf, qgroup_limit, 0);
763 btrfs_set_qgroup_limit_rsv_excl(leaf, qgroup_limit, 0);
765 btrfs_mark_buffer_dirty(trans, leaf);
769 btrfs_free_path(path);
773 static int del_qgroup_item(struct btrfs_trans_handle *trans, u64 qgroupid)
776 struct btrfs_root *quota_root = trans->fs_info->quota_root;
777 struct btrfs_path *path;
778 struct btrfs_key key;
780 path = btrfs_alloc_path();
785 key.type = BTRFS_QGROUP_INFO_KEY;
786 key.offset = qgroupid;
787 ret = btrfs_search_slot(trans, quota_root, &key, path, -1, 1);
796 ret = btrfs_del_item(trans, quota_root, path);
800 btrfs_release_path(path);
802 key.type = BTRFS_QGROUP_LIMIT_KEY;
803 ret = btrfs_search_slot(trans, quota_root, &key, path, -1, 1);
812 ret = btrfs_del_item(trans, quota_root, path);
815 btrfs_free_path(path);
819 static int update_qgroup_limit_item(struct btrfs_trans_handle *trans,
820 struct btrfs_qgroup *qgroup)
822 struct btrfs_root *quota_root = trans->fs_info->quota_root;
823 struct btrfs_path *path;
824 struct btrfs_key key;
825 struct extent_buffer *l;
826 struct btrfs_qgroup_limit_item *qgroup_limit;
831 key.type = BTRFS_QGROUP_LIMIT_KEY;
832 key.offset = qgroup->qgroupid;
834 path = btrfs_alloc_path();
838 ret = btrfs_search_slot(trans, quota_root, &key, path, 0, 1);
846 slot = path->slots[0];
847 qgroup_limit = btrfs_item_ptr(l, slot, struct btrfs_qgroup_limit_item);
848 btrfs_set_qgroup_limit_flags(l, qgroup_limit, qgroup->lim_flags);
849 btrfs_set_qgroup_limit_max_rfer(l, qgroup_limit, qgroup->max_rfer);
850 btrfs_set_qgroup_limit_max_excl(l, qgroup_limit, qgroup->max_excl);
851 btrfs_set_qgroup_limit_rsv_rfer(l, qgroup_limit, qgroup->rsv_rfer);
852 btrfs_set_qgroup_limit_rsv_excl(l, qgroup_limit, qgroup->rsv_excl);
854 btrfs_mark_buffer_dirty(trans, l);
857 btrfs_free_path(path);
861 static int update_qgroup_info_item(struct btrfs_trans_handle *trans,
862 struct btrfs_qgroup *qgroup)
864 struct btrfs_fs_info *fs_info = trans->fs_info;
865 struct btrfs_root *quota_root = fs_info->quota_root;
866 struct btrfs_path *path;
867 struct btrfs_key key;
868 struct extent_buffer *l;
869 struct btrfs_qgroup_info_item *qgroup_info;
873 if (btrfs_is_testing(fs_info))
877 key.type = BTRFS_QGROUP_INFO_KEY;
878 key.offset = qgroup->qgroupid;
880 path = btrfs_alloc_path();
884 ret = btrfs_search_slot(trans, quota_root, &key, path, 0, 1);
892 slot = path->slots[0];
893 qgroup_info = btrfs_item_ptr(l, slot, struct btrfs_qgroup_info_item);
894 btrfs_set_qgroup_info_generation(l, qgroup_info, trans->transid);
895 btrfs_set_qgroup_info_rfer(l, qgroup_info, qgroup->rfer);
896 btrfs_set_qgroup_info_rfer_cmpr(l, qgroup_info, qgroup->rfer_cmpr);
897 btrfs_set_qgroup_info_excl(l, qgroup_info, qgroup->excl);
898 btrfs_set_qgroup_info_excl_cmpr(l, qgroup_info, qgroup->excl_cmpr);
900 btrfs_mark_buffer_dirty(trans, l);
903 btrfs_free_path(path);
907 static int update_qgroup_status_item(struct btrfs_trans_handle *trans)
909 struct btrfs_fs_info *fs_info = trans->fs_info;
910 struct btrfs_root *quota_root = fs_info->quota_root;
911 struct btrfs_path *path;
912 struct btrfs_key key;
913 struct extent_buffer *l;
914 struct btrfs_qgroup_status_item *ptr;
919 key.type = BTRFS_QGROUP_STATUS_KEY;
922 path = btrfs_alloc_path();
926 ret = btrfs_search_slot(trans, quota_root, &key, path, 0, 1);
934 slot = path->slots[0];
935 ptr = btrfs_item_ptr(l, slot, struct btrfs_qgroup_status_item);
936 btrfs_set_qgroup_status_flags(l, ptr, fs_info->qgroup_flags &
937 BTRFS_QGROUP_STATUS_FLAGS_MASK);
938 btrfs_set_qgroup_status_generation(l, ptr, trans->transid);
939 btrfs_set_qgroup_status_rescan(l, ptr,
940 fs_info->qgroup_rescan_progress.objectid);
942 btrfs_mark_buffer_dirty(trans, l);
945 btrfs_free_path(path);
950 * called with qgroup_lock held
952 static int btrfs_clean_quota_tree(struct btrfs_trans_handle *trans,
953 struct btrfs_root *root)
955 struct btrfs_path *path;
956 struct btrfs_key key;
957 struct extent_buffer *leaf = NULL;
961 path = btrfs_alloc_path();
970 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
973 leaf = path->nodes[0];
974 nr = btrfs_header_nritems(leaf);
978 * delete the leaf one by one
979 * since the whole tree is going
983 ret = btrfs_del_items(trans, root, path, 0, nr);
987 btrfs_release_path(path);
991 btrfs_free_path(path);
995 int btrfs_quota_enable(struct btrfs_fs_info *fs_info,
996 struct btrfs_ioctl_quota_ctl_args *quota_ctl_args)
998 struct btrfs_root *quota_root;
999 struct btrfs_root *tree_root = fs_info->tree_root;
1000 struct btrfs_path *path = NULL;
1001 struct btrfs_qgroup_status_item *ptr;
1002 struct extent_buffer *leaf;
1003 struct btrfs_key key;
1004 struct btrfs_key found_key;
1005 struct btrfs_qgroup *qgroup = NULL;
1006 struct btrfs_qgroup *prealloc = NULL;
1007 struct btrfs_trans_handle *trans = NULL;
1008 struct ulist *ulist = NULL;
1009 const bool simple = (quota_ctl_args->cmd == BTRFS_QUOTA_CTL_ENABLE_SIMPLE_QUOTA);
1014 * We need to have subvol_sem write locked, to prevent races between
1015 * concurrent tasks trying to enable quotas, because we will unlock
1016 * and relock qgroup_ioctl_lock before setting fs_info->quota_root
1017 * and before setting BTRFS_FS_QUOTA_ENABLED.
1019 lockdep_assert_held_write(&fs_info->subvol_sem);
1021 if (btrfs_fs_incompat(fs_info, EXTENT_TREE_V2)) {
1023 "qgroups are currently unsupported in extent tree v2");
1027 mutex_lock(&fs_info->qgroup_ioctl_lock);
1028 if (fs_info->quota_root)
1031 ulist = ulist_alloc(GFP_KERNEL);
1037 ret = btrfs_sysfs_add_qgroups(fs_info);
1042 * Unlock qgroup_ioctl_lock before starting the transaction. This is to
1043 * avoid lock acquisition inversion problems (reported by lockdep) between
1044 * qgroup_ioctl_lock and the vfs freeze semaphores, acquired when we
1045 * start a transaction.
1046 * After we started the transaction lock qgroup_ioctl_lock again and
1047 * check if someone else created the quota root in the meanwhile. If so,
1048 * just return success and release the transaction handle.
1050 * Also we don't need to worry about someone else calling
1051 * btrfs_sysfs_add_qgroups() after we unlock and getting an error because
1052 * that function returns 0 (success) when the sysfs entries already exist.
1054 mutex_unlock(&fs_info->qgroup_ioctl_lock);
1057 * 1 for quota root item
1058 * 1 for BTRFS_QGROUP_STATUS item
1060 * Yet we also need 2*n items for a QGROUP_INFO/QGROUP_LIMIT items
1061 * per subvolume. However those are not currently reserved since it
1062 * would be a lot of overkill.
1064 trans = btrfs_start_transaction(tree_root, 2);
1066 mutex_lock(&fs_info->qgroup_ioctl_lock);
1067 if (IS_ERR(trans)) {
1068 ret = PTR_ERR(trans);
1073 if (fs_info->quota_root)
1076 fs_info->qgroup_ulist = ulist;
1080 * initially create the quota tree
1082 quota_root = btrfs_create_tree(trans, BTRFS_QUOTA_TREE_OBJECTID);
1083 if (IS_ERR(quota_root)) {
1084 ret = PTR_ERR(quota_root);
1085 btrfs_abort_transaction(trans, ret);
1089 path = btrfs_alloc_path();
1092 btrfs_abort_transaction(trans, ret);
1097 key.type = BTRFS_QGROUP_STATUS_KEY;
1100 ret = btrfs_insert_empty_item(trans, quota_root, path, &key,
1103 btrfs_abort_transaction(trans, ret);
1107 leaf = path->nodes[0];
1108 ptr = btrfs_item_ptr(leaf, path->slots[0],
1109 struct btrfs_qgroup_status_item);
1110 btrfs_set_qgroup_status_generation(leaf, ptr, trans->transid);
1111 btrfs_set_qgroup_status_version(leaf, ptr, BTRFS_QGROUP_STATUS_VERSION);
1112 fs_info->qgroup_flags = BTRFS_QGROUP_STATUS_FLAG_ON;
1114 fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE;
1115 btrfs_set_qgroup_status_enable_gen(leaf, ptr, trans->transid);
1117 fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT;
1119 btrfs_set_qgroup_status_flags(leaf, ptr, fs_info->qgroup_flags &
1120 BTRFS_QGROUP_STATUS_FLAGS_MASK);
1121 btrfs_set_qgroup_status_rescan(leaf, ptr, 0);
1123 btrfs_mark_buffer_dirty(trans, leaf);
1126 key.type = BTRFS_ROOT_REF_KEY;
1129 btrfs_release_path(path);
1130 ret = btrfs_search_slot_for_read(tree_root, &key, path, 1, 0);
1134 btrfs_abort_transaction(trans, ret);
1139 slot = path->slots[0];
1140 leaf = path->nodes[0];
1141 btrfs_item_key_to_cpu(leaf, &found_key, slot);
1143 if (found_key.type == BTRFS_ROOT_REF_KEY) {
1145 /* Release locks on tree_root before we access quota_root */
1146 btrfs_release_path(path);
1148 /* We should not have a stray @prealloc pointer. */
1149 ASSERT(prealloc == NULL);
1150 prealloc = kzalloc(sizeof(*prealloc), GFP_NOFS);
1153 btrfs_abort_transaction(trans, ret);
1157 ret = add_qgroup_item(trans, quota_root,
1160 btrfs_abort_transaction(trans, ret);
1164 qgroup = add_qgroup_rb(fs_info, prealloc, found_key.offset);
1166 if (IS_ERR(qgroup)) {
1167 ret = PTR_ERR(qgroup);
1168 btrfs_abort_transaction(trans, ret);
1171 ret = btrfs_sysfs_add_one_qgroup(fs_info, qgroup);
1173 btrfs_abort_transaction(trans, ret);
1176 ret = btrfs_search_slot_for_read(tree_root, &found_key,
1179 btrfs_abort_transaction(trans, ret);
1184 * Shouldn't happen, but in case it does we
1185 * don't need to do the btrfs_next_item, just
1191 ret = btrfs_next_item(tree_root, path);
1193 btrfs_abort_transaction(trans, ret);
1201 btrfs_release_path(path);
1202 ret = add_qgroup_item(trans, quota_root, BTRFS_FS_TREE_OBJECTID);
1204 btrfs_abort_transaction(trans, ret);
1208 ASSERT(prealloc == NULL);
1209 prealloc = kzalloc(sizeof(*prealloc), GFP_NOFS);
1214 qgroup = add_qgroup_rb(fs_info, prealloc, BTRFS_FS_TREE_OBJECTID);
1216 ret = btrfs_sysfs_add_one_qgroup(fs_info, qgroup);
1218 btrfs_abort_transaction(trans, ret);
1222 fs_info->qgroup_enable_gen = trans->transid;
1224 mutex_unlock(&fs_info->qgroup_ioctl_lock);
1226 * Commit the transaction while not holding qgroup_ioctl_lock, to avoid
1227 * a deadlock with tasks concurrently doing other qgroup operations, such
1228 * adding/removing qgroups or adding/deleting qgroup relations for example,
1229 * because all qgroup operations first start or join a transaction and then
1230 * lock the qgroup_ioctl_lock mutex.
1231 * We are safe from a concurrent task trying to enable quotas, by calling
1232 * this function, since we are serialized by fs_info->subvol_sem.
1234 ret = btrfs_commit_transaction(trans);
1236 mutex_lock(&fs_info->qgroup_ioctl_lock);
1241 * Set quota enabled flag after committing the transaction, to avoid
1242 * deadlocks on fs_info->qgroup_ioctl_lock with concurrent snapshot
1245 spin_lock(&fs_info->qgroup_lock);
1246 fs_info->quota_root = quota_root;
1247 set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags);
1249 btrfs_set_fs_incompat(fs_info, SIMPLE_QUOTA);
1250 spin_unlock(&fs_info->qgroup_lock);
1252 /* Skip rescan for simple qgroups. */
1253 if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE)
1256 ret = qgroup_rescan_init(fs_info, 0, 1);
1258 qgroup_rescan_zero_tracking(fs_info);
1259 fs_info->qgroup_rescan_running = true;
1260 btrfs_queue_work(fs_info->qgroup_rescan_workers,
1261 &fs_info->qgroup_rescan_work);
1264 * We have set both BTRFS_FS_QUOTA_ENABLED and
1265 * BTRFS_QGROUP_STATUS_FLAG_ON, so we can only fail with
1266 * -EINPROGRESS. That can happen because someone started the
1267 * rescan worker by calling quota rescan ioctl before we
1268 * attempted to initialize the rescan worker. Failure due to
1269 * quotas disabled in the meanwhile is not possible, because
1270 * we are holding a write lock on fs_info->subvol_sem, which
1271 * is also acquired when disabling quotas.
1272 * Ignore such error, and any other error would need to undo
1273 * everything we did in the transaction we just committed.
1275 ASSERT(ret == -EINPROGRESS);
1280 btrfs_free_path(path);
1283 btrfs_put_root(quota_root);
1286 ulist_free(fs_info->qgroup_ulist);
1287 fs_info->qgroup_ulist = NULL;
1288 btrfs_sysfs_del_qgroups(fs_info);
1290 mutex_unlock(&fs_info->qgroup_ioctl_lock);
1292 btrfs_end_transaction(trans);
1294 ret = btrfs_end_transaction(trans);
1301 * It is possible to have outstanding ordered extents which reserved bytes
1302 * before we disabled. We need to fully flush delalloc, ordered extents, and a
1303 * commit to ensure that we don't leak such reservations, only to have them
1304 * come back if we re-enable.
1306 * - enable simple quotas
1308 * - release it, store rsv_bytes in OE
1310 * - enable simple quotas (qgroup rsv are all 0)
1312 * - run delayed refs
1313 * - free rsv_bytes, resulting in miscounting or even underflow
1315 static int flush_reservations(struct btrfs_fs_info *fs_info)
1317 struct btrfs_trans_handle *trans;
1320 ret = btrfs_start_delalloc_roots(fs_info, LONG_MAX, false);
1323 btrfs_wait_ordered_roots(fs_info, U64_MAX, 0, (u64)-1);
1324 trans = btrfs_join_transaction(fs_info->tree_root);
1326 return PTR_ERR(trans);
1327 btrfs_commit_transaction(trans);
1332 int btrfs_quota_disable(struct btrfs_fs_info *fs_info)
1334 struct btrfs_root *quota_root;
1335 struct btrfs_trans_handle *trans = NULL;
1339 * We need to have subvol_sem write locked to prevent races with
1340 * snapshot creation.
1342 lockdep_assert_held_write(&fs_info->subvol_sem);
1345 * Lock the cleaner mutex to prevent races with concurrent relocation,
1346 * because relocation may be building backrefs for blocks of the quota
1347 * root while we are deleting the root. This is like dropping fs roots
1348 * of deleted snapshots/subvolumes, we need the same protection.
1350 * This also prevents races between concurrent tasks trying to disable
1351 * quotas, because we will unlock and relock qgroup_ioctl_lock across
1352 * BTRFS_FS_QUOTA_ENABLED changes.
1354 mutex_lock(&fs_info->cleaner_mutex);
1356 mutex_lock(&fs_info->qgroup_ioctl_lock);
1357 if (!fs_info->quota_root)
1361 * Unlock the qgroup_ioctl_lock mutex before waiting for the rescan worker to
1362 * complete. Otherwise we can deadlock because btrfs_remove_qgroup() needs
1363 * to lock that mutex while holding a transaction handle and the rescan
1364 * worker needs to commit a transaction.
1366 mutex_unlock(&fs_info->qgroup_ioctl_lock);
1369 * Request qgroup rescan worker to complete and wait for it. This wait
1370 * must be done before transaction start for quota disable since it may
1371 * deadlock with transaction by the qgroup rescan worker.
1373 clear_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags);
1374 btrfs_qgroup_wait_for_completion(fs_info, false);
1376 ret = flush_reservations(fs_info);
1378 goto out_unlock_cleaner;
1381 * 1 For the root item
1383 * We should also reserve enough items for the quota tree deletion in
1384 * btrfs_clean_quota_tree but this is not done.
1386 * Also, we must always start a transaction without holding the mutex
1387 * qgroup_ioctl_lock, see btrfs_quota_enable().
1389 trans = btrfs_start_transaction(fs_info->tree_root, 1);
1391 mutex_lock(&fs_info->qgroup_ioctl_lock);
1392 if (IS_ERR(trans)) {
1393 ret = PTR_ERR(trans);
1395 set_bit(BTRFS_FS_QUOTA_ENABLED, &fs_info->flags);
1399 if (!fs_info->quota_root)
1402 spin_lock(&fs_info->qgroup_lock);
1403 quota_root = fs_info->quota_root;
1404 fs_info->quota_root = NULL;
1405 fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_ON;
1406 fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_SIMPLE_MODE;
1407 fs_info->qgroup_drop_subtree_thres = BTRFS_MAX_LEVEL;
1408 spin_unlock(&fs_info->qgroup_lock);
1410 btrfs_free_qgroup_config(fs_info);
1412 ret = btrfs_clean_quota_tree(trans, quota_root);
1414 btrfs_abort_transaction(trans, ret);
1418 ret = btrfs_del_root(trans, "a_root->root_key);
1420 btrfs_abort_transaction(trans, ret);
1424 spin_lock(&fs_info->trans_lock);
1425 list_del("a_root->dirty_list);
1426 spin_unlock(&fs_info->trans_lock);
1428 btrfs_tree_lock(quota_root->node);
1429 btrfs_clear_buffer_dirty(trans, quota_root->node);
1430 btrfs_tree_unlock(quota_root->node);
1431 btrfs_free_tree_block(trans, btrfs_root_id(quota_root),
1432 quota_root->node, 0, 1);
1434 btrfs_put_root(quota_root);
1437 mutex_unlock(&fs_info->qgroup_ioctl_lock);
1439 btrfs_end_transaction(trans);
1441 ret = btrfs_commit_transaction(trans);
1443 mutex_unlock(&fs_info->cleaner_mutex);
1448 static void qgroup_dirty(struct btrfs_fs_info *fs_info,
1449 struct btrfs_qgroup *qgroup)
1451 if (list_empty(&qgroup->dirty))
1452 list_add(&qgroup->dirty, &fs_info->dirty_qgroups);
1455 static void qgroup_iterator_add(struct list_head *head, struct btrfs_qgroup *qgroup)
1457 if (!list_empty(&qgroup->iterator))
1460 list_add_tail(&qgroup->iterator, head);
1463 static void qgroup_iterator_clean(struct list_head *head)
1465 while (!list_empty(head)) {
1466 struct btrfs_qgroup *qgroup;
1468 qgroup = list_first_entry(head, struct btrfs_qgroup, iterator);
1469 list_del_init(&qgroup->iterator);
1474 * The easy accounting, we're updating qgroup relationship whose child qgroup
1475 * only has exclusive extents.
1477 * In this case, all exclusive extents will also be exclusive for parent, so
1478 * excl/rfer just get added/removed.
1480 * So is qgroup reservation space, which should also be added/removed to
1482 * Or when child tries to release reservation space, parent will underflow its
1483 * reservation (for relationship adding case).
1485 * Caller should hold fs_info->qgroup_lock.
1487 static int __qgroup_excl_accounting(struct btrfs_fs_info *fs_info, u64 ref_root,
1488 struct btrfs_qgroup *src, int sign)
1490 struct btrfs_qgroup *qgroup;
1491 struct btrfs_qgroup *cur;
1492 LIST_HEAD(qgroup_list);
1493 u64 num_bytes = src->excl;
1496 qgroup = find_qgroup_rb(fs_info, ref_root);
1500 qgroup_iterator_add(&qgroup_list, qgroup);
1501 list_for_each_entry(cur, &qgroup_list, iterator) {
1502 struct btrfs_qgroup_list *glist;
1504 qgroup->rfer += sign * num_bytes;
1505 qgroup->rfer_cmpr += sign * num_bytes;
1507 WARN_ON(sign < 0 && qgroup->excl < num_bytes);
1508 qgroup->excl += sign * num_bytes;
1509 qgroup->excl_cmpr += sign * num_bytes;
1512 qgroup_rsv_add_by_qgroup(fs_info, qgroup, src);
1514 qgroup_rsv_release_by_qgroup(fs_info, qgroup, src);
1515 qgroup_dirty(fs_info, qgroup);
1517 /* Append parent qgroups to @qgroup_list. */
1518 list_for_each_entry(glist, &qgroup->groups, next_group)
1519 qgroup_iterator_add(&qgroup_list, glist->group);
1523 qgroup_iterator_clean(&qgroup_list);
1529 * Quick path for updating qgroup with only excl refs.
1531 * In that case, just update all parent will be enough.
1532 * Or we needs to do a full rescan.
1533 * Caller should also hold fs_info->qgroup_lock.
1535 * Return 0 for quick update, return >0 for need to full rescan
1536 * and mark INCONSISTENT flag.
1537 * Return < 0 for other error.
1539 static int quick_update_accounting(struct btrfs_fs_info *fs_info,
1540 u64 src, u64 dst, int sign)
1542 struct btrfs_qgroup *qgroup;
1546 qgroup = find_qgroup_rb(fs_info, src);
1549 if (qgroup->excl == qgroup->rfer) {
1551 err = __qgroup_excl_accounting(fs_info, dst, qgroup, sign);
1559 fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT;
1563 int btrfs_add_qgroup_relation(struct btrfs_trans_handle *trans, u64 src, u64 dst)
1565 struct btrfs_fs_info *fs_info = trans->fs_info;
1566 struct btrfs_qgroup *parent;
1567 struct btrfs_qgroup *member;
1568 struct btrfs_qgroup_list *list;
1569 struct btrfs_qgroup_list *prealloc = NULL;
1572 /* Check the level of src and dst first */
1573 if (btrfs_qgroup_level(src) >= btrfs_qgroup_level(dst))
1576 mutex_lock(&fs_info->qgroup_ioctl_lock);
1577 if (!fs_info->quota_root) {
1581 member = find_qgroup_rb(fs_info, src);
1582 parent = find_qgroup_rb(fs_info, dst);
1583 if (!member || !parent) {
1588 /* check if such qgroup relation exist firstly */
1589 list_for_each_entry(list, &member->groups, next_group) {
1590 if (list->group == parent) {
1596 prealloc = kzalloc(sizeof(*list), GFP_NOFS);
1601 ret = add_qgroup_relation_item(trans, src, dst);
1605 ret = add_qgroup_relation_item(trans, dst, src);
1607 del_qgroup_relation_item(trans, src, dst);
1611 spin_lock(&fs_info->qgroup_lock);
1612 ret = __add_relation_rb(prealloc, member, parent);
1615 spin_unlock(&fs_info->qgroup_lock);
1618 ret = quick_update_accounting(fs_info, src, dst, 1);
1619 spin_unlock(&fs_info->qgroup_lock);
1622 mutex_unlock(&fs_info->qgroup_ioctl_lock);
1626 static int __del_qgroup_relation(struct btrfs_trans_handle *trans, u64 src,
1629 struct btrfs_fs_info *fs_info = trans->fs_info;
1630 struct btrfs_qgroup *parent;
1631 struct btrfs_qgroup *member;
1632 struct btrfs_qgroup_list *list;
1637 if (!fs_info->quota_root) {
1642 member = find_qgroup_rb(fs_info, src);
1643 parent = find_qgroup_rb(fs_info, dst);
1645 * The parent/member pair doesn't exist, then try to delete the dead
1646 * relation items only.
1648 if (!member || !parent)
1651 /* check if such qgroup relation exist firstly */
1652 list_for_each_entry(list, &member->groups, next_group) {
1653 if (list->group == parent) {
1660 ret = del_qgroup_relation_item(trans, src, dst);
1661 if (ret < 0 && ret != -ENOENT)
1663 ret2 = del_qgroup_relation_item(trans, dst, src);
1664 if (ret2 < 0 && ret2 != -ENOENT)
1667 /* At least one deletion succeeded, return 0 */
1672 spin_lock(&fs_info->qgroup_lock);
1673 del_relation_rb(fs_info, src, dst);
1674 ret = quick_update_accounting(fs_info, src, dst, -1);
1675 spin_unlock(&fs_info->qgroup_lock);
1681 int btrfs_del_qgroup_relation(struct btrfs_trans_handle *trans, u64 src,
1684 struct btrfs_fs_info *fs_info = trans->fs_info;
1687 mutex_lock(&fs_info->qgroup_ioctl_lock);
1688 ret = __del_qgroup_relation(trans, src, dst);
1689 mutex_unlock(&fs_info->qgroup_ioctl_lock);
1694 int btrfs_create_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid)
1696 struct btrfs_fs_info *fs_info = trans->fs_info;
1697 struct btrfs_root *quota_root;
1698 struct btrfs_qgroup *qgroup;
1699 struct btrfs_qgroup *prealloc = NULL;
1702 if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_DISABLED)
1705 mutex_lock(&fs_info->qgroup_ioctl_lock);
1706 if (!fs_info->quota_root) {
1710 quota_root = fs_info->quota_root;
1711 qgroup = find_qgroup_rb(fs_info, qgroupid);
1717 prealloc = kzalloc(sizeof(*prealloc), GFP_NOFS);
1723 ret = add_qgroup_item(trans, quota_root, qgroupid);
1727 spin_lock(&fs_info->qgroup_lock);
1728 qgroup = add_qgroup_rb(fs_info, prealloc, qgroupid);
1729 spin_unlock(&fs_info->qgroup_lock);
1732 ret = btrfs_sysfs_add_one_qgroup(fs_info, qgroup);
1734 mutex_unlock(&fs_info->qgroup_ioctl_lock);
1739 int btrfs_remove_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid)
1741 struct btrfs_fs_info *fs_info = trans->fs_info;
1742 struct btrfs_qgroup *qgroup;
1743 struct btrfs_qgroup_list *list;
1746 mutex_lock(&fs_info->qgroup_ioctl_lock);
1747 if (!fs_info->quota_root) {
1752 qgroup = find_qgroup_rb(fs_info, qgroupid);
1758 /* Check if there are no children of this qgroup */
1759 if (!list_empty(&qgroup->members)) {
1764 ret = del_qgroup_item(trans, qgroupid);
1765 if (ret && ret != -ENOENT)
1768 while (!list_empty(&qgroup->groups)) {
1769 list = list_first_entry(&qgroup->groups,
1770 struct btrfs_qgroup_list, next_group);
1771 ret = __del_qgroup_relation(trans, qgroupid,
1772 list->group->qgroupid);
1777 spin_lock(&fs_info->qgroup_lock);
1778 del_qgroup_rb(fs_info, qgroupid);
1779 spin_unlock(&fs_info->qgroup_lock);
1782 * Remove the qgroup from sysfs now without holding the qgroup_lock
1783 * spinlock, since the sysfs_remove_group() function needs to take
1784 * the mutex kernfs_mutex through kernfs_remove_by_name_ns().
1786 btrfs_sysfs_del_one_qgroup(fs_info, qgroup);
1789 mutex_unlock(&fs_info->qgroup_ioctl_lock);
1793 int btrfs_limit_qgroup(struct btrfs_trans_handle *trans, u64 qgroupid,
1794 struct btrfs_qgroup_limit *limit)
1796 struct btrfs_fs_info *fs_info = trans->fs_info;
1797 struct btrfs_qgroup *qgroup;
1799 /* Sometimes we would want to clear the limit on this qgroup.
1800 * To meet this requirement, we treat the -1 as a special value
1801 * which tell kernel to clear the limit on this qgroup.
1803 const u64 CLEAR_VALUE = -1;
1805 mutex_lock(&fs_info->qgroup_ioctl_lock);
1806 if (!fs_info->quota_root) {
1811 qgroup = find_qgroup_rb(fs_info, qgroupid);
1817 spin_lock(&fs_info->qgroup_lock);
1818 if (limit->flags & BTRFS_QGROUP_LIMIT_MAX_RFER) {
1819 if (limit->max_rfer == CLEAR_VALUE) {
1820 qgroup->lim_flags &= ~BTRFS_QGROUP_LIMIT_MAX_RFER;
1821 limit->flags &= ~BTRFS_QGROUP_LIMIT_MAX_RFER;
1822 qgroup->max_rfer = 0;
1824 qgroup->max_rfer = limit->max_rfer;
1827 if (limit->flags & BTRFS_QGROUP_LIMIT_MAX_EXCL) {
1828 if (limit->max_excl == CLEAR_VALUE) {
1829 qgroup->lim_flags &= ~BTRFS_QGROUP_LIMIT_MAX_EXCL;
1830 limit->flags &= ~BTRFS_QGROUP_LIMIT_MAX_EXCL;
1831 qgroup->max_excl = 0;
1833 qgroup->max_excl = limit->max_excl;
1836 if (limit->flags & BTRFS_QGROUP_LIMIT_RSV_RFER) {
1837 if (limit->rsv_rfer == CLEAR_VALUE) {
1838 qgroup->lim_flags &= ~BTRFS_QGROUP_LIMIT_RSV_RFER;
1839 limit->flags &= ~BTRFS_QGROUP_LIMIT_RSV_RFER;
1840 qgroup->rsv_rfer = 0;
1842 qgroup->rsv_rfer = limit->rsv_rfer;
1845 if (limit->flags & BTRFS_QGROUP_LIMIT_RSV_EXCL) {
1846 if (limit->rsv_excl == CLEAR_VALUE) {
1847 qgroup->lim_flags &= ~BTRFS_QGROUP_LIMIT_RSV_EXCL;
1848 limit->flags &= ~BTRFS_QGROUP_LIMIT_RSV_EXCL;
1849 qgroup->rsv_excl = 0;
1851 qgroup->rsv_excl = limit->rsv_excl;
1854 qgroup->lim_flags |= limit->flags;
1856 spin_unlock(&fs_info->qgroup_lock);
1858 ret = update_qgroup_limit_item(trans, qgroup);
1860 qgroup_mark_inconsistent(fs_info);
1861 btrfs_info(fs_info, "unable to update quota limit for %llu",
1866 mutex_unlock(&fs_info->qgroup_ioctl_lock);
1871 * Inform qgroup to trace one dirty extent, its info is recorded in @record.
1872 * So qgroup can account it at transaction committing time.
1874 * No lock version, caller must acquire delayed ref lock and allocated memory,
1875 * then call btrfs_qgroup_trace_extent_post() after exiting lock context.
1877 * Return 0 for success insert
1878 * Return >0 for existing record, caller can free @record safely.
1879 * Error is not possible
1881 int btrfs_qgroup_trace_extent_nolock(struct btrfs_fs_info *fs_info,
1882 struct btrfs_delayed_ref_root *delayed_refs,
1883 struct btrfs_qgroup_extent_record *record)
1885 struct rb_node **p = &delayed_refs->dirty_extent_root.rb_node;
1886 struct rb_node *parent_node = NULL;
1887 struct btrfs_qgroup_extent_record *entry;
1888 u64 bytenr = record->bytenr;
1890 if (!btrfs_qgroup_full_accounting(fs_info))
1893 lockdep_assert_held(&delayed_refs->lock);
1894 trace_btrfs_qgroup_trace_extent(fs_info, record);
1898 entry = rb_entry(parent_node, struct btrfs_qgroup_extent_record,
1900 if (bytenr < entry->bytenr) {
1902 } else if (bytenr > entry->bytenr) {
1903 p = &(*p)->rb_right;
1905 if (record->data_rsv && !entry->data_rsv) {
1906 entry->data_rsv = record->data_rsv;
1907 entry->data_rsv_refroot =
1908 record->data_rsv_refroot;
1914 rb_link_node(&record->node, parent_node, p);
1915 rb_insert_color(&record->node, &delayed_refs->dirty_extent_root);
1920 * Post handler after qgroup_trace_extent_nolock().
1922 * NOTE: Current qgroup does the expensive backref walk at transaction
1923 * committing time with TRANS_STATE_COMMIT_DOING, this blocks incoming
1925 * This is designed to allow btrfs_find_all_roots() to get correct new_roots
1928 * However for old_roots there is no need to do backref walk at that time,
1929 * since we search commit roots to walk backref and result will always be
1932 * Due to the nature of no lock version, we can't do backref there.
1933 * So we must call btrfs_qgroup_trace_extent_post() after exiting
1936 * TODO: If we can fix and prove btrfs_find_all_roots() can get correct result
1937 * using current root, then we can move all expensive backref walk out of
1938 * transaction committing, but not now as qgroup accounting will be wrong again.
1940 int btrfs_qgroup_trace_extent_post(struct btrfs_trans_handle *trans,
1941 struct btrfs_qgroup_extent_record *qrecord)
1943 struct btrfs_backref_walk_ctx ctx = { 0 };
1946 if (!btrfs_qgroup_full_accounting(trans->fs_info))
1949 * We are always called in a context where we are already holding a
1950 * transaction handle. Often we are called when adding a data delayed
1951 * reference from btrfs_truncate_inode_items() (truncating or unlinking),
1952 * in which case we will be holding a write lock on extent buffer from a
1953 * subvolume tree. In this case we can't allow btrfs_find_all_roots() to
1954 * acquire fs_info->commit_root_sem, because that is a higher level lock
1955 * that must be acquired before locking any extent buffers.
1957 * So we want btrfs_find_all_roots() to not acquire the commit_root_sem
1958 * but we can't pass it a non-NULL transaction handle, because otherwise
1959 * it would not use commit roots and would lock extent buffers, causing
1960 * a deadlock if it ends up trying to read lock the same extent buffer
1961 * that was previously write locked at btrfs_truncate_inode_items().
1963 * So pass a NULL transaction handle to btrfs_find_all_roots() and
1964 * explicitly tell it to not acquire the commit_root_sem - if we are
1965 * holding a transaction handle we don't need its protection.
1967 ASSERT(trans != NULL);
1969 if (trans->fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING)
1972 ctx.bytenr = qrecord->bytenr;
1973 ctx.fs_info = trans->fs_info;
1975 ret = btrfs_find_all_roots(&ctx, true);
1977 qgroup_mark_inconsistent(trans->fs_info);
1978 btrfs_warn(trans->fs_info,
1979 "error accounting new delayed refs extent (err code: %d), quota inconsistent",
1985 * Here we don't need to get the lock of
1986 * trans->transaction->delayed_refs, since inserted qrecord won't
1987 * be deleted, only qrecord->node may be modified (new qrecord insert)
1989 * So modifying qrecord->old_roots is safe here
1991 qrecord->old_roots = ctx.roots;
1996 * Inform qgroup to trace one dirty extent, specified by @bytenr and
1998 * So qgroup can account it at commit trans time.
2000 * Better encapsulated version, with memory allocation and backref walk for
2002 * So this can sleep.
2004 * Return 0 if the operation is done.
2005 * Return <0 for error, like memory allocation failure or invalid parameter
2008 int btrfs_qgroup_trace_extent(struct btrfs_trans_handle *trans, u64 bytenr,
2011 struct btrfs_fs_info *fs_info = trans->fs_info;
2012 struct btrfs_qgroup_extent_record *record;
2013 struct btrfs_delayed_ref_root *delayed_refs;
2016 if (!btrfs_qgroup_full_accounting(fs_info) || bytenr == 0 || num_bytes == 0)
2018 record = kzalloc(sizeof(*record), GFP_NOFS);
2022 delayed_refs = &trans->transaction->delayed_refs;
2023 record->bytenr = bytenr;
2024 record->num_bytes = num_bytes;
2025 record->old_roots = NULL;
2027 spin_lock(&delayed_refs->lock);
2028 ret = btrfs_qgroup_trace_extent_nolock(fs_info, delayed_refs, record);
2029 spin_unlock(&delayed_refs->lock);
2034 return btrfs_qgroup_trace_extent_post(trans, record);
2038 * Inform qgroup to trace all leaf items of data
2040 * Return 0 for success
2041 * Return <0 for error(ENOMEM)
2043 int btrfs_qgroup_trace_leaf_items(struct btrfs_trans_handle *trans,
2044 struct extent_buffer *eb)
2046 struct btrfs_fs_info *fs_info = trans->fs_info;
2047 int nr = btrfs_header_nritems(eb);
2048 int i, extent_type, ret;
2049 struct btrfs_key key;
2050 struct btrfs_file_extent_item *fi;
2051 u64 bytenr, num_bytes;
2053 /* We can be called directly from walk_up_proc() */
2054 if (!btrfs_qgroup_full_accounting(fs_info))
2057 for (i = 0; i < nr; i++) {
2058 btrfs_item_key_to_cpu(eb, &key, i);
2060 if (key.type != BTRFS_EXTENT_DATA_KEY)
2063 fi = btrfs_item_ptr(eb, i, struct btrfs_file_extent_item);
2064 /* filter out non qgroup-accountable extents */
2065 extent_type = btrfs_file_extent_type(eb, fi);
2067 if (extent_type == BTRFS_FILE_EXTENT_INLINE)
2070 bytenr = btrfs_file_extent_disk_bytenr(eb, fi);
2074 num_bytes = btrfs_file_extent_disk_num_bytes(eb, fi);
2076 ret = btrfs_qgroup_trace_extent(trans, bytenr, num_bytes);
2085 * Walk up the tree from the bottom, freeing leaves and any interior
2086 * nodes which have had all slots visited. If a node (leaf or
2087 * interior) is freed, the node above it will have it's slot
2088 * incremented. The root node will never be freed.
2090 * At the end of this function, we should have a path which has all
2091 * slots incremented to the next position for a search. If we need to
2092 * read a new node it will be NULL and the node above it will have the
2093 * correct slot selected for a later read.
2095 * If we increment the root nodes slot counter past the number of
2096 * elements, 1 is returned to signal completion of the search.
2098 static int adjust_slots_upwards(struct btrfs_path *path, int root_level)
2102 struct extent_buffer *eb;
2104 if (root_level == 0)
2107 while (level <= root_level) {
2108 eb = path->nodes[level];
2109 nr = btrfs_header_nritems(eb);
2110 path->slots[level]++;
2111 slot = path->slots[level];
2112 if (slot >= nr || level == 0) {
2114 * Don't free the root - we will detect this
2115 * condition after our loop and return a
2116 * positive value for caller to stop walking the tree.
2118 if (level != root_level) {
2119 btrfs_tree_unlock_rw(eb, path->locks[level]);
2120 path->locks[level] = 0;
2122 free_extent_buffer(eb);
2123 path->nodes[level] = NULL;
2124 path->slots[level] = 0;
2128 * We have a valid slot to walk back down
2129 * from. Stop here so caller can process these
2138 eb = path->nodes[root_level];
2139 if (path->slots[root_level] >= btrfs_header_nritems(eb))
2146 * Helper function to trace a subtree tree block swap.
2148 * The swap will happen in highest tree block, but there may be a lot of
2149 * tree blocks involved.
2152 * OO = Old tree blocks
2153 * NN = New tree blocks allocated during balance
2155 * File tree (257) Reloc tree for 257
2158 * L1 OO OO (a) OO NN (a)
2160 * L0 OO OO OO OO OO OO NN NN
2163 * When calling qgroup_trace_extent_swap(), we will pass:
2165 * @dst_path = [ nodes[1] = NN(a), nodes[0] = NN(c) ]
2169 * In that case, qgroup_trace_extent_swap() will search from OO(a) to
2170 * reach OO(c), then mark both OO(c) and NN(c) as qgroup dirty.
2172 * The main work of qgroup_trace_extent_swap() can be split into 3 parts:
2174 * 1) Tree search from @src_eb
2175 * It should acts as a simplified btrfs_search_slot().
2176 * The key for search can be extracted from @dst_path->nodes[dst_level]
2179 * 2) Mark the final tree blocks in @src_path and @dst_path qgroup dirty
2180 * NOTE: In above case, OO(a) and NN(a) won't be marked qgroup dirty.
2181 * They should be marked during previous (@dst_level = 1) iteration.
2183 * 3) Mark file extents in leaves dirty
2184 * We don't have good way to pick out new file extents only.
2185 * So we still follow the old method by scanning all file extents in
2188 * This function can free us from keeping two paths, thus later we only need
2189 * to care about how to iterate all new tree blocks in reloc tree.
2191 static int qgroup_trace_extent_swap(struct btrfs_trans_handle* trans,
2192 struct extent_buffer *src_eb,
2193 struct btrfs_path *dst_path,
2194 int dst_level, int root_level,
2197 struct btrfs_key key;
2198 struct btrfs_path *src_path;
2199 struct btrfs_fs_info *fs_info = trans->fs_info;
2200 u32 nodesize = fs_info->nodesize;
2201 int cur_level = root_level;
2204 BUG_ON(dst_level > root_level);
2205 /* Level mismatch */
2206 if (btrfs_header_level(src_eb) != root_level)
2209 src_path = btrfs_alloc_path();
2216 btrfs_node_key_to_cpu(dst_path->nodes[dst_level], &key, 0);
2218 btrfs_item_key_to_cpu(dst_path->nodes[dst_level], &key, 0);
2221 atomic_inc(&src_eb->refs);
2222 src_path->nodes[root_level] = src_eb;
2223 src_path->slots[root_level] = dst_path->slots[root_level];
2224 src_path->locks[root_level] = 0;
2226 /* A simplified version of btrfs_search_slot() */
2227 while (cur_level >= dst_level) {
2228 struct btrfs_key src_key;
2229 struct btrfs_key dst_key;
2231 if (src_path->nodes[cur_level] == NULL) {
2232 struct extent_buffer *eb;
2235 eb = src_path->nodes[cur_level + 1];
2236 parent_slot = src_path->slots[cur_level + 1];
2238 eb = btrfs_read_node_slot(eb, parent_slot);
2244 src_path->nodes[cur_level] = eb;
2246 btrfs_tree_read_lock(eb);
2247 src_path->locks[cur_level] = BTRFS_READ_LOCK;
2250 src_path->slots[cur_level] = dst_path->slots[cur_level];
2252 btrfs_node_key_to_cpu(dst_path->nodes[cur_level],
2253 &dst_key, dst_path->slots[cur_level]);
2254 btrfs_node_key_to_cpu(src_path->nodes[cur_level],
2255 &src_key, src_path->slots[cur_level]);
2257 btrfs_item_key_to_cpu(dst_path->nodes[cur_level],
2258 &dst_key, dst_path->slots[cur_level]);
2259 btrfs_item_key_to_cpu(src_path->nodes[cur_level],
2260 &src_key, src_path->slots[cur_level]);
2262 /* Content mismatch, something went wrong */
2263 if (btrfs_comp_cpu_keys(&dst_key, &src_key)) {
2271 * Now both @dst_path and @src_path have been populated, record the tree
2272 * blocks for qgroup accounting.
2274 ret = btrfs_qgroup_trace_extent(trans, src_path->nodes[dst_level]->start,
2278 ret = btrfs_qgroup_trace_extent(trans, dst_path->nodes[dst_level]->start,
2283 /* Record leaf file extents */
2284 if (dst_level == 0 && trace_leaf) {
2285 ret = btrfs_qgroup_trace_leaf_items(trans, src_path->nodes[0]);
2288 ret = btrfs_qgroup_trace_leaf_items(trans, dst_path->nodes[0]);
2291 btrfs_free_path(src_path);
2296 * Helper function to do recursive generation-aware depth-first search, to
2297 * locate all new tree blocks in a subtree of reloc tree.
2299 * E.g. (OO = Old tree blocks, NN = New tree blocks, whose gen == last_snapshot)
2308 * @dst_path = [ nodes[1] = NN(b), nodes[0] = NULL ],
2312 * We will iterate through tree blocks NN(b), NN(d) and info qgroup to trace
2313 * above tree blocks along with their counter parts in file tree.
2314 * While during search, old tree blocks OO(c) will be skipped as tree block swap
2315 * won't affect OO(c).
2317 static int qgroup_trace_new_subtree_blocks(struct btrfs_trans_handle* trans,
2318 struct extent_buffer *src_eb,
2319 struct btrfs_path *dst_path,
2320 int cur_level, int root_level,
2321 u64 last_snapshot, bool trace_leaf)
2323 struct btrfs_fs_info *fs_info = trans->fs_info;
2324 struct extent_buffer *eb;
2325 bool need_cleanup = false;
2329 /* Level sanity check */
2330 if (cur_level < 0 || cur_level >= BTRFS_MAX_LEVEL - 1 ||
2331 root_level < 0 || root_level >= BTRFS_MAX_LEVEL - 1 ||
2332 root_level < cur_level) {
2333 btrfs_err_rl(fs_info,
2334 "%s: bad levels, cur_level=%d root_level=%d",
2335 __func__, cur_level, root_level);
2339 /* Read the tree block if needed */
2340 if (dst_path->nodes[cur_level] == NULL) {
2345 * dst_path->nodes[root_level] must be initialized before
2346 * calling this function.
2348 if (cur_level == root_level) {
2349 btrfs_err_rl(fs_info,
2350 "%s: dst_path->nodes[%d] not initialized, root_level=%d cur_level=%d",
2351 __func__, root_level, root_level, cur_level);
2356 * We need to get child blockptr/gen from parent before we can
2359 eb = dst_path->nodes[cur_level + 1];
2360 parent_slot = dst_path->slots[cur_level + 1];
2361 child_gen = btrfs_node_ptr_generation(eb, parent_slot);
2363 /* This node is old, no need to trace */
2364 if (child_gen < last_snapshot)
2367 eb = btrfs_read_node_slot(eb, parent_slot);
2373 dst_path->nodes[cur_level] = eb;
2374 dst_path->slots[cur_level] = 0;
2376 btrfs_tree_read_lock(eb);
2377 dst_path->locks[cur_level] = BTRFS_READ_LOCK;
2378 need_cleanup = true;
2381 /* Now record this tree block and its counter part for qgroups */
2382 ret = qgroup_trace_extent_swap(trans, src_eb, dst_path, cur_level,
2383 root_level, trace_leaf);
2387 eb = dst_path->nodes[cur_level];
2389 if (cur_level > 0) {
2390 /* Iterate all child tree blocks */
2391 for (i = 0; i < btrfs_header_nritems(eb); i++) {
2392 /* Skip old tree blocks as they won't be swapped */
2393 if (btrfs_node_ptr_generation(eb, i) < last_snapshot)
2395 dst_path->slots[cur_level] = i;
2397 /* Recursive call (at most 7 times) */
2398 ret = qgroup_trace_new_subtree_blocks(trans, src_eb,
2399 dst_path, cur_level - 1, root_level,
2400 last_snapshot, trace_leaf);
2409 btrfs_tree_unlock_rw(dst_path->nodes[cur_level],
2410 dst_path->locks[cur_level]);
2411 free_extent_buffer(dst_path->nodes[cur_level]);
2412 dst_path->nodes[cur_level] = NULL;
2413 dst_path->slots[cur_level] = 0;
2414 dst_path->locks[cur_level] = 0;
2420 static int qgroup_trace_subtree_swap(struct btrfs_trans_handle *trans,
2421 struct extent_buffer *src_eb,
2422 struct extent_buffer *dst_eb,
2423 u64 last_snapshot, bool trace_leaf)
2425 struct btrfs_fs_info *fs_info = trans->fs_info;
2426 struct btrfs_path *dst_path = NULL;
2430 if (!btrfs_qgroup_full_accounting(fs_info))
2433 /* Wrong parameter order */
2434 if (btrfs_header_generation(src_eb) > btrfs_header_generation(dst_eb)) {
2435 btrfs_err_rl(fs_info,
2436 "%s: bad parameter order, src_gen=%llu dst_gen=%llu", __func__,
2437 btrfs_header_generation(src_eb),
2438 btrfs_header_generation(dst_eb));
2442 if (!extent_buffer_uptodate(src_eb) || !extent_buffer_uptodate(dst_eb)) {
2447 level = btrfs_header_level(dst_eb);
2448 dst_path = btrfs_alloc_path();
2454 atomic_inc(&dst_eb->refs);
2455 dst_path->nodes[level] = dst_eb;
2456 dst_path->slots[level] = 0;
2457 dst_path->locks[level] = 0;
2459 /* Do the generation aware breadth-first search */
2460 ret = qgroup_trace_new_subtree_blocks(trans, src_eb, dst_path, level,
2461 level, last_snapshot, trace_leaf);
2467 btrfs_free_path(dst_path);
2469 qgroup_mark_inconsistent(fs_info);
2474 * Inform qgroup to trace a whole subtree, including all its child tree
2476 * The root tree block is specified by @root_eb.
2478 * Normally used by relocation(tree block swap) and subvolume deletion.
2480 * Return 0 for success
2481 * Return <0 for error(ENOMEM or tree search error)
2483 int btrfs_qgroup_trace_subtree(struct btrfs_trans_handle *trans,
2484 struct extent_buffer *root_eb,
2485 u64 root_gen, int root_level)
2487 struct btrfs_fs_info *fs_info = trans->fs_info;
2490 u8 drop_subptree_thres;
2491 struct extent_buffer *eb = root_eb;
2492 struct btrfs_path *path = NULL;
2494 BUG_ON(root_level < 0 || root_level >= BTRFS_MAX_LEVEL);
2495 BUG_ON(root_eb == NULL);
2497 if (!btrfs_qgroup_full_accounting(fs_info))
2500 spin_lock(&fs_info->qgroup_lock);
2501 drop_subptree_thres = fs_info->qgroup_drop_subtree_thres;
2502 spin_unlock(&fs_info->qgroup_lock);
2505 * This function only gets called for snapshot drop, if we hit a high
2506 * node here, it means we are going to change ownership for quite a lot
2507 * of extents, which will greatly slow down btrfs_commit_transaction().
2509 * So here if we find a high tree here, we just skip the accounting and
2510 * mark qgroup inconsistent.
2512 if (root_level >= drop_subptree_thres) {
2513 qgroup_mark_inconsistent(fs_info);
2517 if (!extent_buffer_uptodate(root_eb)) {
2518 struct btrfs_tree_parent_check check = {
2519 .has_first_key = false,
2520 .transid = root_gen,
2524 ret = btrfs_read_extent_buffer(root_eb, &check);
2529 if (root_level == 0) {
2530 ret = btrfs_qgroup_trace_leaf_items(trans, root_eb);
2534 path = btrfs_alloc_path();
2539 * Walk down the tree. Missing extent blocks are filled in as
2540 * we go. Metadata is accounted every time we read a new
2543 * When we reach a leaf, we account for file extent items in it,
2544 * walk back up the tree (adjusting slot pointers as we go)
2545 * and restart the search process.
2547 atomic_inc(&root_eb->refs); /* For path */
2548 path->nodes[root_level] = root_eb;
2549 path->slots[root_level] = 0;
2550 path->locks[root_level] = 0; /* so release_path doesn't try to unlock */
2553 while (level >= 0) {
2554 if (path->nodes[level] == NULL) {
2559 * We need to get child blockptr from parent before we
2562 eb = path->nodes[level + 1];
2563 parent_slot = path->slots[level + 1];
2564 child_bytenr = btrfs_node_blockptr(eb, parent_slot);
2566 eb = btrfs_read_node_slot(eb, parent_slot);
2572 path->nodes[level] = eb;
2573 path->slots[level] = 0;
2575 btrfs_tree_read_lock(eb);
2576 path->locks[level] = BTRFS_READ_LOCK;
2578 ret = btrfs_qgroup_trace_extent(trans, child_bytenr,
2585 ret = btrfs_qgroup_trace_leaf_items(trans,
2586 path->nodes[level]);
2590 /* Nonzero return here means we completed our search */
2591 ret = adjust_slots_upwards(path, root_level);
2595 /* Restart search with new slots */
2604 btrfs_free_path(path);
2609 static void qgroup_iterator_nested_add(struct list_head *head, struct btrfs_qgroup *qgroup)
2611 if (!list_empty(&qgroup->nested_iterator))
2614 list_add_tail(&qgroup->nested_iterator, head);
2617 static void qgroup_iterator_nested_clean(struct list_head *head)
2619 while (!list_empty(head)) {
2620 struct btrfs_qgroup *qgroup;
2622 qgroup = list_first_entry(head, struct btrfs_qgroup, nested_iterator);
2623 list_del_init(&qgroup->nested_iterator);
2627 #define UPDATE_NEW 0
2628 #define UPDATE_OLD 1
2630 * Walk all of the roots that points to the bytenr and adjust their refcnts.
2632 static void qgroup_update_refcnt(struct btrfs_fs_info *fs_info,
2633 struct ulist *roots, struct list_head *qgroups,
2634 u64 seq, int update_old)
2636 struct ulist_node *unode;
2637 struct ulist_iterator uiter;
2638 struct btrfs_qgroup *qg;
2642 ULIST_ITER_INIT(&uiter);
2643 while ((unode = ulist_next(roots, &uiter))) {
2646 qg = find_qgroup_rb(fs_info, unode->val);
2650 qgroup_iterator_nested_add(qgroups, qg);
2651 qgroup_iterator_add(&tmp, qg);
2652 list_for_each_entry(qg, &tmp, iterator) {
2653 struct btrfs_qgroup_list *glist;
2656 btrfs_qgroup_update_old_refcnt(qg, seq, 1);
2658 btrfs_qgroup_update_new_refcnt(qg, seq, 1);
2660 list_for_each_entry(glist, &qg->groups, next_group) {
2661 qgroup_iterator_nested_add(qgroups, glist->group);
2662 qgroup_iterator_add(&tmp, glist->group);
2665 qgroup_iterator_clean(&tmp);
2670 * Update qgroup rfer/excl counters.
2671 * Rfer update is easy, codes can explain themselves.
2673 * Excl update is tricky, the update is split into 2 parts.
2674 * Part 1: Possible exclusive <-> sharing detect:
2676 * -------------------------------------
2678 * -------------------------------------
2680 * -------------------------------------
2683 * A: cur_old_roots < nr_old_roots (not exclusive before)
2684 * !A: cur_old_roots == nr_old_roots (possible exclusive before)
2685 * B: cur_new_roots < nr_new_roots (not exclusive now)
2686 * !B: cur_new_roots == nr_new_roots (possible exclusive now)
2689 * +: Possible sharing -> exclusive -: Possible exclusive -> sharing
2690 * *: Definitely not changed. **: Possible unchanged.
2692 * For !A and !B condition, the exception is cur_old/new_roots == 0 case.
2694 * To make the logic clear, we first use condition A and B to split
2695 * combination into 4 results.
2697 * Then, for result "+" and "-", check old/new_roots == 0 case, as in them
2698 * only on variant maybe 0.
2700 * Lastly, check result **, since there are 2 variants maybe 0, split them
2702 * But this time we don't need to consider other things, the codes and logic
2703 * is easy to understand now.
2705 static void qgroup_update_counters(struct btrfs_fs_info *fs_info,
2706 struct list_head *qgroups, u64 nr_old_roots,
2707 u64 nr_new_roots, u64 num_bytes, u64 seq)
2709 struct btrfs_qgroup *qg;
2711 list_for_each_entry(qg, qgroups, nested_iterator) {
2712 u64 cur_new_count, cur_old_count;
2715 cur_old_count = btrfs_qgroup_get_old_refcnt(qg, seq);
2716 cur_new_count = btrfs_qgroup_get_new_refcnt(qg, seq);
2718 trace_qgroup_update_counters(fs_info, qg, cur_old_count,
2721 /* Rfer update part */
2722 if (cur_old_count == 0 && cur_new_count > 0) {
2723 qg->rfer += num_bytes;
2724 qg->rfer_cmpr += num_bytes;
2727 if (cur_old_count > 0 && cur_new_count == 0) {
2728 qg->rfer -= num_bytes;
2729 qg->rfer_cmpr -= num_bytes;
2733 /* Excl update part */
2734 /* Exclusive/none -> shared case */
2735 if (cur_old_count == nr_old_roots &&
2736 cur_new_count < nr_new_roots) {
2737 /* Exclusive -> shared */
2738 if (cur_old_count != 0) {
2739 qg->excl -= num_bytes;
2740 qg->excl_cmpr -= num_bytes;
2745 /* Shared -> exclusive/none case */
2746 if (cur_old_count < nr_old_roots &&
2747 cur_new_count == nr_new_roots) {
2748 /* Shared->exclusive */
2749 if (cur_new_count != 0) {
2750 qg->excl += num_bytes;
2751 qg->excl_cmpr += num_bytes;
2756 /* Exclusive/none -> exclusive/none case */
2757 if (cur_old_count == nr_old_roots &&
2758 cur_new_count == nr_new_roots) {
2759 if (cur_old_count == 0) {
2760 /* None -> exclusive/none */
2762 if (cur_new_count != 0) {
2763 /* None -> exclusive */
2764 qg->excl += num_bytes;
2765 qg->excl_cmpr += num_bytes;
2768 /* None -> none, nothing changed */
2770 /* Exclusive -> exclusive/none */
2772 if (cur_new_count == 0) {
2773 /* Exclusive -> none */
2774 qg->excl -= num_bytes;
2775 qg->excl_cmpr -= num_bytes;
2778 /* Exclusive -> exclusive, nothing changed */
2783 qgroup_dirty(fs_info, qg);
2788 * Check if the @roots potentially is a list of fs tree roots
2790 * Return 0 for definitely not a fs/subvol tree roots ulist
2791 * Return 1 for possible fs/subvol tree roots in the list (considering an empty
2794 static int maybe_fs_roots(struct ulist *roots)
2796 struct ulist_node *unode;
2797 struct ulist_iterator uiter;
2799 /* Empty one, still possible for fs roots */
2800 if (!roots || roots->nnodes == 0)
2803 ULIST_ITER_INIT(&uiter);
2804 unode = ulist_next(roots, &uiter);
2809 * If it contains fs tree roots, then it must belong to fs/subvol
2811 * If it contains a non-fs tree, it won't be shared with fs/subvol trees.
2813 return is_fstree(unode->val);
2816 int btrfs_qgroup_account_extent(struct btrfs_trans_handle *trans, u64 bytenr,
2817 u64 num_bytes, struct ulist *old_roots,
2818 struct ulist *new_roots)
2820 struct btrfs_fs_info *fs_info = trans->fs_info;
2823 u64 nr_new_roots = 0;
2824 u64 nr_old_roots = 0;
2828 * If quotas get disabled meanwhile, the resources need to be freed and
2829 * we can't just exit here.
2831 if (!btrfs_qgroup_full_accounting(fs_info) ||
2832 fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING)
2836 if (!maybe_fs_roots(new_roots))
2838 nr_new_roots = new_roots->nnodes;
2841 if (!maybe_fs_roots(old_roots))
2843 nr_old_roots = old_roots->nnodes;
2846 /* Quick exit, either not fs tree roots, or won't affect any qgroup */
2847 if (nr_old_roots == 0 && nr_new_roots == 0)
2850 BUG_ON(!fs_info->quota_root);
2852 trace_btrfs_qgroup_account_extent(fs_info, trans->transid, bytenr,
2853 num_bytes, nr_old_roots, nr_new_roots);
2855 mutex_lock(&fs_info->qgroup_rescan_lock);
2856 if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_RESCAN) {
2857 if (fs_info->qgroup_rescan_progress.objectid <= bytenr) {
2858 mutex_unlock(&fs_info->qgroup_rescan_lock);
2863 mutex_unlock(&fs_info->qgroup_rescan_lock);
2865 spin_lock(&fs_info->qgroup_lock);
2866 seq = fs_info->qgroup_seq;
2868 /* Update old refcnts using old_roots */
2869 qgroup_update_refcnt(fs_info, old_roots, &qgroups, seq, UPDATE_OLD);
2871 /* Update new refcnts using new_roots */
2872 qgroup_update_refcnt(fs_info, new_roots, &qgroups, seq, UPDATE_NEW);
2874 qgroup_update_counters(fs_info, &qgroups, nr_old_roots, nr_new_roots,
2878 * We're done using the iterator, release all its qgroups while holding
2879 * fs_info->qgroup_lock so that we don't race with btrfs_remove_qgroup()
2880 * and trigger use-after-free accesses to qgroups.
2882 qgroup_iterator_nested_clean(&qgroups);
2885 * Bump qgroup_seq to avoid seq overlap
2887 fs_info->qgroup_seq += max(nr_old_roots, nr_new_roots) + 1;
2888 spin_unlock(&fs_info->qgroup_lock);
2890 ulist_free(old_roots);
2891 ulist_free(new_roots);
2895 int btrfs_qgroup_account_extents(struct btrfs_trans_handle *trans)
2897 struct btrfs_fs_info *fs_info = trans->fs_info;
2898 struct btrfs_qgroup_extent_record *record;
2899 struct btrfs_delayed_ref_root *delayed_refs;
2900 struct ulist *new_roots = NULL;
2901 struct rb_node *node;
2902 u64 num_dirty_extents = 0;
2906 if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE)
2909 delayed_refs = &trans->transaction->delayed_refs;
2910 qgroup_to_skip = delayed_refs->qgroup_to_skip;
2911 while ((node = rb_first(&delayed_refs->dirty_extent_root))) {
2912 record = rb_entry(node, struct btrfs_qgroup_extent_record,
2915 num_dirty_extents++;
2916 trace_btrfs_qgroup_account_extents(fs_info, record);
2918 if (!ret && !(fs_info->qgroup_flags &
2919 BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING)) {
2920 struct btrfs_backref_walk_ctx ctx = { 0 };
2922 ctx.bytenr = record->bytenr;
2923 ctx.fs_info = fs_info;
2926 * Old roots should be searched when inserting qgroup
2929 * But for INCONSISTENT (NO_ACCOUNTING) -> rescan case,
2930 * we may have some record inserted during
2931 * NO_ACCOUNTING (thus no old_roots populated), but
2932 * later we start rescan, which clears NO_ACCOUNTING,
2933 * leaving some inserted records without old_roots
2936 * Those cases are rare and should not cause too much
2937 * time spent during commit_transaction().
2939 if (!record->old_roots) {
2940 /* Search commit root to find old_roots */
2941 ret = btrfs_find_all_roots(&ctx, false);
2944 record->old_roots = ctx.roots;
2948 /* Free the reserved data space */
2949 btrfs_qgroup_free_refroot(fs_info,
2950 record->data_rsv_refroot,
2952 BTRFS_QGROUP_RSV_DATA);
2954 * Use BTRFS_SEQ_LAST as time_seq to do special search,
2955 * which doesn't lock tree or delayed_refs and search
2956 * current root. It's safe inside commit_transaction().
2959 ctx.time_seq = BTRFS_SEQ_LAST;
2960 ret = btrfs_find_all_roots(&ctx, false);
2963 new_roots = ctx.roots;
2964 if (qgroup_to_skip) {
2965 ulist_del(new_roots, qgroup_to_skip, 0);
2966 ulist_del(record->old_roots, qgroup_to_skip,
2969 ret = btrfs_qgroup_account_extent(trans, record->bytenr,
2973 record->old_roots = NULL;
2977 ulist_free(record->old_roots);
2978 ulist_free(new_roots);
2980 rb_erase(node, &delayed_refs->dirty_extent_root);
2984 trace_qgroup_num_dirty_extents(fs_info, trans->transid,
2990 * Writes all changed qgroups to disk.
2991 * Called by the transaction commit path and the qgroup assign ioctl.
2993 int btrfs_run_qgroups(struct btrfs_trans_handle *trans)
2995 struct btrfs_fs_info *fs_info = trans->fs_info;
2999 * In case we are called from the qgroup assign ioctl, assert that we
3000 * are holding the qgroup_ioctl_lock, otherwise we can race with a quota
3001 * disable operation (ioctl) and access a freed quota root.
3003 if (trans->transaction->state != TRANS_STATE_COMMIT_DOING)
3004 lockdep_assert_held(&fs_info->qgroup_ioctl_lock);
3006 if (!fs_info->quota_root)
3009 spin_lock(&fs_info->qgroup_lock);
3010 while (!list_empty(&fs_info->dirty_qgroups)) {
3011 struct btrfs_qgroup *qgroup;
3012 qgroup = list_first_entry(&fs_info->dirty_qgroups,
3013 struct btrfs_qgroup, dirty);
3014 list_del_init(&qgroup->dirty);
3015 spin_unlock(&fs_info->qgroup_lock);
3016 ret = update_qgroup_info_item(trans, qgroup);
3018 qgroup_mark_inconsistent(fs_info);
3019 ret = update_qgroup_limit_item(trans, qgroup);
3021 qgroup_mark_inconsistent(fs_info);
3022 spin_lock(&fs_info->qgroup_lock);
3024 if (btrfs_qgroup_enabled(fs_info))
3025 fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_ON;
3027 fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_ON;
3028 spin_unlock(&fs_info->qgroup_lock);
3030 ret = update_qgroup_status_item(trans);
3032 qgroup_mark_inconsistent(fs_info);
3037 static int qgroup_auto_inherit(struct btrfs_fs_info *fs_info,
3039 struct btrfs_qgroup_inherit **inherit)
3042 u64 num_qgroups = 0;
3043 struct btrfs_qgroup *inode_qg;
3044 struct btrfs_qgroup_list *qg_list;
3045 struct btrfs_qgroup_inherit *res;
3052 inode_qg = find_qgroup_rb(fs_info, inode_rootid);
3056 num_qgroups = list_count_nodes(&inode_qg->groups);
3061 struct_sz = struct_size(res, qgroups, num_qgroups);
3062 if (struct_sz == SIZE_MAX)
3065 res = kzalloc(struct_sz, GFP_NOFS);
3068 res->num_qgroups = num_qgroups;
3069 qgids = res->qgroups;
3071 list_for_each_entry(qg_list, &inode_qg->groups, next_group)
3072 qgids[i] = qg_list->group->qgroupid;
3079 * Copy the accounting information between qgroups. This is necessary
3080 * when a snapshot or a subvolume is created. Throwing an error will
3081 * cause a transaction abort so we take extra care here to only error
3082 * when a readonly fs is a reasonable outcome.
3084 int btrfs_qgroup_inherit(struct btrfs_trans_handle *trans, u64 srcid,
3085 u64 objectid, u64 inode_rootid,
3086 struct btrfs_qgroup_inherit *inherit)
3091 bool committing = false;
3092 struct btrfs_fs_info *fs_info = trans->fs_info;
3093 struct btrfs_root *quota_root;
3094 struct btrfs_qgroup *srcgroup;
3095 struct btrfs_qgroup *dstgroup;
3096 struct btrfs_qgroup *prealloc;
3097 struct btrfs_qgroup_list **qlist_prealloc = NULL;
3098 bool free_inherit = false;
3099 bool need_rescan = false;
3103 prealloc = kzalloc(sizeof(*prealloc), GFP_NOFS);
3108 * There are only two callers of this function.
3110 * One in create_subvol() in the ioctl context, which needs to hold
3111 * the qgroup_ioctl_lock.
3113 * The other one in create_pending_snapshot() where no other qgroup
3114 * code can modify the fs as they all need to either start a new trans
3115 * or hold a trans handler, thus we don't need to hold
3116 * qgroup_ioctl_lock.
3117 * This would avoid long and complex lock chain and make lockdep happy.
3119 spin_lock(&fs_info->trans_lock);
3120 if (trans->transaction->state == TRANS_STATE_COMMIT_DOING)
3122 spin_unlock(&fs_info->trans_lock);
3125 mutex_lock(&fs_info->qgroup_ioctl_lock);
3126 if (!btrfs_qgroup_enabled(fs_info))
3129 quota_root = fs_info->quota_root;
3135 if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE && !inherit) {
3136 ret = qgroup_auto_inherit(fs_info, inode_rootid, &inherit);
3139 free_inherit = true;
3143 i_qgroups = (u64 *)(inherit + 1);
3144 nums = inherit->num_qgroups + 2 * inherit->num_ref_copies +
3145 2 * inherit->num_excl_copies;
3146 for (i = 0; i < nums; ++i) {
3147 srcgroup = find_qgroup_rb(fs_info, *i_qgroups);
3150 * Zero out invalid groups so we can ignore
3154 ((srcgroup->qgroupid >> 48) <= (objectid >> 48)))
3162 * create a tracking group for the subvol itself
3164 ret = add_qgroup_item(trans, quota_root, objectid);
3169 * add qgroup to all inherited groups
3172 i_qgroups = (u64 *)(inherit + 1);
3173 for (i = 0; i < inherit->num_qgroups; ++i, ++i_qgroups) {
3174 if (*i_qgroups == 0)
3176 ret = add_qgroup_relation_item(trans, objectid,
3178 if (ret && ret != -EEXIST)
3180 ret = add_qgroup_relation_item(trans, *i_qgroups,
3182 if (ret && ret != -EEXIST)
3187 qlist_prealloc = kcalloc(inherit->num_qgroups,
3188 sizeof(struct btrfs_qgroup_list *),
3190 if (!qlist_prealloc) {
3194 for (int i = 0; i < inherit->num_qgroups; i++) {
3195 qlist_prealloc[i] = kzalloc(sizeof(struct btrfs_qgroup_list),
3197 if (!qlist_prealloc[i]) {
3204 spin_lock(&fs_info->qgroup_lock);
3206 dstgroup = add_qgroup_rb(fs_info, prealloc, objectid);
3209 if (inherit && inherit->flags & BTRFS_QGROUP_INHERIT_SET_LIMITS) {
3210 dstgroup->lim_flags = inherit->lim.flags;
3211 dstgroup->max_rfer = inherit->lim.max_rfer;
3212 dstgroup->max_excl = inherit->lim.max_excl;
3213 dstgroup->rsv_rfer = inherit->lim.rsv_rfer;
3214 dstgroup->rsv_excl = inherit->lim.rsv_excl;
3216 qgroup_dirty(fs_info, dstgroup);
3219 if (srcid && btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_FULL) {
3220 srcgroup = find_qgroup_rb(fs_info, srcid);
3225 * We call inherit after we clone the root in order to make sure
3226 * our counts don't go crazy, so at this point the only
3227 * difference between the two roots should be the root node.
3229 level_size = fs_info->nodesize;
3230 dstgroup->rfer = srcgroup->rfer;
3231 dstgroup->rfer_cmpr = srcgroup->rfer_cmpr;
3232 dstgroup->excl = level_size;
3233 dstgroup->excl_cmpr = level_size;
3234 srcgroup->excl = level_size;
3235 srcgroup->excl_cmpr = level_size;
3237 /* inherit the limit info */
3238 dstgroup->lim_flags = srcgroup->lim_flags;
3239 dstgroup->max_rfer = srcgroup->max_rfer;
3240 dstgroup->max_excl = srcgroup->max_excl;
3241 dstgroup->rsv_rfer = srcgroup->rsv_rfer;
3242 dstgroup->rsv_excl = srcgroup->rsv_excl;
3244 qgroup_dirty(fs_info, dstgroup);
3245 qgroup_dirty(fs_info, srcgroup);
3251 i_qgroups = (u64 *)(inherit + 1);
3252 for (i = 0; i < inherit->num_qgroups; ++i) {
3254 ret = add_relation_rb(fs_info, qlist_prealloc[i], objectid,
3256 qlist_prealloc[i] = NULL;
3263 * If we're doing a snapshot, and adding the snapshot to a new
3264 * qgroup, the numbers are guaranteed to be incorrect.
3270 for (i = 0; i < inherit->num_ref_copies; ++i, i_qgroups += 2) {
3271 struct btrfs_qgroup *src;
3272 struct btrfs_qgroup *dst;
3274 if (!i_qgroups[0] || !i_qgroups[1])
3277 src = find_qgroup_rb(fs_info, i_qgroups[0]);
3278 dst = find_qgroup_rb(fs_info, i_qgroups[1]);
3285 dst->rfer = src->rfer - level_size;
3286 dst->rfer_cmpr = src->rfer_cmpr - level_size;
3288 /* Manually tweaking numbers certainly needs a rescan */
3291 for (i = 0; i < inherit->num_excl_copies; ++i, i_qgroups += 2) {
3292 struct btrfs_qgroup *src;
3293 struct btrfs_qgroup *dst;
3295 if (!i_qgroups[0] || !i_qgroups[1])
3298 src = find_qgroup_rb(fs_info, i_qgroups[0]);
3299 dst = find_qgroup_rb(fs_info, i_qgroups[1]);
3306 dst->excl = src->excl + level_size;
3307 dst->excl_cmpr = src->excl_cmpr + level_size;
3312 spin_unlock(&fs_info->qgroup_lock);
3314 ret = btrfs_sysfs_add_one_qgroup(fs_info, dstgroup);
3317 mutex_unlock(&fs_info->qgroup_ioctl_lock);
3319 qgroup_mark_inconsistent(fs_info);
3320 if (qlist_prealloc) {
3321 for (int i = 0; i < inherit->num_qgroups; i++)
3322 kfree(qlist_prealloc[i]);
3323 kfree(qlist_prealloc);
3331 static bool qgroup_check_limits(const struct btrfs_qgroup *qg, u64 num_bytes)
3333 if ((qg->lim_flags & BTRFS_QGROUP_LIMIT_MAX_RFER) &&
3334 qgroup_rsv_total(qg) + (s64)qg->rfer + num_bytes > qg->max_rfer)
3337 if ((qg->lim_flags & BTRFS_QGROUP_LIMIT_MAX_EXCL) &&
3338 qgroup_rsv_total(qg) + (s64)qg->excl + num_bytes > qg->max_excl)
3344 static int qgroup_reserve(struct btrfs_root *root, u64 num_bytes, bool enforce,
3345 enum btrfs_qgroup_rsv_type type)
3347 struct btrfs_qgroup *qgroup;
3348 struct btrfs_fs_info *fs_info = root->fs_info;
3349 u64 ref_root = root->root_key.objectid;
3351 LIST_HEAD(qgroup_list);
3353 if (!is_fstree(ref_root))
3359 if (test_bit(BTRFS_FS_QUOTA_OVERRIDE, &fs_info->flags) &&
3360 capable(CAP_SYS_RESOURCE))
3363 spin_lock(&fs_info->qgroup_lock);
3364 if (!fs_info->quota_root)
3367 qgroup = find_qgroup_rb(fs_info, ref_root);
3371 qgroup_iterator_add(&qgroup_list, qgroup);
3372 list_for_each_entry(qgroup, &qgroup_list, iterator) {
3373 struct btrfs_qgroup_list *glist;
3375 if (enforce && !qgroup_check_limits(qgroup, num_bytes)) {
3380 list_for_each_entry(glist, &qgroup->groups, next_group)
3381 qgroup_iterator_add(&qgroup_list, glist->group);
3386 * no limits exceeded, now record the reservation into all qgroups
3388 list_for_each_entry(qgroup, &qgroup_list, iterator)
3389 qgroup_rsv_add(fs_info, qgroup, num_bytes, type);
3392 qgroup_iterator_clean(&qgroup_list);
3393 spin_unlock(&fs_info->qgroup_lock);
3398 * Free @num_bytes of reserved space with @type for qgroup. (Normally level 0
3401 * Will handle all higher level qgroup too.
3403 * NOTE: If @num_bytes is (u64)-1, this means to free all bytes of this qgroup.
3404 * This special case is only used for META_PERTRANS type.
3406 void btrfs_qgroup_free_refroot(struct btrfs_fs_info *fs_info,
3407 u64 ref_root, u64 num_bytes,
3408 enum btrfs_qgroup_rsv_type type)
3410 struct btrfs_qgroup *qgroup;
3411 LIST_HEAD(qgroup_list);
3413 if (!is_fstree(ref_root))
3419 if (num_bytes == (u64)-1 && type != BTRFS_QGROUP_RSV_META_PERTRANS) {
3420 WARN(1, "%s: Invalid type to free", __func__);
3423 spin_lock(&fs_info->qgroup_lock);
3425 if (!fs_info->quota_root)
3428 qgroup = find_qgroup_rb(fs_info, ref_root);
3432 if (num_bytes == (u64)-1)
3434 * We're freeing all pertrans rsv, get reserved value from
3435 * level 0 qgroup as real num_bytes to free.
3437 num_bytes = qgroup->rsv.values[type];
3439 qgroup_iterator_add(&qgroup_list, qgroup);
3440 list_for_each_entry(qgroup, &qgroup_list, iterator) {
3441 struct btrfs_qgroup_list *glist;
3443 qgroup_rsv_release(fs_info, qgroup, num_bytes, type);
3444 list_for_each_entry(glist, &qgroup->groups, next_group) {
3445 qgroup_iterator_add(&qgroup_list, glist->group);
3449 qgroup_iterator_clean(&qgroup_list);
3450 spin_unlock(&fs_info->qgroup_lock);
3454 * Check if the leaf is the last leaf. Which means all node pointers
3455 * are at their last position.
3457 static bool is_last_leaf(struct btrfs_path *path)
3461 for (i = 1; i < BTRFS_MAX_LEVEL && path->nodes[i]; i++) {
3462 if (path->slots[i] != btrfs_header_nritems(path->nodes[i]) - 1)
3469 * returns < 0 on error, 0 when more leafs are to be scanned.
3470 * returns 1 when done.
3472 static int qgroup_rescan_leaf(struct btrfs_trans_handle *trans,
3473 struct btrfs_path *path)
3475 struct btrfs_fs_info *fs_info = trans->fs_info;
3476 struct btrfs_root *extent_root;
3477 struct btrfs_key found;
3478 struct extent_buffer *scratch_leaf = NULL;
3484 if (!btrfs_qgroup_full_accounting(fs_info))
3487 mutex_lock(&fs_info->qgroup_rescan_lock);
3488 extent_root = btrfs_extent_root(fs_info,
3489 fs_info->qgroup_rescan_progress.objectid);
3490 ret = btrfs_search_slot_for_read(extent_root,
3491 &fs_info->qgroup_rescan_progress,
3494 btrfs_debug(fs_info,
3495 "current progress key (%llu %u %llu), search_slot ret %d",
3496 fs_info->qgroup_rescan_progress.objectid,
3497 fs_info->qgroup_rescan_progress.type,
3498 fs_info->qgroup_rescan_progress.offset, ret);
3502 * The rescan is about to end, we will not be scanning any
3503 * further blocks. We cannot unset the RESCAN flag here, because
3504 * we want to commit the transaction if everything went well.
3505 * To make the live accounting work in this phase, we set our
3506 * scan progress pointer such that every real extent objectid
3509 fs_info->qgroup_rescan_progress.objectid = (u64)-1;
3510 btrfs_release_path(path);
3511 mutex_unlock(&fs_info->qgroup_rescan_lock);
3514 done = is_last_leaf(path);
3516 btrfs_item_key_to_cpu(path->nodes[0], &found,
3517 btrfs_header_nritems(path->nodes[0]) - 1);
3518 fs_info->qgroup_rescan_progress.objectid = found.objectid + 1;
3520 scratch_leaf = btrfs_clone_extent_buffer(path->nodes[0]);
3521 if (!scratch_leaf) {
3523 mutex_unlock(&fs_info->qgroup_rescan_lock);
3526 slot = path->slots[0];
3527 btrfs_release_path(path);
3528 mutex_unlock(&fs_info->qgroup_rescan_lock);
3530 for (; slot < btrfs_header_nritems(scratch_leaf); ++slot) {
3531 struct btrfs_backref_walk_ctx ctx = { 0 };
3533 btrfs_item_key_to_cpu(scratch_leaf, &found, slot);
3534 if (found.type != BTRFS_EXTENT_ITEM_KEY &&
3535 found.type != BTRFS_METADATA_ITEM_KEY)
3537 if (found.type == BTRFS_METADATA_ITEM_KEY)
3538 num_bytes = fs_info->nodesize;
3540 num_bytes = found.offset;
3542 ctx.bytenr = found.objectid;
3543 ctx.fs_info = fs_info;
3545 ret = btrfs_find_all_roots(&ctx, false);
3548 /* For rescan, just pass old_roots as NULL */
3549 ret = btrfs_qgroup_account_extent(trans, found.objectid,
3550 num_bytes, NULL, ctx.roots);
3556 free_extent_buffer(scratch_leaf);
3560 fs_info->qgroup_rescan_progress.objectid = (u64)-1;
3565 static bool rescan_should_stop(struct btrfs_fs_info *fs_info)
3567 if (btrfs_fs_closing(fs_info))
3569 if (test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state))
3571 if (!btrfs_qgroup_enabled(fs_info))
3573 if (fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN)
3578 static void btrfs_qgroup_rescan_worker(struct btrfs_work *work)
3580 struct btrfs_fs_info *fs_info = container_of(work, struct btrfs_fs_info,
3581 qgroup_rescan_work);
3582 struct btrfs_path *path;
3583 struct btrfs_trans_handle *trans = NULL;
3586 bool stopped = false;
3587 bool did_leaf_rescans = false;
3589 if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE)
3592 path = btrfs_alloc_path();
3596 * Rescan should only search for commit root, and any later difference
3597 * should be recorded by qgroup
3599 path->search_commit_root = 1;
3600 path->skip_locking = 1;
3603 while (!err && !(stopped = rescan_should_stop(fs_info))) {
3604 trans = btrfs_start_transaction(fs_info->fs_root, 0);
3605 if (IS_ERR(trans)) {
3606 err = PTR_ERR(trans);
3610 err = qgroup_rescan_leaf(trans, path);
3611 did_leaf_rescans = true;
3614 btrfs_commit_transaction(trans);
3616 btrfs_end_transaction(trans);
3620 btrfs_free_path(path);
3622 mutex_lock(&fs_info->qgroup_rescan_lock);
3624 fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT) {
3625 fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT;
3626 } else if (err < 0 || stopped) {
3627 fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_INCONSISTENT;
3629 mutex_unlock(&fs_info->qgroup_rescan_lock);
3632 * Only update status, since the previous part has already updated the
3633 * qgroup info, and only if we did any actual work. This also prevents
3634 * race with a concurrent quota disable, which has already set
3635 * fs_info->quota_root to NULL and cleared BTRFS_FS_QUOTA_ENABLED at
3636 * btrfs_quota_disable().
3638 if (did_leaf_rescans) {
3639 trans = btrfs_start_transaction(fs_info->quota_root, 1);
3640 if (IS_ERR(trans)) {
3641 err = PTR_ERR(trans);
3644 "fail to start transaction for status update: %d",
3651 mutex_lock(&fs_info->qgroup_rescan_lock);
3653 fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN)
3654 fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_RESCAN;
3656 ret = update_qgroup_status_item(trans);
3659 btrfs_err(fs_info, "fail to update qgroup status: %d",
3663 fs_info->qgroup_rescan_running = false;
3664 fs_info->qgroup_flags &= ~BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN;
3665 complete_all(&fs_info->qgroup_rescan_completion);
3666 mutex_unlock(&fs_info->qgroup_rescan_lock);
3671 btrfs_end_transaction(trans);
3674 btrfs_info(fs_info, "qgroup scan paused");
3675 } else if (fs_info->qgroup_flags & BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN) {
3676 btrfs_info(fs_info, "qgroup scan cancelled");
3677 } else if (err >= 0) {
3678 btrfs_info(fs_info, "qgroup scan completed%s",
3679 err > 0 ? " (inconsistency flag cleared)" : "");
3681 btrfs_err(fs_info, "qgroup scan failed with %d", err);
3686 * Checks that (a) no rescan is running and (b) quota is enabled. Allocates all
3687 * memory required for the rescan context.
3690 qgroup_rescan_init(struct btrfs_fs_info *fs_info, u64 progress_objectid,
3695 if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_SIMPLE) {
3696 btrfs_warn(fs_info, "qgroup rescan init failed, running in simple mode");
3701 /* we're resuming qgroup rescan at mount time */
3702 if (!(fs_info->qgroup_flags &
3703 BTRFS_QGROUP_STATUS_FLAG_RESCAN)) {
3705 "qgroup rescan init failed, qgroup rescan is not queued");
3707 } else if (!(fs_info->qgroup_flags &
3708 BTRFS_QGROUP_STATUS_FLAG_ON)) {
3710 "qgroup rescan init failed, qgroup is not enabled");
3718 mutex_lock(&fs_info->qgroup_rescan_lock);
3721 if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_RESCAN) {
3723 "qgroup rescan is already in progress");
3725 } else if (!(fs_info->qgroup_flags &
3726 BTRFS_QGROUP_STATUS_FLAG_ON)) {
3728 "qgroup rescan init failed, qgroup is not enabled");
3730 } else if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_DISABLED) {
3731 /* Quota disable is in progress */
3736 mutex_unlock(&fs_info->qgroup_rescan_lock);
3739 fs_info->qgroup_flags |= BTRFS_QGROUP_STATUS_FLAG_RESCAN;
3742 memset(&fs_info->qgroup_rescan_progress, 0,
3743 sizeof(fs_info->qgroup_rescan_progress));
3744 fs_info->qgroup_flags &= ~(BTRFS_QGROUP_RUNTIME_FLAG_CANCEL_RESCAN |
3745 BTRFS_QGROUP_RUNTIME_FLAG_NO_ACCOUNTING);
3746 fs_info->qgroup_rescan_progress.objectid = progress_objectid;
3747 init_completion(&fs_info->qgroup_rescan_completion);
3748 mutex_unlock(&fs_info->qgroup_rescan_lock);
3750 btrfs_init_work(&fs_info->qgroup_rescan_work,
3751 btrfs_qgroup_rescan_worker, NULL);
3756 qgroup_rescan_zero_tracking(struct btrfs_fs_info *fs_info)
3759 struct btrfs_qgroup *qgroup;
3761 spin_lock(&fs_info->qgroup_lock);
3762 /* clear all current qgroup tracking information */
3763 for (n = rb_first(&fs_info->qgroup_tree); n; n = rb_next(n)) {
3764 qgroup = rb_entry(n, struct btrfs_qgroup, node);
3766 qgroup->rfer_cmpr = 0;
3768 qgroup->excl_cmpr = 0;
3769 qgroup_dirty(fs_info, qgroup);
3771 spin_unlock(&fs_info->qgroup_lock);
3775 btrfs_qgroup_rescan(struct btrfs_fs_info *fs_info)
3778 struct btrfs_trans_handle *trans;
3780 ret = qgroup_rescan_init(fs_info, 0, 1);
3785 * We have set the rescan_progress to 0, which means no more
3786 * delayed refs will be accounted by btrfs_qgroup_account_ref.
3787 * However, btrfs_qgroup_account_ref may be right after its call
3788 * to btrfs_find_all_roots, in which case it would still do the
3790 * To solve this, we're committing the transaction, which will
3791 * ensure we run all delayed refs and only after that, we are
3792 * going to clear all tracking information for a clean start.
3795 trans = btrfs_attach_transaction_barrier(fs_info->fs_root);
3796 if (IS_ERR(trans) && trans != ERR_PTR(-ENOENT)) {
3797 fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_RESCAN;
3798 return PTR_ERR(trans);
3799 } else if (trans != ERR_PTR(-ENOENT)) {
3800 ret = btrfs_commit_transaction(trans);
3802 fs_info->qgroup_flags &= ~BTRFS_QGROUP_STATUS_FLAG_RESCAN;
3807 qgroup_rescan_zero_tracking(fs_info);
3809 mutex_lock(&fs_info->qgroup_rescan_lock);
3810 fs_info->qgroup_rescan_running = true;
3811 btrfs_queue_work(fs_info->qgroup_rescan_workers,
3812 &fs_info->qgroup_rescan_work);
3813 mutex_unlock(&fs_info->qgroup_rescan_lock);
3818 int btrfs_qgroup_wait_for_completion(struct btrfs_fs_info *fs_info,
3824 mutex_lock(&fs_info->qgroup_rescan_lock);
3825 running = fs_info->qgroup_rescan_running;
3826 mutex_unlock(&fs_info->qgroup_rescan_lock);
3832 ret = wait_for_completion_interruptible(
3833 &fs_info->qgroup_rescan_completion);
3835 wait_for_completion(&fs_info->qgroup_rescan_completion);
3841 * this is only called from open_ctree where we're still single threaded, thus
3842 * locking is omitted here.
3845 btrfs_qgroup_rescan_resume(struct btrfs_fs_info *fs_info)
3847 if (fs_info->qgroup_flags & BTRFS_QGROUP_STATUS_FLAG_RESCAN) {
3848 mutex_lock(&fs_info->qgroup_rescan_lock);
3849 fs_info->qgroup_rescan_running = true;
3850 btrfs_queue_work(fs_info->qgroup_rescan_workers,
3851 &fs_info->qgroup_rescan_work);
3852 mutex_unlock(&fs_info->qgroup_rescan_lock);
3856 #define rbtree_iterate_from_safe(node, next, start) \
3857 for (node = start; node && ({ next = rb_next(node); 1;}); node = next)
3859 static int qgroup_unreserve_range(struct btrfs_inode *inode,
3860 struct extent_changeset *reserved, u64 start,
3863 struct rb_node *node;
3864 struct rb_node *next;
3865 struct ulist_node *entry;
3868 node = reserved->range_changed.root.rb_node;
3872 entry = rb_entry(node, struct ulist_node, rb_node);
3873 if (entry->val < start)
3874 node = node->rb_right;
3876 node = node->rb_left;
3879 if (entry->val > start && rb_prev(&entry->rb_node))
3880 entry = rb_entry(rb_prev(&entry->rb_node), struct ulist_node,
3883 rbtree_iterate_from_safe(node, next, &entry->rb_node) {
3889 entry = rb_entry(node, struct ulist_node, rb_node);
3890 entry_start = entry->val;
3891 entry_end = entry->aux;
3892 entry_len = entry_end - entry_start + 1;
3894 if (entry_start >= start + len)
3896 if (entry_start + entry_len <= start)
3899 * Now the entry is in [start, start + len), revert the
3900 * EXTENT_QGROUP_RESERVED bit.
3902 clear_ret = clear_extent_bits(&inode->io_tree, entry_start,
3903 entry_end, EXTENT_QGROUP_RESERVED);
3904 if (!ret && clear_ret < 0)
3907 ulist_del(&reserved->range_changed, entry->val, entry->aux);
3908 if (likely(reserved->bytes_changed >= entry_len)) {
3909 reserved->bytes_changed -= entry_len;
3912 reserved->bytes_changed = 0;
3920 * Try to free some space for qgroup.
3922 * For qgroup, there are only 3 ways to free qgroup space:
3923 * - Flush nodatacow write
3924 * Any nodatacow write will free its reserved data space at run_delalloc_range().
3925 * In theory, we should only flush nodatacow inodes, but it's not yet
3926 * possible, so we need to flush the whole root.
3928 * - Wait for ordered extents
3929 * When ordered extents are finished, their reserved metadata is finally
3930 * converted to per_trans status, which can be freed by later commit
3933 * - Commit transaction
3934 * This would free the meta_per_trans space.
3935 * In theory this shouldn't provide much space, but any more qgroup space
3938 static int try_flush_qgroup(struct btrfs_root *root)
3940 struct btrfs_trans_handle *trans;
3943 /* Can't hold an open transaction or we run the risk of deadlocking. */
3944 ASSERT(current->journal_info == NULL);
3945 if (WARN_ON(current->journal_info))
3949 * We don't want to run flush again and again, so if there is a running
3950 * one, we won't try to start a new flush, but exit directly.
3952 if (test_and_set_bit(BTRFS_ROOT_QGROUP_FLUSHING, &root->state)) {
3953 wait_event(root->qgroup_flush_wait,
3954 !test_bit(BTRFS_ROOT_QGROUP_FLUSHING, &root->state));
3958 ret = btrfs_start_delalloc_snapshot(root, true);
3961 btrfs_wait_ordered_extents(root, U64_MAX, 0, (u64)-1);
3963 trans = btrfs_attach_transaction_barrier(root);
3964 if (IS_ERR(trans)) {
3965 ret = PTR_ERR(trans);
3971 ret = btrfs_commit_transaction(trans);
3973 clear_bit(BTRFS_ROOT_QGROUP_FLUSHING, &root->state);
3974 wake_up(&root->qgroup_flush_wait);
3978 static int qgroup_reserve_data(struct btrfs_inode *inode,
3979 struct extent_changeset **reserved_ret, u64 start,
3982 struct btrfs_root *root = inode->root;
3983 struct extent_changeset *reserved;
3984 bool new_reserved = false;
3989 if (btrfs_qgroup_mode(root->fs_info) == BTRFS_QGROUP_MODE_DISABLED ||
3990 !is_fstree(root->root_key.objectid) || len == 0)
3993 /* @reserved parameter is mandatory for qgroup */
3994 if (WARN_ON(!reserved_ret))
3996 if (!*reserved_ret) {
3997 new_reserved = true;
3998 *reserved_ret = extent_changeset_alloc();
4002 reserved = *reserved_ret;
4003 /* Record already reserved space */
4004 orig_reserved = reserved->bytes_changed;
4005 ret = set_record_extent_bits(&inode->io_tree, start,
4006 start + len -1, EXTENT_QGROUP_RESERVED, reserved);
4008 /* Newly reserved space */
4009 to_reserve = reserved->bytes_changed - orig_reserved;
4010 trace_btrfs_qgroup_reserve_data(&inode->vfs_inode, start, len,
4011 to_reserve, QGROUP_RESERVE);
4014 ret = qgroup_reserve(root, to_reserve, true, BTRFS_QGROUP_RSV_DATA);
4021 qgroup_unreserve_range(inode, reserved, start, len);
4024 extent_changeset_free(reserved);
4025 *reserved_ret = NULL;
4031 * Reserve qgroup space for range [start, start + len).
4033 * This function will either reserve space from related qgroups or do nothing
4034 * if the range is already reserved.
4036 * Return 0 for successful reservation
4037 * Return <0 for error (including -EQUOT)
4039 * NOTE: This function may sleep for memory allocation, dirty page flushing and
4040 * commit transaction. So caller should not hold any dirty page locked.
4042 int btrfs_qgroup_reserve_data(struct btrfs_inode *inode,
4043 struct extent_changeset **reserved_ret, u64 start,
4048 ret = qgroup_reserve_data(inode, reserved_ret, start, len);
4049 if (ret <= 0 && ret != -EDQUOT)
4052 ret = try_flush_qgroup(inode->root);
4055 return qgroup_reserve_data(inode, reserved_ret, start, len);
4058 /* Free ranges specified by @reserved, normally in error path */
4059 static int qgroup_free_reserved_data(struct btrfs_inode *inode,
4060 struct extent_changeset *reserved,
4061 u64 start, u64 len, u64 *freed_ret)
4063 struct btrfs_root *root = inode->root;
4064 struct ulist_node *unode;
4065 struct ulist_iterator uiter;
4066 struct extent_changeset changeset;
4070 extent_changeset_init(&changeset);
4071 len = round_up(start + len, root->fs_info->sectorsize);
4072 start = round_down(start, root->fs_info->sectorsize);
4074 ULIST_ITER_INIT(&uiter);
4075 while ((unode = ulist_next(&reserved->range_changed, &uiter))) {
4076 u64 range_start = unode->val;
4077 /* unode->aux is the inclusive end */
4078 u64 range_len = unode->aux - range_start + 1;
4082 extent_changeset_release(&changeset);
4084 /* Only free range in range [start, start + len) */
4085 if (range_start >= start + len ||
4086 range_start + range_len <= start)
4088 free_start = max(range_start, start);
4089 free_len = min(start + len, range_start + range_len) -
4092 * TODO: To also modify reserved->ranges_reserved to reflect
4095 * However as long as we free qgroup reserved according to
4096 * EXTENT_QGROUP_RESERVED, we won't double free.
4097 * So not need to rush.
4099 ret = clear_record_extent_bits(&inode->io_tree, free_start,
4100 free_start + free_len - 1,
4101 EXTENT_QGROUP_RESERVED, &changeset);
4104 freed += changeset.bytes_changed;
4106 btrfs_qgroup_free_refroot(root->fs_info, root->root_key.objectid, freed,
4107 BTRFS_QGROUP_RSV_DATA);
4112 extent_changeset_release(&changeset);
4116 static int __btrfs_qgroup_release_data(struct btrfs_inode *inode,
4117 struct extent_changeset *reserved, u64 start, u64 len,
4118 u64 *released, int free)
4120 struct extent_changeset changeset;
4121 int trace_op = QGROUP_RELEASE;
4124 if (btrfs_qgroup_mode(inode->root->fs_info) == BTRFS_QGROUP_MODE_DISABLED) {
4125 extent_changeset_init(&changeset);
4126 return clear_record_extent_bits(&inode->io_tree, start,
4128 EXTENT_QGROUP_RESERVED, &changeset);
4131 /* In release case, we shouldn't have @reserved */
4132 WARN_ON(!free && reserved);
4133 if (free && reserved)
4134 return qgroup_free_reserved_data(inode, reserved, start, len, released);
4135 extent_changeset_init(&changeset);
4136 ret = clear_record_extent_bits(&inode->io_tree, start, start + len -1,
4137 EXTENT_QGROUP_RESERVED, &changeset);
4142 trace_op = QGROUP_FREE;
4143 trace_btrfs_qgroup_release_data(&inode->vfs_inode, start, len,
4144 changeset.bytes_changed, trace_op);
4146 btrfs_qgroup_free_refroot(inode->root->fs_info,
4147 inode->root->root_key.objectid,
4148 changeset.bytes_changed, BTRFS_QGROUP_RSV_DATA);
4150 *released = changeset.bytes_changed;
4152 extent_changeset_release(&changeset);
4157 * Free a reserved space range from io_tree and related qgroups
4159 * Should be called when a range of pages get invalidated before reaching disk.
4160 * Or for error cleanup case.
4161 * if @reserved is given, only reserved range in [@start, @start + @len) will
4164 * For data written to disk, use btrfs_qgroup_release_data().
4166 * NOTE: This function may sleep for memory allocation.
4168 int btrfs_qgroup_free_data(struct btrfs_inode *inode,
4169 struct extent_changeset *reserved,
4170 u64 start, u64 len, u64 *freed)
4172 return __btrfs_qgroup_release_data(inode, reserved, start, len, freed, 1);
4176 * Release a reserved space range from io_tree only.
4178 * Should be called when a range of pages get written to disk and corresponding
4179 * FILE_EXTENT is inserted into corresponding root.
4181 * Since new qgroup accounting framework will only update qgroup numbers at
4182 * commit_transaction() time, its reserved space shouldn't be freed from
4185 * But we should release the range from io_tree, to allow further write to be
4188 * NOTE: This function may sleep for memory allocation.
4190 int btrfs_qgroup_release_data(struct btrfs_inode *inode, u64 start, u64 len, u64 *released)
4192 return __btrfs_qgroup_release_data(inode, NULL, start, len, released, 0);
4195 static void add_root_meta_rsv(struct btrfs_root *root, int num_bytes,
4196 enum btrfs_qgroup_rsv_type type)
4198 if (type != BTRFS_QGROUP_RSV_META_PREALLOC &&
4199 type != BTRFS_QGROUP_RSV_META_PERTRANS)
4204 spin_lock(&root->qgroup_meta_rsv_lock);
4205 if (type == BTRFS_QGROUP_RSV_META_PREALLOC)
4206 root->qgroup_meta_rsv_prealloc += num_bytes;
4208 root->qgroup_meta_rsv_pertrans += num_bytes;
4209 spin_unlock(&root->qgroup_meta_rsv_lock);
4212 static int sub_root_meta_rsv(struct btrfs_root *root, int num_bytes,
4213 enum btrfs_qgroup_rsv_type type)
4215 if (type != BTRFS_QGROUP_RSV_META_PREALLOC &&
4216 type != BTRFS_QGROUP_RSV_META_PERTRANS)
4221 spin_lock(&root->qgroup_meta_rsv_lock);
4222 if (type == BTRFS_QGROUP_RSV_META_PREALLOC) {
4223 num_bytes = min_t(u64, root->qgroup_meta_rsv_prealloc,
4225 root->qgroup_meta_rsv_prealloc -= num_bytes;
4227 num_bytes = min_t(u64, root->qgroup_meta_rsv_pertrans,
4229 root->qgroup_meta_rsv_pertrans -= num_bytes;
4231 spin_unlock(&root->qgroup_meta_rsv_lock);
4235 int btrfs_qgroup_reserve_meta(struct btrfs_root *root, int num_bytes,
4236 enum btrfs_qgroup_rsv_type type, bool enforce)
4238 struct btrfs_fs_info *fs_info = root->fs_info;
4241 if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_DISABLED ||
4242 !is_fstree(root->root_key.objectid) || num_bytes == 0)
4245 BUG_ON(num_bytes != round_down(num_bytes, fs_info->nodesize));
4246 trace_qgroup_meta_reserve(root, (s64)num_bytes, type);
4247 ret = qgroup_reserve(root, num_bytes, enforce, type);
4251 * Record what we have reserved into root.
4253 * To avoid quota disabled->enabled underflow.
4254 * In that case, we may try to free space we haven't reserved
4255 * (since quota was disabled), so record what we reserved into root.
4256 * And ensure later release won't underflow this number.
4258 add_root_meta_rsv(root, num_bytes, type);
4262 int __btrfs_qgroup_reserve_meta(struct btrfs_root *root, int num_bytes,
4263 enum btrfs_qgroup_rsv_type type, bool enforce,
4268 ret = btrfs_qgroup_reserve_meta(root, num_bytes, type, enforce);
4269 if ((ret <= 0 && ret != -EDQUOT) || noflush)
4272 ret = try_flush_qgroup(root);
4275 return btrfs_qgroup_reserve_meta(root, num_bytes, type, enforce);
4279 * Per-transaction meta reservation should be all freed at transaction commit
4282 void btrfs_qgroup_free_meta_all_pertrans(struct btrfs_root *root)
4284 struct btrfs_fs_info *fs_info = root->fs_info;
4286 if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_DISABLED ||
4287 !is_fstree(root->root_key.objectid))
4290 /* TODO: Update trace point to handle such free */
4291 trace_qgroup_meta_free_all_pertrans(root);
4292 /* Special value -1 means to free all reserved space */
4293 btrfs_qgroup_free_refroot(fs_info, root->root_key.objectid, (u64)-1,
4294 BTRFS_QGROUP_RSV_META_PERTRANS);
4297 void __btrfs_qgroup_free_meta(struct btrfs_root *root, int num_bytes,
4298 enum btrfs_qgroup_rsv_type type)
4300 struct btrfs_fs_info *fs_info = root->fs_info;
4302 if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_DISABLED ||
4303 !is_fstree(root->root_key.objectid))
4307 * reservation for META_PREALLOC can happen before quota is enabled,
4308 * which can lead to underflow.
4309 * Here ensure we will only free what we really have reserved.
4311 num_bytes = sub_root_meta_rsv(root, num_bytes, type);
4312 BUG_ON(num_bytes != round_down(num_bytes, fs_info->nodesize));
4313 trace_qgroup_meta_reserve(root, -(s64)num_bytes, type);
4314 btrfs_qgroup_free_refroot(fs_info, root->root_key.objectid,
4318 static void qgroup_convert_meta(struct btrfs_fs_info *fs_info, u64 ref_root,
4321 struct btrfs_qgroup *qgroup;
4322 LIST_HEAD(qgroup_list);
4326 if (!fs_info->quota_root)
4329 spin_lock(&fs_info->qgroup_lock);
4330 qgroup = find_qgroup_rb(fs_info, ref_root);
4334 qgroup_iterator_add(&qgroup_list, qgroup);
4335 list_for_each_entry(qgroup, &qgroup_list, iterator) {
4336 struct btrfs_qgroup_list *glist;
4338 qgroup_rsv_release(fs_info, qgroup, num_bytes,
4339 BTRFS_QGROUP_RSV_META_PREALLOC);
4340 if (!sb_rdonly(fs_info->sb))
4341 qgroup_rsv_add(fs_info, qgroup, num_bytes,
4342 BTRFS_QGROUP_RSV_META_PERTRANS);
4344 list_for_each_entry(glist, &qgroup->groups, next_group)
4345 qgroup_iterator_add(&qgroup_list, glist->group);
4348 qgroup_iterator_clean(&qgroup_list);
4349 spin_unlock(&fs_info->qgroup_lock);
4353 * Convert @num_bytes of META_PREALLOCATED reservation to META_PERTRANS.
4355 * This is called when preallocated meta reservation needs to be used.
4356 * Normally after btrfs_join_transaction() call.
4358 void btrfs_qgroup_convert_reserved_meta(struct btrfs_root *root, int num_bytes)
4360 struct btrfs_fs_info *fs_info = root->fs_info;
4362 if (btrfs_qgroup_mode(fs_info) == BTRFS_QGROUP_MODE_DISABLED ||
4363 !is_fstree(root->root_key.objectid))
4365 /* Same as btrfs_qgroup_free_meta_prealloc() */
4366 num_bytes = sub_root_meta_rsv(root, num_bytes,
4367 BTRFS_QGROUP_RSV_META_PREALLOC);
4368 trace_qgroup_meta_convert(root, num_bytes);
4369 qgroup_convert_meta(fs_info, root->root_key.objectid, num_bytes);
4373 * Check qgroup reserved space leaking, normally at destroy inode
4376 void btrfs_qgroup_check_reserved_leak(struct btrfs_inode *inode)
4378 struct extent_changeset changeset;
4379 struct ulist_node *unode;
4380 struct ulist_iterator iter;
4383 extent_changeset_init(&changeset);
4384 ret = clear_record_extent_bits(&inode->io_tree, 0, (u64)-1,
4385 EXTENT_QGROUP_RESERVED, &changeset);
4388 if (WARN_ON(changeset.bytes_changed)) {
4389 ULIST_ITER_INIT(&iter);
4390 while ((unode = ulist_next(&changeset.range_changed, &iter))) {
4391 btrfs_warn(inode->root->fs_info,
4392 "leaking qgroup reserved space, ino: %llu, start: %llu, end: %llu",
4393 btrfs_ino(inode), unode->val, unode->aux);
4395 btrfs_qgroup_free_refroot(inode->root->fs_info,
4396 inode->root->root_key.objectid,
4397 changeset.bytes_changed, BTRFS_QGROUP_RSV_DATA);
4400 extent_changeset_release(&changeset);
4403 void btrfs_qgroup_init_swapped_blocks(
4404 struct btrfs_qgroup_swapped_blocks *swapped_blocks)
4408 spin_lock_init(&swapped_blocks->lock);
4409 for (i = 0; i < BTRFS_MAX_LEVEL; i++)
4410 swapped_blocks->blocks[i] = RB_ROOT;
4411 swapped_blocks->swapped = false;
4415 * Delete all swapped blocks record of @root.
4416 * Every record here means we skipped a full subtree scan for qgroup.
4418 * Gets called when committing one transaction.
4420 void btrfs_qgroup_clean_swapped_blocks(struct btrfs_root *root)
4422 struct btrfs_qgroup_swapped_blocks *swapped_blocks;
4425 swapped_blocks = &root->swapped_blocks;
4427 spin_lock(&swapped_blocks->lock);
4428 if (!swapped_blocks->swapped)
4430 for (i = 0; i < BTRFS_MAX_LEVEL; i++) {
4431 struct rb_root *cur_root = &swapped_blocks->blocks[i];
4432 struct btrfs_qgroup_swapped_block *entry;
4433 struct btrfs_qgroup_swapped_block *next;
4435 rbtree_postorder_for_each_entry_safe(entry, next, cur_root,
4438 swapped_blocks->blocks[i] = RB_ROOT;
4440 swapped_blocks->swapped = false;
4442 spin_unlock(&swapped_blocks->lock);
4446 * Add subtree roots record into @subvol_root.
4448 * @subvol_root: tree root of the subvolume tree get swapped
4449 * @bg: block group under balance
4450 * @subvol_parent/slot: pointer to the subtree root in subvolume tree
4451 * @reloc_parent/slot: pointer to the subtree root in reloc tree
4452 * BOTH POINTERS ARE BEFORE TREE SWAP
4453 * @last_snapshot: last snapshot generation of the subvolume tree
4455 int btrfs_qgroup_add_swapped_blocks(struct btrfs_trans_handle *trans,
4456 struct btrfs_root *subvol_root,
4457 struct btrfs_block_group *bg,
4458 struct extent_buffer *subvol_parent, int subvol_slot,
4459 struct extent_buffer *reloc_parent, int reloc_slot,
4462 struct btrfs_fs_info *fs_info = subvol_root->fs_info;
4463 struct btrfs_qgroup_swapped_blocks *blocks = &subvol_root->swapped_blocks;
4464 struct btrfs_qgroup_swapped_block *block;
4465 struct rb_node **cur;
4466 struct rb_node *parent = NULL;
4467 int level = btrfs_header_level(subvol_parent) - 1;
4470 if (!btrfs_qgroup_full_accounting(fs_info))
4473 if (btrfs_node_ptr_generation(subvol_parent, subvol_slot) >
4474 btrfs_node_ptr_generation(reloc_parent, reloc_slot)) {
4475 btrfs_err_rl(fs_info,
4476 "%s: bad parameter order, subvol_gen=%llu reloc_gen=%llu",
4478 btrfs_node_ptr_generation(subvol_parent, subvol_slot),
4479 btrfs_node_ptr_generation(reloc_parent, reloc_slot));
4483 block = kmalloc(sizeof(*block), GFP_NOFS);
4490 * @reloc_parent/slot is still before swap, while @block is going to
4491 * record the bytenr after swap, so we do the swap here.
4493 block->subvol_bytenr = btrfs_node_blockptr(reloc_parent, reloc_slot);
4494 block->subvol_generation = btrfs_node_ptr_generation(reloc_parent,
4496 block->reloc_bytenr = btrfs_node_blockptr(subvol_parent, subvol_slot);
4497 block->reloc_generation = btrfs_node_ptr_generation(subvol_parent,
4499 block->last_snapshot = last_snapshot;
4500 block->level = level;
4503 * If we have bg == NULL, we're called from btrfs_recover_relocation(),
4504 * no one else can modify tree blocks thus we qgroup will not change
4505 * no matter the value of trace_leaf.
4507 if (bg && bg->flags & BTRFS_BLOCK_GROUP_DATA)
4508 block->trace_leaf = true;
4510 block->trace_leaf = false;
4511 btrfs_node_key_to_cpu(reloc_parent, &block->first_key, reloc_slot);
4513 /* Insert @block into @blocks */
4514 spin_lock(&blocks->lock);
4515 cur = &blocks->blocks[level].rb_node;
4517 struct btrfs_qgroup_swapped_block *entry;
4520 entry = rb_entry(parent, struct btrfs_qgroup_swapped_block,
4523 if (entry->subvol_bytenr < block->subvol_bytenr) {
4524 cur = &(*cur)->rb_left;
4525 } else if (entry->subvol_bytenr > block->subvol_bytenr) {
4526 cur = &(*cur)->rb_right;
4528 if (entry->subvol_generation !=
4529 block->subvol_generation ||
4530 entry->reloc_bytenr != block->reloc_bytenr ||
4531 entry->reloc_generation !=
4532 block->reloc_generation) {
4534 * Duplicated but mismatch entry found.
4537 * Marking qgroup inconsistent should be enough
4540 WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
4547 rb_link_node(&block->node, parent, cur);
4548 rb_insert_color(&block->node, &blocks->blocks[level]);
4549 blocks->swapped = true;
4551 spin_unlock(&blocks->lock);
4554 qgroup_mark_inconsistent(fs_info);
4559 * Check if the tree block is a subtree root, and if so do the needed
4560 * delayed subtree trace for qgroup.
4562 * This is called during btrfs_cow_block().
4564 int btrfs_qgroup_trace_subtree_after_cow(struct btrfs_trans_handle *trans,
4565 struct btrfs_root *root,
4566 struct extent_buffer *subvol_eb)
4568 struct btrfs_fs_info *fs_info = root->fs_info;
4569 struct btrfs_tree_parent_check check = { 0 };
4570 struct btrfs_qgroup_swapped_blocks *blocks = &root->swapped_blocks;
4571 struct btrfs_qgroup_swapped_block *block;
4572 struct extent_buffer *reloc_eb = NULL;
4573 struct rb_node *node;
4575 bool swapped = false;
4576 int level = btrfs_header_level(subvol_eb);
4580 if (!btrfs_qgroup_full_accounting(fs_info))
4582 if (!is_fstree(root->root_key.objectid) || !root->reloc_root)
4585 spin_lock(&blocks->lock);
4586 if (!blocks->swapped) {
4587 spin_unlock(&blocks->lock);
4590 node = blocks->blocks[level].rb_node;
4593 block = rb_entry(node, struct btrfs_qgroup_swapped_block, node);
4594 if (block->subvol_bytenr < subvol_eb->start) {
4595 node = node->rb_left;
4596 } else if (block->subvol_bytenr > subvol_eb->start) {
4597 node = node->rb_right;
4604 spin_unlock(&blocks->lock);
4607 /* Found one, remove it from @blocks first and update blocks->swapped */
4608 rb_erase(&block->node, &blocks->blocks[level]);
4609 for (i = 0; i < BTRFS_MAX_LEVEL; i++) {
4610 if (RB_EMPTY_ROOT(&blocks->blocks[i])) {
4615 blocks->swapped = swapped;
4616 spin_unlock(&blocks->lock);
4618 check.level = block->level;
4619 check.transid = block->reloc_generation;
4620 check.has_first_key = true;
4621 memcpy(&check.first_key, &block->first_key, sizeof(check.first_key));
4623 /* Read out reloc subtree root */
4624 reloc_eb = read_tree_block(fs_info, block->reloc_bytenr, &check);
4625 if (IS_ERR(reloc_eb)) {
4626 ret = PTR_ERR(reloc_eb);
4630 if (!extent_buffer_uptodate(reloc_eb)) {
4635 ret = qgroup_trace_subtree_swap(trans, reloc_eb, subvol_eb,
4636 block->last_snapshot, block->trace_leaf);
4639 free_extent_buffer(reloc_eb);
4642 btrfs_err_rl(fs_info,
4643 "failed to account subtree at bytenr %llu: %d",
4644 subvol_eb->start, ret);
4645 qgroup_mark_inconsistent(fs_info);
4650 void btrfs_qgroup_destroy_extent_records(struct btrfs_transaction *trans)
4652 struct btrfs_qgroup_extent_record *entry;
4653 struct btrfs_qgroup_extent_record *next;
4654 struct rb_root *root;
4656 root = &trans->delayed_refs.dirty_extent_root;
4657 rbtree_postorder_for_each_entry_safe(entry, next, root, node) {
4658 ulist_free(entry->old_roots);
4664 void btrfs_free_squota_rsv(struct btrfs_fs_info *fs_info, u64 root, u64 rsv_bytes)
4666 if (btrfs_qgroup_mode(fs_info) != BTRFS_QGROUP_MODE_SIMPLE)
4669 if (!is_fstree(root))
4672 btrfs_qgroup_free_refroot(fs_info, root, rsv_bytes, BTRFS_QGROUP_RSV_DATA);
4675 int btrfs_record_squota_delta(struct btrfs_fs_info *fs_info,
4676 struct btrfs_squota_delta *delta)
4679 struct btrfs_qgroup *qgroup;
4680 struct btrfs_qgroup *qg;
4681 LIST_HEAD(qgroup_list);
4682 u64 root = delta->root;
4683 u64 num_bytes = delta->num_bytes;
4684 const int sign = (delta->is_inc ? 1 : -1);
4686 if (btrfs_qgroup_mode(fs_info) != BTRFS_QGROUP_MODE_SIMPLE)
4689 if (!is_fstree(root))
4692 /* If the extent predates enabling quotas, don't count it. */
4693 if (delta->generation < fs_info->qgroup_enable_gen)
4696 spin_lock(&fs_info->qgroup_lock);
4697 qgroup = find_qgroup_rb(fs_info, root);
4704 qgroup_iterator_add(&qgroup_list, qgroup);
4705 list_for_each_entry(qg, &qgroup_list, iterator) {
4706 struct btrfs_qgroup_list *glist;
4708 qg->excl += num_bytes * sign;
4709 qg->rfer += num_bytes * sign;
4710 qgroup_dirty(fs_info, qg);
4712 list_for_each_entry(glist, &qg->groups, next_group)
4713 qgroup_iterator_add(&qgroup_list, glist->group);
4715 qgroup_iterator_clean(&qgroup_list);
4718 spin_unlock(&fs_info->qgroup_lock);