1 // SPDX-License-Identifier: GPL-2.0
5 #include <linux/types.h>
6 #include <linux/kernel.h>
7 #include <linux/bitops.h>
12 item_tag_set(struct radix_tree_root *root, unsigned long index, int tag)
14 return radix_tree_tag_set(root, index, tag);
18 item_tag_clear(struct radix_tree_root *root, unsigned long index, int tag)
20 return radix_tree_tag_clear(root, index, tag);
23 int item_tag_get(struct radix_tree_root *root, unsigned long index, int tag)
25 return radix_tree_tag_get(root, index, tag);
28 int __item_insert(struct radix_tree_root *root, struct item *item)
30 return __radix_tree_insert(root, item->index, item->order, item);
33 struct item *item_create(unsigned long index, unsigned int order)
35 struct item *ret = malloc(sizeof(*ret));
42 int item_insert_order(struct radix_tree_root *root, unsigned long index,
45 struct item *item = item_create(index, order);
46 int err = __item_insert(root, item);
52 int item_insert(struct radix_tree_root *root, unsigned long index)
54 return item_insert_order(root, index, 0);
57 void item_sanity(struct item *item, unsigned long index)
60 assert(!radix_tree_is_internal_node(item));
61 assert(item->order < BITS_PER_LONG);
62 mask = (1UL << item->order) - 1;
63 assert((item->index | mask) == (index | mask));
66 int item_delete(struct radix_tree_root *root, unsigned long index)
68 struct item *item = radix_tree_delete(root, index);
71 item_sanity(item, index);
78 static void item_free_rcu(struct rcu_head *head)
80 struct item *item = container_of(head, struct item, rcu_head);
85 int item_delete_rcu(struct radix_tree_root *root, unsigned long index)
87 struct item *item = radix_tree_delete(root, index);
90 item_sanity(item, index);
91 call_rcu(&item->rcu_head, item_free_rcu);
97 void item_check_present(struct radix_tree_root *root, unsigned long index)
101 item = radix_tree_lookup(root, index);
102 assert(item != NULL);
103 item_sanity(item, index);
106 struct item *item_lookup(struct radix_tree_root *root, unsigned long index)
108 return radix_tree_lookup(root, index);
111 void item_check_absent(struct radix_tree_root *root, unsigned long index)
115 item = radix_tree_lookup(root, index);
116 assert(item == NULL);
120 * Scan only the passed (start, start+nr] for present items
122 void item_gang_check_present(struct radix_tree_root *root,
123 unsigned long start, unsigned long nr,
126 struct item *items[chunk];
129 for (into = 0; into < nr; ) {
131 int nr_to_find = chunk;
134 if (nr_to_find > (nr - into))
135 nr_to_find = nr - into;
137 nfound = radix_tree_gang_lookup(root, (void **)items,
138 start + into, nr_to_find);
139 assert(nfound == nr_to_find);
140 for (i = 0; i < nfound; i++)
141 assert(items[i]->index == start + into + i);
147 * Scan the entire tree, only expecting present items (start, start+nr]
149 void item_full_scan(struct radix_tree_root *root, unsigned long start,
150 unsigned long nr, int chunk)
152 struct item *items[chunk];
153 unsigned long into = 0;
154 unsigned long this_index = start;
158 // printf("%s(0x%08lx, 0x%08lx, %d)\n", __FUNCTION__, start, nr, chunk);
160 while ((nfound = radix_tree_gang_lookup(root, (void **)items, into,
162 // printf("At 0x%08lx, nfound=%d\n", into, nfound);
163 for (i = 0; i < nfound; i++) {
164 assert(items[i]->index == this_index);
167 // printf("Found 0x%08lx->0x%08lx\n",
168 // items[0]->index, items[nfound-1]->index);
172 assert(this_index == start + nr);
173 nfound = radix_tree_gang_lookup(root, (void **)items,
178 /* Use the same pattern as tag_pages_for_writeback() in mm/page-writeback.c */
179 int tag_tagged_items(struct radix_tree_root *root, pthread_mutex_t *lock,
180 unsigned long start, unsigned long end, unsigned batch,
181 unsigned iftag, unsigned thentag)
183 unsigned long tagged = 0;
184 struct radix_tree_iter iter;
191 pthread_mutex_lock(lock);
192 radix_tree_for_each_tagged(slot, root, &iter, start, iftag) {
193 if (iter.index > end)
195 radix_tree_iter_tag_set(root, &iter, thentag);
197 if ((tagged % batch) != 0)
199 slot = radix_tree_iter_resume(slot, &iter);
201 pthread_mutex_unlock(lock);
203 pthread_mutex_lock(lock);
207 pthread_mutex_unlock(lock);
212 /* Use the same pattern as find_swap_entry() in mm/shmem.c */
213 unsigned long find_item(struct radix_tree_root *root, void *item)
215 struct radix_tree_iter iter;
217 unsigned long found = -1;
218 unsigned long checked = 0;
220 radix_tree_for_each_slot(slot, root, &iter, 0) {
226 if ((checked % 4) != 0)
228 slot = radix_tree_iter_resume(slot, &iter);
234 static int verify_node(struct radix_tree_node *slot, unsigned int tag,
241 slot = entry_to_node(slot);
243 /* Verify consistency at this level */
244 for (i = 0; i < RADIX_TREE_TAG_LONGS; i++) {
245 if (slot->tags[tag][i]) {
250 if (tagged != anyset) {
251 printf("tag: %u, shift %u, tagged: %d, anyset: %d\n",
252 tag, slot->shift, tagged, anyset);
253 for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) {
254 printf("tag %d: ", j);
255 for (i = 0; i < RADIX_TREE_TAG_LONGS; i++)
256 printf("%016lx ", slot->tags[j][i]);
261 assert(tagged == anyset);
263 /* Go for next level */
264 if (slot->shift > 0) {
265 for (i = 0; i < RADIX_TREE_MAP_SIZE; i++)
267 if (verify_node(slot->slots[i], tag,
268 !!test_bit(i, slot->tags[tag]))) {
269 printf("Failure at off %d\n", i);
270 for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) {
271 printf("tag %d: ", j);
272 for (i = 0; i < RADIX_TREE_TAG_LONGS; i++)
273 printf("%016lx ", slot->tags[j][i]);
282 void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag)
284 struct radix_tree_node *node = root->rnode;
285 if (!radix_tree_is_internal_node(node))
287 verify_node(node, tag, !!root_tag_get(root, tag));
290 void item_kill_tree(struct radix_tree_root *root)
292 struct radix_tree_iter iter;
294 struct item *items[32];
297 radix_tree_for_each_slot(slot, root, &iter, 0) {
298 if (radix_tree_exceptional_entry(*slot))
299 radix_tree_delete(root, iter.index);
302 while ((nfound = radix_tree_gang_lookup(root, (void **)items, 0, 32))) {
305 for (i = 0; i < nfound; i++) {
308 ret = radix_tree_delete(root, items[i]->index);
309 assert(ret == items[i]);
313 assert(radix_tree_gang_lookup(root, (void **)items, 0, 32) == 0);
314 assert(root->rnode == NULL);
317 void tree_verify_min_height(struct radix_tree_root *root, int maxindex)
320 struct radix_tree_node *node = root->rnode;
321 if (!radix_tree_is_internal_node(node)) {
322 assert(maxindex == 0);
326 node = entry_to_node(node);
327 assert(maxindex <= node_maxindex(node));
331 assert(maxindex > shift_maxindex(shift - RADIX_TREE_MAP_SHIFT));
333 assert(maxindex > 0);