1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
10 /* info about object pools
11 * note that mr and mw share a single index space
12 * so that one can map an lkey to the correct type of object
14 struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
17 .size = sizeof(struct rxe_ucontext),
18 .flags = RXE_POOL_NO_ALLOC,
22 .size = sizeof(struct rxe_pd),
23 .flags = RXE_POOL_NO_ALLOC,
27 .size = sizeof(struct rxe_ah),
28 .flags = RXE_POOL_ATOMIC | RXE_POOL_NO_ALLOC,
32 .size = sizeof(struct rxe_srq),
33 .flags = RXE_POOL_INDEX | RXE_POOL_NO_ALLOC,
34 .min_index = RXE_MIN_SRQ_INDEX,
35 .max_index = RXE_MAX_SRQ_INDEX,
39 .size = sizeof(struct rxe_qp),
40 .cleanup = rxe_qp_cleanup,
41 .flags = RXE_POOL_INDEX,
42 .min_index = RXE_MIN_QP_INDEX,
43 .max_index = RXE_MAX_QP_INDEX,
47 .size = sizeof(struct rxe_cq),
48 .flags = RXE_POOL_NO_ALLOC,
49 .cleanup = rxe_cq_cleanup,
53 .size = sizeof(struct rxe_mem),
54 .cleanup = rxe_mem_cleanup,
55 .flags = RXE_POOL_INDEX,
56 .max_index = RXE_MAX_MR_INDEX,
57 .min_index = RXE_MIN_MR_INDEX,
61 .size = sizeof(struct rxe_mem),
62 .flags = RXE_POOL_INDEX,
63 .max_index = RXE_MAX_MW_INDEX,
64 .min_index = RXE_MIN_MW_INDEX,
68 .size = sizeof(struct rxe_mc_grp),
69 .cleanup = rxe_mc_cleanup,
70 .flags = RXE_POOL_KEY,
71 .key_offset = offsetof(struct rxe_mc_grp, mgid),
72 .key_size = sizeof(union ib_gid),
74 [RXE_TYPE_MC_ELEM] = {
75 .name = "rxe-mc_elem",
76 .size = sizeof(struct rxe_mc_elem),
77 .flags = RXE_POOL_ATOMIC,
81 static inline const char *pool_name(struct rxe_pool *pool)
83 return rxe_type_info[pool->type].name;
86 static int rxe_pool_init_index(struct rxe_pool *pool, u32 max, u32 min)
91 if ((max - min + 1) < pool->max_elem) {
92 pr_warn("not enough indices for max_elem\n");
97 pool->max_index = max;
98 pool->min_index = min;
100 size = BITS_TO_LONGS(max - min + 1) * sizeof(long);
101 pool->table = kmalloc(size, GFP_KERNEL);
107 pool->table_size = size;
108 bitmap_zero(pool->table, max - min + 1);
116 struct rxe_pool *pool,
117 enum rxe_elem_type type,
118 unsigned int max_elem)
121 size_t size = rxe_type_info[type].size;
123 memset(pool, 0, sizeof(*pool));
127 pool->max_elem = max_elem;
128 pool->elem_size = ALIGN(size, RXE_POOL_ALIGN);
129 pool->flags = rxe_type_info[type].flags;
130 pool->tree = RB_ROOT;
131 pool->cleanup = rxe_type_info[type].cleanup;
133 atomic_set(&pool->num_elem, 0);
135 kref_init(&pool->ref_cnt);
137 rwlock_init(&pool->pool_lock);
139 if (rxe_type_info[type].flags & RXE_POOL_INDEX) {
140 err = rxe_pool_init_index(pool,
141 rxe_type_info[type].max_index,
142 rxe_type_info[type].min_index);
147 if (rxe_type_info[type].flags & RXE_POOL_KEY) {
148 pool->key_offset = rxe_type_info[type].key_offset;
149 pool->key_size = rxe_type_info[type].key_size;
152 pool->state = RXE_POOL_STATE_VALID;
158 static void rxe_pool_release(struct kref *kref)
160 struct rxe_pool *pool = container_of(kref, struct rxe_pool, ref_cnt);
162 pool->state = RXE_POOL_STATE_INVALID;
166 static void rxe_pool_put(struct rxe_pool *pool)
168 kref_put(&pool->ref_cnt, rxe_pool_release);
171 void rxe_pool_cleanup(struct rxe_pool *pool)
175 write_lock_irqsave(&pool->pool_lock, flags);
176 pool->state = RXE_POOL_STATE_INVALID;
177 if (atomic_read(&pool->num_elem) > 0)
178 pr_warn("%s pool destroyed with unfree'd elem\n",
180 write_unlock_irqrestore(&pool->pool_lock, flags);
185 static u32 alloc_index(struct rxe_pool *pool)
188 u32 range = pool->max_index - pool->min_index + 1;
190 index = find_next_zero_bit(pool->table, range, pool->last);
192 index = find_first_zero_bit(pool->table, range);
194 WARN_ON_ONCE(index >= range);
195 set_bit(index, pool->table);
197 return index + pool->min_index;
200 static void insert_index(struct rxe_pool *pool, struct rxe_pool_entry *new)
202 struct rb_node **link = &pool->tree.rb_node;
203 struct rb_node *parent = NULL;
204 struct rxe_pool_entry *elem;
208 elem = rb_entry(parent, struct rxe_pool_entry, node);
210 if (elem->index == new->index) {
211 pr_warn("element already exists!\n");
215 if (elem->index > new->index)
216 link = &(*link)->rb_left;
218 link = &(*link)->rb_right;
221 rb_link_node(&new->node, parent, link);
222 rb_insert_color(&new->node, &pool->tree);
227 static void insert_key(struct rxe_pool *pool, struct rxe_pool_entry *new)
229 struct rb_node **link = &pool->tree.rb_node;
230 struct rb_node *parent = NULL;
231 struct rxe_pool_entry *elem;
236 elem = rb_entry(parent, struct rxe_pool_entry, node);
238 cmp = memcmp((u8 *)elem + pool->key_offset,
239 (u8 *)new + pool->key_offset, pool->key_size);
242 pr_warn("key already exists!\n");
247 link = &(*link)->rb_left;
249 link = &(*link)->rb_right;
252 rb_link_node(&new->node, parent, link);
253 rb_insert_color(&new->node, &pool->tree);
258 void rxe_add_key(void *arg, void *key)
260 struct rxe_pool_entry *elem = arg;
261 struct rxe_pool *pool = elem->pool;
264 write_lock_irqsave(&pool->pool_lock, flags);
265 memcpy((u8 *)elem + pool->key_offset, key, pool->key_size);
266 insert_key(pool, elem);
267 write_unlock_irqrestore(&pool->pool_lock, flags);
270 void rxe_drop_key(void *arg)
272 struct rxe_pool_entry *elem = arg;
273 struct rxe_pool *pool = elem->pool;
276 write_lock_irqsave(&pool->pool_lock, flags);
277 rb_erase(&elem->node, &pool->tree);
278 write_unlock_irqrestore(&pool->pool_lock, flags);
281 void rxe_add_index(void *arg)
283 struct rxe_pool_entry *elem = arg;
284 struct rxe_pool *pool = elem->pool;
287 write_lock_irqsave(&pool->pool_lock, flags);
288 elem->index = alloc_index(pool);
289 insert_index(pool, elem);
290 write_unlock_irqrestore(&pool->pool_lock, flags);
293 void rxe_drop_index(void *arg)
295 struct rxe_pool_entry *elem = arg;
296 struct rxe_pool *pool = elem->pool;
299 write_lock_irqsave(&pool->pool_lock, flags);
300 clear_bit(elem->index - pool->min_index, pool->table);
301 rb_erase(&elem->node, &pool->tree);
302 write_unlock_irqrestore(&pool->pool_lock, flags);
305 void *rxe_alloc(struct rxe_pool *pool)
307 struct rxe_pool_entry *elem;
310 might_sleep_if(!(pool->flags & RXE_POOL_ATOMIC));
312 read_lock_irqsave(&pool->pool_lock, flags);
313 if (pool->state != RXE_POOL_STATE_VALID) {
314 read_unlock_irqrestore(&pool->pool_lock, flags);
317 kref_get(&pool->ref_cnt);
318 read_unlock_irqrestore(&pool->pool_lock, flags);
320 if (!ib_device_try_get(&pool->rxe->ib_dev))
323 if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
326 elem = kzalloc(rxe_type_info[pool->type].size,
327 (pool->flags & RXE_POOL_ATOMIC) ?
328 GFP_ATOMIC : GFP_KERNEL);
333 kref_init(&elem->ref_cnt);
338 atomic_dec(&pool->num_elem);
339 ib_device_put(&pool->rxe->ib_dev);
345 int rxe_add_to_pool(struct rxe_pool *pool, struct rxe_pool_entry *elem)
349 might_sleep_if(!(pool->flags & RXE_POOL_ATOMIC));
351 read_lock_irqsave(&pool->pool_lock, flags);
352 if (pool->state != RXE_POOL_STATE_VALID) {
353 read_unlock_irqrestore(&pool->pool_lock, flags);
356 kref_get(&pool->ref_cnt);
357 read_unlock_irqrestore(&pool->pool_lock, flags);
359 if (!ib_device_try_get(&pool->rxe->ib_dev))
362 if (atomic_inc_return(&pool->num_elem) > pool->max_elem)
366 kref_init(&elem->ref_cnt);
371 atomic_dec(&pool->num_elem);
372 ib_device_put(&pool->rxe->ib_dev);
378 void rxe_elem_release(struct kref *kref)
380 struct rxe_pool_entry *elem =
381 container_of(kref, struct rxe_pool_entry, ref_cnt);
382 struct rxe_pool *pool = elem->pool;
387 if (!(pool->flags & RXE_POOL_NO_ALLOC))
389 atomic_dec(&pool->num_elem);
390 ib_device_put(&pool->rxe->ib_dev);
394 void *rxe_pool_get_index(struct rxe_pool *pool, u32 index)
396 struct rb_node *node = NULL;
397 struct rxe_pool_entry *elem = NULL;
400 read_lock_irqsave(&pool->pool_lock, flags);
402 if (pool->state != RXE_POOL_STATE_VALID)
405 node = pool->tree.rb_node;
408 elem = rb_entry(node, struct rxe_pool_entry, node);
410 if (elem->index > index)
411 node = node->rb_left;
412 else if (elem->index < index)
413 node = node->rb_right;
415 kref_get(&elem->ref_cnt);
421 read_unlock_irqrestore(&pool->pool_lock, flags);
422 return node ? elem : NULL;
425 void *rxe_pool_get_key(struct rxe_pool *pool, void *key)
427 struct rb_node *node = NULL;
428 struct rxe_pool_entry *elem = NULL;
432 read_lock_irqsave(&pool->pool_lock, flags);
434 if (pool->state != RXE_POOL_STATE_VALID)
437 node = pool->tree.rb_node;
440 elem = rb_entry(node, struct rxe_pool_entry, node);
442 cmp = memcmp((u8 *)elem + pool->key_offset,
443 key, pool->key_size);
446 node = node->rb_left;
448 node = node->rb_right;
454 kref_get(&elem->ref_cnt);
457 read_unlock_irqrestore(&pool->pool_lock, flags);
458 return node ? elem : NULL;