2 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 /* Crude resource management */
33 #include <linux/spinlock.h>
34 #include <linux/genalloc.h>
35 #include <linux/ratelimit.h>
38 static int c4iw_init_qid_table(struct c4iw_rdev *rdev)
42 if (c4iw_id_table_alloc(&rdev->resource.qid_table,
43 rdev->lldi.vr->qp.start,
44 rdev->lldi.vr->qp.size,
45 rdev->lldi.vr->qp.size, 0))
48 for (i = rdev->lldi.vr->qp.start;
49 i < rdev->lldi.vr->qp.start + rdev->lldi.vr->qp.size; i++)
50 if (!(i & rdev->qpmask))
51 c4iw_id_free(&rdev->resource.qid_table, i);
55 /* nr_* must be power of 2 */
56 int c4iw_init_resource(struct c4iw_rdev *rdev, u32 nr_tpt,
57 u32 nr_pdid, u32 nr_srqt)
60 err = c4iw_id_table_alloc(&rdev->resource.tpt_table, 0, nr_tpt, 1,
61 C4IW_ID_TABLE_F_RANDOM);
64 err = c4iw_init_qid_table(rdev);
67 err = c4iw_id_table_alloc(&rdev->resource.pdid_table, 0,
72 err = c4iw_id_table_alloc(&rdev->resource.srq_table, 0,
75 err = c4iw_id_table_alloc(&rdev->resource.srq_table, 0,
81 c4iw_id_table_free(&rdev->resource.pdid_table);
83 c4iw_id_table_free(&rdev->resource.qid_table);
85 c4iw_id_table_free(&rdev->resource.tpt_table);
91 * returns 0 if no resource available
93 u32 c4iw_get_resource(struct c4iw_id_table *id_table)
96 entry = c4iw_id_alloc(id_table);
97 if (entry == (u32)(-1))
102 void c4iw_put_resource(struct c4iw_id_table *id_table, u32 entry)
104 pr_debug("entry 0x%x\n", entry);
105 c4iw_id_free(id_table, entry);
108 u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx)
110 struct c4iw_qid_list *entry;
114 mutex_lock(&uctx->lock);
115 if (!list_empty(&uctx->cqids)) {
116 entry = list_entry(uctx->cqids.next, struct c4iw_qid_list,
118 list_del(&entry->entry);
122 qid = c4iw_get_resource(&rdev->resource.qid_table);
125 mutex_lock(&rdev->stats.lock);
126 rdev->stats.qid.cur += rdev->qpmask + 1;
127 mutex_unlock(&rdev->stats.lock);
128 for (i = qid+1; i & rdev->qpmask; i++) {
129 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
133 list_add_tail(&entry->entry, &uctx->cqids);
137 * now put the same ids on the qp list since they all
138 * map to the same db/gts page.
140 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
144 list_add_tail(&entry->entry, &uctx->qpids);
145 for (i = qid+1; i & rdev->qpmask; i++) {
146 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
150 list_add_tail(&entry->entry, &uctx->qpids);
154 mutex_unlock(&uctx->lock);
155 pr_debug("qid 0x%x\n", qid);
156 mutex_lock(&rdev->stats.lock);
157 if (rdev->stats.qid.cur > rdev->stats.qid.max)
158 rdev->stats.qid.max = rdev->stats.qid.cur;
159 mutex_unlock(&rdev->stats.lock);
163 void c4iw_put_cqid(struct c4iw_rdev *rdev, u32 qid,
164 struct c4iw_dev_ucontext *uctx)
166 struct c4iw_qid_list *entry;
168 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
171 pr_debug("qid 0x%x\n", qid);
173 mutex_lock(&uctx->lock);
174 list_add_tail(&entry->entry, &uctx->cqids);
175 mutex_unlock(&uctx->lock);
178 u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx)
180 struct c4iw_qid_list *entry;
184 mutex_lock(&uctx->lock);
185 if (!list_empty(&uctx->qpids)) {
186 entry = list_entry(uctx->qpids.next, struct c4iw_qid_list,
188 list_del(&entry->entry);
192 qid = c4iw_get_resource(&rdev->resource.qid_table);
194 mutex_lock(&rdev->stats.lock);
195 rdev->stats.qid.fail++;
196 mutex_unlock(&rdev->stats.lock);
199 mutex_lock(&rdev->stats.lock);
200 rdev->stats.qid.cur += rdev->qpmask + 1;
201 mutex_unlock(&rdev->stats.lock);
202 for (i = qid+1; i & rdev->qpmask; i++) {
203 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
207 list_add_tail(&entry->entry, &uctx->qpids);
211 * now put the same ids on the cq list since they all
212 * map to the same db/gts page.
214 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
218 list_add_tail(&entry->entry, &uctx->cqids);
219 for (i = qid + 1; i & rdev->qpmask; i++) {
220 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
224 list_add_tail(&entry->entry, &uctx->cqids);
228 mutex_unlock(&uctx->lock);
229 pr_debug("qid 0x%x\n", qid);
230 mutex_lock(&rdev->stats.lock);
231 if (rdev->stats.qid.cur > rdev->stats.qid.max)
232 rdev->stats.qid.max = rdev->stats.qid.cur;
233 mutex_unlock(&rdev->stats.lock);
237 void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qid,
238 struct c4iw_dev_ucontext *uctx)
240 struct c4iw_qid_list *entry;
242 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
245 pr_debug("qid 0x%x\n", qid);
247 mutex_lock(&uctx->lock);
248 list_add_tail(&entry->entry, &uctx->qpids);
249 mutex_unlock(&uctx->lock);
252 void c4iw_destroy_resource(struct c4iw_resource *rscp)
254 c4iw_id_table_free(&rscp->tpt_table);
255 c4iw_id_table_free(&rscp->qid_table);
256 c4iw_id_table_free(&rscp->pdid_table);
260 * PBL Memory Manager. Uses Linux generic allocator.
263 #define MIN_PBL_SHIFT 8 /* 256B == min PBL size (32 entries) */
265 u32 c4iw_pblpool_alloc(struct c4iw_rdev *rdev, int size)
267 unsigned long addr = gen_pool_alloc(rdev->pbl_pool, size);
268 pr_debug("addr 0x%x size %d\n", (u32)addr, size);
269 mutex_lock(&rdev->stats.lock);
271 rdev->stats.pbl.cur += roundup(size, 1 << MIN_PBL_SHIFT);
272 if (rdev->stats.pbl.cur > rdev->stats.pbl.max)
273 rdev->stats.pbl.max = rdev->stats.pbl.cur;
274 kref_get(&rdev->pbl_kref);
276 rdev->stats.pbl.fail++;
277 mutex_unlock(&rdev->stats.lock);
281 static void destroy_pblpool(struct kref *kref)
283 struct c4iw_rdev *rdev;
285 rdev = container_of(kref, struct c4iw_rdev, pbl_kref);
286 gen_pool_destroy(rdev->pbl_pool);
287 complete(&rdev->pbl_compl);
290 void c4iw_pblpool_free(struct c4iw_rdev *rdev, u32 addr, int size)
292 pr_debug("addr 0x%x size %d\n", addr, size);
293 mutex_lock(&rdev->stats.lock);
294 rdev->stats.pbl.cur -= roundup(size, 1 << MIN_PBL_SHIFT);
295 mutex_unlock(&rdev->stats.lock);
296 gen_pool_free(rdev->pbl_pool, (unsigned long)addr, size);
297 kref_put(&rdev->pbl_kref, destroy_pblpool);
300 int c4iw_pblpool_create(struct c4iw_rdev *rdev)
302 unsigned pbl_start, pbl_chunk, pbl_top;
304 rdev->pbl_pool = gen_pool_create(MIN_PBL_SHIFT, -1);
308 pbl_start = rdev->lldi.vr->pbl.start;
309 pbl_chunk = rdev->lldi.vr->pbl.size;
310 pbl_top = pbl_start + pbl_chunk;
312 while (pbl_start < pbl_top) {
313 pbl_chunk = min(pbl_top - pbl_start + 1, pbl_chunk);
314 if (gen_pool_add(rdev->pbl_pool, pbl_start, pbl_chunk, -1)) {
315 pr_debug("failed to add PBL chunk (%x/%x)\n",
316 pbl_start, pbl_chunk);
317 if (pbl_chunk <= 1024 << MIN_PBL_SHIFT) {
318 pr_warn("Failed to add all PBL chunks (%x/%x)\n",
319 pbl_start, pbl_top - pbl_start);
324 pr_debug("added PBL chunk (%x/%x)\n",
325 pbl_start, pbl_chunk);
326 pbl_start += pbl_chunk;
333 void c4iw_pblpool_destroy(struct c4iw_rdev *rdev)
335 kref_put(&rdev->pbl_kref, destroy_pblpool);
339 * RQT Memory Manager. Uses Linux generic allocator.
342 #define MIN_RQT_SHIFT 10 /* 1KB == min RQT size (16 entries) */
344 u32 c4iw_rqtpool_alloc(struct c4iw_rdev *rdev, int size)
346 unsigned long addr = gen_pool_alloc(rdev->rqt_pool, size << 6);
347 pr_debug("addr 0x%x size %d\n", (u32)addr, size << 6);
349 pr_warn_ratelimited("%s: Out of RQT memory\n",
350 pci_name(rdev->lldi.pdev));
351 mutex_lock(&rdev->stats.lock);
353 rdev->stats.rqt.cur += roundup(size << 6, 1 << MIN_RQT_SHIFT);
354 if (rdev->stats.rqt.cur > rdev->stats.rqt.max)
355 rdev->stats.rqt.max = rdev->stats.rqt.cur;
356 kref_get(&rdev->rqt_kref);
358 rdev->stats.rqt.fail++;
359 mutex_unlock(&rdev->stats.lock);
363 static void destroy_rqtpool(struct kref *kref)
365 struct c4iw_rdev *rdev;
367 rdev = container_of(kref, struct c4iw_rdev, rqt_kref);
368 gen_pool_destroy(rdev->rqt_pool);
369 complete(&rdev->rqt_compl);
372 void c4iw_rqtpool_free(struct c4iw_rdev *rdev, u32 addr, int size)
374 pr_debug("addr 0x%x size %d\n", addr, size << 6);
375 mutex_lock(&rdev->stats.lock);
376 rdev->stats.rqt.cur -= roundup(size << 6, 1 << MIN_RQT_SHIFT);
377 mutex_unlock(&rdev->stats.lock);
378 gen_pool_free(rdev->rqt_pool, (unsigned long)addr, size << 6);
379 kref_put(&rdev->rqt_kref, destroy_rqtpool);
382 int c4iw_rqtpool_create(struct c4iw_rdev *rdev)
384 unsigned rqt_start, rqt_chunk, rqt_top;
387 rdev->rqt_pool = gen_pool_create(MIN_RQT_SHIFT, -1);
392 * If SRQs are supported, then never use the first RQE from
393 * the RQT region. This is because HW uses RQT index 0 as NULL.
395 if (rdev->lldi.vr->srq.size)
396 skip = T4_RQT_ENTRY_SIZE;
398 rqt_start = rdev->lldi.vr->rq.start + skip;
399 rqt_chunk = rdev->lldi.vr->rq.size - skip;
400 rqt_top = rqt_start + rqt_chunk;
402 while (rqt_start < rqt_top) {
403 rqt_chunk = min(rqt_top - rqt_start + 1, rqt_chunk);
404 if (gen_pool_add(rdev->rqt_pool, rqt_start, rqt_chunk, -1)) {
405 pr_debug("failed to add RQT chunk (%x/%x)\n",
406 rqt_start, rqt_chunk);
407 if (rqt_chunk <= 1024 << MIN_RQT_SHIFT) {
408 pr_warn("Failed to add all RQT chunks (%x/%x)\n",
409 rqt_start, rqt_top - rqt_start);
414 pr_debug("added RQT chunk (%x/%x)\n",
415 rqt_start, rqt_chunk);
416 rqt_start += rqt_chunk;
422 void c4iw_rqtpool_destroy(struct c4iw_rdev *rdev)
424 kref_put(&rdev->rqt_kref, destroy_rqtpool);
427 int c4iw_alloc_srq_idx(struct c4iw_rdev *rdev)
431 idx = c4iw_id_alloc(&rdev->resource.srq_table);
432 mutex_lock(&rdev->stats.lock);
434 rdev->stats.srqt.fail++;
435 mutex_unlock(&rdev->stats.lock);
438 rdev->stats.srqt.cur++;
439 if (rdev->stats.srqt.cur > rdev->stats.srqt.max)
440 rdev->stats.srqt.max = rdev->stats.srqt.cur;
441 mutex_unlock(&rdev->stats.lock);
445 void c4iw_free_srq_idx(struct c4iw_rdev *rdev, int idx)
447 c4iw_id_free(&rdev->resource.srq_table, idx);
448 mutex_lock(&rdev->stats.lock);
449 rdev->stats.srqt.cur--;
450 mutex_unlock(&rdev->stats.lock);
456 #define MIN_OCQP_SHIFT 12 /* 4KB == min ocqp size */
458 u32 c4iw_ocqp_pool_alloc(struct c4iw_rdev *rdev, int size)
460 unsigned long addr = gen_pool_alloc(rdev->ocqp_pool, size);
461 pr_debug("addr 0x%x size %d\n", (u32)addr, size);
463 mutex_lock(&rdev->stats.lock);
464 rdev->stats.ocqp.cur += roundup(size, 1 << MIN_OCQP_SHIFT);
465 if (rdev->stats.ocqp.cur > rdev->stats.ocqp.max)
466 rdev->stats.ocqp.max = rdev->stats.ocqp.cur;
467 mutex_unlock(&rdev->stats.lock);
472 void c4iw_ocqp_pool_free(struct c4iw_rdev *rdev, u32 addr, int size)
474 pr_debug("addr 0x%x size %d\n", addr, size);
475 mutex_lock(&rdev->stats.lock);
476 rdev->stats.ocqp.cur -= roundup(size, 1 << MIN_OCQP_SHIFT);
477 mutex_unlock(&rdev->stats.lock);
478 gen_pool_free(rdev->ocqp_pool, (unsigned long)addr, size);
481 int c4iw_ocqp_pool_create(struct c4iw_rdev *rdev)
483 unsigned start, chunk, top;
485 rdev->ocqp_pool = gen_pool_create(MIN_OCQP_SHIFT, -1);
486 if (!rdev->ocqp_pool)
489 start = rdev->lldi.vr->ocq.start;
490 chunk = rdev->lldi.vr->ocq.size;
493 while (start < top) {
494 chunk = min(top - start + 1, chunk);
495 if (gen_pool_add(rdev->ocqp_pool, start, chunk, -1)) {
496 pr_debug("failed to add OCQP chunk (%x/%x)\n",
498 if (chunk <= 1024 << MIN_OCQP_SHIFT) {
499 pr_warn("Failed to add all OCQP chunks (%x/%x)\n",
505 pr_debug("added OCQP chunk (%x/%x)\n",
513 void c4iw_ocqp_pool_destroy(struct c4iw_rdev *rdev)
515 gen_pool_destroy(rdev->ocqp_pool);