2 * Copyright (c) 2016 Hisilicon Limited.
3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 #include <linux/platform_device.h>
35 #include "hns_roce_device.h"
36 #include "hns_roce_hem.h"
37 #include "hns_roce_common.h"
39 #define DMA_ADDR_T_SHIFT 12
40 #define BT_BA_SHIFT 32
42 #define HEM_INDEX_BUF BIT(0)
43 #define HEM_INDEX_L0 BIT(1)
44 #define HEM_INDEX_L1 BIT(2)
45 struct hns_roce_hem_index {
49 u32 inited; /* indicate which index is available */
52 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
58 hop_num = hr_dev->caps.qpc_hop_num;
61 hop_num = hr_dev->caps.mpt_hop_num;
64 hop_num = hr_dev->caps.cqc_hop_num;
67 hop_num = hr_dev->caps.srqc_hop_num;
70 hop_num = hr_dev->caps.sccc_hop_num;
72 case HEM_TYPE_QPC_TIMER:
73 hop_num = hr_dev->caps.qpc_timer_hop_num;
75 case HEM_TYPE_CQC_TIMER:
76 hop_num = hr_dev->caps.cqc_timer_hop_num;
82 return hop_num ? true : false;
85 static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 hem_idx,
86 u32 bt_chunk_num, u64 hem_max_num)
88 u64 start_idx = round_down(hem_idx, bt_chunk_num);
89 u64 check_max_num = start_idx + bt_chunk_num;
92 for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++)
93 if (i != hem_idx && hem[i])
99 static bool hns_roce_check_bt_null(u64 **bt, u64 ba_idx, u32 bt_chunk_num)
101 u64 start_idx = round_down(ba_idx, bt_chunk_num);
104 for (i = 0; i < bt_chunk_num; i++)
105 if (i != ba_idx && bt[start_idx + i])
111 static int hns_roce_get_bt_num(u32 table_type, u32 hop_num)
113 if (check_whether_bt_num_3(table_type, hop_num))
115 else if (check_whether_bt_num_2(table_type, hop_num))
117 else if (check_whether_bt_num_1(table_type, hop_num))
123 static int get_hem_table_config(struct hns_roce_dev *hr_dev,
124 struct hns_roce_hem_mhop *mhop,
127 struct device *dev = hr_dev->dev;
131 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
133 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
135 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num;
136 mhop->hop_num = hr_dev->caps.qpc_hop_num;
139 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
141 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
143 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num;
144 mhop->hop_num = hr_dev->caps.mpt_hop_num;
147 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
149 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
151 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num;
152 mhop->hop_num = hr_dev->caps.cqc_hop_num;
155 mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz
157 mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz
159 mhop->ba_l0_num = hr_dev->caps.sccc_bt_num;
160 mhop->hop_num = hr_dev->caps.sccc_hop_num;
162 case HEM_TYPE_QPC_TIMER:
163 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz
165 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz
167 mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num;
168 mhop->hop_num = hr_dev->caps.qpc_timer_hop_num;
170 case HEM_TYPE_CQC_TIMER:
171 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz
173 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz
175 mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num;
176 mhop->hop_num = hr_dev->caps.cqc_timer_hop_num;
179 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
181 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
183 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;
184 mhop->hop_num = hr_dev->caps.srqc_hop_num;
187 dev_err(dev, "table %u not support multi-hop addressing!\n",
195 int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
196 struct hns_roce_hem_table *table, unsigned long *obj,
197 struct hns_roce_hem_mhop *mhop)
199 struct device *dev = hr_dev->dev;
205 if (get_hem_table_config(hr_dev, mhop, table->type))
212 * QPC/MTPT/CQC/SRQC/SCCC alloc hem for buffer pages.
213 * MTT/CQE alloc hem for bt pages.
215 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
216 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
217 chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :
219 table_idx = (*obj & (table->num_obj - 1)) /
220 (chunk_size / table->obj_size);
223 mhop->l2_idx = table_idx & (chunk_ba_num - 1);
224 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1);
225 mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num;
228 mhop->l1_idx = table_idx & (chunk_ba_num - 1);
229 mhop->l0_idx = table_idx / chunk_ba_num;
232 mhop->l0_idx = table_idx;
235 dev_err(dev, "table %u not support hop_num = %u!\n",
236 table->type, mhop->hop_num);
239 if (mhop->l0_idx >= mhop->ba_l0_num)
240 mhop->l0_idx %= mhop->ba_l0_num;
245 static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
247 unsigned long hem_alloc_size,
250 struct hns_roce_hem_chunk *chunk = NULL;
251 struct hns_roce_hem *hem;
252 struct scatterlist *mem;
256 WARN_ON(gfp_mask & __GFP_HIGHMEM);
258 hem = kmalloc(sizeof(*hem),
259 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
264 INIT_LIST_HEAD(&hem->chunk_list);
266 order = get_order(hem_alloc_size);
270 chunk = kmalloc(sizeof(*chunk),
271 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
275 sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN);
278 memset(chunk->buf, 0, sizeof(chunk->buf));
279 list_add_tail(&chunk->list, &hem->chunk_list);
282 while (1 << order > npages)
286 * Alloc memory one time. If failed, don't alloc small block
287 * memory, directly return fail.
289 mem = &chunk->mem[chunk->npages];
290 buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order,
291 &sg_dma_address(mem), gfp_mask);
295 chunk->buf[chunk->npages] = buf;
296 sg_dma_len(mem) = PAGE_SIZE << order;
300 npages -= 1 << order;
306 hns_roce_free_hem(hr_dev, hem);
310 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)
312 struct hns_roce_hem_chunk *chunk, *tmp;
318 list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) {
319 for (i = 0; i < chunk->npages; ++i)
320 dma_free_coherent(hr_dev->dev,
321 sg_dma_len(&chunk->mem[i]),
323 sg_dma_address(&chunk->mem[i]));
330 static int hns_roce_set_hem(struct hns_roce_dev *hr_dev,
331 struct hns_roce_hem_table *table, unsigned long obj)
333 spinlock_t *lock = &hr_dev->bt_cmd_lock;
334 struct device *dev = hr_dev->dev;
337 struct hns_roce_hem_iter iter;
338 void __iomem *bt_cmd;
339 __le32 bt_cmd_val[2];
345 /* Find the HEM(Hardware Entry Memory) entry */
346 unsigned long i = (obj & (table->num_obj - 1)) /
347 (table->table_chunk_size / table->obj_size);
349 switch (table->type) {
354 roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
355 ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, table->type);
361 roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
362 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
363 roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
364 roce_set_bit(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
366 /* Currently iter only a chunk */
367 for (hns_roce_hem_first(table->hem[i], &iter);
368 !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
369 bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT;
371 spin_lock_irqsave(lock, flags);
373 bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
375 end = HW_SYNC_TIMEOUT_MSECS;
377 if (!(readl(bt_cmd) >> BT_CMD_SYNC_SHIFT))
380 mdelay(HW_SYNC_SLEEP_TIME_INTERVAL);
381 end -= HW_SYNC_SLEEP_TIME_INTERVAL;
385 dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
386 spin_unlock_irqrestore(lock, flags);
390 bt_cmd_l = cpu_to_le32(bt_ba);
391 roce_set_field(bt_cmd_h, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
392 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S,
393 bt_ba >> BT_BA_SHIFT);
395 bt_cmd_val[0] = bt_cmd_l;
396 bt_cmd_val[1] = bt_cmd_h;
397 hns_roce_write64_k(bt_cmd_val,
398 hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
399 spin_unlock_irqrestore(lock, flags);
405 static int calc_hem_config(struct hns_roce_dev *hr_dev,
406 struct hns_roce_hem_table *table, unsigned long obj,
407 struct hns_roce_hem_mhop *mhop,
408 struct hns_roce_hem_index *index)
410 struct ib_device *ibdev = &hr_dev->ib_dev;
411 unsigned long mhop_obj = obj;
412 u32 l0_idx, l1_idx, l2_idx;
417 ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop);
421 l0_idx = mhop->l0_idx;
422 l1_idx = mhop->l1_idx;
423 l2_idx = mhop->l2_idx;
424 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
425 bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
428 index->l1 = l0_idx * chunk_ba_num + l1_idx;
430 index->buf = l0_idx * chunk_ba_num * chunk_ba_num +
431 l1_idx * chunk_ba_num + l2_idx;
435 index->buf = l0_idx * chunk_ba_num + l1_idx;
441 ibdev_err(ibdev, "table %u not support mhop.hop_num = %u!\n",
442 table->type, mhop->hop_num);
446 if (unlikely(index->buf >= table->num_hem)) {
447 ibdev_err(ibdev, "table %u exceed hem limt idx %llu, max %lu!\n",
448 table->type, index->buf, table->num_hem);
455 static void free_mhop_hem(struct hns_roce_dev *hr_dev,
456 struct hns_roce_hem_table *table,
457 struct hns_roce_hem_mhop *mhop,
458 struct hns_roce_hem_index *index)
460 u32 bt_size = mhop->bt_chunk_size;
461 struct device *dev = hr_dev->dev;
463 if (index->inited & HEM_INDEX_BUF) {
464 hns_roce_free_hem(hr_dev, table->hem[index->buf]);
465 table->hem[index->buf] = NULL;
468 if (index->inited & HEM_INDEX_L1) {
469 dma_free_coherent(dev, bt_size, table->bt_l1[index->l1],
470 table->bt_l1_dma_addr[index->l1]);
471 table->bt_l1[index->l1] = NULL;
474 if (index->inited & HEM_INDEX_L0) {
475 dma_free_coherent(dev, bt_size, table->bt_l0[index->l0],
476 table->bt_l0_dma_addr[index->l0]);
477 table->bt_l0[index->l0] = NULL;
481 static int alloc_mhop_hem(struct hns_roce_dev *hr_dev,
482 struct hns_roce_hem_table *table,
483 struct hns_roce_hem_mhop *mhop,
484 struct hns_roce_hem_index *index)
486 u32 bt_size = mhop->bt_chunk_size;
487 struct device *dev = hr_dev->dev;
488 struct hns_roce_hem_iter iter;
494 /* alloc L1 BA's chunk */
495 if ((check_whether_bt_num_3(table->type, mhop->hop_num) ||
496 check_whether_bt_num_2(table->type, mhop->hop_num)) &&
497 !table->bt_l0[index->l0]) {
498 table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size,
499 &table->bt_l0_dma_addr[index->l0],
501 if (!table->bt_l0[index->l0]) {
505 index->inited |= HEM_INDEX_L0;
508 /* alloc L2 BA's chunk */
509 if (check_whether_bt_num_3(table->type, mhop->hop_num) &&
510 !table->bt_l1[index->l1]) {
511 table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size,
512 &table->bt_l1_dma_addr[index->l1],
514 if (!table->bt_l1[index->l1]) {
518 index->inited |= HEM_INDEX_L1;
519 *(table->bt_l0[index->l0] + mhop->l1_idx) =
520 table->bt_l1_dma_addr[index->l1];
524 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC.
525 * alloc bt space chunk for MTT/CQE.
527 size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size;
528 flag = (table->lowmem ? GFP_KERNEL : GFP_HIGHUSER) | __GFP_NOWARN;
529 table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT,
531 if (!table->hem[index->buf]) {
536 index->inited |= HEM_INDEX_BUF;
537 hns_roce_hem_first(table->hem[index->buf], &iter);
538 bt_ba = hns_roce_hem_addr(&iter);
539 if (table->type < HEM_TYPE_MTT) {
540 if (mhop->hop_num == 2)
541 *(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba;
542 else if (mhop->hop_num == 1)
543 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
544 } else if (mhop->hop_num == 2) {
545 *(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;
550 free_mhop_hem(hr_dev, table, mhop, index);
555 static int set_mhop_hem(struct hns_roce_dev *hr_dev,
556 struct hns_roce_hem_table *table, unsigned long obj,
557 struct hns_roce_hem_mhop *mhop,
558 struct hns_roce_hem_index *index)
560 struct ib_device *ibdev = &hr_dev->ib_dev;
564 if (index->inited & HEM_INDEX_L0) {
565 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0);
567 ibdev_err(ibdev, "set HEM step 0 failed!\n");
572 if (index->inited & HEM_INDEX_L1) {
573 ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1);
575 ibdev_err(ibdev, "set HEM step 1 failed!\n");
580 if (index->inited & HEM_INDEX_BUF) {
581 if (mhop->hop_num == HNS_ROCE_HOP_NUM_0)
584 step_idx = mhop->hop_num;
585 ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx);
587 ibdev_err(ibdev, "set HEM step last failed!\n");
593 static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,
594 struct hns_roce_hem_table *table,
597 struct ib_device *ibdev = &hr_dev->ib_dev;
598 struct hns_roce_hem_index index = {};
599 struct hns_roce_hem_mhop mhop = {};
602 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
604 ibdev_err(ibdev, "calc hem config failed!\n");
608 mutex_lock(&table->mutex);
609 if (table->hem[index.buf]) {
610 ++table->hem[index.buf]->refcount;
614 ret = alloc_mhop_hem(hr_dev, table, &mhop, &index);
616 ibdev_err(ibdev, "alloc mhop hem failed!\n");
620 /* set HEM base address to hardware */
621 if (table->type < HEM_TYPE_MTT) {
622 ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index);
624 ibdev_err(ibdev, "set HEM address to HW failed!\n");
629 ++table->hem[index.buf]->refcount;
633 free_mhop_hem(hr_dev, table, &mhop, &index);
635 mutex_unlock(&table->mutex);
639 int hns_roce_table_get(struct hns_roce_dev *hr_dev,
640 struct hns_roce_hem_table *table, unsigned long obj)
642 struct device *dev = hr_dev->dev;
646 if (hns_roce_check_whether_mhop(hr_dev, table->type))
647 return hns_roce_table_mhop_get(hr_dev, table, obj);
649 i = (obj & (table->num_obj - 1)) / (table->table_chunk_size /
652 mutex_lock(&table->mutex);
655 ++table->hem[i]->refcount;
659 table->hem[i] = hns_roce_alloc_hem(hr_dev,
660 table->table_chunk_size >> PAGE_SHIFT,
661 table->table_chunk_size,
662 (table->lowmem ? GFP_KERNEL :
663 GFP_HIGHUSER) | __GFP_NOWARN);
664 if (!table->hem[i]) {
669 /* Set HEM base address(128K/page, pa) to Hardware */
670 if (hns_roce_set_hem(hr_dev, table, obj)) {
671 hns_roce_free_hem(hr_dev, table->hem[i]);
672 table->hem[i] = NULL;
674 dev_err(dev, "set HEM base address to HW failed.\n");
678 ++table->hem[i]->refcount;
680 mutex_unlock(&table->mutex);
684 static void clear_mhop_hem(struct hns_roce_dev *hr_dev,
685 struct hns_roce_hem_table *table, unsigned long obj,
686 struct hns_roce_hem_mhop *mhop,
687 struct hns_roce_hem_index *index)
689 struct ib_device *ibdev = &hr_dev->ib_dev;
690 u32 hop_num = mhop->hop_num;
694 index->inited = HEM_INDEX_BUF;
695 chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN;
696 if (check_whether_bt_num_2(table->type, hop_num)) {
697 if (hns_roce_check_hem_null(table->hem, index->buf,
698 chunk_ba_num, table->num_hem))
699 index->inited |= HEM_INDEX_L0;
700 } else if (check_whether_bt_num_3(table->type, hop_num)) {
701 if (hns_roce_check_hem_null(table->hem, index->buf,
702 chunk_ba_num, table->num_hem)) {
703 index->inited |= HEM_INDEX_L1;
704 if (hns_roce_check_bt_null(table->bt_l1, index->l1,
706 index->inited |= HEM_INDEX_L0;
710 if (table->type < HEM_TYPE_MTT) {
711 if (hop_num == HNS_ROCE_HOP_NUM_0)
716 if (hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx))
717 ibdev_warn(ibdev, "failed to clear hop%u HEM.\n", hop_num);
719 if (index->inited & HEM_INDEX_L1)
720 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
721 ibdev_warn(ibdev, "failed to clear HEM step 1.\n");
723 if (index->inited & HEM_INDEX_L0)
724 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
725 ibdev_warn(ibdev, "failed to clear HEM step 0.\n");
729 static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
730 struct hns_roce_hem_table *table,
734 struct ib_device *ibdev = &hr_dev->ib_dev;
735 struct hns_roce_hem_index index = {};
736 struct hns_roce_hem_mhop mhop = {};
739 ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);
741 ibdev_err(ibdev, "calc hem config failed!\n");
745 mutex_lock(&table->mutex);
746 if (check_refcount && (--table->hem[index.buf]->refcount > 0)) {
747 mutex_unlock(&table->mutex);
751 clear_mhop_hem(hr_dev, table, obj, &mhop, &index);
752 free_mhop_hem(hr_dev, table, &mhop, &index);
754 mutex_unlock(&table->mutex);
757 void hns_roce_table_put(struct hns_roce_dev *hr_dev,
758 struct hns_roce_hem_table *table, unsigned long obj)
760 struct device *dev = hr_dev->dev;
763 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
764 hns_roce_table_mhop_put(hr_dev, table, obj, 1);
768 i = (obj & (table->num_obj - 1)) /
769 (table->table_chunk_size / table->obj_size);
771 mutex_lock(&table->mutex);
773 if (--table->hem[i]->refcount == 0) {
774 /* Clear HEM base address */
775 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
776 dev_warn(dev, "Clear HEM base address failed.\n");
778 hns_roce_free_hem(hr_dev, table->hem[i]);
779 table->hem[i] = NULL;
782 mutex_unlock(&table->mutex);
785 void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
786 struct hns_roce_hem_table *table,
787 unsigned long obj, dma_addr_t *dma_handle)
789 struct hns_roce_hem_chunk *chunk;
790 struct hns_roce_hem_mhop mhop;
791 struct hns_roce_hem *hem;
793 unsigned long mhop_obj = obj;
794 unsigned long obj_per_chunk;
795 unsigned long idx_offset;
796 int offset, dma_offset;
804 mutex_lock(&table->mutex);
806 if (!hns_roce_check_whether_mhop(hr_dev, table->type)) {
807 obj_per_chunk = table->table_chunk_size / table->obj_size;
808 hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk];
809 idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk;
810 dma_offset = offset = idx_offset * table->obj_size;
812 u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */
814 if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop))
819 if (mhop.hop_num == 2)
820 hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j;
821 else if (mhop.hop_num == 1 ||
822 mhop.hop_num == HNS_ROCE_HOP_NUM_0)
825 hem = table->hem[hem_idx];
826 dma_offset = offset = (obj & (table->num_obj - 1)) * seg_size %
828 if (mhop.hop_num == 2)
829 dma_offset = offset = 0;
835 list_for_each_entry(chunk, &hem->chunk_list, list) {
836 for (i = 0; i < chunk->npages; ++i) {
837 length = sg_dma_len(&chunk->mem[i]);
838 if (dma_handle && dma_offset >= 0) {
839 if (length > (u32)dma_offset)
840 *dma_handle = sg_dma_address(
841 &chunk->mem[i]) + dma_offset;
842 dma_offset -= length;
845 if (length > (u32)offset) {
846 addr = chunk->buf[i] + offset;
854 mutex_unlock(&table->mutex);
858 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
859 struct hns_roce_hem_table *table, u32 type,
860 unsigned long obj_size, unsigned long nobj,
863 unsigned long obj_per_chunk;
864 unsigned long num_hem;
866 if (!hns_roce_check_whether_mhop(hr_dev, type)) {
867 table->table_chunk_size = hr_dev->caps.chunk_sz;
868 obj_per_chunk = table->table_chunk_size / obj_size;
869 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
871 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);
875 struct hns_roce_hem_mhop mhop = {};
876 unsigned long buf_chunk_size;
877 unsigned long bt_chunk_size;
878 unsigned long bt_chunk_num;
879 unsigned long num_bt_l0 = 0;
882 if (get_hem_table_config(hr_dev, &mhop, type))
885 buf_chunk_size = mhop.buf_chunk_size;
886 bt_chunk_size = mhop.bt_chunk_size;
887 num_bt_l0 = mhop.ba_l0_num;
888 hop_num = mhop.hop_num;
890 obj_per_chunk = buf_chunk_size / obj_size;
891 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
892 bt_chunk_num = bt_chunk_size / BA_BYTE_LEN;
893 if (type >= HEM_TYPE_MTT)
894 num_bt_l0 = bt_chunk_num;
896 table->hem = kcalloc(num_hem, sizeof(*table->hem),
899 goto err_kcalloc_hem_buf;
901 if (check_whether_bt_num_3(type, hop_num)) {
902 unsigned long num_bt_l1;
904 num_bt_l1 = (num_hem + bt_chunk_num - 1) /
906 table->bt_l1 = kcalloc(num_bt_l1,
907 sizeof(*table->bt_l1),
910 goto err_kcalloc_bt_l1;
912 table->bt_l1_dma_addr = kcalloc(num_bt_l1,
913 sizeof(*table->bt_l1_dma_addr),
916 if (!table->bt_l1_dma_addr)
917 goto err_kcalloc_l1_dma;
920 if (check_whether_bt_num_2(type, hop_num) ||
921 check_whether_bt_num_3(type, hop_num)) {
922 table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0),
925 goto err_kcalloc_bt_l0;
927 table->bt_l0_dma_addr = kcalloc(num_bt_l0,
928 sizeof(*table->bt_l0_dma_addr),
930 if (!table->bt_l0_dma_addr)
931 goto err_kcalloc_l0_dma;
936 table->num_hem = num_hem;
937 table->num_obj = nobj;
938 table->obj_size = obj_size;
939 table->lowmem = use_lowmem;
940 mutex_init(&table->mutex);
949 kfree(table->bt_l1_dma_addr);
950 table->bt_l1_dma_addr = NULL;
964 static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev,
965 struct hns_roce_hem_table *table)
967 struct hns_roce_hem_mhop mhop;
972 if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))
974 buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :
977 for (i = 0; i < table->num_hem; ++i) {
978 obj = i * buf_chunk_size / table->obj_size;
980 hns_roce_table_mhop_put(hr_dev, table, obj, 0);
987 kfree(table->bt_l1_dma_addr);
988 table->bt_l1_dma_addr = NULL;
991 kfree(table->bt_l0_dma_addr);
992 table->bt_l0_dma_addr = NULL;
995 void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
996 struct hns_roce_hem_table *table)
998 struct device *dev = hr_dev->dev;
1001 if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
1002 hns_roce_cleanup_mhop_hem_table(hr_dev, table);
1006 for (i = 0; i < table->num_hem; ++i)
1007 if (table->hem[i]) {
1008 if (hr_dev->hw->clear_hem(hr_dev, table,
1009 i * table->table_chunk_size / table->obj_size, 0))
1010 dev_err(dev, "Clear HEM base address failed.\n");
1012 hns_roce_free_hem(hr_dev, table->hem[i]);
1018 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
1020 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ)
1021 hns_roce_cleanup_hem_table(hr_dev,
1022 &hr_dev->srq_table.table);
1023 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
1024 if (hr_dev->caps.qpc_timer_entry_sz)
1025 hns_roce_cleanup_hem_table(hr_dev,
1026 &hr_dev->qpc_timer_table);
1027 if (hr_dev->caps.cqc_timer_entry_sz)
1028 hns_roce_cleanup_hem_table(hr_dev,
1029 &hr_dev->cqc_timer_table);
1030 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL)
1031 hns_roce_cleanup_hem_table(hr_dev,
1032 &hr_dev->qp_table.sccc_table);
1033 if (hr_dev->caps.trrl_entry_sz)
1034 hns_roce_cleanup_hem_table(hr_dev,
1035 &hr_dev->qp_table.trrl_table);
1036 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
1037 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
1038 hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
1041 struct roce_hem_item {
1042 struct list_head list; /* link all hems in the same bt level */
1043 struct list_head sibling; /* link all hems in last hop for mtt */
1045 dma_addr_t dma_addr;
1046 size_t count; /* max ba numbers */
1047 int start; /* start buf offset in this hem */
1048 int end; /* end buf offset in this hem */
1051 static struct roce_hem_item *hem_list_alloc_item(struct hns_roce_dev *hr_dev,
1053 int count, bool exist_bt,
1056 struct roce_hem_item *hem;
1058 hem = kzalloc(sizeof(*hem), GFP_KERNEL);
1063 hem->addr = dma_alloc_coherent(hr_dev->dev,
1064 count * BA_BYTE_LEN,
1065 &hem->dma_addr, GFP_KERNEL);
1075 INIT_LIST_HEAD(&hem->list);
1076 INIT_LIST_HEAD(&hem->sibling);
1081 static void hem_list_free_item(struct hns_roce_dev *hr_dev,
1082 struct roce_hem_item *hem, bool exist_bt)
1085 dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN,
1086 hem->addr, hem->dma_addr);
1090 static void hem_list_free_all(struct hns_roce_dev *hr_dev,
1091 struct list_head *head, bool exist_bt)
1093 struct roce_hem_item *hem, *temp_hem;
1095 list_for_each_entry_safe(hem, temp_hem, head, list) {
1096 list_del(&hem->list);
1097 hem_list_free_item(hr_dev, hem, exist_bt);
1101 static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr,
1104 *(u64 *)(base_addr) = table_addr;
1107 /* assign L0 table address to hem from root bt */
1108 static void hem_list_assign_bt(struct hns_roce_dev *hr_dev,
1109 struct roce_hem_item *hem, void *cpu_addr,
1112 hem->addr = cpu_addr;
1113 hem->dma_addr = (dma_addr_t)phy_addr;
1116 static inline bool hem_list_page_is_in_range(struct roce_hem_item *hem,
1119 return (hem->start <= offset && offset <= hem->end);
1122 static struct roce_hem_item *hem_list_search_item(struct list_head *ba_list,
1125 struct roce_hem_item *hem, *temp_hem;
1126 struct roce_hem_item *found = NULL;
1128 list_for_each_entry_safe(hem, temp_hem, ba_list, list) {
1129 if (hem_list_page_is_in_range(hem, page_offset)) {
1138 static bool hem_list_is_bottom_bt(int hopnum, int bt_level)
1141 * hopnum base address table levels
1145 * 3 L0 -> L1 -> L2 -> buf
1147 return bt_level >= (hopnum ? hopnum - 1 : hopnum);
1151 * calc base address entries num
1152 * @hopnum: num of mutihop addressing
1153 * @bt_level: base address table level
1154 * @unit: ba entries per bt page
1156 static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit)
1162 if (hopnum <= bt_level)
1165 * hopnum bt_level range
1171 * 3 0 unit * unit * unit
1176 max = hopnum - bt_level;
1177 for (i = 0; i < max; i++)
1184 * calc the root ba entries which could cover all regions
1185 * @regions: buf region array
1186 * @region_cnt: array size of @regions
1187 * @unit: ba entries per bt page
1189 int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions,
1190 int region_cnt, int unit)
1192 struct hns_roce_buf_region *r;
1197 for (i = 0; i < region_cnt; i++) {
1198 r = (struct hns_roce_buf_region *)®ions[i];
1199 if (r->hopnum > 1) {
1200 step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1202 total += (r->count + step - 1) / step;
1211 static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev,
1212 const struct hns_roce_buf_region *r, int unit,
1213 int offset, struct list_head *mid_bt,
1214 struct list_head *btm_bt)
1216 struct roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL };
1217 struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL];
1218 struct roce_hem_item *cur, *pre;
1219 const int hopnum = r->hopnum;
1231 if (hopnum > HNS_ROCE_MAX_BT_LEVEL) {
1232 dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum);
1236 if (offset < r->offset) {
1237 dev_err(hr_dev->dev, "invalid offset %d, min %u!\n",
1242 distance = offset - r->offset;
1243 max_ofs = r->offset + r->count - 1;
1244 for (level = 0; level < hopnum; level++)
1245 INIT_LIST_HEAD(&temp_list[level]);
1247 /* config L1 bt to last bt and link them to corresponding parent */
1248 for (level = 1; level < hopnum; level++) {
1249 cur = hem_list_search_item(&mid_bt[level], offset);
1251 hem_ptrs[level] = cur;
1255 step = hem_list_calc_ba_range(hopnum, level, unit);
1261 start_aligned = (distance / step) * step + r->offset;
1262 end = min_t(int, start_aligned + step - 1, max_ofs);
1263 cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit,
1269 hem_ptrs[level] = cur;
1270 list_add(&cur->list, &temp_list[level]);
1271 if (hem_list_is_bottom_bt(hopnum, level))
1272 list_add(&cur->sibling, &temp_list[0]);
1274 /* link bt to parent bt */
1276 pre = hem_ptrs[level - 1];
1277 step = (cur->start - pre->start) / step * BA_BYTE_LEN;
1278 hem_list_link_bt(hr_dev, pre->addr + step,
1283 list_splice(&temp_list[0], btm_bt);
1284 for (level = 1; level < hopnum; level++)
1285 list_splice(&temp_list[level], &mid_bt[level]);
1290 for (level = 1; level < hopnum; level++)
1291 hem_list_free_all(hr_dev, &temp_list[level], true);
1296 static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev,
1297 struct hns_roce_hem_list *hem_list, int unit,
1298 const struct hns_roce_buf_region *regions,
1301 struct roce_hem_item *hem, *temp_hem, *root_hem;
1302 struct list_head temp_list[HNS_ROCE_MAX_BT_REGION];
1303 const struct hns_roce_buf_region *r;
1304 struct list_head temp_root;
1305 struct list_head temp_btm;
1316 root_hem = hem_list_search_item(&hem_list->root_bt, r->offset);
1320 ba_num = hns_roce_hem_list_calc_root_ba(regions, region_cnt, unit);
1324 INIT_LIST_HEAD(&temp_root);
1326 /* indicate to last region */
1327 r = ®ions[region_cnt - 1];
1328 root_hem = hem_list_alloc_item(hr_dev, offset, r->offset + r->count - 1,
1332 list_add(&root_hem->list, &temp_root);
1334 hem_list->root_ba = root_hem->dma_addr;
1336 INIT_LIST_HEAD(&temp_btm);
1337 for (i = 0; i < region_cnt; i++)
1338 INIT_LIST_HEAD(&temp_list[i]);
1341 for (i = 0; i < region_cnt && total < ba_num; i++) {
1346 /* all regions's mid[x][0] shared the root_bt's trunk */
1347 cpu_base = root_hem->addr + total * BA_BYTE_LEN;
1348 phy_base = root_hem->dma_addr + total * BA_BYTE_LEN;
1350 /* if hopnum is 0 or 1, cut a new fake hem from the root bt
1351 * which's address share to all regions.
1353 if (hem_list_is_bottom_bt(r->hopnum, 0)) {
1354 hem = hem_list_alloc_item(hr_dev, r->offset,
1355 r->offset + r->count - 1,
1356 r->count, false, 0);
1361 hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base);
1362 list_add(&hem->list, &temp_list[i]);
1363 list_add(&hem->sibling, &temp_btm);
1366 step = hem_list_calc_ba_range(r->hopnum, 1, unit);
1371 /* if exist mid bt, link L1 to L0 */
1372 list_for_each_entry_safe(hem, temp_hem,
1373 &hem_list->mid_bt[i][1], list) {
1374 offset = (hem->start - r->offset) / step *
1376 hem_list_link_bt(hr_dev, cpu_base + offset,
1383 list_splice(&temp_btm, &hem_list->btm_bt);
1384 list_splice(&temp_root, &hem_list->root_bt);
1385 for (i = 0; i < region_cnt; i++)
1386 list_splice(&temp_list[i], &hem_list->mid_bt[i][0]);
1391 for (i = 0; i < region_cnt; i++)
1392 hem_list_free_all(hr_dev, &temp_list[i], false);
1394 hem_list_free_all(hr_dev, &temp_root, true);
1399 /* construct the base address table and link them by address hop config */
1400 int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev,
1401 struct hns_roce_hem_list *hem_list,
1402 const struct hns_roce_buf_region *regions,
1403 int region_cnt, unsigned int bt_pg_shift)
1405 const struct hns_roce_buf_region *r;
1411 if (region_cnt > HNS_ROCE_MAX_BT_REGION) {
1412 dev_err(hr_dev->dev, "invalid region region_cnt %d!\n",
1417 unit = (1 << bt_pg_shift) / BA_BYTE_LEN;
1418 for (i = 0; i < region_cnt; i++) {
1423 end = r->offset + r->count;
1424 for (ofs = r->offset; ofs < end; ofs += unit) {
1425 ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs,
1426 hem_list->mid_bt[i],
1429 dev_err(hr_dev->dev,
1430 "alloc hem trunk fail ret=%d!\n", ret);
1436 ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions,
1439 dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret);
1444 hns_roce_hem_list_release(hr_dev, hem_list);
1449 void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev,
1450 struct hns_roce_hem_list *hem_list)
1454 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1455 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1456 hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j],
1459 hem_list_free_all(hr_dev, &hem_list->root_bt, true);
1460 INIT_LIST_HEAD(&hem_list->btm_bt);
1461 hem_list->root_ba = 0;
1464 void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list)
1468 INIT_LIST_HEAD(&hem_list->root_bt);
1469 INIT_LIST_HEAD(&hem_list->btm_bt);
1470 for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++)
1471 for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++)
1472 INIT_LIST_HEAD(&hem_list->mid_bt[i][j]);
1475 void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev,
1476 struct hns_roce_hem_list *hem_list,
1477 int offset, int *mtt_cnt, u64 *phy_addr)
1479 struct list_head *head = &hem_list->btm_bt;
1480 struct roce_hem_item *hem, *temp_hem;
1481 void *cpu_base = NULL;
1485 list_for_each_entry_safe(hem, temp_hem, head, sibling) {
1486 if (hem_list_page_is_in_range(hem, offset)) {
1487 nr = offset - hem->start;
1488 cpu_base = hem->addr + nr * BA_BYTE_LEN;
1489 phy_base = hem->dma_addr + nr * BA_BYTE_LEN;
1490 nr = hem->end + 1 - offset;
1499 *phy_addr = phy_base;