drivers/infiniband/hw/hns/hns_roce_hem.c

   1 /*
   2  * Copyright (c) 2016 Hisilicon Limited.
   3  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #include <linux/platform_device.h>
  35 #include "hns_roce_device.h"
  36 #include "hns_roce_hem.h"
  37 #include "hns_roce_common.h"
  38
  39 #define DMA_ADDR_T_SHIFT                12
  40 #define BT_BA_SHIFT                     32
  41
  42 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
  43 {
  44         if ((hr_dev->caps.qpc_hop_num && type == HEM_TYPE_QPC) ||
  45             (hr_dev->caps.mpt_hop_num && type == HEM_TYPE_MTPT) ||
  46             (hr_dev->caps.cqc_hop_num && type == HEM_TYPE_CQC) ||
  47             (hr_dev->caps.srqc_hop_num && type == HEM_TYPE_SRQC) ||
  48             (hr_dev->caps.cqe_hop_num && type == HEM_TYPE_CQE) ||
  49             (hr_dev->caps.mtt_hop_num && type == HEM_TYPE_MTT))
  50                 return true;
  51
  52         return false;
  53 }
  54 EXPORT_SYMBOL_GPL(hns_roce_check_whether_mhop);
  55
  56 static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx,
  57                             u32 bt_chunk_num, u64 hem_max_num)
  58 {
  59         u64 check_max_num = start_idx + bt_chunk_num;
  60         u64 i;
  61
  62         for (i = start_idx; (i < check_max_num) && (i < hem_max_num); i++)
  63                 if (hem[i])
  64                         return false;
  65
  66         return true;
  67 }
  68
  69 static bool hns_roce_check_bt_null(u64 **bt, u64 start_idx, u32 bt_chunk_num)
  70 {
  71         int i;
  72
  73         for (i = 0; i < bt_chunk_num; i++)
  74                 if (bt[start_idx + i])
  75                         return false;
  76
  77         return true;
  78 }
  79
  80 static int hns_roce_get_bt_num(u32 table_type, u32 hop_num)
  81 {
  82         if (check_whether_bt_num_3(table_type, hop_num))
  83                 return 3;
  84         else if (check_whether_bt_num_2(table_type, hop_num))
  85                 return 2;
  86         else if (check_whether_bt_num_1(table_type, hop_num))
  87                 return 1;
  88         else
  89                 return 0;
  90 }
  91
  92 int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
  93                            struct hns_roce_hem_table *table, unsigned long *obj,
  94                            struct hns_roce_hem_mhop *mhop)
  95 {
  96         struct device *dev = hr_dev->dev;
  97         u32 chunk_ba_num;
  98         u32 table_idx;
  99         u32 bt_num;
 100         u32 chunk_size;
 101
 102         switch (table->type) {
 103         case HEM_TYPE_QPC:
 104                 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
 105                                              + PAGE_SHIFT);
 106                 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
 107                                              + PAGE_SHIFT);
 108                 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num;
 109                 mhop->hop_num = hr_dev->caps.qpc_hop_num;
 110                 break;
 111         case HEM_TYPE_MTPT:
 112                 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
 113                                              + PAGE_SHIFT);
 114                 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
 115                                              + PAGE_SHIFT);
 116                 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num;
 117                 mhop->hop_num = hr_dev->caps.mpt_hop_num;
 118                 break;
 119         case HEM_TYPE_CQC:
 120                 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
 121                                              + PAGE_SHIFT);
 122                 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
 123                                             + PAGE_SHIFT);
 124                 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num;
 125                 mhop->hop_num = hr_dev->caps.cqc_hop_num;
 126                 break;
 127         case HEM_TYPE_SRQC:
 128                 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
 129                                              + PAGE_SHIFT);
 130                 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
 131                                              + PAGE_SHIFT);
 132                 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;
 133                 mhop->hop_num = hr_dev->caps.srqc_hop_num;
 134                 break;
 135         case HEM_TYPE_MTT:
 136                 mhop->buf_chunk_size = 1 << (hr_dev->caps.mtt_buf_pg_sz
 137                                              + PAGE_SHIFT);
 138                 mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
 139                                              + PAGE_SHIFT);
 140                 mhop->ba_l0_num = mhop->bt_chunk_size / 8;
 141                 mhop->hop_num = hr_dev->caps.mtt_hop_num;
 142                 break;
 143         case HEM_TYPE_CQE:
 144                 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqe_buf_pg_sz
 145                                              + PAGE_SHIFT);
 146                 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
 147                                              + PAGE_SHIFT);
 148                 mhop->ba_l0_num = mhop->bt_chunk_size / 8;
 149                 mhop->hop_num = hr_dev->caps.cqe_hop_num;
 150                 break;
 151         default:
 152                 dev_err(dev, "Table %d not support multi-hop addressing!\n",
 153                          table->type);
 154                 return -EINVAL;
 155         }
 156
 157         if (!obj)
 158                 return 0;
 159
 160         /*
 161          * QPC/MTPT/CQC/SRQC alloc hem for buffer pages.
 162          * MTT/CQE alloc hem for bt pages.
 163          */
 164         bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
 165         chunk_ba_num = mhop->bt_chunk_size / 8;
 166         chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :
 167                               mhop->bt_chunk_size;
 168         table_idx = (*obj & (table->num_obj - 1)) /
 169                      (chunk_size / table->obj_size);
 170         switch (bt_num) {
 171         case 3:
 172                 mhop->l2_idx = table_idx & (chunk_ba_num - 1);
 173                 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1);
 174                 mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num;
 175                 break;
 176         case 2:
 177                 mhop->l1_idx = table_idx & (chunk_ba_num - 1);
 178                 mhop->l0_idx = table_idx / chunk_ba_num;
 179                 break;
 180         case 1:
 181                 mhop->l0_idx = table_idx;
 182                 break;
 183         default:
 184                 dev_err(dev, "Table %d not support hop_num = %d!\n",
 185                              table->type, mhop->hop_num);
 186                 return -EINVAL;
 187         }
 188         if (mhop->l0_idx >= mhop->ba_l0_num)
 189                 mhop->l0_idx %= mhop->ba_l0_num;
 190
 191         return 0;
 192 }
 193 EXPORT_SYMBOL_GPL(hns_roce_calc_hem_mhop);
 194
 195 static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
 196                                                int npages,
 197                                                unsigned long hem_alloc_size,
 198                                                gfp_t gfp_mask)
 199 {
 200         struct hns_roce_hem_chunk *chunk = NULL;
 201         struct hns_roce_hem *hem;
 202         struct scatterlist *mem;
 203         int order;
 204         void *buf;
 205
 206         WARN_ON(gfp_mask & __GFP_HIGHMEM);
 207
 208         hem = kmalloc(sizeof(*hem),
 209                       gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
 210         if (!hem)
 211                 return NULL;
 212
 213         hem->refcount = 0;
 214         INIT_LIST_HEAD(&hem->chunk_list);
 215
 216         order = get_order(hem_alloc_size);
 217
 218         while (npages > 0) {
 219                 if (!chunk) {
 220                         chunk = kmalloc(sizeof(*chunk),
 221                                 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
 222                         if (!chunk)
 223                                 goto fail;
 224
 225                         sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN);
 226                         chunk->npages = 0;
 227                         chunk->nsg = 0;
 228                         memset(chunk->buf, 0, sizeof(chunk->buf));
 229                         list_add_tail(&chunk->list, &hem->chunk_list);
 230                 }
 231
 232                 while (1 << order > npages)
 233                         --order;
 234
 235                 /*
 236                  * Alloc memory one time. If failed, don't alloc small block
 237                  * memory, directly return fail.
 238                  */
 239                 mem = &chunk->mem[chunk->npages];
 240                 buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order,
 241                                 &sg_dma_address(mem), gfp_mask);
 242                 if (!buf)
 243                         goto fail;
 244
 245                 chunk->buf[chunk->npages] = buf;
 246                 sg_dma_len(mem) = PAGE_SIZE << order;
 247
 248                 ++chunk->npages;
 249                 ++chunk->nsg;
 250                 npages -= 1 << order;
 251         }
 252
 253         return hem;
 254
 255 fail:
 256         hns_roce_free_hem(hr_dev, hem);
 257         return NULL;
 258 }
 259
 260 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)
 261 {
 262         struct hns_roce_hem_chunk *chunk, *tmp;
 263         int i;
 264
 265         if (!hem)
 266                 return;
 267
 268         list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) {
 269                 for (i = 0; i < chunk->npages; ++i)
 270                         dma_free_coherent(hr_dev->dev,
 271                                    sg_dma_len(&chunk->mem[i]),
 272                                    chunk->buf[i],
 273                                    sg_dma_address(&chunk->mem[i]));
 274                 kfree(chunk);
 275         }
 276
 277         kfree(hem);
 278 }
 279
 280 static int hns_roce_set_hem(struct hns_roce_dev *hr_dev,
 281                             struct hns_roce_hem_table *table, unsigned long obj)
 282 {
 283         spinlock_t *lock = &hr_dev->bt_cmd_lock;
 284         struct device *dev = hr_dev->dev;
 285         unsigned long end = 0;
 286         unsigned long flags;
 287         struct hns_roce_hem_iter iter;
 288         void __iomem *bt_cmd;
 289         u32 bt_cmd_h_val = 0;
 290         u32 bt_cmd_val[2];
 291         u32 bt_cmd_l = 0;
 292         u64 bt_ba = 0;
 293         int ret = 0;
 294
 295         /* Find the HEM(Hardware Entry Memory) entry */
 296         unsigned long i = (obj & (table->num_obj - 1)) /
 297                           (table->table_chunk_size / table->obj_size);
 298
 299         switch (table->type) {
 300         case HEM_TYPE_QPC:
 301                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 302                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);
 303                 break;
 304         case HEM_TYPE_MTPT:
 305                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 306                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
 307                                HEM_TYPE_MTPT);
 308                 break;
 309         case HEM_TYPE_CQC:
 310                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 311                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);
 312                 break;
 313         case HEM_TYPE_SRQC:
 314                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 315                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
 316                                HEM_TYPE_SRQC);
 317                 break;
 318         default:
 319                 return ret;
 320         }
 321         roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
 322                        ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
 323         roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
 324         roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
 325
 326         /* Currently iter only a chunk */
 327         for (hns_roce_hem_first(table->hem[i], &iter);
 328              !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
 329                 bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT;
 330
 331                 spin_lock_irqsave(lock, flags);
 332
 333                 bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
 334
 335                 end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;
 336                 while (1) {
 337                         if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
 338                                 if (!(time_before(jiffies, end))) {
 339                                         dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
 340                                         spin_unlock_irqrestore(lock, flags);
 341                                         return -EBUSY;
 342                                 }
 343                         } else {
 344                                 break;
 345                         }
 346                         mdelay(HW_SYNC_SLEEP_TIME_INTERVAL);
 347                 }
 348
 349                 bt_cmd_l = (u32)bt_ba;
 350                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
 351                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S,
 352                                bt_ba >> BT_BA_SHIFT);
 353
 354                 bt_cmd_val[0] = bt_cmd_l;
 355                 bt_cmd_val[1] = bt_cmd_h_val;
 356                 hns_roce_write64_k(bt_cmd_val,
 357                                    hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
 358                 spin_unlock_irqrestore(lock, flags);
 359         }
 360
 361         return ret;
 362 }
 363
 364 static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,
 365                                    struct hns_roce_hem_table *table,
 366                                    unsigned long obj)
 367 {
 368         struct device *dev = hr_dev->dev;
 369         struct hns_roce_hem_mhop mhop;
 370         struct hns_roce_hem_iter iter;
 371         u32 buf_chunk_size;
 372         u32 bt_chunk_size;
 373         u32 chunk_ba_num;
 374         u32 hop_num;
 375         u32 size;
 376         u32 bt_num;
 377         u64 hem_idx;
 378         u64 bt_l1_idx = 0;
 379         u64 bt_l0_idx = 0;
 380         u64 bt_ba;
 381         unsigned long mhop_obj = obj;
 382         int bt_l1_allocated = 0;
 383         int bt_l0_allocated = 0;
 384         int step_idx;
 385         int ret;
 386
 387         ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
 388         if (ret)
 389                 return ret;
 390
 391         buf_chunk_size = mhop.buf_chunk_size;
 392         bt_chunk_size = mhop.bt_chunk_size;
 393         hop_num = mhop.hop_num;
 394         chunk_ba_num = bt_chunk_size / 8;
 395
 396         bt_num = hns_roce_get_bt_num(table->type, hop_num);
 397         switch (bt_num) {
 398         case 3:
 399                 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
 400                           mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
 401                 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
 402                 bt_l0_idx = mhop.l0_idx;
 403                 break;
 404         case 2:
 405                 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
 406                 bt_l0_idx = mhop.l0_idx;
 407                 break;
 408         case 1:
 409                 hem_idx = mhop.l0_idx;
 410                 break;
 411         default:
 412                 dev_err(dev, "Table %d not support hop_num = %d!\n",
 413                              table->type, hop_num);
 414                 return -EINVAL;
 415         }
 416
 417         if (unlikely(hem_idx >= table->num_hem)) {
 418                 dev_err(dev, "Table %d exceed hem limt idx = %llu,max = %lu!\n",
 419                              table->type, hem_idx, table->num_hem);
 420                 return -EINVAL;
 421         }
 422
 423         mutex_lock(&table->mutex);
 424
 425         if (table->hem[hem_idx]) {
 426                 ++table->hem[hem_idx]->refcount;
 427                 goto out;
 428         }
 429
 430         /* alloc L1 BA's chunk */
 431         if ((check_whether_bt_num_3(table->type, hop_num) ||
 432                 check_whether_bt_num_2(table->type, hop_num)) &&
 433                 !table->bt_l0[bt_l0_idx]) {
 434                 table->bt_l0[bt_l0_idx] = dma_alloc_coherent(dev, bt_chunk_size,
 435                                             &(table->bt_l0_dma_addr[bt_l0_idx]),
 436                                             GFP_KERNEL);
 437                 if (!table->bt_l0[bt_l0_idx]) {
 438                         ret = -ENOMEM;
 439                         goto out;
 440                 }
 441                 bt_l0_allocated = 1;
 442
 443                 /* set base address to hardware */
 444                 if (table->type < HEM_TYPE_MTT) {
 445                         step_idx = 0;
 446                         if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
 447                                 ret = -ENODEV;
 448                                 dev_err(dev, "set HEM base address to HW failed!\n");
 449                                 goto err_dma_alloc_l1;
 450                         }
 451                 }
 452         }
 453
 454         /* alloc L2 BA's chunk */
 455         if (check_whether_bt_num_3(table->type, hop_num) &&
 456             !table->bt_l1[bt_l1_idx])  {
 457                 table->bt_l1[bt_l1_idx] = dma_alloc_coherent(dev, bt_chunk_size,
 458                                             &(table->bt_l1_dma_addr[bt_l1_idx]),
 459                                             GFP_KERNEL);
 460                 if (!table->bt_l1[bt_l1_idx]) {
 461                         ret = -ENOMEM;
 462                         goto err_dma_alloc_l1;
 463                 }
 464                 bt_l1_allocated = 1;
 465                 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) =
 466                                                table->bt_l1_dma_addr[bt_l1_idx];
 467
 468                 /* set base address to hardware */
 469                 step_idx = 1;
 470                 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
 471                         ret = -ENODEV;
 472                         dev_err(dev, "set HEM base address to HW failed!\n");
 473                         goto err_alloc_hem_buf;
 474                 }
 475         }
 476
 477         /*
 478          * alloc buffer space chunk for QPC/MTPT/CQC/SRQC.
 479          * alloc bt space chunk for MTT/CQE.
 480          */
 481         size = table->type < HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size;
 482         table->hem[hem_idx] = hns_roce_alloc_hem(hr_dev,
 483                                                 size >> PAGE_SHIFT,
 484                                                 size,
 485                                                 (table->lowmem ? GFP_KERNEL :
 486                                                 GFP_HIGHUSER) | __GFP_NOWARN);
 487         if (!table->hem[hem_idx]) {
 488                 ret = -ENOMEM;
 489                 goto err_alloc_hem_buf;
 490         }
 491
 492         hns_roce_hem_first(table->hem[hem_idx], &iter);
 493         bt_ba = hns_roce_hem_addr(&iter);
 494
 495         if (table->type < HEM_TYPE_MTT) {
 496                 if (hop_num == 2) {
 497                         *(table->bt_l1[bt_l1_idx] + mhop.l2_idx) = bt_ba;
 498                         step_idx = 2;
 499                 } else if (hop_num == 1) {
 500                         *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
 501                         step_idx = 1;
 502                 } else if (hop_num == HNS_ROCE_HOP_NUM_0) {
 503                         step_idx = 0;
 504                 } else {
 505                         ret = -EINVAL;
 506                         goto err_dma_alloc_l1;
 507                 }
 508
 509                 /* set HEM base address to hardware */
 510                 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
 511                         ret = -ENODEV;
 512                         dev_err(dev, "set HEM base address to HW failed!\n");
 513                         goto err_alloc_hem_buf;
 514                 }
 515         } else if (hop_num == 2) {
 516                 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
 517         }
 518
 519         ++table->hem[hem_idx]->refcount;
 520         goto out;
 521
 522 err_alloc_hem_buf:
 523         if (bt_l1_allocated) {
 524                 dma_free_coherent(dev, bt_chunk_size, table->bt_l1[bt_l1_idx],
 525                                   table->bt_l1_dma_addr[bt_l1_idx]);
 526                 table->bt_l1[bt_l1_idx] = NULL;
 527         }
 528
 529 err_dma_alloc_l1:
 530         if (bt_l0_allocated) {
 531                 dma_free_coherent(dev, bt_chunk_size, table->bt_l0[bt_l0_idx],
 532                                   table->bt_l0_dma_addr[bt_l0_idx]);
 533                 table->bt_l0[bt_l0_idx] = NULL;
 534         }
 535
 536 out:
 537         mutex_unlock(&table->mutex);
 538         return ret;
 539 }
 540
 541 int hns_roce_table_get(struct hns_roce_dev *hr_dev,
 542                        struct hns_roce_hem_table *table, unsigned long obj)
 543 {
 544         struct device *dev = hr_dev->dev;
 545         int ret = 0;
 546         unsigned long i;
 547
 548         if (hns_roce_check_whether_mhop(hr_dev, table->type))
 549                 return hns_roce_table_mhop_get(hr_dev, table, obj);
 550
 551         i = (obj & (table->num_obj - 1)) / (table->table_chunk_size /
 552              table->obj_size);
 553
 554         mutex_lock(&table->mutex);
 555
 556         if (table->hem[i]) {
 557                 ++table->hem[i]->refcount;
 558                 goto out;
 559         }
 560
 561         table->hem[i] = hns_roce_alloc_hem(hr_dev,
 562                                        table->table_chunk_size >> PAGE_SHIFT,
 563                                        table->table_chunk_size,
 564                                        (table->lowmem ? GFP_KERNEL :
 565                                         GFP_HIGHUSER) | __GFP_NOWARN);
 566         if (!table->hem[i]) {
 567                 ret = -ENOMEM;
 568                 goto out;
 569         }
 570
 571         /* Set HEM base address(128K/page, pa) to Hardware */
 572         if (hns_roce_set_hem(hr_dev, table, obj)) {
 573                 hns_roce_free_hem(hr_dev, table->hem[i]);
 574                 table->hem[i] = NULL;
 575                 ret = -ENODEV;
 576                 dev_err(dev, "set HEM base address to HW failed.\n");
 577                 goto out;
 578         }
 579
 580         ++table->hem[i]->refcount;
 581 out:
 582         mutex_unlock(&table->mutex);
 583         return ret;
 584 }
 585
 586 static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
 587                                     struct hns_roce_hem_table *table,
 588                                     unsigned long obj,
 589                                     int check_refcount)
 590 {
 591         struct device *dev = hr_dev->dev;
 592         struct hns_roce_hem_mhop mhop;
 593         unsigned long mhop_obj = obj;
 594         u32 bt_chunk_size;
 595         u32 chunk_ba_num;
 596         u32 hop_num;
 597         u32 start_idx;
 598         u32 bt_num;
 599         u64 hem_idx;
 600         u64 bt_l1_idx = 0;
 601         int ret;
 602
 603         ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
 604         if (ret)
 605                 return;
 606
 607         bt_chunk_size = mhop.bt_chunk_size;
 608         hop_num = mhop.hop_num;
 609         chunk_ba_num = bt_chunk_size / 8;
 610
 611         bt_num = hns_roce_get_bt_num(table->type, hop_num);
 612         switch (bt_num) {
 613         case 3:
 614                 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
 615                           mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
 616                 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
 617                 break;
 618         case 2:
 619                 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
 620                 break;
 621         case 1:
 622                 hem_idx = mhop.l0_idx;
 623                 break;
 624         default:
 625                 dev_err(dev, "Table %d not support hop_num = %d!\n",
 626                              table->type, hop_num);
 627                 return;
 628         }
 629
 630         mutex_lock(&table->mutex);
 631
 632         if (check_refcount && (--table->hem[hem_idx]->refcount > 0)) {
 633                 mutex_unlock(&table->mutex);
 634                 return;
 635         }
 636
 637         if (table->type < HEM_TYPE_MTT && hop_num == 1) {
 638                 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
 639                         dev_warn(dev, "Clear HEM base address failed.\n");
 640         } else if (table->type < HEM_TYPE_MTT && hop_num == 2) {
 641                 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 2))
 642                         dev_warn(dev, "Clear HEM base address failed.\n");
 643         } else if (table->type < HEM_TYPE_MTT &&
 644                    hop_num == HNS_ROCE_HOP_NUM_0) {
 645                 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
 646                         dev_warn(dev, "Clear HEM base address failed.\n");
 647         }
 648
 649         /*
 650          * free buffer space chunk for QPC/MTPT/CQC/SRQC.
 651          * free bt space chunk for MTT/CQE.
 652          */
 653         hns_roce_free_hem(hr_dev, table->hem[hem_idx]);
 654         table->hem[hem_idx] = NULL;
 655
 656         if (check_whether_bt_num_2(table->type, hop_num)) {
 657                 start_idx = mhop.l0_idx * chunk_ba_num;
 658                 if (hns_roce_check_hem_null(table->hem, start_idx,
 659                                             chunk_ba_num, table->num_hem)) {
 660                         if (table->type < HEM_TYPE_MTT &&
 661                             hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
 662                                 dev_warn(dev, "Clear HEM base address failed.\n");
 663
 664                         dma_free_coherent(dev, bt_chunk_size,
 665                                           table->bt_l0[mhop.l0_idx],
 666                                           table->bt_l0_dma_addr[mhop.l0_idx]);
 667                         table->bt_l0[mhop.l0_idx] = NULL;
 668                 }
 669         } else if (check_whether_bt_num_3(table->type, hop_num)) {
 670                 start_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
 671                             mhop.l1_idx * chunk_ba_num;
 672                 if (hns_roce_check_hem_null(table->hem, start_idx,
 673                                             chunk_ba_num, table->num_hem)) {
 674                         if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
 675                                 dev_warn(dev, "Clear HEM base address failed.\n");
 676
 677                         dma_free_coherent(dev, bt_chunk_size,
 678                                           table->bt_l1[bt_l1_idx],
 679                                           table->bt_l1_dma_addr[bt_l1_idx]);
 680                         table->bt_l1[bt_l1_idx] = NULL;
 681
 682                         start_idx = mhop.l0_idx * chunk_ba_num;
 683                         if (hns_roce_check_bt_null(table->bt_l1, start_idx,
 684                                                    chunk_ba_num)) {
 685                                 if (hr_dev->hw->clear_hem(hr_dev, table, obj,
 686                                                           0))
 687                                         dev_warn(dev, "Clear HEM base address failed.\n");
 688
 689                                 dma_free_coherent(dev, bt_chunk_size,
 690                                             table->bt_l0[mhop.l0_idx],
 691                                             table->bt_l0_dma_addr[mhop.l0_idx]);
 692                                 table->bt_l0[mhop.l0_idx] = NULL;
 693                         }
 694                 }
 695         }
 696
 697         mutex_unlock(&table->mutex);
 698 }
 699
 700 void hns_roce_table_put(struct hns_roce_dev *hr_dev,
 701                         struct hns_roce_hem_table *table, unsigned long obj)
 702 {
 703         struct device *dev = hr_dev->dev;
 704         unsigned long i;
 705
 706         if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
 707                 hns_roce_table_mhop_put(hr_dev, table, obj, 1);
 708                 return;
 709         }
 710
 711         i = (obj & (table->num_obj - 1)) /
 712             (table->table_chunk_size / table->obj_size);
 713
 714         mutex_lock(&table->mutex);
 715
 716         if (--table->hem[i]->refcount == 0) {
 717                 /* Clear HEM base address */
 718                 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
 719                         dev_warn(dev, "Clear HEM base address failed.\n");
 720
 721                 hns_roce_free_hem(hr_dev, table->hem[i]);
 722                 table->hem[i] = NULL;
 723         }
 724
 725         mutex_unlock(&table->mutex);
 726 }
 727
 728 void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
 729                           struct hns_roce_hem_table *table,
 730                           unsigned long obj, dma_addr_t *dma_handle)
 731 {
 732         struct hns_roce_hem_chunk *chunk;
 733         struct hns_roce_hem_mhop mhop;
 734         struct hns_roce_hem *hem;
 735         void *addr = NULL;
 736         unsigned long mhop_obj = obj;
 737         unsigned long obj_per_chunk;
 738         unsigned long idx_offset;
 739         int offset, dma_offset;
 740         int length;
 741         int i, j;
 742         u32 hem_idx = 0;
 743
 744         if (!table->lowmem)
 745                 return NULL;
 746
 747         mutex_lock(&table->mutex);
 748
 749         if (!hns_roce_check_whether_mhop(hr_dev, table->type)) {
 750                 obj_per_chunk = table->table_chunk_size / table->obj_size;
 751                 hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk];
 752                 idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk;
 753                 dma_offset = offset = idx_offset * table->obj_size;
 754         } else {
 755                 u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */
 756
 757                 hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
 758                 /* mtt mhop */
 759                 i = mhop.l0_idx;
 760                 j = mhop.l1_idx;
 761                 if (mhop.hop_num == 2)
 762                         hem_idx = i * (mhop.bt_chunk_size / 8) + j;
 763                 else if (mhop.hop_num == 1 ||
 764                          mhop.hop_num == HNS_ROCE_HOP_NUM_0)
 765                         hem_idx = i;
 766
 767                 hem = table->hem[hem_idx];
 768                 dma_offset = offset = (obj & (table->num_obj - 1)) * seg_size %
 769                                        mhop.bt_chunk_size;
 770                 if (mhop.hop_num == 2)
 771                         dma_offset = offset = 0;
 772         }
 773
 774         if (!hem)
 775                 goto out;
 776
 777         list_for_each_entry(chunk, &hem->chunk_list, list) {
 778                 for (i = 0; i < chunk->npages; ++i) {
 779                         length = sg_dma_len(&chunk->mem[i]);
 780                         if (dma_handle && dma_offset >= 0) {
 781                                 if (length > (u32)dma_offset)
 782                                         *dma_handle = sg_dma_address(
 783                                                 &chunk->mem[i]) + dma_offset;
 784                                 dma_offset -= length;
 785                         }
 786
 787                         if (length > (u32)offset) {
 788                                 addr = chunk->buf[i] + offset;
 789                                 goto out;
 790                         }
 791                         offset -= length;
 792                 }
 793         }
 794
 795 out:
 796         mutex_unlock(&table->mutex);
 797         return addr;
 798 }
 799 EXPORT_SYMBOL_GPL(hns_roce_table_find);
 800
 801 int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,
 802                              struct hns_roce_hem_table *table,
 803                              unsigned long start, unsigned long end)
 804 {
 805         struct hns_roce_hem_mhop mhop;
 806         unsigned long inc = table->table_chunk_size / table->obj_size;
 807         unsigned long i;
 808         int ret;
 809
 810         if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
 811                 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
 812                 inc = mhop.bt_chunk_size / table->obj_size;
 813         }
 814
 815         /* Allocate MTT entry memory according to chunk(128K) */
 816         for (i = start; i <= end; i += inc) {
 817                 ret = hns_roce_table_get(hr_dev, table, i);
 818                 if (ret)
 819                         goto fail;
 820         }
 821
 822         return 0;
 823
 824 fail:
 825         while (i > start) {
 826                 i -= inc;
 827                 hns_roce_table_put(hr_dev, table, i);
 828         }
 829         return ret;
 830 }
 831
 832 void hns_roce_table_put_range(struct hns_roce_dev *hr_dev,
 833                               struct hns_roce_hem_table *table,
 834                               unsigned long start, unsigned long end)
 835 {
 836         struct hns_roce_hem_mhop mhop;
 837         unsigned long inc = table->table_chunk_size / table->obj_size;
 838         unsigned long i;
 839
 840         if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
 841                 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
 842                 inc = mhop.bt_chunk_size / table->obj_size;
 843         }
 844
 845         for (i = start; i <= end; i += inc)
 846                 hns_roce_table_put(hr_dev, table, i);
 847 }
 848
 849 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
 850                             struct hns_roce_hem_table *table, u32 type,
 851                             unsigned long obj_size, unsigned long nobj,
 852                             int use_lowmem)
 853 {
 854         struct device *dev = hr_dev->dev;
 855         unsigned long obj_per_chunk;
 856         unsigned long num_hem;
 857
 858         if (!hns_roce_check_whether_mhop(hr_dev, type)) {
 859                 table->table_chunk_size = hr_dev->caps.chunk_sz;
 860                 obj_per_chunk = table->table_chunk_size / obj_size;
 861                 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
 862
 863                 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);
 864                 if (!table->hem)
 865                         return -ENOMEM;
 866         } else {
 867                 unsigned long buf_chunk_size;
 868                 unsigned long bt_chunk_size;
 869                 unsigned long bt_chunk_num;
 870                 unsigned long num_bt_l0 = 0;
 871                 u32 hop_num;
 872
 873                 switch (type) {
 874                 case HEM_TYPE_QPC:
 875                         buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
 876                                         + PAGE_SHIFT);
 877                         bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
 878                                         + PAGE_SHIFT);
 879                         num_bt_l0 = hr_dev->caps.qpc_bt_num;
 880                         hop_num = hr_dev->caps.qpc_hop_num;
 881                         break;
 882                 case HEM_TYPE_MTPT:
 883                         buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
 884                                         + PAGE_SHIFT);
 885                         bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
 886                                         + PAGE_SHIFT);
 887                         num_bt_l0 = hr_dev->caps.mpt_bt_num;
 888                         hop_num = hr_dev->caps.mpt_hop_num;
 889                         break;
 890                 case HEM_TYPE_CQC:
 891                         buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
 892                                         + PAGE_SHIFT);
 893                         bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
 894                                         + PAGE_SHIFT);
 895                         num_bt_l0 = hr_dev->caps.cqc_bt_num;
 896                         hop_num = hr_dev->caps.cqc_hop_num;
 897                         break;
 898                 case HEM_TYPE_SRQC:
 899                         buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
 900                                         + PAGE_SHIFT);
 901                         bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
 902                                         + PAGE_SHIFT);
 903                         num_bt_l0 = hr_dev->caps.srqc_bt_num;
 904                         hop_num = hr_dev->caps.srqc_hop_num;
 905                         break;
 906                 case HEM_TYPE_MTT:
 907                         buf_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
 908                                         + PAGE_SHIFT);
 909                         bt_chunk_size = buf_chunk_size;
 910                         hop_num = hr_dev->caps.mtt_hop_num;
 911                         break;
 912                 case HEM_TYPE_CQE:
 913                         buf_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
 914                                         + PAGE_SHIFT);
 915                         bt_chunk_size = buf_chunk_size;
 916                         hop_num = hr_dev->caps.cqe_hop_num;
 917                         break;
 918                 default:
 919                         dev_err(dev,
 920                           "Table %d not support to init hem table here!\n",
 921                           type);
 922                         return -EINVAL;
 923                 }
 924                 obj_per_chunk = buf_chunk_size / obj_size;
 925                 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
 926                 bt_chunk_num = bt_chunk_size / 8;
 927                 if (type >= HEM_TYPE_MTT)
 928                         num_bt_l0 = bt_chunk_num;
 929
 930                 table->hem = kcalloc(num_hem, sizeof(*table->hem),
 931                                          GFP_KERNEL);
 932                 if (!table->hem)
 933                         goto err_kcalloc_hem_buf;
 934
 935                 if (check_whether_bt_num_3(type, hop_num)) {
 936                         unsigned long num_bt_l1;
 937
 938                         num_bt_l1 = (num_hem + bt_chunk_num - 1) /
 939                                              bt_chunk_num;
 940                         table->bt_l1 = kcalloc(num_bt_l1,
 941                                                sizeof(*table->bt_l1),
 942                                                GFP_KERNEL);
 943                         if (!table->bt_l1)
 944                                 goto err_kcalloc_bt_l1;
 945
 946                         table->bt_l1_dma_addr = kcalloc(num_bt_l1,
 947                                                  sizeof(*table->bt_l1_dma_addr),
 948                                                  GFP_KERNEL);
 949
 950                         if (!table->bt_l1_dma_addr)
 951                                 goto err_kcalloc_l1_dma;
 952                 }
 953
 954                 if (check_whether_bt_num_2(type, hop_num) ||
 955                         check_whether_bt_num_3(type, hop_num)) {
 956                         table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0),
 957                                                GFP_KERNEL);
 958                         if (!table->bt_l0)
 959                                 goto err_kcalloc_bt_l0;
 960
 961                         table->bt_l0_dma_addr = kcalloc(num_bt_l0,
 962                                                  sizeof(*table->bt_l0_dma_addr),
 963                                                  GFP_KERNEL);
 964                         if (!table->bt_l0_dma_addr)
 965                                 goto err_kcalloc_l0_dma;
 966                 }
 967         }
 968
 969         table->type = type;
 970         table->num_hem = num_hem;
 971         table->num_obj = nobj;
 972         table->obj_size = obj_size;
 973         table->lowmem = use_lowmem;
 974         mutex_init(&table->mutex);
 975
 976         return 0;
 977
 978 err_kcalloc_l0_dma:
 979         kfree(table->bt_l0);
 980         table->bt_l0 = NULL;
 981
 982 err_kcalloc_bt_l0:
 983         kfree(table->bt_l1_dma_addr);
 984         table->bt_l1_dma_addr = NULL;
 985
 986 err_kcalloc_l1_dma:
 987         kfree(table->bt_l1);
 988         table->bt_l1 = NULL;
 989
 990 err_kcalloc_bt_l1:
 991         kfree(table->hem);
 992         table->hem = NULL;
 993
 994 err_kcalloc_hem_buf:
 995         return -ENOMEM;
 996 }
 997
 998 static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev,
 999                                             struct hns_roce_hem_table *table)
1000 {
1001         struct hns_roce_hem_mhop mhop;
1002         u32 buf_chunk_size;
1003         int i;
1004         u64 obj;
1005
1006         hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
1007         buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :
1008                                         mhop.bt_chunk_size;
1009
1010         for (i = 0; i < table->num_hem; ++i) {
1011                 obj = i * buf_chunk_size / table->obj_size;
1012                 if (table->hem[i])
1013                         hns_roce_table_mhop_put(hr_dev, table, obj, 0);
1014         }
1015
1016         kfree(table->hem);
1017         table->hem = NULL;
1018         kfree(table->bt_l1);
1019         table->bt_l1 = NULL;
1020         kfree(table->bt_l1_dma_addr);
1021         table->bt_l1_dma_addr = NULL;
1022         kfree(table->bt_l0);
1023         table->bt_l0 = NULL;
1024         kfree(table->bt_l0_dma_addr);
1025         table->bt_l0_dma_addr = NULL;
1026 }
1027
1028 void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
1029                                 struct hns_roce_hem_table *table)
1030 {
1031         struct device *dev = hr_dev->dev;
1032         unsigned long i;
1033
1034         if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
1035                 hns_roce_cleanup_mhop_hem_table(hr_dev, table);
1036                 return;
1037         }
1038
1039         for (i = 0; i < table->num_hem; ++i)
1040                 if (table->hem[i]) {
1041                         if (hr_dev->hw->clear_hem(hr_dev, table,
1042                             i * table->table_chunk_size / table->obj_size, 0))
1043                                 dev_err(dev, "Clear HEM base address failed.\n");
1044
1045                         hns_roce_free_hem(hr_dev, table->hem[i]);
1046                 }
1047
1048         kfree(table->hem);
1049 }
1050
1051 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
1052 {
1053         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
1054         if (hr_dev->caps.trrl_entry_sz)
1055                 hns_roce_cleanup_hem_table(hr_dev,
1056                                            &hr_dev->qp_table.trrl_table);
1057         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
1058         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
1059         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
1060         if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
1061                 hns_roce_cleanup_hem_table(hr_dev,
1062                                            &hr_dev->mr_table.mtt_cqe_table);
1063         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table);
1064 }