drivers/infiniband/hw/hns/hns_roce_hw_v1.c

   1 /*
   2  * Copyright (c) 2016 Hisilicon Limited.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <linux/platform_device.h>
  34 #include <linux/acpi.h>
  35 #include <rdma/ib_umem.h>
  36 #include "hns_roce_common.h"
  37 #include "hns_roce_device.h"
  38 #include "hns_roce_cmd.h"
  39 #include "hns_roce_hem.h"
  40 #include "hns_roce_hw_v1.h"
  41
  42 static void set_data_seg(struct hns_roce_wqe_data_seg *dseg, struct ib_sge *sg)
  43 {
  44         dseg->lkey = cpu_to_le32(sg->lkey);
  45         dseg->addr = cpu_to_le64(sg->addr);
  46         dseg->len  = cpu_to_le32(sg->length);
  47 }
  48
  49 static void set_raddr_seg(struct hns_roce_wqe_raddr_seg *rseg, u64 remote_addr,
  50                           u32 rkey)
  51 {
  52         rseg->raddr = cpu_to_le64(remote_addr);
  53         rseg->rkey  = cpu_to_le32(rkey);
  54         rseg->len   = 0;
  55 }
  56
  57 int hns_roce_v1_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
  58                           struct ib_send_wr **bad_wr)
  59 {
  60         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
  61         struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
  62         struct hns_roce_ud_send_wqe *ud_sq_wqe = NULL;
  63         struct hns_roce_wqe_ctrl_seg *ctrl = NULL;
  64         struct hns_roce_wqe_data_seg *dseg = NULL;
  65         struct hns_roce_qp *qp = to_hr_qp(ibqp);
  66         struct device *dev = &hr_dev->pdev->dev;
  67         struct hns_roce_sq_db sq_db;
  68         int ps_opcode = 0, i = 0;
  69         unsigned long flags = 0;
  70         void *wqe = NULL;
  71         u32 doorbell[2];
  72         int nreq = 0;
  73         u32 ind = 0;
  74         int ret = 0;
  75
  76         if (unlikely(ibqp->qp_type != IB_QPT_GSI &&
  77                 ibqp->qp_type != IB_QPT_RC)) {
  78                 dev_err(dev, "un-supported QP type\n");
  79                 *bad_wr = NULL;
  80                 return -EOPNOTSUPP;
  81         }
  82
  83         spin_lock_irqsave(&qp->sq.lock, flags);
  84         ind = qp->sq_next_wqe;
  85         for (nreq = 0; wr; ++nreq, wr = wr->next) {
  86                 if (hns_roce_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
  87                         ret = -ENOMEM;
  88                         *bad_wr = wr;
  89                         goto out;
  90                 }
  91
  92                 if (unlikely(wr->num_sge > qp->sq.max_gs)) {
  93                         dev_err(dev, "num_sge=%d > qp->sq.max_gs=%d\n",
  94                                 wr->num_sge, qp->sq.max_gs);
  95                         ret = -EINVAL;
  96                         *bad_wr = wr;
  97                         goto out;
  98                 }
  99
 100                 wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
 101                 qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] =
 102                                                                       wr->wr_id;
 103
 104                 /* Corresponding to the RC and RD type wqe process separately */
 105                 if (ibqp->qp_type == IB_QPT_GSI) {
 106                         ud_sq_wqe = wqe;
 107                         roce_set_field(ud_sq_wqe->dmac_h,
 108                                        UD_SEND_WQE_U32_4_DMAC_0_M,
 109                                        UD_SEND_WQE_U32_4_DMAC_0_S,
 110                                        ah->av.mac[0]);
 111                         roce_set_field(ud_sq_wqe->dmac_h,
 112                                        UD_SEND_WQE_U32_4_DMAC_1_M,
 113                                        UD_SEND_WQE_U32_4_DMAC_1_S,
 114                                        ah->av.mac[1]);
 115                         roce_set_field(ud_sq_wqe->dmac_h,
 116                                        UD_SEND_WQE_U32_4_DMAC_2_M,
 117                                        UD_SEND_WQE_U32_4_DMAC_2_S,
 118                                        ah->av.mac[2]);
 119                         roce_set_field(ud_sq_wqe->dmac_h,
 120                                        UD_SEND_WQE_U32_4_DMAC_3_M,
 121                                        UD_SEND_WQE_U32_4_DMAC_3_S,
 122                                        ah->av.mac[3]);
 123
 124                         roce_set_field(ud_sq_wqe->u32_8,
 125                                        UD_SEND_WQE_U32_8_DMAC_4_M,
 126                                        UD_SEND_WQE_U32_8_DMAC_4_S,
 127                                        ah->av.mac[4]);
 128                         roce_set_field(ud_sq_wqe->u32_8,
 129                                        UD_SEND_WQE_U32_8_DMAC_5_M,
 130                                        UD_SEND_WQE_U32_8_DMAC_5_S,
 131                                        ah->av.mac[5]);
 132                         roce_set_field(ud_sq_wqe->u32_8,
 133                                        UD_SEND_WQE_U32_8_OPERATION_TYPE_M,
 134                                        UD_SEND_WQE_U32_8_OPERATION_TYPE_S,
 135                                        HNS_ROCE_WQE_OPCODE_SEND);
 136                         roce_set_field(ud_sq_wqe->u32_8,
 137                                        UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_M,
 138                                        UD_SEND_WQE_U32_8_NUMBER_OF_DATA_SEG_S,
 139                                        2);
 140                         roce_set_bit(ud_sq_wqe->u32_8,
 141                                 UD_SEND_WQE_U32_8_SEND_GL_ROUTING_HDR_FLAG_S,
 142                                 1);
 143
 144                         ud_sq_wqe->u32_8 |= (wr->send_flags & IB_SEND_SIGNALED ?
 145                                 cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) |
 146                                 (wr->send_flags & IB_SEND_SOLICITED ?
 147                                 cpu_to_le32(HNS_ROCE_WQE_SE) : 0) |
 148                                 ((wr->opcode == IB_WR_SEND_WITH_IMM) ?
 149                                 cpu_to_le32(HNS_ROCE_WQE_IMM) : 0);
 150
 151                         roce_set_field(ud_sq_wqe->u32_16,
 152                                        UD_SEND_WQE_U32_16_DEST_QP_M,
 153                                        UD_SEND_WQE_U32_16_DEST_QP_S,
 154                                        ud_wr(wr)->remote_qpn);
 155                         roce_set_field(ud_sq_wqe->u32_16,
 156                                        UD_SEND_WQE_U32_16_MAX_STATIC_RATE_M,
 157                                        UD_SEND_WQE_U32_16_MAX_STATIC_RATE_S,
 158                                        ah->av.stat_rate);
 159
 160                         roce_set_field(ud_sq_wqe->u32_36,
 161                                        UD_SEND_WQE_U32_36_FLOW_LABEL_M,
 162                                        UD_SEND_WQE_U32_36_FLOW_LABEL_S, 0);
 163                         roce_set_field(ud_sq_wqe->u32_36,
 164                                        UD_SEND_WQE_U32_36_PRIORITY_M,
 165                                        UD_SEND_WQE_U32_36_PRIORITY_S,
 166                                        ah->av.sl_tclass_flowlabel >>
 167                                        HNS_ROCE_SL_SHIFT);
 168                         roce_set_field(ud_sq_wqe->u32_36,
 169                                        UD_SEND_WQE_U32_36_SGID_INDEX_M,
 170                                        UD_SEND_WQE_U32_36_SGID_INDEX_S,
 171                                        hns_get_gid_index(hr_dev, qp->phy_port,
 172                                                          ah->av.gid_index));
 173
 174                         roce_set_field(ud_sq_wqe->u32_40,
 175                                        UD_SEND_WQE_U32_40_HOP_LIMIT_M,
 176                                        UD_SEND_WQE_U32_40_HOP_LIMIT_S,
 177                                        ah->av.hop_limit);
 178                         roce_set_field(ud_sq_wqe->u32_40,
 179                                        UD_SEND_WQE_U32_40_TRAFFIC_CLASS_M,
 180                                        UD_SEND_WQE_U32_40_TRAFFIC_CLASS_S, 0);
 181
 182                         memcpy(&ud_sq_wqe->dgid[0], &ah->av.dgid[0], GID_LEN);
 183
 184                         ud_sq_wqe->va0_l = (u32)wr->sg_list[0].addr;
 185                         ud_sq_wqe->va0_h = (wr->sg_list[0].addr) >> 32;
 186                         ud_sq_wqe->l_key0 = wr->sg_list[0].lkey;
 187
 188                         ud_sq_wqe->va1_l = (u32)wr->sg_list[1].addr;
 189                         ud_sq_wqe->va1_h = (wr->sg_list[1].addr) >> 32;
 190                         ud_sq_wqe->l_key1 = wr->sg_list[1].lkey;
 191                         ind++;
 192                 } else if (ibqp->qp_type == IB_QPT_RC) {
 193                         ctrl = wqe;
 194                         memset(ctrl, 0, sizeof(struct hns_roce_wqe_ctrl_seg));
 195                         for (i = 0; i < wr->num_sge; i++)
 196                                 ctrl->msg_length += wr->sg_list[i].length;
 197
 198                         ctrl->sgl_pa_h = 0;
 199                         ctrl->flag = 0;
 200                         ctrl->imm_data = send_ieth(wr);
 201
 202                         /*Ctrl field, ctrl set type: sig, solic, imm, fence */
 203                         /* SO wait for conforming application scenarios */
 204                         ctrl->flag |= (wr->send_flags & IB_SEND_SIGNALED ?
 205                                       cpu_to_le32(HNS_ROCE_WQE_CQ_NOTIFY) : 0) |
 206                                       (wr->send_flags & IB_SEND_SOLICITED ?
 207                                       cpu_to_le32(HNS_ROCE_WQE_SE) : 0) |
 208                                       ((wr->opcode == IB_WR_SEND_WITH_IMM ||
 209                                       wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) ?
 210                                       cpu_to_le32(HNS_ROCE_WQE_IMM) : 0) |
 211                                       (wr->send_flags & IB_SEND_FENCE ?
 212                                       (cpu_to_le32(HNS_ROCE_WQE_FENCE)) : 0);
 213
 214                         wqe += sizeof(struct hns_roce_wqe_ctrl_seg);
 215
 216                         switch (wr->opcode) {
 217                         case IB_WR_RDMA_READ:
 218                                 ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_READ;
 219                                 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
 220                                               atomic_wr(wr)->rkey);
 221                                 break;
 222                         case IB_WR_RDMA_WRITE:
 223                         case IB_WR_RDMA_WRITE_WITH_IMM:
 224                                 ps_opcode = HNS_ROCE_WQE_OPCODE_RDMA_WRITE;
 225                                 set_raddr_seg(wqe, atomic_wr(wr)->remote_addr,
 226                                               atomic_wr(wr)->rkey);
 227                                 break;
 228                         case IB_WR_SEND:
 229                         case IB_WR_SEND_WITH_INV:
 230                         case IB_WR_SEND_WITH_IMM:
 231                                 ps_opcode = HNS_ROCE_WQE_OPCODE_SEND;
 232                                 break;
 233                         case IB_WR_LOCAL_INV:
 234                         case IB_WR_ATOMIC_CMP_AND_SWP:
 235                         case IB_WR_ATOMIC_FETCH_AND_ADD:
 236                         case IB_WR_LSO:
 237                         default:
 238                                 ps_opcode = HNS_ROCE_WQE_OPCODE_MASK;
 239                                 break;
 240                         }
 241                         ctrl->flag |= cpu_to_le32(ps_opcode);
 242                         wqe += sizeof(struct hns_roce_wqe_raddr_seg);
 243
 244                         dseg = wqe;
 245                         if (wr->send_flags & IB_SEND_INLINE && wr->num_sge) {
 246                                 if (ctrl->msg_length >
 247                                         hr_dev->caps.max_sq_inline) {
 248                                         ret = -EINVAL;
 249                                         *bad_wr = wr;
 250                                         dev_err(dev, "inline len(1-%d)=%d, illegal",
 251                                                 ctrl->msg_length,
 252                                                 hr_dev->caps.max_sq_inline);
 253                                         goto out;
 254                                 }
 255                                 for (i = 0; i < wr->num_sge; i++) {
 256                                         memcpy(wqe, ((void *) (uintptr_t)
 257                                                wr->sg_list[i].addr),
 258                                                wr->sg_list[i].length);
 259                                         wqe += wr->sg_list[i].length;
 260                                 }
 261                                 ctrl->flag |= HNS_ROCE_WQE_INLINE;
 262                         } else {
 263                                 /*sqe num is two */
 264                                 for (i = 0; i < wr->num_sge; i++)
 265                                         set_data_seg(dseg + i, wr->sg_list + i);
 266
 267                                 ctrl->flag |= cpu_to_le32(wr->num_sge <<
 268                                               HNS_ROCE_WQE_SGE_NUM_BIT);
 269                         }
 270                         ind++;
 271                 }
 272         }
 273
 274 out:
 275         /* Set DB return */
 276         if (likely(nreq)) {
 277                 qp->sq.head += nreq;
 278                 /* Memory barrier */
 279                 wmb();
 280
 281                 sq_db.u32_4 = 0;
 282                 sq_db.u32_8 = 0;
 283                 roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_SQ_HEAD_M,
 284                                SQ_DOORBELL_U32_4_SQ_HEAD_S,
 285                               (qp->sq.head & ((qp->sq.wqe_cnt << 1) - 1)));
 286                 roce_set_field(sq_db.u32_4, SQ_DOORBELL_U32_4_PORT_M,
 287                                SQ_DOORBELL_U32_4_PORT_S, qp->phy_port);
 288                 roce_set_field(sq_db.u32_8, SQ_DOORBELL_U32_8_QPN_M,
 289                                SQ_DOORBELL_U32_8_QPN_S, qp->doorbell_qpn);
 290                 roce_set_bit(sq_db.u32_8, SQ_DOORBELL_HW_SYNC_S, 1);
 291
 292                 doorbell[0] = sq_db.u32_4;
 293                 doorbell[1] = sq_db.u32_8;
 294
 295                 hns_roce_write64_k(doorbell, qp->sq.db_reg_l);
 296                 qp->sq_next_wqe = ind;
 297         }
 298
 299         spin_unlock_irqrestore(&qp->sq.lock, flags);
 300
 301         return ret;
 302 }
 303
 304 int hns_roce_v1_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
 305                           struct ib_recv_wr **bad_wr)
 306 {
 307         int ret = 0;
 308         int nreq = 0;
 309         int ind = 0;
 310         int i = 0;
 311         u32 reg_val = 0;
 312         unsigned long flags = 0;
 313         struct hns_roce_rq_wqe_ctrl *ctrl = NULL;
 314         struct hns_roce_wqe_data_seg *scat = NULL;
 315         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
 316         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 317         struct device *dev = &hr_dev->pdev->dev;
 318         struct hns_roce_rq_db rq_db;
 319         uint32_t doorbell[2] = {0};
 320
 321         spin_lock_irqsave(&hr_qp->rq.lock, flags);
 322         ind = hr_qp->rq.head & (hr_qp->rq.wqe_cnt - 1);
 323
 324         for (nreq = 0; wr; ++nreq, wr = wr->next) {
 325                 if (hns_roce_wq_overflow(&hr_qp->rq, nreq,
 326                         hr_qp->ibqp.recv_cq)) {
 327                         ret = -ENOMEM;
 328                         *bad_wr = wr;
 329                         goto out;
 330                 }
 331
 332                 if (unlikely(wr->num_sge > hr_qp->rq.max_gs)) {
 333                         dev_err(dev, "rq:num_sge=%d > qp->sq.max_gs=%d\n",
 334                                 wr->num_sge, hr_qp->rq.max_gs);
 335                         ret = -EINVAL;
 336                         *bad_wr = wr;
 337                         goto out;
 338                 }
 339
 340                 ctrl = get_recv_wqe(hr_qp, ind);
 341
 342                 roce_set_field(ctrl->rwqe_byte_12,
 343                                RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_M,
 344                                RQ_WQE_CTRL_RWQE_BYTE_12_RWQE_SGE_NUM_S,
 345                                wr->num_sge);
 346
 347                 scat = (struct hns_roce_wqe_data_seg *)(ctrl + 1);
 348
 349                 for (i = 0; i < wr->num_sge; i++)
 350                         set_data_seg(scat + i, wr->sg_list + i);
 351
 352                 hr_qp->rq.wrid[ind] = wr->wr_id;
 353
 354                 ind = (ind + 1) & (hr_qp->rq.wqe_cnt - 1);
 355         }
 356
 357 out:
 358         if (likely(nreq)) {
 359                 hr_qp->rq.head += nreq;
 360                 /* Memory barrier */
 361                 wmb();
 362
 363                 if (ibqp->qp_type == IB_QPT_GSI) {
 364                         /* SW update GSI rq header */
 365                         reg_val = roce_read(to_hr_dev(ibqp->device),
 366                                             ROCEE_QP1C_CFG3_0_REG +
 367                                             QP1C_CFGN_OFFSET * hr_qp->phy_port);
 368                         roce_set_field(reg_val,
 369                                        ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_M,
 370                                        ROCEE_QP1C_CFG3_0_ROCEE_QP1C_RQ_HEAD_S,
 371                                        hr_qp->rq.head);
 372                         roce_write(to_hr_dev(ibqp->device),
 373                                    ROCEE_QP1C_CFG3_0_REG +
 374                                    QP1C_CFGN_OFFSET * hr_qp->phy_port, reg_val);
 375                 } else {
 376                         rq_db.u32_4 = 0;
 377                         rq_db.u32_8 = 0;
 378
 379                         roce_set_field(rq_db.u32_4, RQ_DOORBELL_U32_4_RQ_HEAD_M,
 380                                        RQ_DOORBELL_U32_4_RQ_HEAD_S,
 381                                        hr_qp->rq.head);
 382                         roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_QPN_M,
 383                                        RQ_DOORBELL_U32_8_QPN_S, hr_qp->qpn);
 384                         roce_set_field(rq_db.u32_8, RQ_DOORBELL_U32_8_CMD_M,
 385                                        RQ_DOORBELL_U32_8_CMD_S, 1);
 386                         roce_set_bit(rq_db.u32_8, RQ_DOORBELL_U32_8_HW_SYNC_S,
 387                                      1);
 388
 389                         doorbell[0] = rq_db.u32_4;
 390                         doorbell[1] = rq_db.u32_8;
 391
 392                         hns_roce_write64_k(doorbell, hr_qp->rq.db_reg_l);
 393                 }
 394         }
 395         spin_unlock_irqrestore(&hr_qp->rq.lock, flags);
 396
 397         return ret;
 398 }
 399
 400 static void hns_roce_set_db_event_mode(struct hns_roce_dev *hr_dev,
 401                                        int sdb_mode, int odb_mode)
 402 {
 403         u32 val;
 404
 405         val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
 406         roce_set_bit(val, ROCEE_GLB_CFG_ROCEE_DB_SQ_MODE_S, sdb_mode);
 407         roce_set_bit(val, ROCEE_GLB_CFG_ROCEE_DB_OTH_MODE_S, odb_mode);
 408         roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
 409 }
 410
 411 static void hns_roce_set_db_ext_mode(struct hns_roce_dev *hr_dev, u32 sdb_mode,
 412                                      u32 odb_mode)
 413 {
 414         u32 val;
 415
 416         /* Configure SDB/ODB extend mode */
 417         val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
 418         roce_set_bit(val, ROCEE_GLB_CFG_SQ_EXT_DB_MODE_S, sdb_mode);
 419         roce_set_bit(val, ROCEE_GLB_CFG_OTH_EXT_DB_MODE_S, odb_mode);
 420         roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
 421 }
 422
 423 static void hns_roce_set_sdb(struct hns_roce_dev *hr_dev, u32 sdb_alept,
 424                              u32 sdb_alful)
 425 {
 426         u32 val;
 427
 428         /* Configure SDB */
 429         val = roce_read(hr_dev, ROCEE_DB_SQ_WL_REG);
 430         roce_set_field(val, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_M,
 431                        ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_S, sdb_alful);
 432         roce_set_field(val, ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_M,
 433                        ROCEE_DB_SQ_WL_ROCEE_DB_SQ_WL_EMPTY_S, sdb_alept);
 434         roce_write(hr_dev, ROCEE_DB_SQ_WL_REG, val);
 435 }
 436
 437 static void hns_roce_set_odb(struct hns_roce_dev *hr_dev, u32 odb_alept,
 438                              u32 odb_alful)
 439 {
 440         u32 val;
 441
 442         /* Configure ODB */
 443         val = roce_read(hr_dev, ROCEE_DB_OTHERS_WL_REG);
 444         roce_set_field(val, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_M,
 445                        ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_S, odb_alful);
 446         roce_set_field(val, ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_M,
 447                        ROCEE_DB_OTHERS_WL_ROCEE_DB_OTH_WL_EMPTY_S, odb_alept);
 448         roce_write(hr_dev, ROCEE_DB_OTHERS_WL_REG, val);
 449 }
 450
 451 static void hns_roce_set_sdb_ext(struct hns_roce_dev *hr_dev, u32 ext_sdb_alept,
 452                                  u32 ext_sdb_alful)
 453 {
 454         struct device *dev = &hr_dev->pdev->dev;
 455         struct hns_roce_v1_priv *priv;
 456         struct hns_roce_db_table *db;
 457         dma_addr_t sdb_dma_addr;
 458         u32 val;
 459
 460         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 461         db = &priv->db_table;
 462
 463         /* Configure extend SDB threshold */
 464         roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_EMPTY_REG, ext_sdb_alept);
 465         roce_write(hr_dev, ROCEE_EXT_DB_SQ_WL_REG, ext_sdb_alful);
 466
 467         /* Configure extend SDB base addr */
 468         sdb_dma_addr = db->ext_db->sdb_buf_list->map;
 469         roce_write(hr_dev, ROCEE_EXT_DB_SQ_REG, (u32)(sdb_dma_addr >> 12));
 470
 471         /* Configure extend SDB depth */
 472         val = roce_read(hr_dev, ROCEE_EXT_DB_SQ_H_REG);
 473         roce_set_field(val, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_M,
 474                        ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_SHIFT_S,
 475                        db->ext_db->esdb_dep);
 476         /*
 477          * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
 478          * using 4K page, and shift more 32 because of
 479          * caculating the high 32 bit value evaluated to hardware.
 480          */
 481         roce_set_field(val, ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_M,
 482                        ROCEE_EXT_DB_SQ_H_EXT_DB_SQ_BA_H_S, sdb_dma_addr >> 44);
 483         roce_write(hr_dev, ROCEE_EXT_DB_SQ_H_REG, val);
 484
 485         dev_dbg(dev, "ext SDB depth: 0x%x\n", db->ext_db->esdb_dep);
 486         dev_dbg(dev, "ext SDB threshold: epmty: 0x%x, ful: 0x%x\n",
 487                 ext_sdb_alept, ext_sdb_alful);
 488 }
 489
 490 static void hns_roce_set_odb_ext(struct hns_roce_dev *hr_dev, u32 ext_odb_alept,
 491                                  u32 ext_odb_alful)
 492 {
 493         struct device *dev = &hr_dev->pdev->dev;
 494         struct hns_roce_v1_priv *priv;
 495         struct hns_roce_db_table *db;
 496         dma_addr_t odb_dma_addr;
 497         u32 val;
 498
 499         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 500         db = &priv->db_table;
 501
 502         /* Configure extend ODB threshold */
 503         roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_EMPTY_REG, ext_odb_alept);
 504         roce_write(hr_dev, ROCEE_EXT_DB_OTHERS_WL_REG, ext_odb_alful);
 505
 506         /* Configure extend ODB base addr */
 507         odb_dma_addr = db->ext_db->odb_buf_list->map;
 508         roce_write(hr_dev, ROCEE_EXT_DB_OTH_REG, (u32)(odb_dma_addr >> 12));
 509
 510         /* Configure extend ODB depth */
 511         val = roce_read(hr_dev, ROCEE_EXT_DB_OTH_H_REG);
 512         roce_set_field(val, ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_M,
 513                        ROCEE_EXT_DB_OTH_H_EXT_DB_OTH_SHIFT_S,
 514                        db->ext_db->eodb_dep);
 515         roce_set_field(val, ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_M,
 516                        ROCEE_EXT_DB_SQ_H_EXT_DB_OTH_BA_H_S,
 517                        db->ext_db->eodb_dep);
 518         roce_write(hr_dev, ROCEE_EXT_DB_OTH_H_REG, val);
 519
 520         dev_dbg(dev, "ext ODB depth: 0x%x\n", db->ext_db->eodb_dep);
 521         dev_dbg(dev, "ext ODB threshold: empty: 0x%x, ful: 0x%x\n",
 522                 ext_odb_alept, ext_odb_alful);
 523 }
 524
 525 static int hns_roce_db_ext_init(struct hns_roce_dev *hr_dev, u32 sdb_ext_mod,
 526                                 u32 odb_ext_mod)
 527 {
 528         struct device *dev = &hr_dev->pdev->dev;
 529         struct hns_roce_v1_priv *priv;
 530         struct hns_roce_db_table *db;
 531         dma_addr_t sdb_dma_addr;
 532         dma_addr_t odb_dma_addr;
 533         int ret = 0;
 534
 535         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 536         db = &priv->db_table;
 537
 538         db->ext_db = kmalloc(sizeof(*db->ext_db), GFP_KERNEL);
 539         if (!db->ext_db)
 540                 return -ENOMEM;
 541
 542         if (sdb_ext_mod) {
 543                 db->ext_db->sdb_buf_list = kmalloc(
 544                                 sizeof(*db->ext_db->sdb_buf_list), GFP_KERNEL);
 545                 if (!db->ext_db->sdb_buf_list) {
 546                         ret = -ENOMEM;
 547                         goto ext_sdb_buf_fail_out;
 548                 }
 549
 550                 db->ext_db->sdb_buf_list->buf = dma_alloc_coherent(dev,
 551                                                      HNS_ROCE_V1_EXT_SDB_SIZE,
 552                                                      &sdb_dma_addr, GFP_KERNEL);
 553                 if (!db->ext_db->sdb_buf_list->buf) {
 554                         ret = -ENOMEM;
 555                         goto alloc_sq_db_buf_fail;
 556                 }
 557                 db->ext_db->sdb_buf_list->map = sdb_dma_addr;
 558
 559                 db->ext_db->esdb_dep = ilog2(HNS_ROCE_V1_EXT_SDB_DEPTH);
 560                 hns_roce_set_sdb_ext(hr_dev, HNS_ROCE_V1_EXT_SDB_ALEPT,
 561                                      HNS_ROCE_V1_EXT_SDB_ALFUL);
 562         } else
 563                 hns_roce_set_sdb(hr_dev, HNS_ROCE_V1_SDB_ALEPT,
 564                                  HNS_ROCE_V1_SDB_ALFUL);
 565
 566         if (odb_ext_mod) {
 567                 db->ext_db->odb_buf_list = kmalloc(
 568                                 sizeof(*db->ext_db->odb_buf_list), GFP_KERNEL);
 569                 if (!db->ext_db->odb_buf_list) {
 570                         ret = -ENOMEM;
 571                         goto ext_odb_buf_fail_out;
 572                 }
 573
 574                 db->ext_db->odb_buf_list->buf = dma_alloc_coherent(dev,
 575                                                      HNS_ROCE_V1_EXT_ODB_SIZE,
 576                                                      &odb_dma_addr, GFP_KERNEL);
 577                 if (!db->ext_db->odb_buf_list->buf) {
 578                         ret = -ENOMEM;
 579                         goto alloc_otr_db_buf_fail;
 580                 }
 581                 db->ext_db->odb_buf_list->map = odb_dma_addr;
 582
 583                 db->ext_db->eodb_dep = ilog2(HNS_ROCE_V1_EXT_ODB_DEPTH);
 584                 hns_roce_set_odb_ext(hr_dev, HNS_ROCE_V1_EXT_ODB_ALEPT,
 585                                      HNS_ROCE_V1_EXT_ODB_ALFUL);
 586         } else
 587                 hns_roce_set_odb(hr_dev, HNS_ROCE_V1_ODB_ALEPT,
 588                                  HNS_ROCE_V1_ODB_ALFUL);
 589
 590         hns_roce_set_db_ext_mode(hr_dev, sdb_ext_mod, odb_ext_mod);
 591
 592         return 0;
 593
 594 alloc_otr_db_buf_fail:
 595         kfree(db->ext_db->odb_buf_list);
 596
 597 ext_odb_buf_fail_out:
 598         if (sdb_ext_mod) {
 599                 dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE,
 600                                   db->ext_db->sdb_buf_list->buf,
 601                                   db->ext_db->sdb_buf_list->map);
 602         }
 603
 604 alloc_sq_db_buf_fail:
 605         if (sdb_ext_mod)
 606                 kfree(db->ext_db->sdb_buf_list);
 607
 608 ext_sdb_buf_fail_out:
 609         kfree(db->ext_db);
 610         return ret;
 611 }
 612
 613 static int hns_roce_db_init(struct hns_roce_dev *hr_dev)
 614 {
 615         struct device *dev = &hr_dev->pdev->dev;
 616         struct hns_roce_v1_priv *priv;
 617         struct hns_roce_db_table *db;
 618         u32 sdb_ext_mod;
 619         u32 odb_ext_mod;
 620         u32 sdb_evt_mod;
 621         u32 odb_evt_mod;
 622         int ret = 0;
 623
 624         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 625         db = &priv->db_table;
 626
 627         memset(db, 0, sizeof(*db));
 628
 629         /* Default DB mode */
 630         sdb_ext_mod = HNS_ROCE_SDB_EXTEND_MODE;
 631         odb_ext_mod = HNS_ROCE_ODB_EXTEND_MODE;
 632         sdb_evt_mod = HNS_ROCE_SDB_NORMAL_MODE;
 633         odb_evt_mod = HNS_ROCE_ODB_POLL_MODE;
 634
 635         db->sdb_ext_mod = sdb_ext_mod;
 636         db->odb_ext_mod = odb_ext_mod;
 637
 638         /* Init extend DB */
 639         ret = hns_roce_db_ext_init(hr_dev, sdb_ext_mod, odb_ext_mod);
 640         if (ret) {
 641                 dev_err(dev, "Failed in extend DB configuration.\n");
 642                 return ret;
 643         }
 644
 645         hns_roce_set_db_event_mode(hr_dev, sdb_evt_mod, odb_evt_mod);
 646
 647         return 0;
 648 }
 649
 650 static void hns_roce_db_free(struct hns_roce_dev *hr_dev)
 651 {
 652         struct device *dev = &hr_dev->pdev->dev;
 653         struct hns_roce_v1_priv *priv;
 654         struct hns_roce_db_table *db;
 655
 656         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 657         db = &priv->db_table;
 658
 659         if (db->sdb_ext_mod) {
 660                 dma_free_coherent(dev, HNS_ROCE_V1_EXT_SDB_SIZE,
 661                                   db->ext_db->sdb_buf_list->buf,
 662                                   db->ext_db->sdb_buf_list->map);
 663                 kfree(db->ext_db->sdb_buf_list);
 664         }
 665
 666         if (db->odb_ext_mod) {
 667                 dma_free_coherent(dev, HNS_ROCE_V1_EXT_ODB_SIZE,
 668                                   db->ext_db->odb_buf_list->buf,
 669                                   db->ext_db->odb_buf_list->map);
 670                 kfree(db->ext_db->odb_buf_list);
 671         }
 672
 673         kfree(db->ext_db);
 674 }
 675
 676 static int hns_roce_raq_init(struct hns_roce_dev *hr_dev)
 677 {
 678         int ret;
 679         int raq_shift = 0;
 680         dma_addr_t addr;
 681         u32 val;
 682         struct hns_roce_v1_priv *priv;
 683         struct hns_roce_raq_table *raq;
 684         struct device *dev = &hr_dev->pdev->dev;
 685
 686         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 687         raq = &priv->raq_table;
 688
 689         raq->e_raq_buf = kzalloc(sizeof(*(raq->e_raq_buf)), GFP_KERNEL);
 690         if (!raq->e_raq_buf)
 691                 return -ENOMEM;
 692
 693         raq->e_raq_buf->buf = dma_alloc_coherent(dev, HNS_ROCE_V1_RAQ_SIZE,
 694                                                  &addr, GFP_KERNEL);
 695         if (!raq->e_raq_buf->buf) {
 696                 ret = -ENOMEM;
 697                 goto err_dma_alloc_raq;
 698         }
 699         raq->e_raq_buf->map = addr;
 700
 701         /* Configure raq extended address. 48bit 4K align*/
 702         roce_write(hr_dev, ROCEE_EXT_RAQ_REG, raq->e_raq_buf->map >> 12);
 703
 704         /* Configure raq_shift */
 705         raq_shift = ilog2(HNS_ROCE_V1_RAQ_SIZE / HNS_ROCE_V1_RAQ_ENTRY);
 706         val = roce_read(hr_dev, ROCEE_EXT_RAQ_H_REG);
 707         roce_set_field(val, ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_M,
 708                        ROCEE_EXT_RAQ_H_EXT_RAQ_SHIFT_S, raq_shift);
 709         /*
 710          * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
 711          * using 4K page, and shift more 32 because of
 712          * caculating the high 32 bit value evaluated to hardware.
 713          */
 714         roce_set_field(val, ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_M,
 715                        ROCEE_EXT_RAQ_H_EXT_RAQ_BA_H_S,
 716                        raq->e_raq_buf->map >> 44);
 717         roce_write(hr_dev, ROCEE_EXT_RAQ_H_REG, val);
 718         dev_dbg(dev, "Configure raq_shift 0x%x.\n", val);
 719
 720         /* Configure raq threshold */
 721         val = roce_read(hr_dev, ROCEE_RAQ_WL_REG);
 722         roce_set_field(val, ROCEE_RAQ_WL_ROCEE_RAQ_WL_M,
 723                        ROCEE_RAQ_WL_ROCEE_RAQ_WL_S,
 724                        HNS_ROCE_V1_EXT_RAQ_WF);
 725         roce_write(hr_dev, ROCEE_RAQ_WL_REG, val);
 726         dev_dbg(dev, "Configure raq_wl 0x%x.\n", val);
 727
 728         /* Enable extend raq */
 729         val = roce_read(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG);
 730         roce_set_field(val,
 731                        ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_M,
 732                        ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_POL_TIME_INTERVAL_S,
 733                        POL_TIME_INTERVAL_VAL);
 734         roce_set_bit(val, ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_EXT_RAQ_MODE, 1);
 735         roce_set_field(val,
 736                        ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_M,
 737                        ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_CFG_S,
 738                        2);
 739         roce_set_bit(val,
 740                      ROCEE_WRMS_POL_TIME_INTERVAL_WRMS_RAQ_TIMEOUT_CHK_EN_S, 1);
 741         roce_write(hr_dev, ROCEE_WRMS_POL_TIME_INTERVAL_REG, val);
 742         dev_dbg(dev, "Configure WrmsPolTimeInterval 0x%x.\n", val);
 743
 744         /* Enable raq drop */
 745         val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
 746         roce_set_bit(val, ROCEE_GLB_CFG_TRP_RAQ_DROP_EN_S, 1);
 747         roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
 748         dev_dbg(dev, "Configure GlbCfg = 0x%x.\n", val);
 749
 750         return 0;
 751
 752 err_dma_alloc_raq:
 753         kfree(raq->e_raq_buf);
 754         return ret;
 755 }
 756
 757 static void hns_roce_raq_free(struct hns_roce_dev *hr_dev)
 758 {
 759         struct device *dev = &hr_dev->pdev->dev;
 760         struct hns_roce_v1_priv *priv;
 761         struct hns_roce_raq_table *raq;
 762
 763         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 764         raq = &priv->raq_table;
 765
 766         dma_free_coherent(dev, HNS_ROCE_V1_RAQ_SIZE, raq->e_raq_buf->buf,
 767                           raq->e_raq_buf->map);
 768         kfree(raq->e_raq_buf);
 769 }
 770
 771 static void hns_roce_port_enable(struct hns_roce_dev *hr_dev, int enable_flag)
 772 {
 773         u32 val;
 774
 775         if (enable_flag) {
 776                 val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
 777                  /* Open all ports */
 778                 roce_set_field(val, ROCEE_GLB_CFG_ROCEE_PORT_ST_M,
 779                                ROCEE_GLB_CFG_ROCEE_PORT_ST_S,
 780                                ALL_PORT_VAL_OPEN);
 781                 roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
 782         } else {
 783                 val = roce_read(hr_dev, ROCEE_GLB_CFG_REG);
 784                 /* Close all ports */
 785                 roce_set_field(val, ROCEE_GLB_CFG_ROCEE_PORT_ST_M,
 786                                ROCEE_GLB_CFG_ROCEE_PORT_ST_S, 0x0);
 787                 roce_write(hr_dev, ROCEE_GLB_CFG_REG, val);
 788         }
 789 }
 790
 791 static int hns_roce_bt_init(struct hns_roce_dev *hr_dev)
 792 {
 793         struct device *dev = &hr_dev->pdev->dev;
 794         struct hns_roce_v1_priv *priv;
 795         int ret;
 796
 797         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 798
 799         priv->bt_table.qpc_buf.buf = dma_alloc_coherent(dev,
 800                 HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.qpc_buf.map,
 801                 GFP_KERNEL);
 802         if (!priv->bt_table.qpc_buf.buf)
 803                 return -ENOMEM;
 804
 805         priv->bt_table.mtpt_buf.buf = dma_alloc_coherent(dev,
 806                 HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.mtpt_buf.map,
 807                 GFP_KERNEL);
 808         if (!priv->bt_table.mtpt_buf.buf) {
 809                 ret = -ENOMEM;
 810                 goto err_failed_alloc_mtpt_buf;
 811         }
 812
 813         priv->bt_table.cqc_buf.buf = dma_alloc_coherent(dev,
 814                 HNS_ROCE_BT_RSV_BUF_SIZE, &priv->bt_table.cqc_buf.map,
 815                 GFP_KERNEL);
 816         if (!priv->bt_table.cqc_buf.buf) {
 817                 ret = -ENOMEM;
 818                 goto err_failed_alloc_cqc_buf;
 819         }
 820
 821         return 0;
 822
 823 err_failed_alloc_cqc_buf:
 824         dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
 825                 priv->bt_table.mtpt_buf.buf, priv->bt_table.mtpt_buf.map);
 826
 827 err_failed_alloc_mtpt_buf:
 828         dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
 829                 priv->bt_table.qpc_buf.buf, priv->bt_table.qpc_buf.map);
 830
 831         return ret;
 832 }
 833
 834 static void hns_roce_bt_free(struct hns_roce_dev *hr_dev)
 835 {
 836         struct device *dev = &hr_dev->pdev->dev;
 837         struct hns_roce_v1_priv *priv;
 838
 839         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
 840
 841         dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
 842                 priv->bt_table.cqc_buf.buf, priv->bt_table.cqc_buf.map);
 843
 844         dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
 845                 priv->bt_table.mtpt_buf.buf, priv->bt_table.mtpt_buf.map);
 846
 847         dma_free_coherent(dev, HNS_ROCE_BT_RSV_BUF_SIZE,
 848                 priv->bt_table.qpc_buf.buf, priv->bt_table.qpc_buf.map);
 849 }
 850
 851 /**
 852  * hns_roce_v1_reset - reset RoCE
 853  * @hr_dev: RoCE device struct pointer
 854  * @enable: true -- drop reset, false -- reset
 855  * return 0 - success , negative --fail
 856  */
 857 int hns_roce_v1_reset(struct hns_roce_dev *hr_dev, bool dereset)
 858 {
 859         struct device_node *dsaf_node;
 860         struct device *dev = &hr_dev->pdev->dev;
 861         struct device_node *np = dev->of_node;
 862         struct fwnode_handle *fwnode;
 863         int ret;
 864
 865         /* check if this is DT/ACPI case */
 866         if (dev_of_node(dev)) {
 867                 dsaf_node = of_parse_phandle(np, "dsaf-handle", 0);
 868                 if (!dsaf_node) {
 869                         dev_err(dev, "could not find dsaf-handle\n");
 870                         return -EINVAL;
 871                 }
 872                 fwnode = &dsaf_node->fwnode;
 873         } else if (is_acpi_device_node(dev->fwnode)) {
 874                 struct acpi_reference_args args;
 875
 876                 ret = acpi_node_get_property_reference(dev->fwnode,
 877                                                        "dsaf-handle", 0, &args);
 878                 if (ret) {
 879                         dev_err(dev, "could not find dsaf-handle\n");
 880                         return ret;
 881                 }
 882                 fwnode = acpi_fwnode_handle(args.adev);
 883         } else {
 884                 dev_err(dev, "cannot read data from DT or ACPI\n");
 885                 return -ENXIO;
 886         }
 887
 888         ret = hns_dsaf_roce_reset(fwnode, false);
 889         if (ret)
 890                 return ret;
 891
 892         if (dereset) {
 893                 msleep(SLEEP_TIME_INTERVAL);
 894                 ret = hns_dsaf_roce_reset(fwnode, true);
 895         }
 896
 897         return ret;
 898 }
 899
 900 void hns_roce_v1_profile(struct hns_roce_dev *hr_dev)
 901 {
 902         int i = 0;
 903         struct hns_roce_caps *caps = &hr_dev->caps;
 904
 905         hr_dev->vendor_id = le32_to_cpu(roce_read(hr_dev, ROCEE_VENDOR_ID_REG));
 906         hr_dev->vendor_part_id = le32_to_cpu(roce_read(hr_dev,
 907                                              ROCEE_VENDOR_PART_ID_REG));
 908         hr_dev->hw_rev = le32_to_cpu(roce_read(hr_dev, ROCEE_HW_VERSION_REG));
 909
 910         hr_dev->sys_image_guid = le32_to_cpu(roce_read(hr_dev,
 911                                              ROCEE_SYS_IMAGE_GUID_L_REG)) |
 912                                 ((u64)le32_to_cpu(roce_read(hr_dev,
 913                                             ROCEE_SYS_IMAGE_GUID_H_REG)) << 32);
 914
 915         caps->num_qps           = HNS_ROCE_V1_MAX_QP_NUM;
 916         caps->max_wqes          = HNS_ROCE_V1_MAX_WQE_NUM;
 917         caps->num_cqs           = HNS_ROCE_V1_MAX_CQ_NUM;
 918         caps->max_cqes          = HNS_ROCE_V1_MAX_CQE_NUM;
 919         caps->max_sq_sg         = HNS_ROCE_V1_SG_NUM;
 920         caps->max_rq_sg         = HNS_ROCE_V1_SG_NUM;
 921         caps->max_sq_inline     = HNS_ROCE_V1_INLINE_SIZE;
 922         caps->num_uars          = HNS_ROCE_V1_UAR_NUM;
 923         caps->phy_num_uars      = HNS_ROCE_V1_PHY_UAR_NUM;
 924         caps->num_aeq_vectors   = HNS_ROCE_AEQE_VEC_NUM;
 925         caps->num_comp_vectors  = HNS_ROCE_COMP_VEC_NUM;
 926         caps->num_other_vectors = HNS_ROCE_AEQE_OF_VEC_NUM;
 927         caps->num_mtpts         = HNS_ROCE_V1_MAX_MTPT_NUM;
 928         caps->num_mtt_segs      = HNS_ROCE_V1_MAX_MTT_SEGS;
 929         caps->num_pds           = HNS_ROCE_V1_MAX_PD_NUM;
 930         caps->max_qp_init_rdma  = HNS_ROCE_V1_MAX_QP_INIT_RDMA;
 931         caps->max_qp_dest_rdma  = HNS_ROCE_V1_MAX_QP_DEST_RDMA;
 932         caps->max_sq_desc_sz    = HNS_ROCE_V1_MAX_SQ_DESC_SZ;
 933         caps->max_rq_desc_sz    = HNS_ROCE_V1_MAX_RQ_DESC_SZ;
 934         caps->qpc_entry_sz      = HNS_ROCE_V1_QPC_ENTRY_SIZE;
 935         caps->irrl_entry_sz     = HNS_ROCE_V1_IRRL_ENTRY_SIZE;
 936         caps->cqc_entry_sz      = HNS_ROCE_V1_CQC_ENTRY_SIZE;
 937         caps->mtpt_entry_sz     = HNS_ROCE_V1_MTPT_ENTRY_SIZE;
 938         caps->mtt_entry_sz      = HNS_ROCE_V1_MTT_ENTRY_SIZE;
 939         caps->cq_entry_sz       = HNS_ROCE_V1_CQE_ENTRY_SIZE;
 940         caps->page_size_cap     = HNS_ROCE_V1_PAGE_SIZE_SUPPORT;
 941         caps->reserved_lkey     = 0;
 942         caps->reserved_pds      = 0;
 943         caps->reserved_mrws     = 1;
 944         caps->reserved_uars     = 0;
 945         caps->reserved_cqs      = 0;
 946
 947         for (i = 0; i < caps->num_ports; i++)
 948                 caps->pkey_table_len[i] = 1;
 949
 950         for (i = 0; i < caps->num_ports; i++) {
 951                 /* Six ports shared 16 GID in v1 engine */
 952                 if (i >= (HNS_ROCE_V1_GID_NUM % caps->num_ports))
 953                         caps->gid_table_len[i] = HNS_ROCE_V1_GID_NUM /
 954                                                  caps->num_ports;
 955                 else
 956                         caps->gid_table_len[i] = HNS_ROCE_V1_GID_NUM /
 957                                                  caps->num_ports + 1;
 958         }
 959
 960         for (i = 0; i < caps->num_comp_vectors; i++)
 961                 caps->ceqe_depth[i] = HNS_ROCE_V1_NUM_COMP_EQE;
 962
 963         caps->aeqe_depth = HNS_ROCE_V1_NUM_ASYNC_EQE;
 964         caps->local_ca_ack_delay = le32_to_cpu(roce_read(hr_dev,
 965                                                          ROCEE_ACK_DELAY_REG));
 966         caps->max_mtu = IB_MTU_2048;
 967 }
 968
 969 int hns_roce_v1_init(struct hns_roce_dev *hr_dev)
 970 {
 971         int ret;
 972         u32 val;
 973         struct device *dev = &hr_dev->pdev->dev;
 974
 975         /* DMAE user config */
 976         val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG1_REG);
 977         roce_set_field(val, ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_M,
 978                        ROCEE_DMAE_USER_CFG1_ROCEE_CACHE_TB_CFG_S, 0xf);
 979         roce_set_field(val, ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_M,
 980                        ROCEE_DMAE_USER_CFG1_ROCEE_STREAM_ID_TB_CFG_S,
 981                        1 << PAGES_SHIFT_16);
 982         roce_write(hr_dev, ROCEE_DMAE_USER_CFG1_REG, val);
 983
 984         val = roce_read(hr_dev, ROCEE_DMAE_USER_CFG2_REG);
 985         roce_set_field(val, ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_M,
 986                        ROCEE_DMAE_USER_CFG2_ROCEE_CACHE_PKT_CFG_S, 0xf);
 987         roce_set_field(val, ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_M,
 988                        ROCEE_DMAE_USER_CFG2_ROCEE_STREAM_ID_PKT_CFG_S,
 989                        1 << PAGES_SHIFT_16);
 990
 991         ret = hns_roce_db_init(hr_dev);
 992         if (ret) {
 993                 dev_err(dev, "doorbell init failed!\n");
 994                 return ret;
 995         }
 996
 997         ret = hns_roce_raq_init(hr_dev);
 998         if (ret) {
 999                 dev_err(dev, "raq init failed!\n");
1000                 goto error_failed_raq_init;
1001         }
1002
1003         hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_UP);
1004
1005         ret = hns_roce_bt_init(hr_dev);
1006         if (ret) {
1007                 dev_err(dev, "bt init failed!\n");
1008                 goto error_failed_bt_init;
1009         }
1010
1011         return 0;
1012
1013 error_failed_bt_init:
1014         hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_DOWN);
1015         hns_roce_raq_free(hr_dev);
1016
1017 error_failed_raq_init:
1018         hns_roce_db_free(hr_dev);
1019         return ret;
1020 }
1021
1022 void hns_roce_v1_exit(struct hns_roce_dev *hr_dev)
1023 {
1024         hns_roce_bt_free(hr_dev);
1025         hns_roce_port_enable(hr_dev, HNS_ROCE_PORT_DOWN);
1026         hns_roce_raq_free(hr_dev);
1027         hns_roce_db_free(hr_dev);
1028 }
1029
1030 void hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port, int gid_index,
1031                          union ib_gid *gid)
1032 {
1033         u32 *p = NULL;
1034         u8 gid_idx = 0;
1035
1036         gid_idx = hns_get_gid_index(hr_dev, port, gid_index);
1037
1038         p = (u32 *)&gid->raw[0];
1039         roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_L_0_REG +
1040                        (HNS_ROCE_V1_GID_NUM * gid_idx));
1041
1042         p = (u32 *)&gid->raw[4];
1043         roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_ML_0_REG +
1044                        (HNS_ROCE_V1_GID_NUM * gid_idx));
1045
1046         p = (u32 *)&gid->raw[8];
1047         roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_MH_0_REG +
1048                        (HNS_ROCE_V1_GID_NUM * gid_idx));
1049
1050         p = (u32 *)&gid->raw[0xc];
1051         roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_H_0_REG +
1052                        (HNS_ROCE_V1_GID_NUM * gid_idx));
1053 }
1054
1055 void hns_roce_v1_set_mac(struct hns_roce_dev *hr_dev, u8 phy_port, u8 *addr)
1056 {
1057         u32 reg_smac_l;
1058         u16 reg_smac_h;
1059         u16 *p_h;
1060         u32 *p;
1061         u32 val;
1062
1063         p = (u32 *)(&addr[0]);
1064         reg_smac_l = *p;
1065         roce_raw_write(reg_smac_l, hr_dev->reg_base + ROCEE_SMAC_L_0_REG +
1066                        PHY_PORT_OFFSET * phy_port);
1067
1068         val = roce_read(hr_dev,
1069                         ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET);
1070         p_h = (u16 *)(&addr[4]);
1071         reg_smac_h  = *p_h;
1072         roce_set_field(val, ROCEE_SMAC_H_ROCEE_SMAC_H_M,
1073                        ROCEE_SMAC_H_ROCEE_SMAC_H_S, reg_smac_h);
1074         roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET,
1075                    val);
1076 }
1077
1078 void hns_roce_v1_set_mtu(struct hns_roce_dev *hr_dev, u8 phy_port,
1079                          enum ib_mtu mtu)
1080 {
1081         u32 val;
1082
1083         val = roce_read(hr_dev,
1084                         ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET);
1085         roce_set_field(val, ROCEE_SMAC_H_ROCEE_PORT_MTU_M,
1086                        ROCEE_SMAC_H_ROCEE_PORT_MTU_S, mtu);
1087         roce_write(hr_dev, ROCEE_SMAC_H_0_REG + phy_port * PHY_PORT_OFFSET,
1088                    val);
1089 }
1090
1091 int hns_roce_v1_write_mtpt(void *mb_buf, struct hns_roce_mr *mr,
1092                            unsigned long mtpt_idx)
1093 {
1094         struct hns_roce_v1_mpt_entry *mpt_entry;
1095         struct scatterlist *sg;
1096         u64 *pages;
1097         int entry;
1098         int i;
1099
1100         /* MPT filled into mailbox buf */
1101         mpt_entry = (struct hns_roce_v1_mpt_entry *)mb_buf;
1102         memset(mpt_entry, 0, sizeof(*mpt_entry));
1103
1104         roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_KEY_STATE_M,
1105                        MPT_BYTE_4_KEY_STATE_S, KEY_VALID);
1106         roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_KEY_M,
1107                        MPT_BYTE_4_KEY_S, mr->key);
1108         roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_PAGE_SIZE_M,
1109                        MPT_BYTE_4_PAGE_SIZE_S, MR_SIZE_4K);
1110         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_MW_TYPE_S, 0);
1111         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_MW_BIND_ENABLE_S,
1112                      (mr->access & IB_ACCESS_MW_BIND ? 1 : 0));
1113         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_OWN_S, 0);
1114         roce_set_field(mpt_entry->mpt_byte_4, MPT_BYTE_4_MEMORY_LOCATION_TYPE_M,
1115                        MPT_BYTE_4_MEMORY_LOCATION_TYPE_S, mr->type);
1116         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_ATOMIC_S, 0);
1117         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_LOCAL_WRITE_S,
1118                      (mr->access & IB_ACCESS_LOCAL_WRITE ? 1 : 0));
1119         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_WRITE_S,
1120                      (mr->access & IB_ACCESS_REMOTE_WRITE ? 1 : 0));
1121         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_READ_S,
1122                      (mr->access & IB_ACCESS_REMOTE_READ ? 1 : 0));
1123         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_REMOTE_INVAL_ENABLE_S,
1124                      0);
1125         roce_set_bit(mpt_entry->mpt_byte_4, MPT_BYTE_4_ADDRESS_TYPE_S, 0);
1126
1127         roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M,
1128                        MPT_BYTE_12_PBL_ADDR_H_S, 0);
1129         roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_MW_BIND_COUNTER_M,
1130                        MPT_BYTE_12_MW_BIND_COUNTER_S, 0);
1131
1132         mpt_entry->virt_addr_l = (u32)mr->iova;
1133         mpt_entry->virt_addr_h = (u32)(mr->iova >> 32);
1134         mpt_entry->length = (u32)mr->size;
1135
1136         roce_set_field(mpt_entry->mpt_byte_28, MPT_BYTE_28_PD_M,
1137                        MPT_BYTE_28_PD_S, mr->pd);
1138         roce_set_field(mpt_entry->mpt_byte_28, MPT_BYTE_28_L_KEY_IDX_L_M,
1139                        MPT_BYTE_28_L_KEY_IDX_L_S, mtpt_idx);
1140         roce_set_field(mpt_entry->mpt_byte_64, MPT_BYTE_64_L_KEY_IDX_H_M,
1141                        MPT_BYTE_64_L_KEY_IDX_H_S, mtpt_idx >> MTPT_IDX_SHIFT);
1142
1143         /* DMA momery regsiter */
1144         if (mr->type == MR_TYPE_DMA)
1145                 return 0;
1146
1147         pages = (u64 *) __get_free_page(GFP_KERNEL);
1148         if (!pages)
1149                 return -ENOMEM;
1150
1151         i = 0;
1152         for_each_sg(mr->umem->sg_head.sgl, sg, mr->umem->nmap, entry) {
1153                 pages[i] = ((u64)sg_dma_address(sg)) >> 12;
1154
1155                 /* Directly record to MTPT table firstly 7 entry */
1156                 if (i >= HNS_ROCE_MAX_INNER_MTPT_NUM)
1157                         break;
1158                 i++;
1159         }
1160
1161         /* Register user mr */
1162         for (i = 0; i < HNS_ROCE_MAX_INNER_MTPT_NUM; i++) {
1163                 switch (i) {
1164                 case 0:
1165                         mpt_entry->pa0_l = cpu_to_le32((u32)(pages[i]));
1166                         roce_set_field(mpt_entry->mpt_byte_36,
1167                                 MPT_BYTE_36_PA0_H_M,
1168                                 MPT_BYTE_36_PA0_H_S,
1169                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_32)));
1170                         break;
1171                 case 1:
1172                         roce_set_field(mpt_entry->mpt_byte_36,
1173                                        MPT_BYTE_36_PA1_L_M,
1174                                        MPT_BYTE_36_PA1_L_S,
1175                                        cpu_to_le32((u32)(pages[i])));
1176                         roce_set_field(mpt_entry->mpt_byte_40,
1177                                 MPT_BYTE_40_PA1_H_M,
1178                                 MPT_BYTE_40_PA1_H_S,
1179                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_24)));
1180                         break;
1181                 case 2:
1182                         roce_set_field(mpt_entry->mpt_byte_40,
1183                                        MPT_BYTE_40_PA2_L_M,
1184                                        MPT_BYTE_40_PA2_L_S,
1185                                        cpu_to_le32((u32)(pages[i])));
1186                         roce_set_field(mpt_entry->mpt_byte_44,
1187                                 MPT_BYTE_44_PA2_H_M,
1188                                 MPT_BYTE_44_PA2_H_S,
1189                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_16)));
1190                         break;
1191                 case 3:
1192                         roce_set_field(mpt_entry->mpt_byte_44,
1193                                        MPT_BYTE_44_PA3_L_M,
1194                                        MPT_BYTE_44_PA3_L_S,
1195                                        cpu_to_le32((u32)(pages[i])));
1196                         roce_set_field(mpt_entry->mpt_byte_48,
1197                                 MPT_BYTE_48_PA3_H_M,
1198                                 MPT_BYTE_48_PA3_H_S,
1199                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_8)));
1200                         break;
1201                 case 4:
1202                         mpt_entry->pa4_l = cpu_to_le32((u32)(pages[i]));
1203                         roce_set_field(mpt_entry->mpt_byte_56,
1204                                 MPT_BYTE_56_PA4_H_M,
1205                                 MPT_BYTE_56_PA4_H_S,
1206                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_32)));
1207                         break;
1208                 case 5:
1209                         roce_set_field(mpt_entry->mpt_byte_56,
1210                                        MPT_BYTE_56_PA5_L_M,
1211                                        MPT_BYTE_56_PA5_L_S,
1212                                        cpu_to_le32((u32)(pages[i])));
1213                         roce_set_field(mpt_entry->mpt_byte_60,
1214                                 MPT_BYTE_60_PA5_H_M,
1215                                 MPT_BYTE_60_PA5_H_S,
1216                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_24)));
1217                         break;
1218                 case 6:
1219                         roce_set_field(mpt_entry->mpt_byte_60,
1220                                        MPT_BYTE_60_PA6_L_M,
1221                                        MPT_BYTE_60_PA6_L_S,
1222                                        cpu_to_le32((u32)(pages[i])));
1223                         roce_set_field(mpt_entry->mpt_byte_64,
1224                                 MPT_BYTE_64_PA6_H_M,
1225                                 MPT_BYTE_64_PA6_H_S,
1226                                 cpu_to_le32((u32)(pages[i] >> PAGES_SHIFT_16)));
1227                         break;
1228                 default:
1229                         break;
1230                 }
1231         }
1232
1233         free_page((unsigned long) pages);
1234
1235         mpt_entry->pbl_addr_l = (u32)(mr->pbl_dma_addr);
1236
1237         roce_set_field(mpt_entry->mpt_byte_12, MPT_BYTE_12_PBL_ADDR_H_M,
1238                        MPT_BYTE_12_PBL_ADDR_H_S,
1239                        ((u32)(mr->pbl_dma_addr >> 32)));
1240
1241         return 0;
1242 }
1243
1244 static void *get_cqe(struct hns_roce_cq *hr_cq, int n)
1245 {
1246         return hns_roce_buf_offset(&hr_cq->hr_buf.hr_buf,
1247                                    n * HNS_ROCE_V1_CQE_ENTRY_SIZE);
1248 }
1249
1250 static void *get_sw_cqe(struct hns_roce_cq *hr_cq, int n)
1251 {
1252         struct hns_roce_cqe *hr_cqe = get_cqe(hr_cq, n & hr_cq->ib_cq.cqe);
1253
1254         /* Get cqe when Owner bit is Conversely with the MSB of cons_idx */
1255         return (roce_get_bit(hr_cqe->cqe_byte_4, CQE_BYTE_4_OWNER_S) ^
1256                 !!(n & (hr_cq->ib_cq.cqe + 1))) ? hr_cqe : NULL;
1257 }
1258
1259 static struct hns_roce_cqe *next_cqe_sw(struct hns_roce_cq *hr_cq)
1260 {
1261         return get_sw_cqe(hr_cq, hr_cq->cons_index);
1262 }
1263
1264 void hns_roce_v1_cq_set_ci(struct hns_roce_cq *hr_cq, u32 cons_index)
1265 {
1266         u32 doorbell[2];
1267
1268         doorbell[0] = cons_index & ((hr_cq->cq_depth << 1) - 1);
1269         doorbell[1] = 0;
1270         roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1);
1271         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M,
1272                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3);
1273         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M,
1274                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 0);
1275         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M,
1276                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S, hr_cq->cqn);
1277
1278         hns_roce_write64_k(doorbell, hr_cq->cq_db_l);
1279 }
1280
1281 static void __hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
1282                                    struct hns_roce_srq *srq)
1283 {
1284         struct hns_roce_cqe *cqe, *dest;
1285         u32 prod_index;
1286         int nfreed = 0;
1287         u8 owner_bit;
1288
1289         for (prod_index = hr_cq->cons_index; get_sw_cqe(hr_cq, prod_index);
1290              ++prod_index) {
1291                 if (prod_index == hr_cq->cons_index + hr_cq->ib_cq.cqe)
1292                         break;
1293         }
1294
1295         /*
1296         * Now backwards through the CQ, removing CQ entries
1297         * that match our QP by overwriting them with next entries.
1298         */
1299         while ((int) --prod_index - (int) hr_cq->cons_index >= 0) {
1300                 cqe = get_cqe(hr_cq, prod_index & hr_cq->ib_cq.cqe);
1301                 if ((roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
1302                                      CQE_BYTE_16_LOCAL_QPN_S) &
1303                                      HNS_ROCE_CQE_QPN_MASK) == qpn) {
1304                         /* In v1 engine, not support SRQ */
1305                         ++nfreed;
1306                 } else if (nfreed) {
1307                         dest = get_cqe(hr_cq, (prod_index + nfreed) &
1308                                        hr_cq->ib_cq.cqe);
1309                         owner_bit = roce_get_bit(dest->cqe_byte_4,
1310                                                  CQE_BYTE_4_OWNER_S);
1311                         memcpy(dest, cqe, sizeof(*cqe));
1312                         roce_set_bit(dest->cqe_byte_4, CQE_BYTE_4_OWNER_S,
1313                                      owner_bit);
1314                 }
1315         }
1316
1317         if (nfreed) {
1318                 hr_cq->cons_index += nfreed;
1319                 /*
1320                 * Make sure update of buffer contents is done before
1321                 * updating consumer index.
1322                 */
1323                 wmb();
1324
1325                 hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index);
1326         }
1327 }
1328
1329 static void hns_roce_v1_cq_clean(struct hns_roce_cq *hr_cq, u32 qpn,
1330                                  struct hns_roce_srq *srq)
1331 {
1332         spin_lock_irq(&hr_cq->lock);
1333         __hns_roce_v1_cq_clean(hr_cq, qpn, srq);
1334         spin_unlock_irq(&hr_cq->lock);
1335 }
1336
1337 void hns_roce_v1_write_cqc(struct hns_roce_dev *hr_dev,
1338                            struct hns_roce_cq *hr_cq, void *mb_buf, u64 *mtts,
1339                            dma_addr_t dma_handle, int nent, u32 vector)
1340 {
1341         struct hns_roce_cq_context *cq_context = NULL;
1342         void __iomem *tptr_addr;
1343
1344         cq_context = mb_buf;
1345         memset(cq_context, 0, sizeof(*cq_context));
1346
1347         tptr_addr = 0;
1348         hr_dev->priv_addr = tptr_addr;
1349         hr_cq->tptr_addr = tptr_addr;
1350
1351         /* Register cq_context members */
1352         roce_set_field(cq_context->cqc_byte_4,
1353                        CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_M,
1354                        CQ_CONTEXT_CQC_BYTE_4_CQC_STATE_S, CQ_STATE_VALID);
1355         roce_set_field(cq_context->cqc_byte_4, CQ_CONTEXT_CQC_BYTE_4_CQN_M,
1356                        CQ_CONTEXT_CQC_BYTE_4_CQN_S, hr_cq->cqn);
1357         cq_context->cqc_byte_4 = cpu_to_le32(cq_context->cqc_byte_4);
1358
1359         cq_context->cq_bt_l = (u32)dma_handle;
1360         cq_context->cq_bt_l = cpu_to_le32(cq_context->cq_bt_l);
1361
1362         roce_set_field(cq_context->cqc_byte_12,
1363                        CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_M,
1364                        CQ_CONTEXT_CQC_BYTE_12_CQ_BT_H_S,
1365                        ((u64)dma_handle >> 32));
1366         roce_set_field(cq_context->cqc_byte_12,
1367                        CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_M,
1368                        CQ_CONTEXT_CQC_BYTE_12_CQ_CQE_SHIFT_S,
1369                        ilog2((unsigned int)nent));
1370         roce_set_field(cq_context->cqc_byte_12, CQ_CONTEXT_CQC_BYTE_12_CEQN_M,
1371                        CQ_CONTEXT_CQC_BYTE_12_CEQN_S, vector);
1372         cq_context->cqc_byte_12 = cpu_to_le32(cq_context->cqc_byte_12);
1373
1374         cq_context->cur_cqe_ba0_l = (u32)(mtts[0]);
1375         cq_context->cur_cqe_ba0_l = cpu_to_le32(cq_context->cur_cqe_ba0_l);
1376
1377         roce_set_field(cq_context->cqc_byte_20,
1378                        CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_M,
1379                        CQ_CONTEXT_CQC_BYTE_20_CUR_CQE_BA0_H_S,
1380                        cpu_to_le32((mtts[0]) >> 32));
1381         /* Dedicated hardware, directly set 0 */
1382         roce_set_field(cq_context->cqc_byte_20,
1383                        CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_M,
1384                        CQ_CONTEXT_CQC_BYTE_20_CQ_CUR_INDEX_S, 0);
1385         /**
1386          * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
1387          * using 4K page, and shift more 32 because of
1388          * caculating the high 32 bit value evaluated to hardware.
1389          */
1390         roce_set_field(cq_context->cqc_byte_20,
1391                        CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_M,
1392                        CQ_CONTEXT_CQC_BYTE_20_CQE_TPTR_ADDR_H_S,
1393                        (u64)tptr_addr >> 44);
1394         cq_context->cqc_byte_20 = cpu_to_le32(cq_context->cqc_byte_20);
1395
1396         cq_context->cqe_tptr_addr_l = (u32)((u64)tptr_addr >> 12);
1397
1398         roce_set_field(cq_context->cqc_byte_32,
1399                        CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_M,
1400                        CQ_CONTEXT_CQC_BYTE_32_CUR_CQE_BA1_H_S, 0);
1401         roce_set_bit(cq_context->cqc_byte_32,
1402                      CQ_CONTEXT_CQC_BYTE_32_SE_FLAG_S, 0);
1403         roce_set_bit(cq_context->cqc_byte_32,
1404                      CQ_CONTEXT_CQC_BYTE_32_CE_FLAG_S, 0);
1405         roce_set_bit(cq_context->cqc_byte_32,
1406                      CQ_CONTEXT_CQC_BYTE_32_NOTIFICATION_FLAG_S, 0);
1407         roce_set_bit(cq_context->cqc_byte_32,
1408                      CQ_CQNTEXT_CQC_BYTE_32_TYPE_OF_COMPLETION_NOTIFICATION_S,
1409                      0);
1410         /*The initial value of cq's ci is 0 */
1411         roce_set_field(cq_context->cqc_byte_32,
1412                        CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_M,
1413                        CQ_CONTEXT_CQC_BYTE_32_CQ_CONS_IDX_S, 0);
1414         cq_context->cqc_byte_32 = cpu_to_le32(cq_context->cqc_byte_32);
1415 }
1416
1417 int hns_roce_v1_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
1418 {
1419         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
1420         u32 notification_flag;
1421         u32 doorbell[2];
1422         int ret = 0;
1423
1424         notification_flag = (flags & IB_CQ_SOLICITED_MASK) ==
1425                             IB_CQ_SOLICITED ? CQ_DB_REQ_NOT : CQ_DB_REQ_NOT_SOL;
1426         /*
1427         * flags = 0; Notification Flag = 1, next
1428         * flags = 1; Notification Flag = 0, solocited
1429         */
1430         doorbell[0] = hr_cq->cons_index & ((hr_cq->cq_depth << 1) - 1);
1431         roce_set_bit(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_HW_SYNS_S, 1);
1432         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_M,
1433                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_S, 3);
1434         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_M,
1435                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_CMD_MDF_S, 1);
1436         roce_set_field(doorbell[1], ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_M,
1437                        ROCEE_DB_OTHERS_H_ROCEE_DB_OTH_INP_H_S,
1438                        hr_cq->cqn | notification_flag);
1439
1440         hns_roce_write64_k(doorbell, hr_cq->cq_db_l);
1441
1442         return ret;
1443 }
1444
1445 static int hns_roce_v1_poll_one(struct hns_roce_cq *hr_cq,
1446                                 struct hns_roce_qp **cur_qp, struct ib_wc *wc)
1447 {
1448         int qpn;
1449         int is_send;
1450         u16 wqe_ctr;
1451         u32 status;
1452         u32 opcode;
1453         struct hns_roce_cqe *cqe;
1454         struct hns_roce_qp *hr_qp;
1455         struct hns_roce_wq *wq;
1456         struct hns_roce_wqe_ctrl_seg *sq_wqe;
1457         struct hns_roce_dev *hr_dev = to_hr_dev(hr_cq->ib_cq.device);
1458         struct device *dev = &hr_dev->pdev->dev;
1459
1460         /* Find cqe according consumer index */
1461         cqe = next_cqe_sw(hr_cq);
1462         if (!cqe)
1463                 return -EAGAIN;
1464
1465         ++hr_cq->cons_index;
1466         /* Memory barrier */
1467         rmb();
1468         /* 0->SQ, 1->RQ */
1469         is_send  = !(roce_get_bit(cqe->cqe_byte_4, CQE_BYTE_4_SQ_RQ_FLAG_S));
1470
1471         /* Local_qpn in UD cqe is always 1, so it needs to compute new qpn */
1472         if (roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
1473                            CQE_BYTE_16_LOCAL_QPN_S) <= 1) {
1474                 qpn = roce_get_field(cqe->cqe_byte_20, CQE_BYTE_20_PORT_NUM_M,
1475                                      CQE_BYTE_20_PORT_NUM_S) +
1476                       roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
1477                                      CQE_BYTE_16_LOCAL_QPN_S) *
1478                                      HNS_ROCE_MAX_PORTS;
1479         } else {
1480                 qpn = roce_get_field(cqe->cqe_byte_16, CQE_BYTE_16_LOCAL_QPN_M,
1481                                      CQE_BYTE_16_LOCAL_QPN_S);
1482         }
1483
1484         if (!*cur_qp || (qpn & HNS_ROCE_CQE_QPN_MASK) != (*cur_qp)->qpn) {
1485                 hr_qp = __hns_roce_qp_lookup(hr_dev, qpn);
1486                 if (unlikely(!hr_qp)) {
1487                         dev_err(dev, "CQ %06lx with entry for unknown QPN %06x\n",
1488                                 hr_cq->cqn, (qpn & HNS_ROCE_CQE_QPN_MASK));
1489                         return -EINVAL;
1490                 }
1491
1492                 *cur_qp = hr_qp;
1493         }
1494
1495         wc->qp = &(*cur_qp)->ibqp;
1496         wc->vendor_err = 0;
1497
1498         status = roce_get_field(cqe->cqe_byte_4,
1499                                 CQE_BYTE_4_STATUS_OF_THE_OPERATION_M,
1500                                 CQE_BYTE_4_STATUS_OF_THE_OPERATION_S) &
1501                                 HNS_ROCE_CQE_STATUS_MASK;
1502         switch (status) {
1503         case HNS_ROCE_CQE_SUCCESS:
1504                 wc->status = IB_WC_SUCCESS;
1505                 break;
1506         case HNS_ROCE_CQE_SYNDROME_LOCAL_LENGTH_ERR:
1507                 wc->status = IB_WC_LOC_LEN_ERR;
1508                 break;
1509         case HNS_ROCE_CQE_SYNDROME_LOCAL_QP_OP_ERR:
1510                 wc->status = IB_WC_LOC_QP_OP_ERR;
1511                 break;
1512         case HNS_ROCE_CQE_SYNDROME_LOCAL_PROT_ERR:
1513                 wc->status = IB_WC_LOC_PROT_ERR;
1514                 break;
1515         case HNS_ROCE_CQE_SYNDROME_WR_FLUSH_ERR:
1516                 wc->status = IB_WC_WR_FLUSH_ERR;
1517                 break;
1518         case HNS_ROCE_CQE_SYNDROME_MEM_MANAGE_OPERATE_ERR:
1519                 wc->status = IB_WC_MW_BIND_ERR;
1520                 break;
1521         case HNS_ROCE_CQE_SYNDROME_BAD_RESP_ERR:
1522                 wc->status = IB_WC_BAD_RESP_ERR;
1523                 break;
1524         case HNS_ROCE_CQE_SYNDROME_LOCAL_ACCESS_ERR:
1525                 wc->status = IB_WC_LOC_ACCESS_ERR;
1526                 break;
1527         case HNS_ROCE_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:
1528                 wc->status = IB_WC_REM_INV_REQ_ERR;
1529                 break;
1530         case HNS_ROCE_CQE_SYNDROME_REMOTE_ACCESS_ERR:
1531                 wc->status = IB_WC_REM_ACCESS_ERR;
1532                 break;
1533         case HNS_ROCE_CQE_SYNDROME_REMOTE_OP_ERR:
1534                 wc->status = IB_WC_REM_OP_ERR;
1535                 break;
1536         case HNS_ROCE_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:
1537                 wc->status = IB_WC_RETRY_EXC_ERR;
1538                 break;
1539         case HNS_ROCE_CQE_SYNDROME_RNR_RETRY_EXC_ERR:
1540                 wc->status = IB_WC_RNR_RETRY_EXC_ERR;
1541                 break;
1542         default:
1543                 wc->status = IB_WC_GENERAL_ERR;
1544                 break;
1545         }
1546
1547         /* CQE status error, directly return */
1548         if (wc->status != IB_WC_SUCCESS)
1549                 return 0;
1550
1551         if (is_send) {
1552                 /* SQ conrespond to CQE */
1553                 sq_wqe = get_send_wqe(*cur_qp, roce_get_field(cqe->cqe_byte_4,
1554                                                 CQE_BYTE_4_WQE_INDEX_M,
1555                                                 CQE_BYTE_4_WQE_INDEX_S)&
1556                                                 ((*cur_qp)->sq.wqe_cnt-1));
1557                 switch (sq_wqe->flag & HNS_ROCE_WQE_OPCODE_MASK) {
1558                 case HNS_ROCE_WQE_OPCODE_SEND:
1559                         wc->opcode = IB_WC_SEND;
1560                         break;
1561                 case HNS_ROCE_WQE_OPCODE_RDMA_READ:
1562                         wc->opcode = IB_WC_RDMA_READ;
1563                         wc->byte_len = le32_to_cpu(cqe->byte_cnt);
1564                         break;
1565                 case HNS_ROCE_WQE_OPCODE_RDMA_WRITE:
1566                         wc->opcode = IB_WC_RDMA_WRITE;
1567                         break;
1568                 case HNS_ROCE_WQE_OPCODE_LOCAL_INV:
1569                         wc->opcode = IB_WC_LOCAL_INV;
1570                         break;
1571                 case HNS_ROCE_WQE_OPCODE_UD_SEND:
1572                         wc->opcode = IB_WC_SEND;
1573                         break;
1574                 default:
1575                         wc->status = IB_WC_GENERAL_ERR;
1576                         break;
1577                 }
1578                 wc->wc_flags = (sq_wqe->flag & HNS_ROCE_WQE_IMM ?
1579                                 IB_WC_WITH_IMM : 0);
1580
1581                 wq = &(*cur_qp)->sq;
1582                 if ((*cur_qp)->sq_signal_bits) {
1583                         /*
1584                         * If sg_signal_bit is 1,
1585                         * firstly tail pointer updated to wqe
1586                         * which current cqe correspond to
1587                         */
1588                         wqe_ctr = (u16)roce_get_field(cqe->cqe_byte_4,
1589                                                       CQE_BYTE_4_WQE_INDEX_M,
1590                                                       CQE_BYTE_4_WQE_INDEX_S);
1591                         wq->tail += (wqe_ctr - (u16)wq->tail) &
1592                                     (wq->wqe_cnt - 1);
1593                 }
1594                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
1595                 ++wq->tail;
1596                 } else {
1597                 /* RQ conrespond to CQE */
1598                 wc->byte_len = le32_to_cpu(cqe->byte_cnt);
1599                 opcode = roce_get_field(cqe->cqe_byte_4,
1600                                         CQE_BYTE_4_OPERATION_TYPE_M,
1601                                         CQE_BYTE_4_OPERATION_TYPE_S) &
1602                                         HNS_ROCE_CQE_OPCODE_MASK;
1603                 switch (opcode) {
1604                 case HNS_ROCE_OPCODE_RDMA_WITH_IMM_RECEIVE:
1605                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
1606                         wc->wc_flags = IB_WC_WITH_IMM;
1607                         wc->ex.imm_data = le32_to_cpu(cqe->immediate_data);
1608                         break;
1609                 case HNS_ROCE_OPCODE_SEND_DATA_RECEIVE:
1610                         if (roce_get_bit(cqe->cqe_byte_4,
1611                                          CQE_BYTE_4_IMM_INDICATOR_S)) {
1612                                 wc->opcode = IB_WC_RECV;
1613                                 wc->wc_flags = IB_WC_WITH_IMM;
1614                                 wc->ex.imm_data = le32_to_cpu(
1615                                                   cqe->immediate_data);
1616                         } else {
1617                                 wc->opcode = IB_WC_RECV;
1618                                 wc->wc_flags = 0;
1619                         }
1620                         break;
1621                 default:
1622                         wc->status = IB_WC_GENERAL_ERR;
1623                         break;
1624                 }
1625
1626                 /* Update tail pointer, record wr_id */
1627                 wq = &(*cur_qp)->rq;
1628                 wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
1629                 ++wq->tail;
1630                 wc->sl = (u8)roce_get_field(cqe->cqe_byte_20, CQE_BYTE_20_SL_M,
1631                                             CQE_BYTE_20_SL_S);
1632                 wc->src_qp = (u8)roce_get_field(cqe->cqe_byte_20,
1633                                                 CQE_BYTE_20_REMOTE_QPN_M,
1634                                                 CQE_BYTE_20_REMOTE_QPN_S);
1635                 wc->wc_flags |= (roce_get_bit(cqe->cqe_byte_20,
1636                                               CQE_BYTE_20_GRH_PRESENT_S) ?
1637                                               IB_WC_GRH : 0);
1638                 wc->pkey_index = (u16)roce_get_field(cqe->cqe_byte_28,
1639                                                      CQE_BYTE_28_P_KEY_IDX_M,
1640                                                      CQE_BYTE_28_P_KEY_IDX_S);
1641         }
1642
1643         return 0;
1644 }
1645
1646 int hns_roce_v1_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
1647 {
1648         struct hns_roce_cq *hr_cq = to_hr_cq(ibcq);
1649         struct hns_roce_qp *cur_qp = NULL;
1650         unsigned long flags;
1651         int npolled;
1652         int ret = 0;
1653
1654         spin_lock_irqsave(&hr_cq->lock, flags);
1655
1656         for (npolled = 0; npolled < num_entries; ++npolled) {
1657                 ret = hns_roce_v1_poll_one(hr_cq, &cur_qp, wc + npolled);
1658                 if (ret)
1659                         break;
1660         }
1661
1662         if (npolled)
1663                 hns_roce_v1_cq_set_ci(hr_cq, hr_cq->cons_index);
1664
1665         spin_unlock_irqrestore(&hr_cq->lock, flags);
1666
1667         if (ret == 0 || ret == -EAGAIN)
1668                 return npolled;
1669         else
1670                 return ret;
1671 }
1672
1673 int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev,
1674                 struct hns_roce_hem_table *table, int obj)
1675 {
1676         struct device *dev = &hr_dev->pdev->dev;
1677         struct hns_roce_v1_priv *priv;
1678         unsigned long end = 0, flags = 0;
1679         uint32_t bt_cmd_val[2] = {0};
1680         void __iomem *bt_cmd;
1681         u64 bt_ba = 0;
1682
1683         priv = (struct hns_roce_v1_priv *)hr_dev->hw->priv;
1684
1685         switch (table->type) {
1686         case HEM_TYPE_QPC:
1687                 roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
1688                         ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);
1689                 bt_ba = priv->bt_table.qpc_buf.map >> 12;
1690                 break;
1691         case HEM_TYPE_MTPT:
1692                 roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
1693                         ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_MTPT);
1694                 bt_ba = priv->bt_table.mtpt_buf.map >> 12;
1695                 break;
1696         case HEM_TYPE_CQC:
1697                 roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
1698                         ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);
1699                 bt_ba = priv->bt_table.cqc_buf.map >> 12;
1700                 break;
1701         case HEM_TYPE_SRQC:
1702                 dev_dbg(dev, "HEM_TYPE_SRQC not support.\n");
1703                 return -EINVAL;
1704         default:
1705                 return 0;
1706         }
1707         roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
1708                 ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
1709         roce_set_bit(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
1710         roce_set_bit(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
1711
1712         spin_lock_irqsave(&hr_dev->bt_cmd_lock, flags);
1713
1714         bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
1715
1716         end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;
1717         while (1) {
1718                 if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
1719                         if (!(time_before(jiffies, end))) {
1720                                 dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
1721                                 spin_unlock_irqrestore(&hr_dev->bt_cmd_lock,
1722                                         flags);
1723                                 return -EBUSY;
1724                         }
1725                 } else {
1726                         break;
1727                 }
1728                 msleep(HW_SYNC_SLEEP_TIME_INTERVAL);
1729         }
1730
1731         bt_cmd_val[0] = (uint32_t)bt_ba;
1732         roce_set_field(bt_cmd_val[1], ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
1733                 ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S, bt_ba >> 32);
1734         hns_roce_write64_k(bt_cmd_val, hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
1735
1736         spin_unlock_irqrestore(&hr_dev->bt_cmd_lock, flags);
1737
1738         return 0;
1739 }
1740
1741 static int hns_roce_v1_qp_modify(struct hns_roce_dev *hr_dev,
1742                                  struct hns_roce_mtt *mtt,
1743                                  enum hns_roce_qp_state cur_state,
1744                                  enum hns_roce_qp_state new_state,
1745                                  struct hns_roce_qp_context *context,
1746                                  struct hns_roce_qp *hr_qp)
1747 {
1748         static const u16
1749         op[HNS_ROCE_QP_NUM_STATE][HNS_ROCE_QP_NUM_STATE] = {
1750                 [HNS_ROCE_QP_STATE_RST] = {
1751                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
1752                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
1753                 [HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP,
1754                 },
1755                 [HNS_ROCE_QP_STATE_INIT] = {
1756                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
1757                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
1758                 /* Note: In v1 engine, HW doesn't support RST2INIT.
1759                  * We use RST2INIT cmd instead of INIT2INIT.
1760                  */
1761                 [HNS_ROCE_QP_STATE_INIT] = HNS_ROCE_CMD_RST2INIT_QP,
1762                 [HNS_ROCE_QP_STATE_RTR] = HNS_ROCE_CMD_INIT2RTR_QP,
1763                 },
1764                 [HNS_ROCE_QP_STATE_RTR] = {
1765                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
1766                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
1767                 [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTR2RTS_QP,
1768                 },
1769                 [HNS_ROCE_QP_STATE_RTS] = {
1770                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
1771                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
1772                 [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_RTS2RTS_QP,
1773                 [HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_RTS2SQD_QP,
1774                 },
1775                 [HNS_ROCE_QP_STATE_SQD] = {
1776                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
1777                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
1778                 [HNS_ROCE_QP_STATE_RTS] = HNS_ROCE_CMD_SQD2RTS_QP,
1779                 [HNS_ROCE_QP_STATE_SQD] = HNS_ROCE_CMD_SQD2SQD_QP,
1780                 },
1781                 [HNS_ROCE_QP_STATE_ERR] = {
1782                 [HNS_ROCE_QP_STATE_RST] = HNS_ROCE_CMD_2RST_QP,
1783                 [HNS_ROCE_QP_STATE_ERR] = HNS_ROCE_CMD_2ERR_QP,
1784                 }
1785         };
1786
1787         struct hns_roce_cmd_mailbox *mailbox;
1788         struct device *dev = &hr_dev->pdev->dev;
1789         int ret = 0;
1790
1791         if (cur_state >= HNS_ROCE_QP_NUM_STATE ||
1792             new_state >= HNS_ROCE_QP_NUM_STATE ||
1793             !op[cur_state][new_state]) {
1794                 dev_err(dev, "[modify_qp]not support state %d to %d\n",
1795                         cur_state, new_state);
1796                 return -EINVAL;
1797         }
1798
1799         if (op[cur_state][new_state] == HNS_ROCE_CMD_2RST_QP)
1800                 return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2,
1801                                          HNS_ROCE_CMD_2RST_QP,
1802                                          HNS_ROCE_CMD_TIME_CLASS_A);
1803
1804         if (op[cur_state][new_state] == HNS_ROCE_CMD_2ERR_QP)
1805                 return hns_roce_cmd_mbox(hr_dev, 0, 0, hr_qp->qpn, 2,
1806                                          HNS_ROCE_CMD_2ERR_QP,
1807                                          HNS_ROCE_CMD_TIME_CLASS_A);
1808
1809         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
1810         if (IS_ERR(mailbox))
1811                 return PTR_ERR(mailbox);
1812
1813         memcpy(mailbox->buf, context, sizeof(*context));
1814
1815         ret = hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, hr_qp->qpn, 0,
1816                                 op[cur_state][new_state],
1817                                 HNS_ROCE_CMD_TIME_CLASS_C);
1818
1819         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
1820         return ret;
1821 }
1822
1823 static int hns_roce_v1_m_sqp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
1824                              int attr_mask, enum ib_qp_state cur_state,
1825                              enum ib_qp_state new_state)
1826 {
1827         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1828         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1829         struct hns_roce_sqp_context *context;
1830         struct device *dev = &hr_dev->pdev->dev;
1831         dma_addr_t dma_handle = 0;
1832         int rq_pa_start;
1833         u32 reg_val;
1834         u64 *mtts;
1835         u32 *addr;
1836
1837         context = kzalloc(sizeof(*context), GFP_KERNEL);
1838         if (!context)
1839                 return -ENOMEM;
1840
1841         /* Search QP buf's MTTs */
1842         mtts = hns_roce_table_find(&hr_dev->mr_table.mtt_table,
1843                                    hr_qp->mtt.first_seg, &dma_handle);
1844         if (!mtts) {
1845                 dev_err(dev, "qp buf pa find failed\n");
1846                 goto out;
1847         }
1848
1849         if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
1850                 roce_set_field(context->qp1c_bytes_4,
1851                                QP1C_BYTES_4_SQ_WQE_SHIFT_M,
1852                                QP1C_BYTES_4_SQ_WQE_SHIFT_S,
1853                                ilog2((unsigned int)hr_qp->sq.wqe_cnt));
1854                 roce_set_field(context->qp1c_bytes_4,
1855                                QP1C_BYTES_4_RQ_WQE_SHIFT_M,
1856                                QP1C_BYTES_4_RQ_WQE_SHIFT_S,
1857                                ilog2((unsigned int)hr_qp->rq.wqe_cnt));
1858                 roce_set_field(context->qp1c_bytes_4, QP1C_BYTES_4_PD_M,
1859                                QP1C_BYTES_4_PD_S, to_hr_pd(ibqp->pd)->pdn);
1860
1861                 context->sq_rq_bt_l = (u32)(dma_handle);
1862                 roce_set_field(context->qp1c_bytes_12,
1863                                QP1C_BYTES_12_SQ_RQ_BT_H_M,
1864                                QP1C_BYTES_12_SQ_RQ_BT_H_S,
1865                                ((u32)(dma_handle >> 32)));
1866
1867                 roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_HEAD_M,
1868                                QP1C_BYTES_16_RQ_HEAD_S, hr_qp->rq.head);
1869                 roce_set_field(context->qp1c_bytes_16, QP1C_BYTES_16_PORT_NUM_M,
1870                                QP1C_BYTES_16_PORT_NUM_S, hr_qp->phy_port);
1871                 roce_set_bit(context->qp1c_bytes_16,
1872                              QP1C_BYTES_16_SIGNALING_TYPE_S,
1873                              hr_qp->sq_signal_bits);
1874                 roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_RQ_BA_FLG_S,
1875                              1);
1876                 roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_SQ_BA_FLG_S,
1877                              1);
1878                 roce_set_bit(context->qp1c_bytes_16, QP1C_BYTES_16_QP1_ERR_S,
1879                              0);
1880
1881                 roce_set_field(context->qp1c_bytes_20, QP1C_BYTES_20_SQ_HEAD_M,
1882                                QP1C_BYTES_20_SQ_HEAD_S, hr_qp->sq.head);
1883                 roce_set_field(context->qp1c_bytes_20, QP1C_BYTES_20_PKEY_IDX_M,
1884                                QP1C_BYTES_20_PKEY_IDX_S, attr->pkey_index);
1885
1886                 rq_pa_start = (u32)hr_qp->rq.offset / PAGE_SIZE;
1887                 context->cur_rq_wqe_ba_l = (u32)(mtts[rq_pa_start]);
1888
1889                 roce_set_field(context->qp1c_bytes_28,
1890                                QP1C_BYTES_28_CUR_RQ_WQE_BA_H_M,
1891                                QP1C_BYTES_28_CUR_RQ_WQE_BA_H_S,
1892                                (mtts[rq_pa_start]) >> 32);
1893                 roce_set_field(context->qp1c_bytes_28,
1894                                QP1C_BYTES_28_RQ_CUR_IDX_M,
1895                                QP1C_BYTES_28_RQ_CUR_IDX_S, 0);
1896
1897                 roce_set_field(context->qp1c_bytes_32,
1898                                QP1C_BYTES_32_RX_CQ_NUM_M,
1899                                QP1C_BYTES_32_RX_CQ_NUM_S,
1900                                to_hr_cq(ibqp->recv_cq)->cqn);
1901                 roce_set_field(context->qp1c_bytes_32,
1902                                QP1C_BYTES_32_TX_CQ_NUM_M,
1903                                QP1C_BYTES_32_TX_CQ_NUM_S,
1904                                to_hr_cq(ibqp->send_cq)->cqn);
1905
1906                 context->cur_sq_wqe_ba_l  = (u32)mtts[0];
1907
1908                 roce_set_field(context->qp1c_bytes_40,
1909                                QP1C_BYTES_40_CUR_SQ_WQE_BA_H_M,
1910                                QP1C_BYTES_40_CUR_SQ_WQE_BA_H_S,
1911                                (mtts[0]) >> 32);
1912                 roce_set_field(context->qp1c_bytes_40,
1913                                QP1C_BYTES_40_SQ_CUR_IDX_M,
1914                                QP1C_BYTES_40_SQ_CUR_IDX_S, 0);
1915
1916                 /* Copy context to QP1C register */
1917                 addr = (u32 *)(hr_dev->reg_base + ROCEE_QP1C_CFG0_0_REG +
1918                         hr_qp->phy_port * sizeof(*context));
1919
1920                 writel(context->qp1c_bytes_4, addr);
1921                 writel(context->sq_rq_bt_l, addr + 1);
1922                 writel(context->qp1c_bytes_12, addr + 2);
1923                 writel(context->qp1c_bytes_16, addr + 3);
1924                 writel(context->qp1c_bytes_20, addr + 4);
1925                 writel(context->cur_rq_wqe_ba_l, addr + 5);
1926                 writel(context->qp1c_bytes_28, addr + 6);
1927                 writel(context->qp1c_bytes_32, addr + 7);
1928                 writel(context->cur_sq_wqe_ba_l, addr + 8);
1929                 writel(context->qp1c_bytes_40, addr + 9);
1930         }
1931
1932         /* Modify QP1C status */
1933         reg_val = roce_read(hr_dev, ROCEE_QP1C_CFG0_0_REG +
1934                             hr_qp->phy_port * sizeof(*context));
1935         roce_set_field(reg_val, ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_M,
1936                        ROCEE_QP1C_CFG0_0_ROCEE_QP1C_QP_ST_S, new_state);
1937         roce_write(hr_dev, ROCEE_QP1C_CFG0_0_REG +
1938                     hr_qp->phy_port * sizeof(*context), reg_val);
1939
1940         hr_qp->state = new_state;
1941         if (new_state == IB_QPS_RESET) {
1942                 hns_roce_v1_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
1943                                      ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
1944                 if (ibqp->send_cq != ibqp->recv_cq)
1945                         hns_roce_v1_cq_clean(to_hr_cq(ibqp->send_cq),
1946                                              hr_qp->qpn, NULL);
1947
1948                 hr_qp->rq.head = 0;
1949                 hr_qp->rq.tail = 0;
1950                 hr_qp->sq.head = 0;
1951                 hr_qp->sq.tail = 0;
1952                 hr_qp->sq_next_wqe = 0;
1953         }
1954
1955         kfree(context);
1956         return 0;
1957
1958 out:
1959         kfree(context);
1960         return -EINVAL;
1961 }
1962
1963 static int hns_roce_v1_m_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
1964                             int attr_mask, enum ib_qp_state cur_state,
1965                             enum ib_qp_state new_state)
1966 {
1967         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1968         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1969         struct device *dev = &hr_dev->pdev->dev;
1970         struct hns_roce_qp_context *context;
1971         dma_addr_t dma_handle_2 = 0;
1972         dma_addr_t dma_handle = 0;
1973         uint32_t doorbell[2] = {0};
1974         int rq_pa_start = 0;
1975         u64 *mtts_2 = NULL;
1976         int ret = -EINVAL;
1977         u64 *mtts = NULL;
1978         int port;
1979         u8 *dmac;
1980         u8 *smac;
1981
1982         context = kzalloc(sizeof(*context), GFP_KERNEL);
1983         if (!context)
1984                 return -ENOMEM;
1985
1986         /* Search qp buf's mtts */
1987         mtts = hns_roce_table_find(&hr_dev->mr_table.mtt_table,
1988                                    hr_qp->mtt.first_seg, &dma_handle);
1989         if (mtts == NULL) {
1990                 dev_err(dev, "qp buf pa find failed\n");
1991                 goto out;
1992         }
1993
1994         /* Search IRRL's mtts */
1995         mtts_2 = hns_roce_table_find(&hr_dev->qp_table.irrl_table, hr_qp->qpn,
1996                                      &dma_handle_2);
1997         if (mtts_2 == NULL) {
1998                 dev_err(dev, "qp irrl_table find failed\n");
1999                 goto out;
2000         }
2001
2002         /*
2003         *Reset to init
2004         *       Mandatory param:
2005         *       IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT | IB_QP_ACCESS_FLAGS
2006         *       Optional param: NA
2007         */
2008         if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
2009                 roce_set_field(context->qpc_bytes_4,
2010                                QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M,
2011                                QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S,
2012                                to_hr_qp_type(hr_qp->ibqp.qp_type));
2013
2014                 roce_set_bit(context->qpc_bytes_4,
2015                              QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0);
2016                 roce_set_bit(context->qpc_bytes_4,
2017                              QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
2018                              !!(attr->qp_access_flags & IB_ACCESS_REMOTE_READ));
2019                 roce_set_bit(context->qpc_bytes_4,
2020                              QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
2021                              !!(attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
2022                              );
2023                 roce_set_bit(context->qpc_bytes_4,
2024                              QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S,
2025                              !!(attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)
2026                              );
2027                 roce_set_bit(context->qpc_bytes_4,
2028                              QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1);
2029                 roce_set_field(context->qpc_bytes_4,
2030                                QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M,
2031                                QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S,
2032                                ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2033                 roce_set_field(context->qpc_bytes_4,
2034                                QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M,
2035                                QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S,
2036                                ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2037                 roce_set_field(context->qpc_bytes_4,
2038                                QP_CONTEXT_QPC_BYTES_4_PD_M,
2039                                QP_CONTEXT_QPC_BYTES_4_PD_S,
2040                                to_hr_pd(ibqp->pd)->pdn);
2041                 hr_qp->access_flags = attr->qp_access_flags;
2042                 roce_set_field(context->qpc_bytes_8,
2043                                QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M,
2044                                QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S,
2045                                to_hr_cq(ibqp->send_cq)->cqn);
2046                 roce_set_field(context->qpc_bytes_8,
2047                                QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M,
2048                                QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S,
2049                                to_hr_cq(ibqp->recv_cq)->cqn);
2050
2051                 if (ibqp->srq)
2052                         roce_set_field(context->qpc_bytes_12,
2053                                        QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M,
2054                                        QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S,
2055                                        to_hr_srq(ibqp->srq)->srqn);
2056
2057                 roce_set_field(context->qpc_bytes_12,
2058                                QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
2059                                QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,
2060                                attr->pkey_index);
2061                 hr_qp->pkey_index = attr->pkey_index;
2062                 roce_set_field(context->qpc_bytes_16,
2063                                QP_CONTEXT_QPC_BYTES_16_QP_NUM_M,
2064                                QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp->qpn);
2065
2066         } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
2067                 roce_set_field(context->qpc_bytes_4,
2068                                QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_M,
2069                                QP_CONTEXT_QPC_BYTES_4_TRANSPORT_SERVICE_TYPE_S,
2070                                to_hr_qp_type(hr_qp->ibqp.qp_type));
2071                 roce_set_bit(context->qpc_bytes_4,
2072                              QP_CONTEXT_QPC_BYTE_4_ENABLE_FPMR_S, 0);
2073                 if (attr_mask & IB_QP_ACCESS_FLAGS) {
2074                         roce_set_bit(context->qpc_bytes_4,
2075                                      QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
2076                                      !!(attr->qp_access_flags &
2077                                      IB_ACCESS_REMOTE_READ));
2078                         roce_set_bit(context->qpc_bytes_4,
2079                                      QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
2080                                      !!(attr->qp_access_flags &
2081                                      IB_ACCESS_REMOTE_WRITE));
2082                 } else {
2083                         roce_set_bit(context->qpc_bytes_4,
2084                                      QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S,
2085                                      !!(hr_qp->access_flags &
2086                                      IB_ACCESS_REMOTE_READ));
2087                         roce_set_bit(context->qpc_bytes_4,
2088                                      QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S,
2089                                      !!(hr_qp->access_flags &
2090                                      IB_ACCESS_REMOTE_WRITE));
2091                 }
2092
2093                 roce_set_bit(context->qpc_bytes_4,
2094                              QP_CONTEXT_QPC_BYTE_4_RDMAR_USE_S, 1);
2095                 roce_set_field(context->qpc_bytes_4,
2096                                QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_M,
2097                                QP_CONTEXT_QPC_BYTES_4_SQ_WQE_SHIFT_S,
2098                                ilog2((unsigned int)hr_qp->sq.wqe_cnt));
2099                 roce_set_field(context->qpc_bytes_4,
2100                                QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_M,
2101                                QP_CONTEXT_QPC_BYTES_4_RQ_WQE_SHIFT_S,
2102                                ilog2((unsigned int)hr_qp->rq.wqe_cnt));
2103                 roce_set_field(context->qpc_bytes_4,
2104                                QP_CONTEXT_QPC_BYTES_4_PD_M,
2105                                QP_CONTEXT_QPC_BYTES_4_PD_S,
2106                                to_hr_pd(ibqp->pd)->pdn);
2107
2108                 roce_set_field(context->qpc_bytes_8,
2109                                QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_M,
2110                                QP_CONTEXT_QPC_BYTES_8_TX_COMPLETION_S,
2111                                to_hr_cq(ibqp->send_cq)->cqn);
2112                 roce_set_field(context->qpc_bytes_8,
2113                                QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_M,
2114                                QP_CONTEXT_QPC_BYTES_8_RX_COMPLETION_S,
2115                                to_hr_cq(ibqp->recv_cq)->cqn);
2116
2117                 if (ibqp->srq)
2118                         roce_set_field(context->qpc_bytes_12,
2119                                        QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_M,
2120                                        QP_CONTEXT_QPC_BYTES_12_SRQ_NUMBER_S,
2121                                        to_hr_srq(ibqp->srq)->srqn);
2122                 if (attr_mask & IB_QP_PKEY_INDEX)
2123                         roce_set_field(context->qpc_bytes_12,
2124                                        QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
2125                                        QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,
2126                                        attr->pkey_index);
2127                 else
2128                         roce_set_field(context->qpc_bytes_12,
2129                                        QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
2130                                        QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S,
2131                                        hr_qp->pkey_index);
2132
2133                 roce_set_field(context->qpc_bytes_16,
2134                                QP_CONTEXT_QPC_BYTES_16_QP_NUM_M,
2135                                QP_CONTEXT_QPC_BYTES_16_QP_NUM_S, hr_qp->qpn);
2136         } else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
2137                 if ((attr_mask & IB_QP_ALT_PATH) ||
2138                     (attr_mask & IB_QP_ACCESS_FLAGS) ||
2139                     (attr_mask & IB_QP_PKEY_INDEX) ||
2140                     (attr_mask & IB_QP_QKEY)) {
2141                         dev_err(dev, "INIT2RTR attr_mask error\n");
2142                         goto out;
2143                 }
2144
2145                 dmac = (u8 *)attr->ah_attr.dmac;
2146
2147                 context->sq_rq_bt_l = (u32)(dma_handle);
2148                 roce_set_field(context->qpc_bytes_24,
2149                                QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_M,
2150                                QP_CONTEXT_QPC_BYTES_24_SQ_RQ_BT_H_S,
2151                                ((u32)(dma_handle >> 32)));
2152                 roce_set_bit(context->qpc_bytes_24,
2153                              QP_CONTEXT_QPC_BYTE_24_REMOTE_ENABLE_E2E_CREDITS_S,
2154                              1);
2155                 roce_set_field(context->qpc_bytes_24,
2156                                QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M,
2157                                QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S,
2158                                attr->min_rnr_timer);
2159                 context->irrl_ba_l = (u32)(dma_handle_2);
2160                 roce_set_field(context->qpc_bytes_32,
2161                                QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M,
2162                                QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_S,
2163                                ((u32)(dma_handle_2 >> 32)) &
2164                                 QP_CONTEXT_QPC_BYTES_32_IRRL_BA_H_M);
2165                 roce_set_field(context->qpc_bytes_32,
2166                                QP_CONTEXT_QPC_BYTES_32_MIG_STATE_M,
2167                                QP_CONTEXT_QPC_BYTES_32_MIG_STATE_S, 0);
2168                 roce_set_bit(context->qpc_bytes_32,
2169                              QP_CONTEXT_QPC_BYTE_32_LOCAL_ENABLE_E2E_CREDITS_S,
2170                              1);
2171                 roce_set_bit(context->qpc_bytes_32,
2172                              QP_CONTEXT_QPC_BYTE_32_SIGNALING_TYPE_S,
2173                              hr_qp->sq_signal_bits);
2174
2175                 for (port = 0; port < hr_dev->caps.num_ports; port++) {
2176                         smac = (u8 *)hr_dev->dev_addr[port];
2177                         dev_dbg(dev, "smac: %2x: %2x: %2x: %2x: %2x: %2x\n",
2178                                 smac[0], smac[1], smac[2], smac[3], smac[4],
2179                                 smac[5]);
2180                         if ((dmac[0] == smac[0]) && (dmac[1] == smac[1]) &&
2181                             (dmac[2] == smac[2]) && (dmac[3] == smac[3]) &&
2182                             (dmac[4] == smac[4]) && (dmac[5] == smac[5])) {
2183                                 roce_set_bit(context->qpc_bytes_32,
2184                                     QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S,
2185                                     1);
2186                                 break;
2187                         }
2188                 }
2189
2190                 if (hr_dev->loop_idc == 0x1)
2191                         roce_set_bit(context->qpc_bytes_32,
2192                                 QP_CONTEXT_QPC_BYTE_32_LOOPBACK_INDICATOR_S, 1);
2193
2194                 roce_set_bit(context->qpc_bytes_32,
2195                              QP_CONTEXT_QPC_BYTE_32_GLOBAL_HEADER_S,
2196                              attr->ah_attr.ah_flags);
2197                 roce_set_field(context->qpc_bytes_32,
2198                                QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M,
2199                                QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S,
2200                                ilog2((unsigned int)attr->max_dest_rd_atomic));
2201
2202                 roce_set_field(context->qpc_bytes_36,
2203                                QP_CONTEXT_QPC_BYTES_36_DEST_QP_M,
2204                                QP_CONTEXT_QPC_BYTES_36_DEST_QP_S,
2205                                attr->dest_qp_num);
2206
2207                 /* Configure GID index */
2208                 roce_set_field(context->qpc_bytes_36,
2209                                QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M,
2210                                QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S,
2211                                hns_get_gid_index(hr_dev,
2212                                                  attr->ah_attr.port_num - 1,
2213                                                  attr->ah_attr.grh.sgid_index));
2214
2215                 memcpy(&(context->dmac_l), dmac, 4);
2216
2217                 roce_set_field(context->qpc_bytes_44,
2218                                QP_CONTEXT_QPC_BYTES_44_DMAC_H_M,
2219                                QP_CONTEXT_QPC_BYTES_44_DMAC_H_S,
2220                                *((u16 *)(&dmac[4])));
2221                 roce_set_field(context->qpc_bytes_44,
2222                                QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_M,
2223                                QP_CONTEXT_QPC_BYTES_44_MAXIMUM_STATIC_RATE_S,
2224                                attr->ah_attr.static_rate);
2225                 roce_set_field(context->qpc_bytes_44,
2226                                QP_CONTEXT_QPC_BYTES_44_HOPLMT_M,
2227                                QP_CONTEXT_QPC_BYTES_44_HOPLMT_S,
2228                                attr->ah_attr.grh.hop_limit);
2229
2230                 roce_set_field(context->qpc_bytes_48,
2231                                QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M,
2232                                QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S,
2233                                attr->ah_attr.grh.flow_label);
2234                 roce_set_field(context->qpc_bytes_48,
2235                                QP_CONTEXT_QPC_BYTES_48_TCLASS_M,
2236                                QP_CONTEXT_QPC_BYTES_48_TCLASS_S,
2237                                attr->ah_attr.grh.traffic_class);
2238                 roce_set_field(context->qpc_bytes_48,
2239                                QP_CONTEXT_QPC_BYTES_48_MTU_M,
2240                                QP_CONTEXT_QPC_BYTES_48_MTU_S, attr->path_mtu);
2241
2242                 memcpy(context->dgid, attr->ah_attr.grh.dgid.raw,
2243                        sizeof(attr->ah_attr.grh.dgid.raw));
2244
2245                 dev_dbg(dev, "dmac:%x :%lx\n", context->dmac_l,
2246                         roce_get_field(context->qpc_bytes_44,
2247                                        QP_CONTEXT_QPC_BYTES_44_DMAC_H_M,
2248                                        QP_CONTEXT_QPC_BYTES_44_DMAC_H_S));
2249
2250                 roce_set_field(context->qpc_bytes_68,
2251                                QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_M,
2252                                QP_CONTEXT_QPC_BYTES_68_RQ_HEAD_S,
2253                                hr_qp->rq.head);
2254                 roce_set_field(context->qpc_bytes_68,
2255                                QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_M,
2256                                QP_CONTEXT_QPC_BYTES_68_RQ_CUR_INDEX_S, 0);
2257
2258                 rq_pa_start = (u32)hr_qp->rq.offset / PAGE_SIZE;
2259                 context->cur_rq_wqe_ba_l = (u32)(mtts[rq_pa_start]);
2260
2261                 roce_set_field(context->qpc_bytes_76,
2262                         QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_M,
2263                         QP_CONTEXT_QPC_BYTES_76_CUR_RQ_WQE_BA_H_S,
2264                         mtts[rq_pa_start] >> 32);
2265                 roce_set_field(context->qpc_bytes_76,
2266                                QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_M,
2267                                QP_CONTEXT_QPC_BYTES_76_RX_REQ_MSN_S, 0);
2268
2269                 context->rx_rnr_time = 0;
2270
2271                 roce_set_field(context->qpc_bytes_84,
2272                                QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_M,
2273                                QP_CONTEXT_QPC_BYTES_84_LAST_ACK_PSN_S,
2274                                attr->rq_psn - 1);
2275                 roce_set_field(context->qpc_bytes_84,
2276                                QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_M,
2277                                QP_CONTEXT_QPC_BYTES_84_TRRL_HEAD_S, 0);
2278
2279                 roce_set_field(context->qpc_bytes_88,
2280                                QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M,
2281                                QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S,
2282                                attr->rq_psn);
2283                 roce_set_bit(context->qpc_bytes_88,
2284                              QP_CONTEXT_QPC_BYTES_88_RX_REQ_PSN_ERR_FLAG_S, 0);
2285                 roce_set_bit(context->qpc_bytes_88,
2286                              QP_CONTEXT_QPC_BYTES_88_RX_LAST_OPCODE_FLG_S, 0);
2287                 roce_set_field(context->qpc_bytes_88,
2288                         QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_M,
2289                         QP_CONTEXT_QPC_BYTES_88_RQ_REQ_LAST_OPERATION_TYPE_S,
2290                         0);
2291                 roce_set_field(context->qpc_bytes_88,
2292                                QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_M,
2293                                QP_CONTEXT_QPC_BYTES_88_RQ_REQ_RDMA_WR_FLAG_S,
2294                                0);
2295
2296                 context->dma_length = 0;
2297                 context->r_key = 0;
2298                 context->va_l = 0;
2299                 context->va_h = 0;
2300
2301                 roce_set_field(context->qpc_bytes_108,
2302                                QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_M,
2303                                QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_S, 0);
2304                 roce_set_bit(context->qpc_bytes_108,
2305                              QP_CONTEXT_QPC_BYTES_108_TRRL_SDB_PSN_FLG_S, 0);
2306                 roce_set_bit(context->qpc_bytes_108,
2307                              QP_CONTEXT_QPC_BYTES_108_TRRL_TDB_PSN_FLG_S, 0);
2308
2309                 roce_set_field(context->qpc_bytes_112,
2310                                QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_M,
2311                                QP_CONTEXT_QPC_BYTES_112_TRRL_TDB_PSN_S, 0);
2312                 roce_set_field(context->qpc_bytes_112,
2313                                QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_M,
2314                                QP_CONTEXT_QPC_BYTES_112_TRRL_TAIL_S, 0);
2315
2316                 /* For chip resp ack */
2317                 roce_set_field(context->qpc_bytes_156,
2318                                QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M,
2319                                QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S,
2320                                hr_qp->phy_port);
2321                 roce_set_field(context->qpc_bytes_156,
2322                                QP_CONTEXT_QPC_BYTES_156_SL_M,
2323                                QP_CONTEXT_QPC_BYTES_156_SL_S, attr->ah_attr.sl);
2324                 hr_qp->sl = attr->ah_attr.sl;
2325         } else if (cur_state == IB_QPS_RTR &&
2326                 new_state == IB_QPS_RTS) {
2327                 /* If exist optional param, return error */
2328                 if ((attr_mask & IB_QP_ALT_PATH) ||
2329                     (attr_mask & IB_QP_ACCESS_FLAGS) ||
2330                     (attr_mask & IB_QP_QKEY) ||
2331                     (attr_mask & IB_QP_PATH_MIG_STATE) ||
2332                     (attr_mask & IB_QP_CUR_STATE) ||
2333                     (attr_mask & IB_QP_MIN_RNR_TIMER)) {
2334                         dev_err(dev, "RTR2RTS attr_mask error\n");
2335                         goto out;
2336                 }
2337
2338                 context->rx_cur_sq_wqe_ba_l = (u32)(mtts[0]);
2339
2340                 roce_set_field(context->qpc_bytes_120,
2341                                QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_M,
2342                                QP_CONTEXT_QPC_BYTES_120_RX_CUR_SQ_WQE_BA_H_S,
2343                                (mtts[0]) >> 32);
2344
2345                 roce_set_field(context->qpc_bytes_124,
2346                                QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_M,
2347                                QP_CONTEXT_QPC_BYTES_124_RX_ACK_MSN_S, 0);
2348                 roce_set_field(context->qpc_bytes_124,
2349                                QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_M,
2350                                QP_CONTEXT_QPC_BYTES_124_IRRL_MSG_IDX_S, 0);
2351
2352                 roce_set_field(context->qpc_bytes_128,
2353                                QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_M,
2354                                QP_CONTEXT_QPC_BYTES_128_RX_ACK_EPSN_S,
2355                                attr->sq_psn);
2356                 roce_set_bit(context->qpc_bytes_128,
2357                              QP_CONTEXT_QPC_BYTES_128_RX_ACK_PSN_ERR_FLG_S, 0);
2358                 roce_set_field(context->qpc_bytes_128,
2359                              QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_M,
2360                              QP_CONTEXT_QPC_BYTES_128_ACK_LAST_OPERATION_TYPE_S,
2361                              0);
2362                 roce_set_bit(context->qpc_bytes_128,
2363                              QP_CONTEXT_QPC_BYTES_128_IRRL_PSN_VLD_FLG_S, 0);
2364
2365                 roce_set_field(context->qpc_bytes_132,
2366                                QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_M,
2367                                QP_CONTEXT_QPC_BYTES_132_IRRL_PSN_S, 0);
2368                 roce_set_field(context->qpc_bytes_132,
2369                                QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_M,
2370                                QP_CONTEXT_QPC_BYTES_132_IRRL_TAIL_S, 0);
2371
2372                 roce_set_field(context->qpc_bytes_136,
2373                                QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_M,
2374                                QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_PSN_S,
2375                                attr->sq_psn);
2376                 roce_set_field(context->qpc_bytes_136,
2377                                QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_M,
2378                                QP_CONTEXT_QPC_BYTES_136_RETRY_MSG_FPKT_PSN_L_S,
2379                                attr->sq_psn);
2380
2381                 roce_set_field(context->qpc_bytes_140,
2382                                QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_M,
2383                                QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_FPKT_PSN_H_S,
2384                                (attr->sq_psn >> SQ_PSN_SHIFT));
2385                 roce_set_field(context->qpc_bytes_140,
2386                                QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_M,
2387                                QP_CONTEXT_QPC_BYTES_140_RETRY_MSG_MSN_S, 0);
2388                 roce_set_bit(context->qpc_bytes_140,
2389                              QP_CONTEXT_QPC_BYTES_140_RNR_RETRY_FLG_S, 0);
2390
2391                 roce_set_field(context->qpc_bytes_148,
2392                                QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_M,
2393                                QP_CONTEXT_QPC_BYTES_148_CHECK_FLAG_S, 0);
2394                 roce_set_field(context->qpc_bytes_148,
2395                                QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M,
2396                                QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S,
2397                                attr->retry_cnt);
2398                 roce_set_field(context->qpc_bytes_148,
2399                                QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_M,
2400                                QP_CONTEXT_QPC_BYTES_148_RNR_RETRY_COUNT_S,
2401                                attr->rnr_retry);
2402                 roce_set_field(context->qpc_bytes_148,
2403                                QP_CONTEXT_QPC_BYTES_148_LSN_M,
2404                                QP_CONTEXT_QPC_BYTES_148_LSN_S, 0x100);
2405
2406                 context->rnr_retry = 0;
2407
2408                 roce_set_field(context->qpc_bytes_156,
2409                                QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_M,
2410                                QP_CONTEXT_QPC_BYTES_156_RETRY_COUNT_INIT_S,
2411                                attr->retry_cnt);
2412                 if (attr->timeout < 0x12) {
2413                         dev_info(dev, "ack timeout value(0x%x) must bigger than 0x12.\n",
2414                                  attr->timeout);
2415                         roce_set_field(context->qpc_bytes_156,
2416                                        QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M,
2417                                        QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S,
2418                                        0x12);
2419                 } else {
2420                         roce_set_field(context->qpc_bytes_156,
2421                                        QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M,
2422                                        QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S,
2423                                        attr->timeout);
2424                 }
2425                 roce_set_field(context->qpc_bytes_156,
2426                                QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_M,
2427                                QP_CONTEXT_QPC_BYTES_156_RNR_RETRY_COUNT_INIT_S,
2428                                attr->rnr_retry);
2429                 roce_set_field(context->qpc_bytes_156,
2430                                QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M,
2431                                QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S,
2432                                hr_qp->phy_port);
2433                 roce_set_field(context->qpc_bytes_156,
2434                                QP_CONTEXT_QPC_BYTES_156_SL_M,
2435                                QP_CONTEXT_QPC_BYTES_156_SL_S, attr->ah_attr.sl);
2436                 hr_qp->sl = attr->ah_attr.sl;
2437                 roce_set_field(context->qpc_bytes_156,
2438                                QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M,
2439                                QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S,
2440                                ilog2((unsigned int)attr->max_rd_atomic));
2441                 roce_set_field(context->qpc_bytes_156,
2442                                QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_M,
2443                                QP_CONTEXT_QPC_BYTES_156_ACK_REQ_IND_S, 0);
2444                 context->pkt_use_len = 0;
2445
2446                 roce_set_field(context->qpc_bytes_164,
2447                                QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M,
2448                                QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S, attr->sq_psn);
2449                 roce_set_field(context->qpc_bytes_164,
2450                                QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_M,
2451                                QP_CONTEXT_QPC_BYTES_164_IRRL_HEAD_S, 0);
2452
2453                 roce_set_field(context->qpc_bytes_168,
2454                                QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_M,
2455                                QP_CONTEXT_QPC_BYTES_168_RETRY_SQ_PSN_S,
2456                                attr->sq_psn);
2457                 roce_set_field(context->qpc_bytes_168,
2458                                QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_M,
2459                                QP_CONTEXT_QPC_BYTES_168_SGE_USE_FLA_S, 0);
2460                 roce_set_field(context->qpc_bytes_168,
2461                                QP_CONTEXT_QPC_BYTES_168_DB_TYPE_M,
2462                                QP_CONTEXT_QPC_BYTES_168_DB_TYPE_S, 0);
2463                 roce_set_bit(context->qpc_bytes_168,
2464                              QP_CONTEXT_QPC_BYTES_168_MSG_LP_IND_S, 0);
2465                 roce_set_bit(context->qpc_bytes_168,
2466                              QP_CONTEXT_QPC_BYTES_168_CSDB_LP_IND_S, 0);
2467                 roce_set_bit(context->qpc_bytes_168,
2468                              QP_CONTEXT_QPC_BYTES_168_QP_ERR_FLG_S, 0);
2469                 context->sge_use_len = 0;
2470
2471                 roce_set_field(context->qpc_bytes_176,
2472                                QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_M,
2473                                QP_CONTEXT_QPC_BYTES_176_DB_CUR_INDEX_S, 0);
2474                 roce_set_field(context->qpc_bytes_176,
2475                                QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_M,
2476                                QP_CONTEXT_QPC_BYTES_176_RETRY_DB_CUR_INDEX_S,
2477                                0);
2478                 roce_set_field(context->qpc_bytes_180,
2479                                QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_M,
2480                                QP_CONTEXT_QPC_BYTES_180_SQ_CUR_INDEX_S, 0);
2481                 roce_set_field(context->qpc_bytes_180,
2482                                QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_M,
2483                                QP_CONTEXT_QPC_BYTES_180_SQ_HEAD_S, 0);
2484
2485                 context->tx_cur_sq_wqe_ba_l = (u32)(mtts[0]);
2486
2487                 roce_set_field(context->qpc_bytes_188,
2488                                QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_M,
2489                                QP_CONTEXT_QPC_BYTES_188_TX_CUR_SQ_WQE_BA_H_S,
2490                                (mtts[0]) >> 32);
2491                 roce_set_bit(context->qpc_bytes_188,
2492                              QP_CONTEXT_QPC_BYTES_188_PKT_RETRY_FLG_S, 0);
2493                 roce_set_field(context->qpc_bytes_188,
2494                                QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_M,
2495                                QP_CONTEXT_QPC_BYTES_188_TX_RETRY_CUR_INDEX_S,
2496                                0);
2497         } else if (!((cur_state == IB_QPS_INIT && new_state == IB_QPS_RESET) ||
2498                    (cur_state == IB_QPS_INIT && new_state == IB_QPS_ERR) ||
2499                    (cur_state == IB_QPS_RTR && new_state == IB_QPS_RESET) ||
2500                    (cur_state == IB_QPS_RTR && new_state == IB_QPS_ERR) ||
2501                    (cur_state == IB_QPS_RTS && new_state == IB_QPS_RESET) ||
2502                    (cur_state == IB_QPS_RTS && new_state == IB_QPS_ERR) ||
2503                    (cur_state == IB_QPS_ERR && new_state == IB_QPS_RESET) ||
2504                    (cur_state == IB_QPS_ERR && new_state == IB_QPS_ERR))) {
2505                 dev_err(dev, "not support this status migration\n");
2506                 goto out;
2507         }
2508
2509         /* Every status migrate must change state */
2510         roce_set_field(context->qpc_bytes_144,
2511                        QP_CONTEXT_QPC_BYTES_144_QP_STATE_M,
2512                        QP_CONTEXT_QPC_BYTES_144_QP_STATE_S, attr->qp_state);
2513
2514         /* SW pass context to HW */
2515         ret = hns_roce_v1_qp_modify(hr_dev, &hr_qp->mtt,
2516                                     to_hns_roce_state(cur_state),
2517                                     to_hns_roce_state(new_state), context,
2518                                     hr_qp);
2519         if (ret) {
2520                 dev_err(dev, "hns_roce_qp_modify failed\n");
2521                 goto out;
2522         }
2523
2524         /*
2525         * Use rst2init to instead of init2init with drv,
2526         * need to hw to flash RQ HEAD by DB again
2527         */
2528         if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
2529                 /* Memory barrier */
2530                 wmb();
2531
2532                 roce_set_field(doorbell[0], RQ_DOORBELL_U32_4_RQ_HEAD_M,
2533                                RQ_DOORBELL_U32_4_RQ_HEAD_S, hr_qp->rq.head);
2534                 roce_set_field(doorbell[1], RQ_DOORBELL_U32_8_QPN_M,
2535                                RQ_DOORBELL_U32_8_QPN_S, hr_qp->qpn);
2536                 roce_set_field(doorbell[1], RQ_DOORBELL_U32_8_CMD_M,
2537                                RQ_DOORBELL_U32_8_CMD_S, 1);
2538                 roce_set_bit(doorbell[1], RQ_DOORBELL_U32_8_HW_SYNC_S, 1);
2539
2540                 if (ibqp->uobject) {
2541                         hr_qp->rq.db_reg_l = hr_dev->reg_base +
2542                                      ROCEE_DB_OTHERS_L_0_REG +
2543                                      DB_REG_OFFSET * hr_dev->priv_uar.index;
2544                 }
2545
2546                 hns_roce_write64_k(doorbell, hr_qp->rq.db_reg_l);
2547         }
2548
2549         hr_qp->state = new_state;
2550
2551         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
2552                 hr_qp->resp_depth = attr->max_dest_rd_atomic;
2553         if (attr_mask & IB_QP_PORT) {
2554                 hr_qp->port = attr->port_num - 1;
2555                 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
2556         }
2557
2558         if (new_state == IB_QPS_RESET && !ibqp->uobject) {
2559                 hns_roce_v1_cq_clean(to_hr_cq(ibqp->recv_cq), hr_qp->qpn,
2560                                      ibqp->srq ? to_hr_srq(ibqp->srq) : NULL);
2561                 if (ibqp->send_cq != ibqp->recv_cq)
2562                         hns_roce_v1_cq_clean(to_hr_cq(ibqp->send_cq),
2563                                              hr_qp->qpn, NULL);
2564
2565                 hr_qp->rq.head = 0;
2566                 hr_qp->rq.tail = 0;
2567                 hr_qp->sq.head = 0;
2568                 hr_qp->sq.tail = 0;
2569                 hr_qp->sq_next_wqe = 0;
2570         }
2571 out:
2572         kfree(context);
2573         return ret;
2574 }
2575
2576 int hns_roce_v1_modify_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr,
2577                           int attr_mask, enum ib_qp_state cur_state,
2578                           enum ib_qp_state new_state)
2579 {
2580
2581         if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI)
2582                 return hns_roce_v1_m_sqp(ibqp, attr, attr_mask, cur_state,
2583                                          new_state);
2584         else
2585                 return hns_roce_v1_m_qp(ibqp, attr, attr_mask, cur_state,
2586                                         new_state);
2587 }
2588
2589 static enum ib_qp_state to_ib_qp_state(enum hns_roce_qp_state state)
2590 {
2591         switch (state) {
2592         case HNS_ROCE_QP_STATE_RST:
2593                 return IB_QPS_RESET;
2594         case HNS_ROCE_QP_STATE_INIT:
2595                 return IB_QPS_INIT;
2596         case HNS_ROCE_QP_STATE_RTR:
2597                 return IB_QPS_RTR;
2598         case HNS_ROCE_QP_STATE_RTS:
2599                 return IB_QPS_RTS;
2600         case HNS_ROCE_QP_STATE_SQD:
2601                 return IB_QPS_SQD;
2602         case HNS_ROCE_QP_STATE_ERR:
2603                 return IB_QPS_ERR;
2604         default:
2605                 return IB_QPS_ERR;
2606         }
2607 }
2608
2609 static int hns_roce_v1_query_qpc(struct hns_roce_dev *hr_dev,
2610                                  struct hns_roce_qp *hr_qp,
2611                                  struct hns_roce_qp_context *hr_context)
2612 {
2613         struct hns_roce_cmd_mailbox *mailbox;
2614         int ret;
2615
2616         mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
2617         if (IS_ERR(mailbox))
2618                 return PTR_ERR(mailbox);
2619
2620         ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, hr_qp->qpn, 0,
2621                                 HNS_ROCE_CMD_QUERY_QP,
2622                                 HNS_ROCE_CMD_TIME_CLASS_A);
2623         if (!ret)
2624                 memcpy(hr_context, mailbox->buf, sizeof(*hr_context));
2625         else
2626                 dev_err(&hr_dev->pdev->dev, "QUERY QP cmd process error\n");
2627
2628         hns_roce_free_cmd_mailbox(hr_dev, mailbox);
2629
2630         return ret;
2631 }
2632
2633 int hns_roce_v1_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
2634                          int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
2635 {
2636         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
2637         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2638         struct device *dev = &hr_dev->pdev->dev;
2639         struct hns_roce_qp_context *context;
2640         int tmp_qp_state = 0;
2641         int ret = 0;
2642         int state;
2643
2644         context = kzalloc(sizeof(*context), GFP_KERNEL);
2645         if (!context)
2646                 return -ENOMEM;
2647
2648         memset(qp_attr, 0, sizeof(*qp_attr));
2649         memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2650
2651         mutex_lock(&hr_qp->mutex);
2652
2653         if (hr_qp->state == IB_QPS_RESET) {
2654                 qp_attr->qp_state = IB_QPS_RESET;
2655                 goto done;
2656         }
2657
2658         ret = hns_roce_v1_query_qpc(hr_dev, hr_qp, context);
2659         if (ret) {
2660                 dev_err(dev, "query qpc error\n");
2661                 ret = -EINVAL;
2662                 goto out;
2663         }
2664
2665         state = roce_get_field(context->qpc_bytes_144,
2666                                QP_CONTEXT_QPC_BYTES_144_QP_STATE_M,
2667                                QP_CONTEXT_QPC_BYTES_144_QP_STATE_S);
2668         tmp_qp_state = (int)to_ib_qp_state((enum hns_roce_qp_state)state);
2669         if (tmp_qp_state == -1) {
2670                 dev_err(dev, "to_ib_qp_state error\n");
2671                 ret = -EINVAL;
2672                 goto out;
2673         }
2674         hr_qp->state = (u8)tmp_qp_state;
2675         qp_attr->qp_state = (enum ib_qp_state)hr_qp->state;
2676         qp_attr->path_mtu = (enum ib_mtu)roce_get_field(context->qpc_bytes_48,
2677                                                QP_CONTEXT_QPC_BYTES_48_MTU_M,
2678                                                QP_CONTEXT_QPC_BYTES_48_MTU_S);
2679         qp_attr->path_mig_state = IB_MIG_ARMED;
2680         if (hr_qp->ibqp.qp_type == IB_QPT_UD)
2681                 qp_attr->qkey = QKEY_VAL;
2682
2683         qp_attr->rq_psn = roce_get_field(context->qpc_bytes_88,
2684                                          QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_M,
2685                                          QP_CONTEXT_QPC_BYTES_88_RX_REQ_EPSN_S);
2686         qp_attr->sq_psn = (u32)roce_get_field(context->qpc_bytes_164,
2687                                              QP_CONTEXT_QPC_BYTES_164_SQ_PSN_M,
2688                                              QP_CONTEXT_QPC_BYTES_164_SQ_PSN_S);
2689         qp_attr->dest_qp_num = (u8)roce_get_field(context->qpc_bytes_36,
2690                                         QP_CONTEXT_QPC_BYTES_36_DEST_QP_M,
2691                                         QP_CONTEXT_QPC_BYTES_36_DEST_QP_S);
2692         qp_attr->qp_access_flags = ((roce_get_bit(context->qpc_bytes_4,
2693                         QP_CONTEXT_QPC_BYTE_4_RDMA_READ_ENABLE_S)) << 2) |
2694                                    ((roce_get_bit(context->qpc_bytes_4,
2695                         QP_CONTEXT_QPC_BYTE_4_RDMA_WRITE_ENABLE_S)) << 1) |
2696                                    ((roce_get_bit(context->qpc_bytes_4,
2697                         QP_CONTEXT_QPC_BYTE_4_ATOMIC_OPERATION_ENABLE_S)) << 3);
2698
2699         if (hr_qp->ibqp.qp_type == IB_QPT_RC ||
2700             hr_qp->ibqp.qp_type == IB_QPT_UC) {
2701                 qp_attr->ah_attr.sl = roce_get_field(context->qpc_bytes_156,
2702                                                 QP_CONTEXT_QPC_BYTES_156_SL_M,
2703                                                 QP_CONTEXT_QPC_BYTES_156_SL_S);
2704                 qp_attr->ah_attr.grh.flow_label = roce_get_field(
2705                                         context->qpc_bytes_48,
2706                                         QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_M,
2707                                         QP_CONTEXT_QPC_BYTES_48_FLOWLABEL_S);
2708                 qp_attr->ah_attr.grh.sgid_index = roce_get_field(
2709                                         context->qpc_bytes_36,
2710                                         QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_M,
2711                                         QP_CONTEXT_QPC_BYTES_36_SGID_INDEX_S);
2712                 qp_attr->ah_attr.grh.hop_limit = roce_get_field(
2713                                         context->qpc_bytes_44,
2714                                         QP_CONTEXT_QPC_BYTES_44_HOPLMT_M,
2715                                         QP_CONTEXT_QPC_BYTES_44_HOPLMT_S);
2716                 qp_attr->ah_attr.grh.traffic_class = roce_get_field(
2717                                         context->qpc_bytes_48,
2718                                         QP_CONTEXT_QPC_BYTES_48_TCLASS_M,
2719                                         QP_CONTEXT_QPC_BYTES_48_TCLASS_S);
2720
2721                 memcpy(qp_attr->ah_attr.grh.dgid.raw, context->dgid,
2722                        sizeof(qp_attr->ah_attr.grh.dgid.raw));
2723         }
2724
2725         qp_attr->pkey_index = roce_get_field(context->qpc_bytes_12,
2726                               QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_M,
2727                               QP_CONTEXT_QPC_BYTES_12_P_KEY_INDEX_S);
2728         qp_attr->port_num = (u8)roce_get_field(context->qpc_bytes_156,
2729                              QP_CONTEXT_QPC_BYTES_156_PORT_NUM_M,
2730                              QP_CONTEXT_QPC_BYTES_156_PORT_NUM_S) + 1;
2731         qp_attr->sq_draining = 0;
2732         qp_attr->max_rd_atomic = roce_get_field(context->qpc_bytes_156,
2733                                  QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_M,
2734                                  QP_CONTEXT_QPC_BYTES_156_INITIATOR_DEPTH_S);
2735         qp_attr->max_dest_rd_atomic = roce_get_field(context->qpc_bytes_32,
2736                                  QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_M,
2737                                  QP_CONTEXT_QPC_BYTES_32_RESPONDER_RESOURCES_S);
2738         qp_attr->min_rnr_timer = (u8)(roce_get_field(context->qpc_bytes_24,
2739                         QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_M,
2740                         QP_CONTEXT_QPC_BYTES_24_MINIMUM_RNR_NAK_TIMER_S));
2741         qp_attr->timeout = (u8)(roce_get_field(context->qpc_bytes_156,
2742                             QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_M,
2743                             QP_CONTEXT_QPC_BYTES_156_ACK_TIMEOUT_S));
2744         qp_attr->retry_cnt = roce_get_field(context->qpc_bytes_148,
2745                              QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_M,
2746                              QP_CONTEXT_QPC_BYTES_148_RETRY_COUNT_S);
2747         qp_attr->rnr_retry = context->rnr_retry;
2748
2749 done:
2750         qp_attr->cur_qp_state = qp_attr->qp_state;
2751         qp_attr->cap.max_recv_wr = hr_qp->rq.wqe_cnt;
2752         qp_attr->cap.max_recv_sge = hr_qp->rq.max_gs;
2753
2754         if (!ibqp->uobject) {
2755                 qp_attr->cap.max_send_wr = hr_qp->sq.wqe_cnt;
2756                 qp_attr->cap.max_send_sge = hr_qp->sq.max_gs;
2757         } else {
2758                 qp_attr->cap.max_send_wr = 0;
2759                 qp_attr->cap.max_send_sge = 0;
2760         }
2761
2762         qp_init_attr->cap = qp_attr->cap;
2763
2764 out:
2765         mutex_unlock(&hr_qp->mutex);
2766         kfree(context);
2767         return ret;
2768 }
2769
2770 static void hns_roce_v1_destroy_qp_common(struct hns_roce_dev *hr_dev,
2771                                           struct hns_roce_qp *hr_qp,
2772                                           int is_user)
2773 {
2774         u32 sdbinvcnt;
2775         unsigned long end = 0;
2776         u32 sdbinvcnt_val;
2777         u32 sdbsendptr_val;
2778         u32 sdbisusepr_val;
2779         struct hns_roce_cq *send_cq, *recv_cq;
2780         struct device *dev = &hr_dev->pdev->dev;
2781
2782         if (hr_qp->ibqp.qp_type == IB_QPT_RC) {
2783                 if (hr_qp->state != IB_QPS_RESET) {
2784                         /*
2785                         * Set qp to ERR,
2786                         * waiting for hw complete processing all dbs
2787                         */
2788                         if (hns_roce_v1_qp_modify(hr_dev, NULL,
2789                                         to_hns_roce_state(
2790                                                 (enum ib_qp_state)hr_qp->state),
2791                                                 HNS_ROCE_QP_STATE_ERR, NULL,
2792                                                 hr_qp))
2793                                 dev_err(dev, "modify QP %06lx to ERR failed.\n",
2794                                         hr_qp->qpn);
2795
2796                         /* Record issued doorbell */
2797                         sdbisusepr_val = roce_read(hr_dev,
2798                                          ROCEE_SDB_ISSUE_PTR_REG);
2799                         /*
2800                         * Query db process status,
2801                         * until hw process completely
2802                         */
2803                         end = msecs_to_jiffies(
2804                               HNS_ROCE_QP_DESTROY_TIMEOUT_MSECS) + jiffies;
2805                         do {
2806                                 sdbsendptr_val = roce_read(hr_dev,
2807                                                  ROCEE_SDB_SEND_PTR_REG);
2808                                 if (!time_before(jiffies, end)) {
2809                                         dev_err(dev, "destroy qp(0x%lx) timeout!!!",
2810                                                 hr_qp->qpn);
2811                                         break;
2812                                 }
2813                         } while ((short)(roce_get_field(sdbsendptr_val,
2814                                         ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_M,
2815                                         ROCEE_SDB_SEND_PTR_SDB_SEND_PTR_S) -
2816                                 roce_get_field(sdbisusepr_val,
2817                                         ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_M,
2818                                         ROCEE_SDB_ISSUE_PTR_SDB_ISSUE_PTR_S)
2819                                 ) < 0);
2820
2821                         /* Get list pointer */
2822                         sdbinvcnt = roce_read(hr_dev, ROCEE_SDB_INV_CNT_REG);
2823
2824                         /* Query db's list status, until hw reversal */
2825                         do {
2826                                 sdbinvcnt_val = roce_read(hr_dev,
2827                                                 ROCEE_SDB_INV_CNT_REG);
2828                                 if (!time_before(jiffies, end)) {
2829                                         dev_err(dev, "destroy qp(0x%lx) timeout!!!",
2830                                                 hr_qp->qpn);
2831                                         dev_err(dev, "SdbInvCnt = 0x%x\n",
2832                                                 sdbinvcnt_val);
2833                                         break;
2834                                 }
2835                         } while ((short)(roce_get_field(sdbinvcnt_val,
2836                                   ROCEE_SDB_INV_CNT_SDB_INV_CNT_M,
2837                                   ROCEE_SDB_INV_CNT_SDB_INV_CNT_S) -
2838                                   (sdbinvcnt + SDB_INV_CNT_OFFSET)) < 0);
2839
2840                         /* Modify qp to reset before destroying qp */
2841                         if (hns_roce_v1_qp_modify(hr_dev, NULL,
2842                                         to_hns_roce_state(
2843                                         (enum ib_qp_state)hr_qp->state),
2844                                         HNS_ROCE_QP_STATE_RST, NULL, hr_qp))
2845                                 dev_err(dev, "modify QP %06lx to RESET failed.\n",
2846                                         hr_qp->qpn);
2847                 }
2848         }
2849
2850         send_cq = to_hr_cq(hr_qp->ibqp.send_cq);
2851         recv_cq = to_hr_cq(hr_qp->ibqp.recv_cq);
2852
2853         hns_roce_lock_cqs(send_cq, recv_cq);
2854
2855         if (!is_user) {
2856                 __hns_roce_v1_cq_clean(recv_cq, hr_qp->qpn, hr_qp->ibqp.srq ?
2857                                        to_hr_srq(hr_qp->ibqp.srq) : NULL);
2858                 if (send_cq != recv_cq)
2859                         __hns_roce_v1_cq_clean(send_cq, hr_qp->qpn, NULL);
2860         }
2861
2862         hns_roce_qp_remove(hr_dev, hr_qp);
2863
2864         hns_roce_unlock_cqs(send_cq, recv_cq);
2865
2866         hns_roce_qp_free(hr_dev, hr_qp);
2867
2868         /* Not special_QP, free their QPN */
2869         if ((hr_qp->ibqp.qp_type == IB_QPT_RC) ||
2870             (hr_qp->ibqp.qp_type == IB_QPT_UC) ||
2871             (hr_qp->ibqp.qp_type == IB_QPT_UD))
2872                 hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1);
2873
2874         hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt);
2875
2876         if (is_user) {
2877                 ib_umem_release(hr_qp->umem);
2878         } else {
2879                 kfree(hr_qp->sq.wrid);
2880                 kfree(hr_qp->rq.wrid);
2881                 hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf);
2882         }
2883 }
2884
2885 int hns_roce_v1_destroy_qp(struct ib_qp *ibqp)
2886 {
2887         struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
2888         struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
2889
2890         hns_roce_v1_destroy_qp_common(hr_dev, hr_qp, !!ibqp->pd->uobject);
2891
2892         if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
2893                 kfree(hr_to_hr_sqp(hr_qp));
2894         else
2895                 kfree(hr_qp);
2896
2897         return 0;
2898 }
2899
2900 struct hns_roce_v1_priv hr_v1_priv;
2901
2902 struct hns_roce_hw hns_roce_hw_v1 = {
2903         .reset = hns_roce_v1_reset,
2904         .hw_profile = hns_roce_v1_profile,
2905         .hw_init = hns_roce_v1_init,
2906         .hw_exit = hns_roce_v1_exit,
2907         .set_gid = hns_roce_v1_set_gid,
2908         .set_mac = hns_roce_v1_set_mac,
2909         .set_mtu = hns_roce_v1_set_mtu,
2910         .write_mtpt = hns_roce_v1_write_mtpt,
2911         .write_cqc = hns_roce_v1_write_cqc,
2912         .clear_hem = hns_roce_v1_clear_hem,
2913         .modify_qp = hns_roce_v1_modify_qp,
2914         .query_qp = hns_roce_v1_query_qp,
2915         .destroy_qp = hns_roce_v1_destroy_qp,
2916         .post_send = hns_roce_v1_post_send,
2917         .post_recv = hns_roce_v1_post_recv,
2918         .req_notify_cq = hns_roce_v1_req_notify_cq,
2919         .poll_cq = hns_roce_v1_poll_cq,
2920         .priv = &hr_v1_priv,
2921 };