drivers/infiniband/hw/i40iw/i40iw_ctrl.c

   1 /*******************************************************************************
   2 *
   3 * Copyright (c) 2015-2016 Intel Corporation.  All rights reserved.
   4 *
   5 * This software is available to you under a choice of one of two
   6 * licenses.  You may choose to be licensed under the terms of the GNU
   7 * General Public License (GPL) Version 2, available from the file
   8 * COPYING in the main directory of this source tree, or the
   9 * OpenFabrics.org BSD license below:
  10 *
  11 *   Redistribution and use in source and binary forms, with or
  12 *   without modification, are permitted provided that the following
  13 *   conditions are met:
  14 *
  15 *    - Redistributions of source code must retain the above
  16 *       copyright notice, this list of conditions and the following
  17 *       disclaimer.
  18 *
  19 *    - Redistributions in binary form must reproduce the above
  20 *       copyright notice, this list of conditions and the following
  21 *       disclaimer in the documentation and/or other materials
  22 *       provided with the distribution.
  23 *
  24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31 * SOFTWARE.
  32 *
  33 *******************************************************************************/
  34
  35 #include "i40iw_osdep.h"
  36 #include "i40iw_register.h"
  37 #include "i40iw_status.h"
  38 #include "i40iw_hmc.h"
  39
  40 #include "i40iw_d.h"
  41 #include "i40iw_type.h"
  42 #include "i40iw_p.h"
  43 #include "i40iw_vf.h"
  44 #include "i40iw_virtchnl.h"
  45
  46 /**
  47  * i40iw_insert_wqe_hdr - write wqe header
  48  * @wqe: cqp wqe for header
  49  * @header: header for the cqp wqe
  50  */
  51 void i40iw_insert_wqe_hdr(u64 *wqe, u64 header)
  52 {
  53         wmb();            /* make sure WQE is populated before polarity is set */
  54         set_64bit_val(wqe, 24, header);
  55 }
  56
  57 void i40iw_check_cqp_progress(struct i40iw_cqp_timeout *cqp_timeout, struct i40iw_sc_dev *dev)
  58 {
  59         if (cqp_timeout->compl_cqp_cmds != dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]) {
  60                 cqp_timeout->compl_cqp_cmds = dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS];
  61                 cqp_timeout->count = 0;
  62         } else {
  63                 if (dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS] != cqp_timeout->compl_cqp_cmds)
  64                         cqp_timeout->count++;
  65         }
  66 }
  67
  68 /**
  69  * i40iw_get_cqp_reg_info - get head and tail for cqp using registers
  70  * @cqp: struct for cqp hw
  71  * @val: cqp tail register value
  72  * @tail:wqtail register value
  73  * @error: cqp processing err
  74  */
  75 static inline void i40iw_get_cqp_reg_info(struct i40iw_sc_cqp *cqp,
  76                                           u32 *val,
  77                                           u32 *tail,
  78                                           u32 *error)
  79 {
  80         if (cqp->dev->is_pf) {
  81                 *val = i40iw_rd32(cqp->dev->hw, I40E_PFPE_CQPTAIL);
  82                 *tail = RS_32(*val, I40E_PFPE_CQPTAIL_WQTAIL);
  83                 *error = RS_32(*val, I40E_PFPE_CQPTAIL_CQP_OP_ERR);
  84         } else {
  85                 *val = i40iw_rd32(cqp->dev->hw, I40E_VFPE_CQPTAIL1);
  86                 *tail = RS_32(*val, I40E_VFPE_CQPTAIL_WQTAIL);
  87                 *error = RS_32(*val, I40E_VFPE_CQPTAIL_CQP_OP_ERR);
  88         }
  89 }
  90
  91 /**
  92  * i40iw_cqp_poll_registers - poll cqp registers
  93  * @cqp: struct for cqp hw
  94  * @tail:wqtail register value
  95  * @count: how many times to try for completion
  96  */
  97 static enum i40iw_status_code i40iw_cqp_poll_registers(
  98                                                 struct i40iw_sc_cqp *cqp,
  99                                                 u32 tail,
 100                                                 u32 count)
 101 {
 102         u32 i = 0;
 103         u32 newtail, error, val;
 104
 105         while (i < count) {
 106                 i++;
 107                 i40iw_get_cqp_reg_info(cqp, &val, &newtail, &error);
 108                 if (error) {
 109                         error = (cqp->dev->is_pf) ?
 110                                  i40iw_rd32(cqp->dev->hw, I40E_PFPE_CQPERRCODES) :
 111                                  i40iw_rd32(cqp->dev->hw, I40E_VFPE_CQPERRCODES1);
 112                         return I40IW_ERR_CQP_COMPL_ERROR;
 113                 }
 114                 if (newtail != tail) {
 115                         /* SUCCESS */
 116                         I40IW_RING_MOVE_TAIL(cqp->sq_ring);
 117                         cqp->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]++;
 118                         return 0;
 119                 }
 120                 udelay(I40IW_SLEEP_COUNT);
 121         }
 122         return I40IW_ERR_TIMEOUT;
 123 }
 124
 125 /**
 126  * i40iw_sc_parse_fpm_commit_buf - parse fpm commit buffer
 127  * @buf: ptr to fpm commit buffer
 128  * @info: ptr to i40iw_hmc_obj_info struct
 129  * @sd: number of SDs for HMC objects
 130  *
 131  * parses fpm commit info and copy base value
 132  * of hmc objects in hmc_info
 133  */
 134 static enum i40iw_status_code i40iw_sc_parse_fpm_commit_buf(
 135                                 u64 *buf,
 136                                 struct i40iw_hmc_obj_info *info,
 137                                 u32 *sd)
 138 {
 139         u64 temp;
 140         u64 size;
 141         u64 base = 0;
 142         u32 i, j;
 143         u32 k = 0;
 144
 145         /* copy base values in obj_info */
 146         for (i = I40IW_HMC_IW_QP, j = 0; i <= I40IW_HMC_IW_PBLE; i++, j += 8) {
 147                 if ((i == I40IW_HMC_IW_SRQ) ||
 148                         (i == I40IW_HMC_IW_FSIMC) ||
 149                         (i == I40IW_HMC_IW_FSIAV)) {
 150                         info[i].base = 0;
 151                         info[i].cnt = 0;
 152                         continue;
 153                 }
 154                 get_64bit_val(buf, j, &temp);
 155                 info[i].base = RS_64_1(temp, 32) * 512;
 156                 if (info[i].base > base) {
 157                         base = info[i].base;
 158                         k = i;
 159                 }
 160                 if (i == I40IW_HMC_IW_APBVT_ENTRY) {
 161                         info[i].cnt = 1;
 162                         continue;
 163                 }
 164                 if (i == I40IW_HMC_IW_QP)
 165                         info[i].cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_QPS);
 166                 else if (i == I40IW_HMC_IW_CQ)
 167                         info[i].cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_CQS);
 168                 else
 169                         info[i].cnt = (u32)(temp);
 170         }
 171         size = info[k].cnt * info[k].size + info[k].base;
 172         if (size & 0x1FFFFF)
 173                 *sd = (u32)((size >> 21) + 1); /* add 1 for remainder */
 174         else
 175                 *sd = (u32)(size >> 21);
 176
 177         return 0;
 178 }
 179
 180 /**
 181  * i40iw_sc_decode_fpm_query() - Decode a 64 bit value into max count and size
 182  * @buf: ptr to fpm query buffer
 183  * @buf_idx: index into buf
 184  * @info: ptr to i40iw_hmc_obj_info struct
 185  * @rsrc_idx: resource index into info
 186  *
 187  * Decode a 64 bit value from fpm query buffer into max count and size
 188  */
 189 static u64 i40iw_sc_decode_fpm_query(u64 *buf,
 190                                             u32 buf_idx,
 191                                             struct i40iw_hmc_obj_info *obj_info,
 192                                             u32 rsrc_idx)
 193 {
 194         u64 temp;
 195         u32 size;
 196
 197         get_64bit_val(buf, buf_idx, &temp);
 198         obj_info[rsrc_idx].max_cnt = (u32)temp;
 199         size = (u32)RS_64_1(temp, 32);
 200         obj_info[rsrc_idx].size = LS_64_1(1, size);
 201
 202         return temp;
 203 }
 204
 205 /**
 206  * i40iw_sc_parse_fpm_query_buf() - parses fpm query buffer
 207  * @buf: ptr to fpm query buffer
 208  * @info: ptr to i40iw_hmc_obj_info struct
 209  * @hmc_fpm_misc: ptr to fpm data
 210  *
 211  * parses fpm query buffer and copy max_cnt and
 212  * size value of hmc objects in hmc_info
 213  */
 214 static enum i40iw_status_code i40iw_sc_parse_fpm_query_buf(
 215                                 u64 *buf,
 216                                 struct i40iw_hmc_info *hmc_info,
 217                                 struct i40iw_hmc_fpm_misc *hmc_fpm_misc)
 218 {
 219         struct i40iw_hmc_obj_info *obj_info;
 220         u64 temp;
 221         u32 size;
 222         u16 max_pe_sds;
 223
 224         obj_info = hmc_info->hmc_obj;
 225
 226         get_64bit_val(buf, 0, &temp);
 227         hmc_info->first_sd_index = (u16)RS_64(temp, I40IW_QUERY_FPM_FIRST_PE_SD_INDEX);
 228         max_pe_sds = (u16)RS_64(temp, I40IW_QUERY_FPM_MAX_PE_SDS);
 229
 230         /* Reduce SD count for VFs by 1 to account for PBLE backing page rounding */
 231         if (hmc_info->hmc_fn_id >= I40IW_FIRST_VF_FPM_ID)
 232                 max_pe_sds--;
 233         hmc_fpm_misc->max_sds = max_pe_sds;
 234         hmc_info->sd_table.sd_cnt = max_pe_sds + hmc_info->first_sd_index;
 235
 236         get_64bit_val(buf, 8, &temp);
 237         obj_info[I40IW_HMC_IW_QP].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_QPS);
 238         size = (u32)RS_64_1(temp, 32);
 239         obj_info[I40IW_HMC_IW_QP].size = LS_64_1(1, size);
 240
 241         get_64bit_val(buf, 16, &temp);
 242         obj_info[I40IW_HMC_IW_CQ].max_cnt = (u32)RS_64(temp, I40IW_QUERY_FPM_MAX_CQS);
 243         size = (u32)RS_64_1(temp, 32);
 244         obj_info[I40IW_HMC_IW_CQ].size = LS_64_1(1, size);
 245
 246         i40iw_sc_decode_fpm_query(buf, 32, obj_info, I40IW_HMC_IW_HTE);
 247         i40iw_sc_decode_fpm_query(buf, 40, obj_info, I40IW_HMC_IW_ARP);
 248
 249         obj_info[I40IW_HMC_IW_APBVT_ENTRY].size = 8192;
 250         obj_info[I40IW_HMC_IW_APBVT_ENTRY].max_cnt = 1;
 251
 252         i40iw_sc_decode_fpm_query(buf, 48, obj_info, I40IW_HMC_IW_MR);
 253         i40iw_sc_decode_fpm_query(buf, 56, obj_info, I40IW_HMC_IW_XF);
 254
 255         get_64bit_val(buf, 64, &temp);
 256         obj_info[I40IW_HMC_IW_XFFL].max_cnt = (u32)temp;
 257         obj_info[I40IW_HMC_IW_XFFL].size = 4;
 258         hmc_fpm_misc->xf_block_size = RS_64(temp, I40IW_QUERY_FPM_XFBLOCKSIZE);
 259         if (!hmc_fpm_misc->xf_block_size)
 260                 return I40IW_ERR_INVALID_SIZE;
 261
 262         i40iw_sc_decode_fpm_query(buf, 72, obj_info, I40IW_HMC_IW_Q1);
 263
 264         get_64bit_val(buf, 80, &temp);
 265         obj_info[I40IW_HMC_IW_Q1FL].max_cnt = (u32)temp;
 266         obj_info[I40IW_HMC_IW_Q1FL].size = 4;
 267         hmc_fpm_misc->q1_block_size = RS_64(temp, I40IW_QUERY_FPM_Q1BLOCKSIZE);
 268         if (!hmc_fpm_misc->q1_block_size)
 269                 return I40IW_ERR_INVALID_SIZE;
 270
 271         i40iw_sc_decode_fpm_query(buf, 88, obj_info, I40IW_HMC_IW_TIMER);
 272
 273         get_64bit_val(buf, 112, &temp);
 274         obj_info[I40IW_HMC_IW_PBLE].max_cnt = (u32)temp;
 275         obj_info[I40IW_HMC_IW_PBLE].size = 8;
 276
 277         get_64bit_val(buf, 120, &temp);
 278         hmc_fpm_misc->max_ceqs = (u8)RS_64(temp, I40IW_QUERY_FPM_MAX_CEQS);
 279         hmc_fpm_misc->ht_multiplier = RS_64(temp, I40IW_QUERY_FPM_HTMULTIPLIER);
 280         hmc_fpm_misc->timer_bucket = RS_64(temp, I40IW_QUERY_FPM_TIMERBUCKET);
 281
 282         return 0;
 283 }
 284
 285 /**
 286  * i40iw_fill_qos_list - Change all unknown qs handles to available ones
 287  * @qs_list: list of qs_handles to be fixed with valid qs_handles
 288  */
 289 static void i40iw_fill_qos_list(u16 *qs_list)
 290 {
 291         u16 qshandle = qs_list[0];
 292         int i;
 293
 294         for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) {
 295                 if (qs_list[i] == QS_HANDLE_UNKNOWN)
 296                         qs_list[i] = qshandle;
 297                 else
 298                         qshandle = qs_list[i];
 299         }
 300 }
 301
 302 /**
 303  * i40iw_qp_from_entry - Given entry, get to the qp structure
 304  * @entry: Points to list of qp structure
 305  */
 306 static struct i40iw_sc_qp *i40iw_qp_from_entry(struct list_head *entry)
 307 {
 308         if (!entry)
 309                 return NULL;
 310
 311         return (struct i40iw_sc_qp *)((char *)entry - offsetof(struct i40iw_sc_qp, list));
 312 }
 313
 314 /**
 315  * i40iw_get_qp - get the next qp from the list given current qp
 316  * @head: Listhead of qp's
 317  * @qp: current qp
 318  */
 319 static struct i40iw_sc_qp *i40iw_get_qp(struct list_head *head, struct i40iw_sc_qp *qp)
 320 {
 321         struct list_head *entry = NULL;
 322         struct list_head *lastentry;
 323
 324         if (list_empty(head))
 325                 return NULL;
 326
 327         if (!qp) {
 328                 entry = head->next;
 329         } else {
 330                 lastentry = &qp->list;
 331                 entry = (lastentry != head) ? lastentry->next : NULL;
 332         }
 333
 334         return i40iw_qp_from_entry(entry);
 335 }
 336
 337 /**
 338  * i40iw_change_l2params - given the new l2 parameters, change all qp
 339  * @vsi: pointer to the vsi structure
 340  * @l2params: New paramaters from l2
 341  */
 342 void i40iw_change_l2params(struct i40iw_sc_vsi *vsi, struct i40iw_l2params *l2params)
 343 {
 344         struct i40iw_sc_dev *dev = vsi->dev;
 345         struct i40iw_sc_qp *qp = NULL;
 346         bool qs_handle_change = false;
 347         unsigned long flags;
 348         u16 qs_handle;
 349         int i;
 350
 351         if (vsi->mtu != l2params->mtu) {
 352                 vsi->mtu = l2params->mtu;
 353                 i40iw_reinitialize_ieq(dev);
 354         }
 355
 356         i40iw_fill_qos_list(l2params->qs_handle_list);
 357         for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) {
 358                 qs_handle = l2params->qs_handle_list[i];
 359                 if (vsi->qos[i].qs_handle != qs_handle)
 360                         qs_handle_change = true;
 361                 spin_lock_irqsave(&vsi->qos[i].lock, flags);
 362                 qp = i40iw_get_qp(&vsi->qos[i].qplist, qp);
 363                 while (qp) {
 364                         if (qs_handle_change) {
 365                                 qp->qs_handle = qs_handle;
 366                                 /* issue cqp suspend command */
 367                                 i40iw_qp_suspend_resume(dev, qp, true);
 368                         }
 369                         qp = i40iw_get_qp(&vsi->qos[i].qplist, qp);
 370                 }
 371                 spin_unlock_irqrestore(&vsi->qos[i].lock, flags);
 372                 vsi->qos[i].qs_handle = qs_handle;
 373         }
 374 }
 375
 376 /**
 377  * i40iw_qp_rem_qos - remove qp from qos lists during destroy qp
 378  * @qp: qp to be removed from qos
 379  */
 380 void i40iw_qp_rem_qos(struct i40iw_sc_qp *qp)
 381 {
 382         struct i40iw_sc_vsi *vsi = qp->vsi;
 383         unsigned long flags;
 384
 385         if (!qp->on_qoslist)
 386                 return;
 387         spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags);
 388         list_del(&qp->list);
 389         spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags);
 390 }
 391
 392 /**
 393  * i40iw_qp_add_qos - called during setctx fot qp to be added to qos
 394  * @qp: qp to be added to qos
 395  */
 396 void i40iw_qp_add_qos(struct i40iw_sc_qp *qp)
 397 {
 398         struct i40iw_sc_vsi *vsi = qp->vsi;
 399         unsigned long flags;
 400
 401         if (qp->on_qoslist)
 402                 return;
 403         spin_lock_irqsave(&vsi->qos[qp->user_pri].lock, flags);
 404         qp->qs_handle = vsi->qos[qp->user_pri].qs_handle;
 405         list_add(&qp->list, &vsi->qos[qp->user_pri].qplist);
 406         qp->on_qoslist = true;
 407         spin_unlock_irqrestore(&vsi->qos[qp->user_pri].lock, flags);
 408 }
 409
 410 /**
 411  * i40iw_sc_pd_init - initialize sc pd struct
 412  * @dev: sc device struct
 413  * @pd: sc pd ptr
 414  * @pd_id: pd_id for allocated pd
 415  * @abi_ver: ABI version from user context, -1 if not valid
 416  */
 417 static void i40iw_sc_pd_init(struct i40iw_sc_dev *dev,
 418                              struct i40iw_sc_pd *pd,
 419                              u16 pd_id,
 420                              int abi_ver)
 421 {
 422         pd->size = sizeof(*pd);
 423         pd->pd_id = pd_id;
 424         pd->abi_ver = abi_ver;
 425         pd->dev = dev;
 426 }
 427
 428 /**
 429  * i40iw_get_encoded_wqe_size - given wq size, returns hardware encoded size
 430  * @wqsize: size of the wq (sq, rq, srq) to encoded_size
 431  * @cqpsq: encoded size for sq for cqp as its encoded size is 1+ other wq's
 432  */
 433 u8 i40iw_get_encoded_wqe_size(u32 wqsize, bool cqpsq)
 434 {
 435         u8 encoded_size = 0;
 436
 437         /* cqp sq's hw coded value starts from 1 for size of 4
 438          * while it starts from 0 for qp' wq's.
 439          */
 440         if (cqpsq)
 441                 encoded_size = 1;
 442         wqsize >>= 2;
 443         while (wqsize >>= 1)
 444                 encoded_size++;
 445         return encoded_size;
 446 }
 447
 448 /**
 449  * i40iw_sc_cqp_init - Initialize buffers for a control Queue Pair
 450  * @cqp: IWARP control queue pair pointer
 451  * @info: IWARP control queue pair init info pointer
 452  *
 453  * Initializes the object and context buffers for a control Queue Pair.
 454  */
 455 static enum i40iw_status_code i40iw_sc_cqp_init(struct i40iw_sc_cqp *cqp,
 456                                                 struct i40iw_cqp_init_info *info)
 457 {
 458         u8 hw_sq_size;
 459
 460         if ((info->sq_size > I40IW_CQP_SW_SQSIZE_2048) ||
 461             (info->sq_size < I40IW_CQP_SW_SQSIZE_4) ||
 462             ((info->sq_size & (info->sq_size - 1))))
 463                 return I40IW_ERR_INVALID_SIZE;
 464
 465         hw_sq_size = i40iw_get_encoded_wqe_size(info->sq_size, true);
 466         cqp->size = sizeof(*cqp);
 467         cqp->sq_size = info->sq_size;
 468         cqp->hw_sq_size = hw_sq_size;
 469         cqp->sq_base = info->sq;
 470         cqp->host_ctx = info->host_ctx;
 471         cqp->sq_pa = info->sq_pa;
 472         cqp->host_ctx_pa = info->host_ctx_pa;
 473         cqp->dev = info->dev;
 474         cqp->struct_ver = info->struct_ver;
 475         cqp->scratch_array = info->scratch_array;
 476         cqp->polarity = 0;
 477         cqp->en_datacenter_tcp = info->en_datacenter_tcp;
 478         cqp->enabled_vf_count = info->enabled_vf_count;
 479         cqp->hmc_profile = info->hmc_profile;
 480         info->dev->cqp = cqp;
 481
 482         I40IW_RING_INIT(cqp->sq_ring, cqp->sq_size);
 483         cqp->dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS] = 0;
 484         cqp->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS] = 0;
 485         INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head);               /* for the cqp commands backlog. */
 486
 487         i40iw_wr32(cqp->dev->hw, I40E_PFPE_CQPTAIL, 0);
 488         i40iw_wr32(cqp->dev->hw, I40E_PFPE_CQPDB, 0);
 489
 490         i40iw_debug(cqp->dev, I40IW_DEBUG_WQE,
 491                     "%s: sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%llxh] cqp[%p] polarity[x%04X]\n",
 492                     __func__, cqp->sq_size, cqp->hw_sq_size,
 493                     cqp->sq_base, cqp->sq_pa, cqp, cqp->polarity);
 494         return 0;
 495 }
 496
 497 /**
 498  * i40iw_sc_cqp_create - create cqp during bringup
 499  * @cqp: struct for cqp hw
 500  * @maj_err: If error, major err number
 501  * @min_err: If error, minor err number
 502  */
 503 static enum i40iw_status_code i40iw_sc_cqp_create(struct i40iw_sc_cqp *cqp,
 504                                                   u16 *maj_err,
 505                                                   u16 *min_err)
 506 {
 507         u64 temp;
 508         u32 cnt = 0, p1, p2, val = 0, err_code;
 509         enum i40iw_status_code ret_code;
 510
 511         *maj_err = 0;
 512         *min_err = 0;
 513
 514         ret_code = i40iw_allocate_dma_mem(cqp->dev->hw,
 515                                           &cqp->sdbuf,
 516                                           I40IW_UPDATE_SD_BUF_SIZE * cqp->sq_size,
 517                                           I40IW_SD_BUF_ALIGNMENT);
 518
 519         if (ret_code)
 520                 goto exit;
 521
 522         temp = LS_64(cqp->hw_sq_size, I40IW_CQPHC_SQSIZE) |
 523                LS_64(cqp->struct_ver, I40IW_CQPHC_SVER);
 524
 525         set_64bit_val(cqp->host_ctx, 0, temp);
 526         set_64bit_val(cqp->host_ctx, 8, cqp->sq_pa);
 527         temp = LS_64(cqp->enabled_vf_count, I40IW_CQPHC_ENABLED_VFS) |
 528                LS_64(cqp->hmc_profile, I40IW_CQPHC_HMC_PROFILE);
 529         set_64bit_val(cqp->host_ctx, 16, temp);
 530         set_64bit_val(cqp->host_ctx, 24, (uintptr_t)cqp);
 531         set_64bit_val(cqp->host_ctx, 32, 0);
 532         set_64bit_val(cqp->host_ctx, 40, 0);
 533         set_64bit_val(cqp->host_ctx, 48, 0);
 534         set_64bit_val(cqp->host_ctx, 56, 0);
 535
 536         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQP_HOST_CTX",
 537                         cqp->host_ctx, I40IW_CQP_CTX_SIZE * 8);
 538
 539         p1 = RS_32_1(cqp->host_ctx_pa, 32);
 540         p2 = (u32)cqp->host_ctx_pa;
 541
 542         if (cqp->dev->is_pf) {
 543                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPHIGH, p1);
 544                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPLOW, p2);
 545         } else {
 546                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPHIGH1, p1);
 547                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPLOW1, p2);
 548         }
 549         do {
 550                 if (cnt++ > I40IW_DONE_COUNT) {
 551                         i40iw_free_dma_mem(cqp->dev->hw, &cqp->sdbuf);
 552                         ret_code = I40IW_ERR_TIMEOUT;
 553                         /*
 554                          * read PFPE_CQPERRORCODES register to get the minor
 555                          * and major error code
 556                          */
 557                         if (cqp->dev->is_pf)
 558                                 err_code = i40iw_rd32(cqp->dev->hw, I40E_PFPE_CQPERRCODES);
 559                         else
 560                                 err_code = i40iw_rd32(cqp->dev->hw, I40E_VFPE_CQPERRCODES1);
 561                         *min_err = RS_32(err_code, I40E_PFPE_CQPERRCODES_CQP_MINOR_CODE);
 562                         *maj_err = RS_32(err_code, I40E_PFPE_CQPERRCODES_CQP_MAJOR_CODE);
 563                         goto exit;
 564                 }
 565                 udelay(I40IW_SLEEP_COUNT);
 566                 if (cqp->dev->is_pf)
 567                         val = i40iw_rd32(cqp->dev->hw, I40E_PFPE_CCQPSTATUS);
 568                 else
 569                         val = i40iw_rd32(cqp->dev->hw, I40E_VFPE_CCQPSTATUS1);
 570         } while (!val);
 571
 572 exit:
 573         if (!ret_code)
 574                 cqp->process_cqp_sds = i40iw_update_sds_noccq;
 575         return ret_code;
 576 }
 577
 578 /**
 579  * i40iw_sc_cqp_post_sq - post of cqp's sq
 580  * @cqp: struct for cqp hw
 581  */
 582 void i40iw_sc_cqp_post_sq(struct i40iw_sc_cqp *cqp)
 583 {
 584         if (cqp->dev->is_pf)
 585                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CQPDB, I40IW_RING_GETCURRENT_HEAD(cqp->sq_ring));
 586         else
 587                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CQPDB1, I40IW_RING_GETCURRENT_HEAD(cqp->sq_ring));
 588
 589         i40iw_debug(cqp->dev,
 590                     I40IW_DEBUG_WQE,
 591                     "%s: HEAD_TAIL[%04d,%04d,%04d]\n",
 592                     __func__,
 593                     cqp->sq_ring.head,
 594                     cqp->sq_ring.tail,
 595                     cqp->sq_ring.size);
 596 }
 597
 598 /**
 599  * i40iw_sc_cqp_get_next_send_wqe_idx - get next WQE on CQP SQ and pass back the index
 600  * @cqp: pointer to CQP structure
 601  * @scratch: private data for CQP WQE
 602  * @wqe_idx: WQE index for next WQE on CQP SQ
 603  */
 604 static u64 *i40iw_sc_cqp_get_next_send_wqe_idx(struct i40iw_sc_cqp *cqp,
 605                                                u64 scratch, u32 *wqe_idx)
 606 {
 607         u64 *wqe = NULL;
 608         enum i40iw_status_code ret_code;
 609
 610         if (I40IW_RING_FULL_ERR(cqp->sq_ring)) {
 611                 i40iw_debug(cqp->dev,
 612                             I40IW_DEBUG_WQE,
 613                             "%s: ring is full head %x tail %x size %x\n",
 614                             __func__,
 615                             cqp->sq_ring.head,
 616                             cqp->sq_ring.tail,
 617                             cqp->sq_ring.size);
 618                 return NULL;
 619         }
 620         I40IW_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, *wqe_idx, ret_code);
 621         cqp->dev->cqp_cmd_stats[OP_REQUESTED_COMMANDS]++;
 622         if (ret_code)
 623                 return NULL;
 624         if (!*wqe_idx)
 625                 cqp->polarity = !cqp->polarity;
 626
 627         wqe = cqp->sq_base[*wqe_idx].elem;
 628         cqp->scratch_array[*wqe_idx] = scratch;
 629         I40IW_CQP_INIT_WQE(wqe);
 630
 631         return wqe;
 632 }
 633
 634 /**
 635  * i40iw_sc_cqp_get_next_send_wqe - get next wqe on cqp sq
 636  * @cqp: struct for cqp hw
 637  * @scratch: private data for CQP WQE
 638  */
 639 u64 *i40iw_sc_cqp_get_next_send_wqe(struct i40iw_sc_cqp *cqp, u64 scratch)
 640 {
 641         u32 wqe_idx;
 642
 643         return i40iw_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
 644 }
 645
 646 /**
 647  * i40iw_sc_cqp_destroy - destroy cqp during close
 648  * @cqp: struct for cqp hw
 649  */
 650 static enum i40iw_status_code i40iw_sc_cqp_destroy(struct i40iw_sc_cqp *cqp)
 651 {
 652         u32 cnt = 0, val = 1;
 653         enum i40iw_status_code ret_code = 0;
 654         u32 cqpstat_addr;
 655
 656         if (cqp->dev->is_pf) {
 657                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPHIGH, 0);
 658                 i40iw_wr32(cqp->dev->hw, I40E_PFPE_CCQPLOW, 0);
 659                 cqpstat_addr = I40E_PFPE_CCQPSTATUS;
 660         } else {
 661                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPHIGH1, 0);
 662                 i40iw_wr32(cqp->dev->hw, I40E_VFPE_CCQPLOW1, 0);
 663                 cqpstat_addr = I40E_VFPE_CCQPSTATUS1;
 664         }
 665         do {
 666                 if (cnt++ > I40IW_DONE_COUNT) {
 667                         ret_code = I40IW_ERR_TIMEOUT;
 668                         break;
 669                 }
 670                 udelay(I40IW_SLEEP_COUNT);
 671                 val = i40iw_rd32(cqp->dev->hw, cqpstat_addr);
 672         } while (val);
 673
 674         i40iw_free_dma_mem(cqp->dev->hw, &cqp->sdbuf);
 675         return ret_code;
 676 }
 677
 678 /**
 679  * i40iw_sc_ccq_arm - enable intr for control cq
 680  * @ccq: ccq sc struct
 681  */
 682 static void i40iw_sc_ccq_arm(struct i40iw_sc_cq *ccq)
 683 {
 684         u64 temp_val;
 685         u16 sw_cq_sel;
 686         u8 arm_next_se;
 687         u8 arm_seq_num;
 688
 689         /* write to cq doorbell shadow area */
 690         /* arm next se should always be zero */
 691         get_64bit_val(ccq->cq_uk.shadow_area, 32, &temp_val);
 692
 693         sw_cq_sel = (u16)RS_64(temp_val, I40IW_CQ_DBSA_SW_CQ_SELECT);
 694         arm_next_se = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_NEXT_SE);
 695
 696         arm_seq_num = (u8)RS_64(temp_val, I40IW_CQ_DBSA_ARM_SEQ_NUM);
 697         arm_seq_num++;
 698
 699         temp_val = LS_64(arm_seq_num, I40IW_CQ_DBSA_ARM_SEQ_NUM) |
 700                    LS_64(sw_cq_sel, I40IW_CQ_DBSA_SW_CQ_SELECT) |
 701                    LS_64(arm_next_se, I40IW_CQ_DBSA_ARM_NEXT_SE) |
 702                    LS_64(1, I40IW_CQ_DBSA_ARM_NEXT);
 703
 704         set_64bit_val(ccq->cq_uk.shadow_area, 32, temp_val);
 705
 706         wmb();       /* make sure shadow area is updated before arming */
 707
 708         if (ccq->dev->is_pf)
 709                 i40iw_wr32(ccq->dev->hw, I40E_PFPE_CQARM, ccq->cq_uk.cq_id);
 710         else
 711                 i40iw_wr32(ccq->dev->hw, I40E_VFPE_CQARM1, ccq->cq_uk.cq_id);
 712 }
 713
 714 /**
 715  * i40iw_sc_ccq_get_cqe_info - get ccq's cq entry
 716  * @ccq: ccq sc struct
 717  * @info: completion q entry to return
 718  */
 719 static enum i40iw_status_code i40iw_sc_ccq_get_cqe_info(
 720                                         struct i40iw_sc_cq *ccq,
 721                                         struct i40iw_ccq_cqe_info *info)
 722 {
 723         u64 qp_ctx, temp, temp1;
 724         u64 *cqe;
 725         struct i40iw_sc_cqp *cqp;
 726         u32 wqe_idx;
 727         u8 polarity;
 728         enum i40iw_status_code ret_code = 0;
 729
 730         if (ccq->cq_uk.avoid_mem_cflct)
 731                 cqe = (u64 *)I40IW_GET_CURRENT_EXTENDED_CQ_ELEMENT(&ccq->cq_uk);
 732         else
 733                 cqe = (u64 *)I40IW_GET_CURRENT_CQ_ELEMENT(&ccq->cq_uk);
 734
 735         get_64bit_val(cqe, 24, &temp);
 736         polarity = (u8)RS_64(temp, I40IW_CQ_VALID);
 737         if (polarity != ccq->cq_uk.polarity)
 738                 return I40IW_ERR_QUEUE_EMPTY;
 739
 740         get_64bit_val(cqe, 8, &qp_ctx);
 741         cqp = (struct i40iw_sc_cqp *)(unsigned long)qp_ctx;
 742         info->error = (bool)RS_64(temp, I40IW_CQ_ERROR);
 743         info->min_err_code = (u16)RS_64(temp, I40IW_CQ_MINERR);
 744         if (info->error) {
 745                 info->maj_err_code = (u16)RS_64(temp, I40IW_CQ_MAJERR);
 746                 info->min_err_code = (u16)RS_64(temp, I40IW_CQ_MINERR);
 747         }
 748         wqe_idx = (u32)RS_64(temp, I40IW_CQ_WQEIDX);
 749         info->scratch = cqp->scratch_array[wqe_idx];
 750
 751         get_64bit_val(cqe, 16, &temp1);
 752         info->op_ret_val = (u32)RS_64(temp1, I40IW_CCQ_OPRETVAL);
 753         get_64bit_val(cqp->sq_base[wqe_idx].elem, 24, &temp1);
 754         info->op_code = (u8)RS_64(temp1, I40IW_CQPSQ_OPCODE);
 755         info->cqp = cqp;
 756
 757         /*  move the head for cq */
 758         I40IW_RING_MOVE_HEAD(ccq->cq_uk.cq_ring, ret_code);
 759         if (I40IW_RING_GETCURRENT_HEAD(ccq->cq_uk.cq_ring) == 0)
 760                 ccq->cq_uk.polarity ^= 1;
 761
 762         /* update cq tail in cq shadow memory also */
 763         I40IW_RING_MOVE_TAIL(ccq->cq_uk.cq_ring);
 764         set_64bit_val(ccq->cq_uk.shadow_area,
 765                       0,
 766                       I40IW_RING_GETCURRENT_HEAD(ccq->cq_uk.cq_ring));
 767         wmb(); /* write shadow area before tail */
 768         I40IW_RING_MOVE_TAIL(cqp->sq_ring);
 769         ccq->dev->cqp_cmd_stats[OP_COMPLETED_COMMANDS]++;
 770
 771         return ret_code;
 772 }
 773
 774 /**
 775  * i40iw_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ
 776  * @cqp: struct for cqp hw
 777  * @op_code: cqp opcode for completion
 778  * @info: completion q entry to return
 779  */
 780 static enum i40iw_status_code i40iw_sc_poll_for_cqp_op_done(
 781                                         struct i40iw_sc_cqp *cqp,
 782                                         u8 op_code,
 783                                         struct i40iw_ccq_cqe_info *compl_info)
 784 {
 785         struct i40iw_ccq_cqe_info info;
 786         struct i40iw_sc_cq *ccq;
 787         enum i40iw_status_code ret_code = 0;
 788         u32 cnt = 0;
 789
 790         memset(&info, 0, sizeof(info));
 791         ccq = cqp->dev->ccq;
 792         while (1) {
 793                 if (cnt++ > I40IW_DONE_COUNT)
 794                         return I40IW_ERR_TIMEOUT;
 795
 796                 if (i40iw_sc_ccq_get_cqe_info(ccq, &info)) {
 797                         udelay(I40IW_SLEEP_COUNT);
 798                         continue;
 799                 }
 800
 801                 if (info.error) {
 802                         ret_code = I40IW_ERR_CQP_COMPL_ERROR;
 803                         break;
 804                 }
 805                 /* check if opcode is cq create */
 806                 if (op_code != info.op_code) {
 807                         i40iw_debug(cqp->dev, I40IW_DEBUG_WQE,
 808                                     "%s: opcode mismatch for my op code 0x%x, returned opcode %x\n",
 809                                     __func__, op_code, info.op_code);
 810                 }
 811                 /* success, exit out of the loop */
 812                 if (op_code == info.op_code)
 813                         break;
 814         }
 815
 816         if (compl_info)
 817                 memcpy(compl_info, &info, sizeof(*compl_info));
 818
 819         return ret_code;
 820 }
 821
 822 /**
 823  * i40iw_sc_manage_push_page - Handle push page
 824  * @cqp: struct for cqp hw
 825  * @info: push page info
 826  * @scratch: u64 saved to be used during cqp completion
 827  * @post_sq: flag for cqp db to ring
 828  */
 829 static enum i40iw_status_code i40iw_sc_manage_push_page(
 830                                 struct i40iw_sc_cqp *cqp,
 831                                 struct i40iw_cqp_manage_push_page_info *info,
 832                                 u64 scratch,
 833                                 bool post_sq)
 834 {
 835         u64 *wqe;
 836         u64 header;
 837
 838         if (info->push_idx >= I40IW_MAX_PUSH_PAGE_COUNT)
 839                 return I40IW_ERR_INVALID_PUSH_PAGE_INDEX;
 840
 841         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
 842         if (!wqe)
 843                 return I40IW_ERR_RING_FULL;
 844
 845         set_64bit_val(wqe, 16, info->qs_handle);
 846
 847         header = LS_64(info->push_idx, I40IW_CQPSQ_MPP_PPIDX) |
 848                  LS_64(I40IW_CQP_OP_MANAGE_PUSH_PAGES, I40IW_CQPSQ_OPCODE) |
 849                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) |
 850                  LS_64(info->free_page, I40IW_CQPSQ_MPP_FREE_PAGE);
 851
 852         i40iw_insert_wqe_hdr(wqe, header);
 853
 854         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_PUSH_PAGES WQE",
 855                         wqe, I40IW_CQP_WQE_SIZE * 8);
 856
 857         if (post_sq)
 858                 i40iw_sc_cqp_post_sq(cqp);
 859         return 0;
 860 }
 861
 862 /**
 863  * i40iw_sc_manage_hmc_pm_func_table - manage of function table
 864  * @cqp: struct for cqp hw
 865  * @scratch: u64 saved to be used during cqp completion
 866  * @vf_index: vf index for cqp
 867  * @free_pm_fcn: function number
 868  * @post_sq: flag for cqp db to ring
 869  */
 870 static enum i40iw_status_code i40iw_sc_manage_hmc_pm_func_table(
 871                                 struct i40iw_sc_cqp *cqp,
 872                                 u64 scratch,
 873                                 u8 vf_index,
 874                                 bool free_pm_fcn,
 875                                 bool post_sq)
 876 {
 877         u64 *wqe;
 878         u64 header;
 879
 880         if (vf_index >= I40IW_MAX_VF_PER_PF)
 881                 return I40IW_ERR_INVALID_VF_ID;
 882         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
 883         if (!wqe)
 884                 return I40IW_ERR_RING_FULL;
 885
 886         header = LS_64(vf_index, I40IW_CQPSQ_MHMC_VFIDX) |
 887                  LS_64(I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE, I40IW_CQPSQ_OPCODE) |
 888                  LS_64(free_pm_fcn, I40IW_CQPSQ_MHMC_FREEPMFN) |
 889                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
 890
 891         i40iw_insert_wqe_hdr(wqe, header);
 892         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_HMC_PM_FUNC_TABLE WQE",
 893                         wqe, I40IW_CQP_WQE_SIZE * 8);
 894         if (post_sq)
 895                 i40iw_sc_cqp_post_sq(cqp);
 896         return 0;
 897 }
 898
 899 /**
 900  * i40iw_sc_set_hmc_resource_profile - cqp wqe for hmc profile
 901  * @cqp: struct for cqp hw
 902  * @scratch: u64 saved to be used during cqp completion
 903  * @hmc_profile_type: type of profile to set
 904  * @vf_num: vf number for profile
 905  * @post_sq: flag for cqp db to ring
 906  * @poll_registers: flag to poll register for cqp completion
 907  */
 908 static enum i40iw_status_code i40iw_sc_set_hmc_resource_profile(
 909                                 struct i40iw_sc_cqp *cqp,
 910                                 u64 scratch,
 911                                 u8 hmc_profile_type,
 912                                 u8 vf_num, bool post_sq,
 913                                 bool poll_registers)
 914 {
 915         u64 *wqe;
 916         u64 header;
 917         u32 val, tail, error;
 918         enum i40iw_status_code ret_code = 0;
 919
 920         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
 921         if (!wqe)
 922                 return I40IW_ERR_RING_FULL;
 923
 924         set_64bit_val(wqe, 16,
 925                       (LS_64(hmc_profile_type, I40IW_CQPSQ_SHMCRP_HMC_PROFILE) |
 926                                 LS_64(vf_num, I40IW_CQPSQ_SHMCRP_VFNUM)));
 927
 928         header = LS_64(I40IW_CQP_OP_SET_HMC_RESOURCE_PROFILE, I40IW_CQPSQ_OPCODE) |
 929                        LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
 930
 931         i40iw_insert_wqe_hdr(wqe, header);
 932
 933         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_HMC_PM_FUNC_TABLE WQE",
 934                         wqe, I40IW_CQP_WQE_SIZE * 8);
 935
 936         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
 937         if (error)
 938                 return I40IW_ERR_CQP_COMPL_ERROR;
 939
 940         if (post_sq) {
 941                 i40iw_sc_cqp_post_sq(cqp);
 942                 if (poll_registers)
 943                         ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000000);
 944                 else
 945                         ret_code = i40iw_sc_poll_for_cqp_op_done(cqp,
 946                                                                  I40IW_CQP_OP_SHMC_PAGES_ALLOCATED,
 947                                                                  NULL);
 948         }
 949
 950         return ret_code;
 951 }
 952
 953 /**
 954  * i40iw_sc_manage_hmc_pm_func_table_done - wait for cqp wqe completion for function table
 955  * @cqp: struct for cqp hw
 956  */
 957 static enum i40iw_status_code i40iw_sc_manage_hmc_pm_func_table_done(struct i40iw_sc_cqp *cqp)
 958 {
 959         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE, NULL);
 960 }
 961
 962 /**
 963  * i40iw_sc_commit_fpm_values_done - wait for cqp eqe completion for fpm commit
 964  * @cqp: struct for cqp hw
 965  */
 966 static enum i40iw_status_code i40iw_sc_commit_fpm_values_done(struct i40iw_sc_cqp *cqp)
 967 {
 968         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_COMMIT_FPM_VALUES, NULL);
 969 }
 970
 971 /**
 972  * i40iw_sc_commit_fpm_values - cqp wqe for commit fpm values
 973  * @cqp: struct for cqp hw
 974  * @scratch: u64 saved to be used during cqp completion
 975  * @hmc_fn_id: hmc function id
 976  * @commit_fpm_mem; Memory for fpm values
 977  * @post_sq: flag for cqp db to ring
 978  * @wait_type: poll ccq or cqp registers for cqp completion
 979  */
 980 static enum i40iw_status_code i40iw_sc_commit_fpm_values(
 981                                         struct i40iw_sc_cqp *cqp,
 982                                         u64 scratch,
 983                                         u8 hmc_fn_id,
 984                                         struct i40iw_dma_mem *commit_fpm_mem,
 985                                         bool post_sq,
 986                                         u8 wait_type)
 987 {
 988         u64 *wqe;
 989         u64 header;
 990         u32 tail, val, error;
 991         enum i40iw_status_code ret_code = 0;
 992
 993         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
 994         if (!wqe)
 995                 return I40IW_ERR_RING_FULL;
 996
 997         set_64bit_val(wqe, 16, hmc_fn_id);
 998         set_64bit_val(wqe, 32, commit_fpm_mem->pa);
 999
1000         header = LS_64(I40IW_CQP_OP_COMMIT_FPM_VALUES, I40IW_CQPSQ_OPCODE) |
1001                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1002
1003         i40iw_insert_wqe_hdr(wqe, header);
1004
1005         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "COMMIT_FPM_VALUES WQE",
1006                         wqe, I40IW_CQP_WQE_SIZE * 8);
1007
1008         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
1009         if (error)
1010                 return I40IW_ERR_CQP_COMPL_ERROR;
1011
1012         if (post_sq) {
1013                 i40iw_sc_cqp_post_sq(cqp);
1014
1015                 if (wait_type == I40IW_CQP_WAIT_POLL_REGS)
1016                         ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);
1017                 else if (wait_type == I40IW_CQP_WAIT_POLL_CQ)
1018                         ret_code = i40iw_sc_commit_fpm_values_done(cqp);
1019         }
1020
1021         return ret_code;
1022 }
1023
1024 /**
1025  * i40iw_sc_query_rdma_features_done - poll cqp for query features done
1026  * @cqp: struct for cqp hw
1027  */
1028 static enum i40iw_status_code
1029 i40iw_sc_query_rdma_features_done(struct i40iw_sc_cqp *cqp)
1030 {
1031         return i40iw_sc_poll_for_cqp_op_done(
1032                 cqp, I40IW_CQP_OP_QUERY_RDMA_FEATURES, NULL);
1033 }
1034
1035 /**
1036  * i40iw_sc_query_rdma_features - query rdma features
1037  * @cqp: struct for cqp hw
1038  * @feat_mem: holds PA for HW to use
1039  * @scratch: u64 saved to be used during cqp completion
1040  */
1041 static enum i40iw_status_code
1042 i40iw_sc_query_rdma_features(struct i40iw_sc_cqp *cqp,
1043                              struct i40iw_dma_mem *feat_mem, u64 scratch)
1044 {
1045         u64 *wqe;
1046         u64 header;
1047
1048         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1049         if (!wqe)
1050                 return I40IW_ERR_RING_FULL;
1051
1052         set_64bit_val(wqe, 32, feat_mem->pa);
1053
1054         header = LS_64(I40IW_CQP_OP_QUERY_RDMA_FEATURES, I40IW_CQPSQ_OPCODE) |
1055                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) | feat_mem->size;
1056
1057         i40iw_insert_wqe_hdr(wqe, header);
1058
1059         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QUERY RDMA FEATURES WQE",
1060                         wqe, I40IW_CQP_WQE_SIZE * 8);
1061
1062         i40iw_sc_cqp_post_sq(cqp);
1063
1064         return 0;
1065 }
1066
1067 /**
1068  * i40iw_get_rdma_features - get RDMA features
1069  * @dev - sc device struct
1070  */
1071 enum i40iw_status_code i40iw_get_rdma_features(struct i40iw_sc_dev *dev)
1072 {
1073         enum i40iw_status_code ret_code;
1074         struct i40iw_dma_mem feat_buf;
1075         u64 temp;
1076         u16 byte_idx, feat_type, feat_cnt;
1077
1078         ret_code = i40iw_allocate_dma_mem(dev->hw, &feat_buf,
1079                                           I40IW_FEATURE_BUF_SIZE,
1080                                           I40IW_FEATURE_BUF_ALIGNMENT);
1081
1082         if (ret_code)
1083                 return I40IW_ERR_NO_MEMORY;
1084
1085         ret_code = i40iw_sc_query_rdma_features(dev->cqp, &feat_buf, 0);
1086         if (!ret_code)
1087                 ret_code = i40iw_sc_query_rdma_features_done(dev->cqp);
1088
1089         if (ret_code)
1090                 goto exit;
1091
1092         get_64bit_val(feat_buf.va, 0, &temp);
1093         feat_cnt = RS_64(temp, I40IW_FEATURE_CNT);
1094         if (feat_cnt < I40IW_MAX_FEATURES) {
1095                 ret_code = I40IW_ERR_INVALID_FEAT_CNT;
1096                 goto exit;
1097         } else if (feat_cnt > I40IW_MAX_FEATURES) {
1098                 i40iw_debug(dev, I40IW_DEBUG_CQP,
1099                             "features buf size insufficient\n");
1100         }
1101
1102         for (byte_idx = 0, feat_type = 0; feat_type < I40IW_MAX_FEATURES;
1103              feat_type++, byte_idx += 8) {
1104                 get_64bit_val((u64 *)feat_buf.va, byte_idx, &temp);
1105                 dev->feature_info[feat_type] = RS_64(temp, I40IW_FEATURE_INFO);
1106         }
1107 exit:
1108         i40iw_free_dma_mem(dev->hw, &feat_buf);
1109
1110         return ret_code;
1111 }
1112
1113 /**
1114  * i40iw_sc_query_fpm_values_done - poll for cqp wqe completion for query fpm
1115  * @cqp: struct for cqp hw
1116  */
1117 static enum i40iw_status_code i40iw_sc_query_fpm_values_done(struct i40iw_sc_cqp *cqp)
1118 {
1119         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_QUERY_FPM_VALUES, NULL);
1120 }
1121
1122 /**
1123  * i40iw_sc_query_fpm_values - cqp wqe query fpm values
1124  * @cqp: struct for cqp hw
1125  * @scratch: u64 saved to be used during cqp completion
1126  * @hmc_fn_id: hmc function id
1127  * @query_fpm_mem: memory for return fpm values
1128  * @post_sq: flag for cqp db to ring
1129  * @wait_type: poll ccq or cqp registers for cqp completion
1130  */
1131 static enum i40iw_status_code i40iw_sc_query_fpm_values(
1132                                         struct i40iw_sc_cqp *cqp,
1133                                         u64 scratch,
1134                                         u8 hmc_fn_id,
1135                                         struct i40iw_dma_mem *query_fpm_mem,
1136                                         bool post_sq,
1137                                         u8 wait_type)
1138 {
1139         u64 *wqe;
1140         u64 header;
1141         u32 tail, val, error;
1142         enum i40iw_status_code ret_code = 0;
1143
1144         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1145         if (!wqe)
1146                 return I40IW_ERR_RING_FULL;
1147
1148         set_64bit_val(wqe, 16, hmc_fn_id);
1149         set_64bit_val(wqe, 32, query_fpm_mem->pa);
1150
1151         header = LS_64(I40IW_CQP_OP_QUERY_FPM_VALUES, I40IW_CQPSQ_OPCODE) |
1152                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1153
1154         i40iw_insert_wqe_hdr(wqe, header);
1155
1156         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QUERY_FPM WQE",
1157                         wqe, I40IW_CQP_WQE_SIZE * 8);
1158
1159         /* read the tail from CQP_TAIL register */
1160         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
1161
1162         if (error)
1163                 return I40IW_ERR_CQP_COMPL_ERROR;
1164
1165         if (post_sq) {
1166                 i40iw_sc_cqp_post_sq(cqp);
1167                 if (wait_type == I40IW_CQP_WAIT_POLL_REGS)
1168                         ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);
1169                 else if (wait_type == I40IW_CQP_WAIT_POLL_CQ)
1170                         ret_code = i40iw_sc_query_fpm_values_done(cqp);
1171         }
1172
1173         return ret_code;
1174 }
1175
1176 /**
1177  * i40iw_sc_add_arp_cache_entry - cqp wqe add arp cache entry
1178  * @cqp: struct for cqp hw
1179  * @info: arp entry information
1180  * @scratch: u64 saved to be used during cqp completion
1181  * @post_sq: flag for cqp db to ring
1182  */
1183 static enum i40iw_status_code i40iw_sc_add_arp_cache_entry(
1184                                 struct i40iw_sc_cqp *cqp,
1185                                 struct i40iw_add_arp_cache_entry_info *info,
1186                                 u64 scratch,
1187                                 bool post_sq)
1188 {
1189         u64 *wqe;
1190         u64 temp, header;
1191
1192         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1193         if (!wqe)
1194                 return I40IW_ERR_RING_FULL;
1195         set_64bit_val(wqe, 8, info->reach_max);
1196
1197         temp = info->mac_addr[5] |
1198                LS_64_1(info->mac_addr[4], 8) |
1199                LS_64_1(info->mac_addr[3], 16) |
1200                LS_64_1(info->mac_addr[2], 24) |
1201                LS_64_1(info->mac_addr[1], 32) |
1202                LS_64_1(info->mac_addr[0], 40);
1203
1204         set_64bit_val(wqe, 16, temp);
1205
1206         header = info->arp_index |
1207                  LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) |
1208                  LS_64((info->permanent ? 1 : 0), I40IW_CQPSQ_MAT_PERMANENT) |
1209                  LS_64(1, I40IW_CQPSQ_MAT_ENTRYVALID) |
1210                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1211
1212         i40iw_insert_wqe_hdr(wqe, header);
1213
1214         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ARP_CACHE_ENTRY WQE",
1215                         wqe, I40IW_CQP_WQE_SIZE * 8);
1216
1217         if (post_sq)
1218                 i40iw_sc_cqp_post_sq(cqp);
1219         return 0;
1220 }
1221
1222 /**
1223  * i40iw_sc_del_arp_cache_entry - dele arp cache entry
1224  * @cqp: struct for cqp hw
1225  * @scratch: u64 saved to be used during cqp completion
1226  * @arp_index: arp index to delete arp entry
1227  * @post_sq: flag for cqp db to ring
1228  */
1229 static enum i40iw_status_code i40iw_sc_del_arp_cache_entry(
1230                                         struct i40iw_sc_cqp *cqp,
1231                                         u64 scratch,
1232                                         u16 arp_index,
1233                                         bool post_sq)
1234 {
1235         u64 *wqe;
1236         u64 header;
1237
1238         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1239         if (!wqe)
1240                 return I40IW_ERR_RING_FULL;
1241
1242         header = arp_index |
1243                  LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) |
1244                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1245         i40iw_insert_wqe_hdr(wqe, header);
1246
1247         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ARP_CACHE_DEL_ENTRY WQE",
1248                         wqe, I40IW_CQP_WQE_SIZE * 8);
1249
1250         if (post_sq)
1251                 i40iw_sc_cqp_post_sq(cqp);
1252         return 0;
1253 }
1254
1255 /**
1256  * i40iw_sc_query_arp_cache_entry - cqp wqe to query arp and arp index
1257  * @cqp: struct for cqp hw
1258  * @scratch: u64 saved to be used during cqp completion
1259  * @arp_index: arp index to delete arp entry
1260  * @post_sq: flag for cqp db to ring
1261  */
1262 static enum i40iw_status_code i40iw_sc_query_arp_cache_entry(
1263                                 struct i40iw_sc_cqp *cqp,
1264                                 u64 scratch,
1265                                 u16 arp_index,
1266                                 bool post_sq)
1267 {
1268         u64 *wqe;
1269         u64 header;
1270
1271         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1272         if (!wqe)
1273                 return I40IW_ERR_RING_FULL;
1274
1275         header = arp_index |
1276                  LS_64(I40IW_CQP_OP_MANAGE_ARP, I40IW_CQPSQ_OPCODE) |
1277                  LS_64(1, I40IW_CQPSQ_MAT_QUERY) |
1278                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1279
1280         i40iw_insert_wqe_hdr(wqe, header);
1281
1282         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QUERY_ARP_CACHE_ENTRY WQE",
1283                         wqe, I40IW_CQP_WQE_SIZE * 8);
1284
1285         if (post_sq)
1286                 i40iw_sc_cqp_post_sq(cqp);
1287         return 0;
1288 }
1289
1290 /**
1291  * i40iw_sc_manage_apbvt_entry - for adding and deleting apbvt entries
1292  * @cqp: struct for cqp hw
1293  * @info: info for apbvt entry to add or delete
1294  * @scratch: u64 saved to be used during cqp completion
1295  * @post_sq: flag for cqp db to ring
1296  */
1297 static enum i40iw_status_code i40iw_sc_manage_apbvt_entry(
1298                                 struct i40iw_sc_cqp *cqp,
1299                                 struct i40iw_apbvt_info *info,
1300                                 u64 scratch,
1301                                 bool post_sq)
1302 {
1303         u64 *wqe;
1304         u64 header;
1305
1306         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1307         if (!wqe)
1308                 return I40IW_ERR_RING_FULL;
1309
1310         set_64bit_val(wqe, 16, info->port);
1311
1312         header = LS_64(I40IW_CQP_OP_MANAGE_APBVT, I40IW_CQPSQ_OPCODE) |
1313                  LS_64(info->add, I40IW_CQPSQ_MAPT_ADDPORT) |
1314                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1315
1316         i40iw_insert_wqe_hdr(wqe, header);
1317
1318         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_APBVT WQE",
1319                         wqe, I40IW_CQP_WQE_SIZE * 8);
1320
1321         if (post_sq)
1322                 i40iw_sc_cqp_post_sq(cqp);
1323         return 0;
1324 }
1325
1326 /**
1327  * i40iw_sc_manage_qhash_table_entry - manage quad hash entries
1328  * @cqp: struct for cqp hw
1329  * @info: info for quad hash to manage
1330  * @scratch: u64 saved to be used during cqp completion
1331  * @post_sq: flag for cqp db to ring
1332  *
1333  * This is called before connection establishment is started. For passive connections, when
1334  * listener is created, it will call with entry type of  I40IW_QHASH_TYPE_TCP_SYN with local
1335  * ip address and tcp port. When SYN is received (passive connections) or
1336  * sent (active connections), this routine is called with entry type of
1337  * I40IW_QHASH_TYPE_TCP_ESTABLISHED and quad is passed in info.
1338  *
1339  * When iwarp connection is done and its state moves to RTS, the quad hash entry in
1340  * the hardware will point to iwarp's qp number and requires no calls from the driver.
1341  */
1342 static enum i40iw_status_code i40iw_sc_manage_qhash_table_entry(
1343                                         struct i40iw_sc_cqp *cqp,
1344                                         struct i40iw_qhash_table_info *info,
1345                                         u64 scratch,
1346                                         bool post_sq)
1347 {
1348         u64 *wqe;
1349         u64 qw1 = 0;
1350         u64 qw2 = 0;
1351         u64 temp;
1352         struct i40iw_sc_vsi *vsi = info->vsi;
1353
1354         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1355         if (!wqe)
1356                 return I40IW_ERR_RING_FULL;
1357
1358         temp = info->mac_addr[5] |
1359                 LS_64_1(info->mac_addr[4], 8) |
1360                 LS_64_1(info->mac_addr[3], 16) |
1361                 LS_64_1(info->mac_addr[2], 24) |
1362                 LS_64_1(info->mac_addr[1], 32) |
1363                 LS_64_1(info->mac_addr[0], 40);
1364
1365         set_64bit_val(wqe, 0, temp);
1366
1367         qw1 = LS_64(info->qp_num, I40IW_CQPSQ_QHASH_QPN) |
1368               LS_64(info->dest_port, I40IW_CQPSQ_QHASH_DEST_PORT);
1369         if (info->ipv4_valid) {
1370                 set_64bit_val(wqe,
1371                               48,
1372                               LS_64(info->dest_ip[0], I40IW_CQPSQ_QHASH_ADDR3));
1373         } else {
1374                 set_64bit_val(wqe,
1375                               56,
1376                               LS_64(info->dest_ip[0], I40IW_CQPSQ_QHASH_ADDR0) |
1377                               LS_64(info->dest_ip[1], I40IW_CQPSQ_QHASH_ADDR1));
1378
1379                 set_64bit_val(wqe,
1380                               48,
1381                               LS_64(info->dest_ip[2], I40IW_CQPSQ_QHASH_ADDR2) |
1382                               LS_64(info->dest_ip[3], I40IW_CQPSQ_QHASH_ADDR3));
1383         }
1384         qw2 = LS_64(vsi->qos[info->user_pri].qs_handle, I40IW_CQPSQ_QHASH_QS_HANDLE);
1385         if (info->vlan_valid)
1386                 qw2 |= LS_64(info->vlan_id, I40IW_CQPSQ_QHASH_VLANID);
1387         set_64bit_val(wqe, 16, qw2);
1388         if (info->entry_type == I40IW_QHASH_TYPE_TCP_ESTABLISHED) {
1389                 qw1 |= LS_64(info->src_port, I40IW_CQPSQ_QHASH_SRC_PORT);
1390                 if (!info->ipv4_valid) {
1391                         set_64bit_val(wqe,
1392                                       40,
1393                                       LS_64(info->src_ip[0], I40IW_CQPSQ_QHASH_ADDR0) |
1394                                       LS_64(info->src_ip[1], I40IW_CQPSQ_QHASH_ADDR1));
1395                         set_64bit_val(wqe,
1396                                       32,
1397                                       LS_64(info->src_ip[2], I40IW_CQPSQ_QHASH_ADDR2) |
1398                                       LS_64(info->src_ip[3], I40IW_CQPSQ_QHASH_ADDR3));
1399                 } else {
1400                         set_64bit_val(wqe,
1401                                       32,
1402                                       LS_64(info->src_ip[0], I40IW_CQPSQ_QHASH_ADDR3));
1403                 }
1404         }
1405
1406         set_64bit_val(wqe, 8, qw1);
1407         temp = LS_64(cqp->polarity, I40IW_CQPSQ_QHASH_WQEVALID) |
1408                LS_64(I40IW_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY, I40IW_CQPSQ_QHASH_OPCODE) |
1409                LS_64(info->manage, I40IW_CQPSQ_QHASH_MANAGE) |
1410                LS_64(info->ipv4_valid, I40IW_CQPSQ_QHASH_IPV4VALID) |
1411                LS_64(info->vlan_valid, I40IW_CQPSQ_QHASH_VLANVALID) |
1412                LS_64(info->entry_type, I40IW_CQPSQ_QHASH_ENTRYTYPE);
1413
1414         i40iw_insert_wqe_hdr(wqe, temp);
1415
1416         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "MANAGE_QHASH WQE",
1417                         wqe, I40IW_CQP_WQE_SIZE * 8);
1418
1419         if (post_sq)
1420                 i40iw_sc_cqp_post_sq(cqp);
1421         return 0;
1422 }
1423
1424 /**
1425  * i40iw_sc_alloc_local_mac_ipaddr_entry - cqp wqe for loc mac entry
1426  * @cqp: struct for cqp hw
1427  * @scratch: u64 saved to be used during cqp completion
1428  * @post_sq: flag for cqp db to ring
1429  */
1430 static enum i40iw_status_code i40iw_sc_alloc_local_mac_ipaddr_entry(
1431                                         struct i40iw_sc_cqp *cqp,
1432                                         u64 scratch,
1433                                         bool post_sq)
1434 {
1435         u64 *wqe;
1436         u64 header;
1437
1438         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1439         if (!wqe)
1440                 return I40IW_ERR_RING_FULL;
1441         header = LS_64(I40IW_CQP_OP_ALLOCATE_LOC_MAC_IP_TABLE_ENTRY, I40IW_CQPSQ_OPCODE) |
1442                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1443
1444         i40iw_insert_wqe_hdr(wqe, header);
1445         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ALLOCATE_LOCAL_MAC_IPADDR WQE",
1446                         wqe, I40IW_CQP_WQE_SIZE * 8);
1447         if (post_sq)
1448                 i40iw_sc_cqp_post_sq(cqp);
1449         return 0;
1450 }
1451
1452 /**
1453  * i40iw_sc_add_local_mac_ipaddr_entry - add mac enry
1454  * @cqp: struct for cqp hw
1455  * @info:mac addr info
1456  * @scratch: u64 saved to be used during cqp completion
1457  * @post_sq: flag for cqp db to ring
1458  */
1459 static enum i40iw_status_code i40iw_sc_add_local_mac_ipaddr_entry(
1460                                 struct i40iw_sc_cqp *cqp,
1461                                 struct i40iw_local_mac_ipaddr_entry_info *info,
1462                                 u64 scratch,
1463                                 bool post_sq)
1464 {
1465         u64 *wqe;
1466         u64 temp, header;
1467
1468         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1469         if (!wqe)
1470                 return I40IW_ERR_RING_FULL;
1471         temp = info->mac_addr[5] |
1472                 LS_64_1(info->mac_addr[4], 8) |
1473                 LS_64_1(info->mac_addr[3], 16) |
1474                 LS_64_1(info->mac_addr[2], 24) |
1475                 LS_64_1(info->mac_addr[1], 32) |
1476                 LS_64_1(info->mac_addr[0], 40);
1477
1478         set_64bit_val(wqe, 32, temp);
1479
1480         header = LS_64(info->entry_idx, I40IW_CQPSQ_MLIPA_IPTABLEIDX) |
1481                  LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE, I40IW_CQPSQ_OPCODE) |
1482                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1483
1484         i40iw_insert_wqe_hdr(wqe, header);
1485
1486         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "ADD_LOCAL_MAC_IPADDR WQE",
1487                         wqe, I40IW_CQP_WQE_SIZE * 8);
1488
1489         if (post_sq)
1490                 i40iw_sc_cqp_post_sq(cqp);
1491         return 0;
1492 }
1493
1494 /**
1495  * i40iw_sc_del_local_mac_ipaddr_entry - cqp wqe to dele local mac
1496  * @cqp: struct for cqp hw
1497  * @scratch: u64 saved to be used during cqp completion
1498  * @entry_idx: index of mac entry
1499  * @ ignore_ref_count: to force mac adde delete
1500  * @post_sq: flag for cqp db to ring
1501  */
1502 static enum i40iw_status_code i40iw_sc_del_local_mac_ipaddr_entry(
1503                                 struct i40iw_sc_cqp *cqp,
1504                                 u64 scratch,
1505                                 u8 entry_idx,
1506                                 u8 ignore_ref_count,
1507                                 bool post_sq)
1508 {
1509         u64 *wqe;
1510         u64 header;
1511
1512         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1513         if (!wqe)
1514                 return I40IW_ERR_RING_FULL;
1515         header = LS_64(entry_idx, I40IW_CQPSQ_MLIPA_IPTABLEIDX) |
1516                  LS_64(I40IW_CQP_OP_MANAGE_LOC_MAC_IP_TABLE, I40IW_CQPSQ_OPCODE) |
1517                  LS_64(1, I40IW_CQPSQ_MLIPA_FREEENTRY) |
1518                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) |
1519                  LS_64(ignore_ref_count, I40IW_CQPSQ_MLIPA_IGNORE_REF_CNT);
1520
1521         i40iw_insert_wqe_hdr(wqe, header);
1522
1523         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "DEL_LOCAL_MAC_IPADDR WQE",
1524                         wqe, I40IW_CQP_WQE_SIZE * 8);
1525
1526         if (post_sq)
1527                 i40iw_sc_cqp_post_sq(cqp);
1528         return 0;
1529 }
1530
1531 /**
1532  * i40iw_sc_cqp_nop - send a nop wqe
1533  * @cqp: struct for cqp hw
1534  * @scratch: u64 saved to be used during cqp completion
1535  * @post_sq: flag for cqp db to ring
1536  */
1537 static enum i40iw_status_code i40iw_sc_cqp_nop(struct i40iw_sc_cqp *cqp,
1538                                                u64 scratch,
1539                                                bool post_sq)
1540 {
1541         u64 *wqe;
1542         u64 header;
1543
1544         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1545         if (!wqe)
1546                 return I40IW_ERR_RING_FULL;
1547         header = LS_64(I40IW_CQP_OP_NOP, I40IW_CQPSQ_OPCODE) |
1548                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1549         i40iw_insert_wqe_hdr(wqe, header);
1550         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "NOP WQE",
1551                         wqe, I40IW_CQP_WQE_SIZE * 8);
1552
1553         if (post_sq)
1554                 i40iw_sc_cqp_post_sq(cqp);
1555         return 0;
1556 }
1557
1558 /**
1559  * i40iw_sc_ceq_init - initialize ceq
1560  * @ceq: ceq sc structure
1561  * @info: ceq initialization info
1562  */
1563 static enum i40iw_status_code i40iw_sc_ceq_init(struct i40iw_sc_ceq *ceq,
1564                                                 struct i40iw_ceq_init_info *info)
1565 {
1566         u32 pble_obj_cnt;
1567
1568         if ((info->elem_cnt < I40IW_MIN_CEQ_ENTRIES) ||
1569             (info->elem_cnt > I40IW_MAX_CEQ_ENTRIES))
1570                 return I40IW_ERR_INVALID_SIZE;
1571
1572         if (info->ceq_id >= I40IW_MAX_CEQID)
1573                 return I40IW_ERR_INVALID_CEQ_ID;
1574
1575         pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
1576
1577         if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
1578                 return I40IW_ERR_INVALID_PBLE_INDEX;
1579
1580         ceq->size = sizeof(*ceq);
1581         ceq->ceqe_base = (struct i40iw_ceqe *)info->ceqe_base;
1582         ceq->ceq_id = info->ceq_id;
1583         ceq->dev = info->dev;
1584         ceq->elem_cnt = info->elem_cnt;
1585         ceq->ceq_elem_pa = info->ceqe_pa;
1586         ceq->virtual_map = info->virtual_map;
1587
1588         ceq->pbl_chunk_size = (ceq->virtual_map ? info->pbl_chunk_size : 0);
1589         ceq->first_pm_pbl_idx = (ceq->virtual_map ? info->first_pm_pbl_idx : 0);
1590         ceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL);
1591
1592         ceq->tph_en = info->tph_en;
1593         ceq->tph_val = info->tph_val;
1594         ceq->polarity = 1;
1595         I40IW_RING_INIT(ceq->ceq_ring, ceq->elem_cnt);
1596         ceq->dev->ceq[info->ceq_id] = ceq;
1597
1598         return 0;
1599 }
1600
1601 /**
1602  * i40iw_sc_ceq_create - create ceq wqe
1603  * @ceq: ceq sc structure
1604  * @scratch: u64 saved to be used during cqp completion
1605  * @post_sq: flag for cqp db to ring
1606  */
1607 static enum i40iw_status_code i40iw_sc_ceq_create(struct i40iw_sc_ceq *ceq,
1608                                                   u64 scratch,
1609                                                   bool post_sq)
1610 {
1611         struct i40iw_sc_cqp *cqp;
1612         u64 *wqe;
1613         u64 header;
1614
1615         cqp = ceq->dev->cqp;
1616         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1617         if (!wqe)
1618                 return I40IW_ERR_RING_FULL;
1619         set_64bit_val(wqe, 16, ceq->elem_cnt);
1620         set_64bit_val(wqe, 32, (ceq->virtual_map ? 0 : ceq->ceq_elem_pa));
1621         set_64bit_val(wqe, 48, (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0));
1622         set_64bit_val(wqe, 56, LS_64(ceq->tph_val, I40IW_CQPSQ_TPHVAL));
1623
1624         header = ceq->ceq_id |
1625                  LS_64(I40IW_CQP_OP_CREATE_CEQ, I40IW_CQPSQ_OPCODE) |
1626                  LS_64(ceq->pbl_chunk_size, I40IW_CQPSQ_CEQ_LPBLSIZE) |
1627                  LS_64(ceq->virtual_map, I40IW_CQPSQ_CEQ_VMAP) |
1628                  LS_64(ceq->tph_en, I40IW_CQPSQ_TPHEN) |
1629                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1630
1631         i40iw_insert_wqe_hdr(wqe, header);
1632
1633         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CEQ_CREATE WQE",
1634                         wqe, I40IW_CQP_WQE_SIZE * 8);
1635
1636         if (post_sq)
1637                 i40iw_sc_cqp_post_sq(cqp);
1638         return 0;
1639 }
1640
1641 /**
1642  * i40iw_sc_cceq_create_done - poll for control ceq wqe to complete
1643  * @ceq: ceq sc structure
1644  */
1645 static enum i40iw_status_code i40iw_sc_cceq_create_done(struct i40iw_sc_ceq *ceq)
1646 {
1647         struct i40iw_sc_cqp *cqp;
1648
1649         cqp = ceq->dev->cqp;
1650         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_CEQ, NULL);
1651 }
1652
1653 /**
1654  * i40iw_sc_cceq_destroy_done - poll for destroy cceq to complete
1655  * @ceq: ceq sc structure
1656  */
1657 static enum i40iw_status_code i40iw_sc_cceq_destroy_done(struct i40iw_sc_ceq *ceq)
1658 {
1659         struct i40iw_sc_cqp *cqp;
1660
1661         cqp = ceq->dev->cqp;
1662         cqp->process_cqp_sds = i40iw_update_sds_noccq;
1663         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_DESTROY_CEQ, NULL);
1664 }
1665
1666 /**
1667  * i40iw_sc_cceq_create - create cceq
1668  * @ceq: ceq sc structure
1669  * @scratch: u64 saved to be used during cqp completion
1670  */
1671 static enum i40iw_status_code i40iw_sc_cceq_create(struct i40iw_sc_ceq *ceq, u64 scratch)
1672 {
1673         enum i40iw_status_code ret_code;
1674
1675         ret_code = i40iw_sc_ceq_create(ceq, scratch, true);
1676         if (!ret_code)
1677                 ret_code = i40iw_sc_cceq_create_done(ceq);
1678         return ret_code;
1679 }
1680
1681 /**
1682  * i40iw_sc_ceq_destroy - destroy ceq
1683  * @ceq: ceq sc structure
1684  * @scratch: u64 saved to be used during cqp completion
1685  * @post_sq: flag for cqp db to ring
1686  */
1687 static enum i40iw_status_code i40iw_sc_ceq_destroy(struct i40iw_sc_ceq *ceq,
1688                                                    u64 scratch,
1689                                                    bool post_sq)
1690 {
1691         struct i40iw_sc_cqp *cqp;
1692         u64 *wqe;
1693         u64 header;
1694
1695         cqp = ceq->dev->cqp;
1696         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1697         if (!wqe)
1698                 return I40IW_ERR_RING_FULL;
1699         set_64bit_val(wqe, 16, ceq->elem_cnt);
1700         set_64bit_val(wqe, 48, ceq->first_pm_pbl_idx);
1701         header = ceq->ceq_id |
1702                  LS_64(I40IW_CQP_OP_DESTROY_CEQ, I40IW_CQPSQ_OPCODE) |
1703                  LS_64(ceq->pbl_chunk_size, I40IW_CQPSQ_CEQ_LPBLSIZE) |
1704                  LS_64(ceq->virtual_map, I40IW_CQPSQ_CEQ_VMAP) |
1705                  LS_64(ceq->tph_en, I40IW_CQPSQ_TPHEN) |
1706                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1707         i40iw_insert_wqe_hdr(wqe, header);
1708         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CEQ_DESTROY WQE",
1709                         wqe, I40IW_CQP_WQE_SIZE * 8);
1710
1711         if (post_sq)
1712                 i40iw_sc_cqp_post_sq(cqp);
1713         return 0;
1714 }
1715
1716 /**
1717  * i40iw_sc_process_ceq - process ceq
1718  * @dev: sc device struct
1719  * @ceq: ceq sc structure
1720  */
1721 static void *i40iw_sc_process_ceq(struct i40iw_sc_dev *dev, struct i40iw_sc_ceq *ceq)
1722 {
1723         u64 temp;
1724         u64 *ceqe;
1725         struct i40iw_sc_cq *cq = NULL;
1726         u8 polarity;
1727
1728         ceqe = (u64 *)I40IW_GET_CURRENT_CEQ_ELEMENT(ceq);
1729         get_64bit_val(ceqe, 0, &temp);
1730         polarity = (u8)RS_64(temp, I40IW_CEQE_VALID);
1731         if (polarity != ceq->polarity)
1732                 return cq;
1733
1734         cq = (struct i40iw_sc_cq *)(unsigned long)LS_64_1(temp, 1);
1735
1736         I40IW_RING_MOVE_TAIL(ceq->ceq_ring);
1737         if (I40IW_RING_GETCURRENT_TAIL(ceq->ceq_ring) == 0)
1738                 ceq->polarity ^= 1;
1739
1740         if (dev->is_pf)
1741                 i40iw_wr32(dev->hw, I40E_PFPE_CQACK, cq->cq_uk.cq_id);
1742         else
1743                 i40iw_wr32(dev->hw, I40E_VFPE_CQACK1, cq->cq_uk.cq_id);
1744
1745         return cq;
1746 }
1747
1748 /**
1749  * i40iw_sc_aeq_init - initialize aeq
1750  * @aeq: aeq structure ptr
1751  * @info: aeq initialization info
1752  */
1753 static enum i40iw_status_code i40iw_sc_aeq_init(struct i40iw_sc_aeq *aeq,
1754                                                 struct i40iw_aeq_init_info *info)
1755 {
1756         u32 pble_obj_cnt;
1757
1758         if ((info->elem_cnt < I40IW_MIN_AEQ_ENTRIES) ||
1759             (info->elem_cnt > I40IW_MAX_AEQ_ENTRIES))
1760                 return I40IW_ERR_INVALID_SIZE;
1761         pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
1762
1763         if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
1764                 return I40IW_ERR_INVALID_PBLE_INDEX;
1765
1766         aeq->size = sizeof(*aeq);
1767         aeq->polarity = 1;
1768         aeq->aeqe_base = (struct i40iw_sc_aeqe *)info->aeqe_base;
1769         aeq->dev = info->dev;
1770         aeq->elem_cnt = info->elem_cnt;
1771
1772         aeq->aeq_elem_pa = info->aeq_elem_pa;
1773         I40IW_RING_INIT(aeq->aeq_ring, aeq->elem_cnt);
1774         info->dev->aeq = aeq;
1775
1776         aeq->virtual_map = info->virtual_map;
1777         aeq->pbl_list = (aeq->virtual_map ? info->pbl_list : NULL);
1778         aeq->pbl_chunk_size = (aeq->virtual_map ? info->pbl_chunk_size : 0);
1779         aeq->first_pm_pbl_idx = (aeq->virtual_map ? info->first_pm_pbl_idx : 0);
1780         info->dev->aeq = aeq;
1781         return 0;
1782 }
1783
1784 /**
1785  * i40iw_sc_aeq_create - create aeq
1786  * @aeq: aeq structure ptr
1787  * @scratch: u64 saved to be used during cqp completion
1788  * @post_sq: flag for cqp db to ring
1789  */
1790 static enum i40iw_status_code i40iw_sc_aeq_create(struct i40iw_sc_aeq *aeq,
1791                                                   u64 scratch,
1792                                                   bool post_sq)
1793 {
1794         u64 *wqe;
1795         struct i40iw_sc_cqp *cqp;
1796         u64 header;
1797
1798         cqp = aeq->dev->cqp;
1799         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1800         if (!wqe)
1801                 return I40IW_ERR_RING_FULL;
1802         set_64bit_val(wqe, 16, aeq->elem_cnt);
1803         set_64bit_val(wqe, 32,
1804                       (aeq->virtual_map ? 0 : aeq->aeq_elem_pa));
1805         set_64bit_val(wqe, 48,
1806                       (aeq->virtual_map ? aeq->first_pm_pbl_idx : 0));
1807
1808         header = LS_64(I40IW_CQP_OP_CREATE_AEQ, I40IW_CQPSQ_OPCODE) |
1809                  LS_64(aeq->pbl_chunk_size, I40IW_CQPSQ_AEQ_LPBLSIZE) |
1810                  LS_64(aeq->virtual_map, I40IW_CQPSQ_AEQ_VMAP) |
1811                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1812
1813         i40iw_insert_wqe_hdr(wqe, header);
1814         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "AEQ_CREATE WQE",
1815                         wqe, I40IW_CQP_WQE_SIZE * 8);
1816         if (post_sq)
1817                 i40iw_sc_cqp_post_sq(cqp);
1818         return 0;
1819 }
1820
1821 /**
1822  * i40iw_sc_aeq_destroy - destroy aeq during close
1823  * @aeq: aeq structure ptr
1824  * @scratch: u64 saved to be used during cqp completion
1825  * @post_sq: flag for cqp db to ring
1826  */
1827 static enum i40iw_status_code i40iw_sc_aeq_destroy(struct i40iw_sc_aeq *aeq,
1828                                                    u64 scratch,
1829                                                    bool post_sq)
1830 {
1831         u64 *wqe;
1832         struct i40iw_sc_cqp *cqp;
1833         u64 header;
1834
1835         cqp = aeq->dev->cqp;
1836         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
1837         if (!wqe)
1838                 return I40IW_ERR_RING_FULL;
1839         set_64bit_val(wqe, 16, aeq->elem_cnt);
1840         set_64bit_val(wqe, 48, aeq->first_pm_pbl_idx);
1841         header = LS_64(I40IW_CQP_OP_DESTROY_AEQ, I40IW_CQPSQ_OPCODE) |
1842                  LS_64(aeq->pbl_chunk_size, I40IW_CQPSQ_AEQ_LPBLSIZE) |
1843                  LS_64(aeq->virtual_map, I40IW_CQPSQ_AEQ_VMAP) |
1844                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
1845         i40iw_insert_wqe_hdr(wqe, header);
1846
1847         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "AEQ_DESTROY WQE",
1848                         wqe, I40IW_CQP_WQE_SIZE * 8);
1849         if (post_sq)
1850                 i40iw_sc_cqp_post_sq(cqp);
1851         return 0;
1852 }
1853
1854 /**
1855  * i40iw_sc_get_next_aeqe - get next aeq entry
1856  * @aeq: aeq structure ptr
1857  * @info: aeqe info to be returned
1858  */
1859 static enum i40iw_status_code i40iw_sc_get_next_aeqe(struct i40iw_sc_aeq *aeq,
1860                                                      struct i40iw_aeqe_info *info)
1861 {
1862         u64 temp, compl_ctx;
1863         u64 *aeqe;
1864         u16 wqe_idx;
1865         u8 ae_src;
1866         u8 polarity;
1867
1868         aeqe = (u64 *)I40IW_GET_CURRENT_AEQ_ELEMENT(aeq);
1869         get_64bit_val(aeqe, 0, &compl_ctx);
1870         get_64bit_val(aeqe, 8, &temp);
1871         polarity = (u8)RS_64(temp, I40IW_AEQE_VALID);
1872
1873         if (aeq->polarity != polarity)
1874                 return I40IW_ERR_QUEUE_EMPTY;
1875
1876         i40iw_debug_buf(aeq->dev, I40IW_DEBUG_WQE, "AEQ_ENTRY", aeqe, 16);
1877
1878         ae_src = (u8)RS_64(temp, I40IW_AEQE_AESRC);
1879         wqe_idx = (u16)RS_64(temp, I40IW_AEQE_WQDESCIDX);
1880         info->qp_cq_id = (u32)RS_64(temp, I40IW_AEQE_QPCQID);
1881         info->ae_id = (u16)RS_64(temp, I40IW_AEQE_AECODE);
1882         info->tcp_state = (u8)RS_64(temp, I40IW_AEQE_TCPSTATE);
1883         info->iwarp_state = (u8)RS_64(temp, I40IW_AEQE_IWSTATE);
1884         info->q2_data_written = (u8)RS_64(temp, I40IW_AEQE_Q2DATA);
1885         info->aeqe_overflow = (bool)RS_64(temp, I40IW_AEQE_OVERFLOW);
1886
1887         switch (info->ae_id) {
1888         case I40IW_AE_PRIV_OPERATION_DENIED:
1889         case I40IW_AE_UDA_XMIT_DGRAM_TOO_LONG:
1890         case I40IW_AE_UDA_XMIT_DGRAM_TOO_SHORT:
1891         case I40IW_AE_BAD_CLOSE:
1892         case I40IW_AE_RDMAP_ROE_BAD_LLP_CLOSE:
1893         case I40IW_AE_RDMA_READ_WHILE_ORD_ZERO:
1894         case I40IW_AE_STAG_ZERO_INVALID:
1895         case I40IW_AE_IB_RREQ_AND_Q1_FULL:
1896         case I40IW_AE_WQE_UNEXPECTED_OPCODE:
1897         case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION:
1898         case I40IW_AE_DDP_UBE_INVALID_MO:
1899         case I40IW_AE_DDP_UBE_INVALID_QN:
1900         case I40IW_AE_DDP_NO_L_BIT:
1901         case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
1902         case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE:
1903         case I40IW_AE_ROE_INVALID_RDMA_READ_REQUEST:
1904         case I40IW_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
1905         case I40IW_AE_INVALID_ARP_ENTRY:
1906         case I40IW_AE_INVALID_TCP_OPTION_RCVD:
1907         case I40IW_AE_STALE_ARP_ENTRY:
1908         case I40IW_AE_LLP_CLOSE_COMPLETE:
1909         case I40IW_AE_LLP_CONNECTION_RESET:
1910         case I40IW_AE_LLP_FIN_RECEIVED:
1911         case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR:
1912         case I40IW_AE_LLP_SEGMENT_TOO_SMALL:
1913         case I40IW_AE_LLP_SYN_RECEIVED:
1914         case I40IW_AE_LLP_TERMINATE_RECEIVED:
1915         case I40IW_AE_LLP_TOO_MANY_RETRIES:
1916         case I40IW_AE_LLP_DOUBT_REACHABILITY:
1917         case I40IW_AE_RESET_SENT:
1918         case I40IW_AE_TERMINATE_SENT:
1919         case I40IW_AE_RESET_NOT_SENT:
1920         case I40IW_AE_LCE_QP_CATASTROPHIC:
1921         case I40IW_AE_QP_SUSPEND_COMPLETE:
1922                 info->qp = true;
1923                 info->compl_ctx = compl_ctx;
1924                 ae_src = I40IW_AE_SOURCE_RSVD;
1925                 break;
1926         case I40IW_AE_LCE_CQ_CATASTROPHIC:
1927                 info->cq = true;
1928                 info->compl_ctx = LS_64_1(compl_ctx, 1);
1929                 ae_src = I40IW_AE_SOURCE_RSVD;
1930                 break;
1931         }
1932
1933         switch (ae_src) {
1934         case I40IW_AE_SOURCE_RQ:
1935         case I40IW_AE_SOURCE_RQ_0011:
1936                 info->qp = true;
1937                 info->wqe_idx = wqe_idx;
1938                 info->compl_ctx = compl_ctx;
1939                 break;
1940         case I40IW_AE_SOURCE_CQ:
1941         case I40IW_AE_SOURCE_CQ_0110:
1942         case I40IW_AE_SOURCE_CQ_1010:
1943         case I40IW_AE_SOURCE_CQ_1110:
1944                 info->cq = true;
1945                 info->compl_ctx = LS_64_1(compl_ctx, 1);
1946                 break;
1947         case I40IW_AE_SOURCE_SQ:
1948         case I40IW_AE_SOURCE_SQ_0111:
1949                 info->qp = true;
1950                 info->sq = true;
1951                 info->wqe_idx = wqe_idx;
1952                 info->compl_ctx = compl_ctx;
1953                 break;
1954         case I40IW_AE_SOURCE_IN_RR_WR:
1955         case I40IW_AE_SOURCE_IN_RR_WR_1011:
1956                 info->qp = true;
1957                 info->compl_ctx = compl_ctx;
1958                 info->in_rdrsp_wr = true;
1959                 break;
1960         case I40IW_AE_SOURCE_OUT_RR:
1961         case I40IW_AE_SOURCE_OUT_RR_1111:
1962                 info->qp = true;
1963                 info->compl_ctx = compl_ctx;
1964                 info->out_rdrsp = true;
1965                 break;
1966         case I40IW_AE_SOURCE_RSVD:
1967         default:
1968                 break;
1969         }
1970         I40IW_RING_MOVE_TAIL(aeq->aeq_ring);
1971         if (I40IW_RING_GETCURRENT_TAIL(aeq->aeq_ring) == 0)
1972                 aeq->polarity ^= 1;
1973         return 0;
1974 }
1975
1976 /**
1977  * i40iw_sc_repost_aeq_entries - repost completed aeq entries
1978  * @dev: sc device struct
1979  * @count: allocate count
1980  */
1981 static enum i40iw_status_code i40iw_sc_repost_aeq_entries(struct i40iw_sc_dev *dev,
1982                                                           u32 count)
1983 {
1984
1985         if (dev->is_pf)
1986                 i40iw_wr32(dev->hw, I40E_PFPE_AEQALLOC, count);
1987         else
1988                 i40iw_wr32(dev->hw, I40E_VFPE_AEQALLOC1, count);
1989
1990         return 0;
1991 }
1992
1993 /**
1994  * i40iw_sc_aeq_create_done - create aeq
1995  * @aeq: aeq structure ptr
1996  */
1997 static enum i40iw_status_code i40iw_sc_aeq_create_done(struct i40iw_sc_aeq *aeq)
1998 {
1999         struct i40iw_sc_cqp *cqp;
2000
2001         cqp = aeq->dev->cqp;
2002         return i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_AEQ, NULL);
2003 }
2004
2005 /**
2006  * i40iw_sc_aeq_destroy_done - destroy of aeq during close
2007  * @aeq: aeq structure ptr
2008  */
2009 static enum i40iw_status_code i40iw_sc_aeq_destroy_done(struct i40iw_sc_aeq *aeq)
2010 {
2011         struct i40iw_sc_cqp *cqp;
2012
2013         cqp = aeq->dev->cqp;
2014         return  i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_DESTROY_AEQ, NULL);
2015 }
2016
2017 /**
2018  * i40iw_sc_ccq_init - initialize control cq
2019  * @cq: sc's cq ctruct
2020  * @info: info for control cq initialization
2021  */
2022 static enum i40iw_status_code i40iw_sc_ccq_init(struct i40iw_sc_cq *cq,
2023                                                 struct i40iw_ccq_init_info *info)
2024 {
2025         u32 pble_obj_cnt;
2026
2027         if (info->num_elem < I40IW_MIN_CQ_SIZE || info->num_elem > I40IW_MAX_CQ_SIZE)
2028                 return I40IW_ERR_INVALID_SIZE;
2029
2030         if (info->ceq_id > I40IW_MAX_CEQID)
2031                 return I40IW_ERR_INVALID_CEQ_ID;
2032
2033         pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
2034
2035         if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
2036                 return I40IW_ERR_INVALID_PBLE_INDEX;
2037
2038         cq->cq_pa = info->cq_pa;
2039         cq->cq_uk.cq_base = info->cq_base;
2040         cq->shadow_area_pa = info->shadow_area_pa;
2041         cq->cq_uk.shadow_area = info->shadow_area;
2042         cq->shadow_read_threshold = info->shadow_read_threshold;
2043         cq->dev = info->dev;
2044         cq->ceq_id = info->ceq_id;
2045         cq->cq_uk.cq_size = info->num_elem;
2046         cq->cq_type = I40IW_CQ_TYPE_CQP;
2047         cq->ceqe_mask = info->ceqe_mask;
2048         I40IW_RING_INIT(cq->cq_uk.cq_ring, info->num_elem);
2049
2050         cq->cq_uk.cq_id = 0;    /* control cq is id 0 always */
2051         cq->ceq_id_valid = info->ceq_id_valid;
2052         cq->tph_en = info->tph_en;
2053         cq->tph_val = info->tph_val;
2054         cq->cq_uk.avoid_mem_cflct = info->avoid_mem_cflct;
2055
2056         cq->pbl_list = info->pbl_list;
2057         cq->virtual_map = info->virtual_map;
2058         cq->pbl_chunk_size = info->pbl_chunk_size;
2059         cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
2060         cq->cq_uk.polarity = true;
2061
2062         /* following are only for iw cqs so initialize them to zero */
2063         cq->cq_uk.cqe_alloc_reg = NULL;
2064         info->dev->ccq = cq;
2065         return 0;
2066 }
2067
2068 /**
2069  * i40iw_sc_ccq_create_done - poll cqp for ccq create
2070  * @ccq: ccq sc struct
2071  */
2072 static enum i40iw_status_code i40iw_sc_ccq_create_done(struct i40iw_sc_cq *ccq)
2073 {
2074         struct i40iw_sc_cqp *cqp;
2075
2076         cqp = ccq->dev->cqp;
2077         return  i40iw_sc_poll_for_cqp_op_done(cqp, I40IW_CQP_OP_CREATE_CQ, NULL);
2078 }
2079
2080 /**
2081  * i40iw_sc_ccq_create - create control cq
2082  * @ccq: ccq sc struct
2083  * @scratch: u64 saved to be used during cqp completion
2084  * @check_overflow: overlow flag for ccq
2085  * @post_sq: flag for cqp db to ring
2086  */
2087 static enum i40iw_status_code i40iw_sc_ccq_create(struct i40iw_sc_cq *ccq,
2088                                                   u64 scratch,
2089                                                   bool check_overflow,
2090                                                   bool post_sq)
2091 {
2092         u64 *wqe;
2093         struct i40iw_sc_cqp *cqp;
2094         u64 header;
2095         enum i40iw_status_code ret_code;
2096
2097         cqp = ccq->dev->cqp;
2098         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2099         if (!wqe)
2100                 return I40IW_ERR_RING_FULL;
2101         set_64bit_val(wqe, 0, ccq->cq_uk.cq_size);
2102         set_64bit_val(wqe, 8, RS_64_1(ccq, 1));
2103         set_64bit_val(wqe, 16,
2104                       LS_64(ccq->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
2105         set_64bit_val(wqe, 32, (ccq->virtual_map ? 0 : ccq->cq_pa));
2106         set_64bit_val(wqe, 40, ccq->shadow_area_pa);
2107         set_64bit_val(wqe, 48,
2108                       (ccq->virtual_map ? ccq->first_pm_pbl_idx : 0));
2109         set_64bit_val(wqe, 56,
2110                       LS_64(ccq->tph_val, I40IW_CQPSQ_TPHVAL));
2111
2112         header = ccq->cq_uk.cq_id |
2113                  LS_64((ccq->ceq_id_valid ? ccq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
2114                  LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) |
2115                  LS_64(ccq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
2116                  LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) |
2117                  LS_64(ccq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
2118                  LS_64(ccq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
2119                  LS_64(ccq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
2120                  LS_64(ccq->tph_en, I40IW_CQPSQ_TPHEN) |
2121                  LS_64(ccq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
2122                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2123
2124         i40iw_insert_wqe_hdr(wqe, header);
2125
2126         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CCQ_CREATE WQE",
2127                         wqe, I40IW_CQP_WQE_SIZE * 8);
2128
2129         if (post_sq) {
2130                 i40iw_sc_cqp_post_sq(cqp);
2131                 ret_code = i40iw_sc_ccq_create_done(ccq);
2132                 if (ret_code)
2133                         return ret_code;
2134         }
2135         cqp->process_cqp_sds = i40iw_cqp_sds_cmd;
2136
2137         return 0;
2138 }
2139
2140 /**
2141  * i40iw_sc_ccq_destroy - destroy ccq during close
2142  * @ccq: ccq sc struct
2143  * @scratch: u64 saved to be used during cqp completion
2144  * @post_sq: flag for cqp db to ring
2145  */
2146 static enum i40iw_status_code i40iw_sc_ccq_destroy(struct i40iw_sc_cq *ccq,
2147                                                    u64 scratch,
2148                                                    bool post_sq)
2149 {
2150         struct i40iw_sc_cqp *cqp;
2151         u64 *wqe;
2152         u64 header;
2153         enum i40iw_status_code ret_code = 0;
2154         u32 tail, val, error;
2155
2156         cqp = ccq->dev->cqp;
2157         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2158         if (!wqe)
2159                 return I40IW_ERR_RING_FULL;
2160         set_64bit_val(wqe, 0, ccq->cq_uk.cq_size);
2161         set_64bit_val(wqe, 8, RS_64_1(ccq, 1));
2162         set_64bit_val(wqe, 40, ccq->shadow_area_pa);
2163
2164         header = ccq->cq_uk.cq_id |
2165                  LS_64((ccq->ceq_id_valid ? ccq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
2166                  LS_64(I40IW_CQP_OP_DESTROY_CQ, I40IW_CQPSQ_OPCODE) |
2167                  LS_64(ccq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
2168                  LS_64(ccq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
2169                  LS_64(ccq->tph_en, I40IW_CQPSQ_TPHEN) |
2170                  LS_64(ccq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
2171                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2172
2173         i40iw_insert_wqe_hdr(wqe, header);
2174
2175         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CCQ_DESTROY WQE",
2176                         wqe, I40IW_CQP_WQE_SIZE * 8);
2177
2178         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
2179         if (error)
2180                 return I40IW_ERR_CQP_COMPL_ERROR;
2181
2182         if (post_sq) {
2183                 i40iw_sc_cqp_post_sq(cqp);
2184                 ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000);
2185         }
2186
2187         cqp->process_cqp_sds = i40iw_update_sds_noccq;
2188
2189         return ret_code;
2190 }
2191
2192 /**
2193  * i40iw_sc_cq_init - initialize completion q
2194  * @cq: cq struct
2195  * @info: cq initialization info
2196  */
2197 static enum i40iw_status_code i40iw_sc_cq_init(struct i40iw_sc_cq *cq,
2198                                                struct i40iw_cq_init_info *info)
2199 {
2200         u32 __iomem *cqe_alloc_reg = NULL;
2201         enum i40iw_status_code ret_code;
2202         u32 pble_obj_cnt;
2203         u32 arm_offset;
2204
2205         pble_obj_cnt = info->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
2206
2207         if (info->virtual_map && (info->first_pm_pbl_idx >= pble_obj_cnt))
2208                 return I40IW_ERR_INVALID_PBLE_INDEX;
2209
2210         cq->cq_pa = info->cq_base_pa;
2211         cq->dev = info->dev;
2212         cq->ceq_id = info->ceq_id;
2213         arm_offset = (info->dev->is_pf) ? I40E_PFPE_CQARM : I40E_VFPE_CQARM1;
2214         if (i40iw_get_hw_addr(cq->dev))
2215                 cqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(cq->dev) +
2216                                               arm_offset);
2217         info->cq_uk_init_info.cqe_alloc_reg = cqe_alloc_reg;
2218         ret_code = i40iw_cq_uk_init(&cq->cq_uk, &info->cq_uk_init_info);
2219         if (ret_code)
2220                 return ret_code;
2221         cq->virtual_map = info->virtual_map;
2222         cq->pbl_chunk_size = info->pbl_chunk_size;
2223         cq->ceqe_mask = info->ceqe_mask;
2224         cq->cq_type = (info->type) ? info->type : I40IW_CQ_TYPE_IWARP;
2225
2226         cq->shadow_area_pa = info->shadow_area_pa;
2227         cq->shadow_read_threshold = info->shadow_read_threshold;
2228
2229         cq->ceq_id_valid = info->ceq_id_valid;
2230         cq->tph_en = info->tph_en;
2231         cq->tph_val = info->tph_val;
2232
2233         cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
2234
2235         return 0;
2236 }
2237
2238 /**
2239  * i40iw_sc_cq_create - create completion q
2240  * @cq: cq struct
2241  * @scratch: u64 saved to be used during cqp completion
2242  * @check_overflow: flag for overflow check
2243  * @post_sq: flag for cqp db to ring
2244  */
2245 static enum i40iw_status_code i40iw_sc_cq_create(struct i40iw_sc_cq *cq,
2246                                                  u64 scratch,
2247                                                  bool check_overflow,
2248                                                  bool post_sq)
2249 {
2250         u64 *wqe;
2251         struct i40iw_sc_cqp *cqp;
2252         u64 header;
2253
2254         if (cq->cq_uk.cq_id > I40IW_MAX_CQID)
2255                 return I40IW_ERR_INVALID_CQ_ID;
2256
2257         if (cq->ceq_id > I40IW_MAX_CEQID)
2258                 return I40IW_ERR_INVALID_CEQ_ID;
2259
2260         cqp = cq->dev->cqp;
2261         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2262         if (!wqe)
2263                 return I40IW_ERR_RING_FULL;
2264
2265         set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
2266         set_64bit_val(wqe, 8, RS_64_1(cq, 1));
2267         set_64bit_val(wqe,
2268                       16,
2269                       LS_64(cq->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
2270
2271         set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa));
2272
2273         set_64bit_val(wqe, 40, cq->shadow_area_pa);
2274         set_64bit_val(wqe, 48, (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
2275         set_64bit_val(wqe, 56, LS_64(cq->tph_val, I40IW_CQPSQ_TPHVAL));
2276
2277         header = cq->cq_uk.cq_id |
2278                  LS_64((cq->ceq_id_valid ? cq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
2279                  LS_64(I40IW_CQP_OP_CREATE_CQ, I40IW_CQPSQ_OPCODE) |
2280                  LS_64(cq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
2281                  LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) |
2282                  LS_64(cq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
2283                  LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
2284                  LS_64(cq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
2285                  LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) |
2286                  LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
2287                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2288
2289         i40iw_insert_wqe_hdr(wqe, header);
2290
2291         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_CREATE WQE",
2292                         wqe, I40IW_CQP_WQE_SIZE * 8);
2293
2294         if (post_sq)
2295                 i40iw_sc_cqp_post_sq(cqp);
2296         return 0;
2297 }
2298
2299 /**
2300  * i40iw_sc_cq_destroy - destroy completion q
2301  * @cq: cq struct
2302  * @scratch: u64 saved to be used during cqp completion
2303  * @post_sq: flag for cqp db to ring
2304  */
2305 static enum i40iw_status_code i40iw_sc_cq_destroy(struct i40iw_sc_cq *cq,
2306                                                   u64 scratch,
2307                                                   bool post_sq)
2308 {
2309         struct i40iw_sc_cqp *cqp;
2310         u64 *wqe;
2311         u64 header;
2312
2313         cqp = cq->dev->cqp;
2314         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2315         if (!wqe)
2316                 return I40IW_ERR_RING_FULL;
2317         set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
2318         set_64bit_val(wqe, 8, RS_64_1(cq, 1));
2319         set_64bit_val(wqe, 40, cq->shadow_area_pa);
2320         set_64bit_val(wqe, 48, (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
2321
2322         header = cq->cq_uk.cq_id |
2323                  LS_64((cq->ceq_id_valid ? cq->ceq_id : 0), I40IW_CQPSQ_CQ_CEQID) |
2324                  LS_64(I40IW_CQP_OP_DESTROY_CQ, I40IW_CQPSQ_OPCODE) |
2325                  LS_64(cq->pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
2326                  LS_64(cq->virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
2327                  LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
2328                  LS_64(cq->ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
2329                  LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) |
2330                  LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
2331                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2332
2333         i40iw_insert_wqe_hdr(wqe, header);
2334
2335         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_DESTROY WQE",
2336                         wqe, I40IW_CQP_WQE_SIZE * 8);
2337
2338         if (post_sq)
2339                 i40iw_sc_cqp_post_sq(cqp);
2340         return 0;
2341 }
2342
2343 /**
2344  * i40iw_sc_cq_modify - modify a Completion Queue
2345  * @cq: cq struct
2346  * @info: modification info struct
2347  * @scratch:
2348  * @post_sq: flag to post to sq
2349  */
2350 static enum i40iw_status_code i40iw_sc_cq_modify(struct i40iw_sc_cq *cq,
2351                                                  struct i40iw_modify_cq_info *info,
2352                                                  u64 scratch,
2353                                                  bool post_sq)
2354 {
2355         struct i40iw_sc_cqp *cqp;
2356         u64 *wqe;
2357         u64 header;
2358         u32 cq_size, ceq_id, first_pm_pbl_idx;
2359         u8 pbl_chunk_size;
2360         bool virtual_map, ceq_id_valid, check_overflow;
2361         u32 pble_obj_cnt;
2362
2363         if (info->ceq_valid && (info->ceq_id > I40IW_MAX_CEQID))
2364                 return I40IW_ERR_INVALID_CEQ_ID;
2365
2366         pble_obj_cnt = cq->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
2367
2368         if (info->cq_resize && info->virtual_map &&
2369             (info->first_pm_pbl_idx >= pble_obj_cnt))
2370                 return I40IW_ERR_INVALID_PBLE_INDEX;
2371
2372         cqp = cq->dev->cqp;
2373         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2374         if (!wqe)
2375                 return I40IW_ERR_RING_FULL;
2376
2377         cq->pbl_list = info->pbl_list;
2378         cq->cq_pa = info->cq_pa;
2379         cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
2380
2381         cq_size = info->cq_resize ? info->cq_size : cq->cq_uk.cq_size;
2382         if (info->ceq_change) {
2383                 ceq_id_valid = true;
2384                 ceq_id = info->ceq_id;
2385         } else {
2386                 ceq_id_valid = cq->ceq_id_valid;
2387                 ceq_id = ceq_id_valid ? cq->ceq_id : 0;
2388         }
2389         virtual_map = info->cq_resize ? info->virtual_map : cq->virtual_map;
2390         first_pm_pbl_idx = (info->cq_resize ?
2391                             (info->virtual_map ? info->first_pm_pbl_idx : 0) :
2392                             (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
2393         pbl_chunk_size = (info->cq_resize ?
2394                           (info->virtual_map ? info->pbl_chunk_size : 0) :
2395                           (cq->virtual_map ? cq->pbl_chunk_size : 0));
2396         check_overflow = info->check_overflow_change ? info->check_overflow :
2397                          cq->check_overflow;
2398         cq->cq_uk.cq_size = cq_size;
2399         cq->ceq_id_valid = ceq_id_valid;
2400         cq->ceq_id = ceq_id;
2401         cq->virtual_map = virtual_map;
2402         cq->first_pm_pbl_idx = first_pm_pbl_idx;
2403         cq->pbl_chunk_size = pbl_chunk_size;
2404         cq->check_overflow = check_overflow;
2405
2406         set_64bit_val(wqe, 0, cq_size);
2407         set_64bit_val(wqe, 8, RS_64_1(cq, 1));
2408         set_64bit_val(wqe, 16,
2409                       LS_64(info->shadow_read_threshold, I40IW_CQPSQ_CQ_SHADOW_READ_THRESHOLD));
2410         set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa));
2411         set_64bit_val(wqe, 40, cq->shadow_area_pa);
2412         set_64bit_val(wqe, 48, (cq->virtual_map ? first_pm_pbl_idx : 0));
2413         set_64bit_val(wqe, 56, LS_64(cq->tph_val, I40IW_CQPSQ_TPHVAL));
2414
2415         header = cq->cq_uk.cq_id |
2416                  LS_64(ceq_id, I40IW_CQPSQ_CQ_CEQID) |
2417                  LS_64(I40IW_CQP_OP_MODIFY_CQ, I40IW_CQPSQ_OPCODE) |
2418                  LS_64(info->cq_resize, I40IW_CQPSQ_CQ_CQRESIZE) |
2419                  LS_64(pbl_chunk_size, I40IW_CQPSQ_CQ_LPBLSIZE) |
2420                  LS_64(check_overflow, I40IW_CQPSQ_CQ_CHKOVERFLOW) |
2421                  LS_64(virtual_map, I40IW_CQPSQ_CQ_VIRTMAP) |
2422                  LS_64(cq->ceqe_mask, I40IW_CQPSQ_CQ_ENCEQEMASK) |
2423                  LS_64(ceq_id_valid, I40IW_CQPSQ_CQ_CEQIDVALID) |
2424                  LS_64(cq->tph_en, I40IW_CQPSQ_TPHEN) |
2425                  LS_64(cq->cq_uk.avoid_mem_cflct, I40IW_CQPSQ_CQ_AVOIDMEMCNFLCT) |
2426                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2427
2428         i40iw_insert_wqe_hdr(wqe, header);
2429
2430         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "CQ_MODIFY WQE",
2431                         wqe, I40IW_CQP_WQE_SIZE * 8);
2432
2433         if (post_sq)
2434                 i40iw_sc_cqp_post_sq(cqp);
2435         return 0;
2436 }
2437
2438 /**
2439  * i40iw_sc_qp_init - initialize qp
2440  * @qp: sc qp
2441  * @info: initialization qp info
2442  */
2443 static enum i40iw_status_code i40iw_sc_qp_init(struct i40iw_sc_qp *qp,
2444                                                struct i40iw_qp_init_info *info)
2445 {
2446         u32 __iomem *wqe_alloc_reg = NULL;
2447         enum i40iw_status_code ret_code;
2448         u32 pble_obj_cnt;
2449         u8 wqe_size;
2450         u32 offset;
2451
2452         qp->dev = info->pd->dev;
2453         qp->vsi = info->vsi;
2454         qp->sq_pa = info->sq_pa;
2455         qp->rq_pa = info->rq_pa;
2456         qp->hw_host_ctx_pa = info->host_ctx_pa;
2457         qp->q2_pa = info->q2_pa;
2458         qp->shadow_area_pa = info->shadow_area_pa;
2459
2460         qp->q2_buf = info->q2;
2461         qp->pd = info->pd;
2462         qp->hw_host_ctx = info->host_ctx;
2463         offset = (qp->pd->dev->is_pf) ? I40E_PFPE_WQEALLOC : I40E_VFPE_WQEALLOC1;
2464         if (i40iw_get_hw_addr(qp->pd->dev))
2465                 wqe_alloc_reg = (u32 __iomem *)(i40iw_get_hw_addr(qp->pd->dev) +
2466                                               offset);
2467
2468         info->qp_uk_init_info.wqe_alloc_reg = wqe_alloc_reg;
2469         info->qp_uk_init_info.abi_ver = qp->pd->abi_ver;
2470         ret_code = i40iw_qp_uk_init(&qp->qp_uk, &info->qp_uk_init_info);
2471         if (ret_code)
2472                 return ret_code;
2473         qp->virtual_map = info->virtual_map;
2474
2475         pble_obj_cnt = info->pd->dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
2476
2477         if ((info->virtual_map && (info->sq_pa >= pble_obj_cnt)) ||
2478             (info->virtual_map && (info->rq_pa >= pble_obj_cnt)))
2479                 return I40IW_ERR_INVALID_PBLE_INDEX;
2480
2481         qp->llp_stream_handle = (void *)(-1);
2482         qp->qp_type = (info->type) ? info->type : I40IW_QP_TYPE_IWARP;
2483
2484         qp->hw_sq_size = i40iw_get_encoded_wqe_size(qp->qp_uk.sq_ring.size,
2485                                                     false);
2486         i40iw_debug(qp->dev, I40IW_DEBUG_WQE, "%s: hw_sq_size[%04d] sq_ring.size[%04d]\n",
2487                     __func__, qp->hw_sq_size, qp->qp_uk.sq_ring.size);
2488
2489         switch (qp->pd->abi_ver) {
2490         case 4:
2491                 ret_code = i40iw_fragcnt_to_wqesize_rq(qp->qp_uk.max_rq_frag_cnt,
2492                                                        &wqe_size);
2493                 if (ret_code)
2494                         return ret_code;
2495                 break;
2496         case 5: /* fallthrough until next ABI version */
2497         default:
2498                 if (qp->qp_uk.max_rq_frag_cnt > I40IW_MAX_WQ_FRAGMENT_COUNT)
2499                         return I40IW_ERR_INVALID_FRAG_COUNT;
2500                 wqe_size = I40IW_MAX_WQE_SIZE_RQ;
2501                 break;
2502         }
2503         qp->hw_rq_size = i40iw_get_encoded_wqe_size(qp->qp_uk.rq_size *
2504                                 (wqe_size / I40IW_QP_WQE_MIN_SIZE), false);
2505         i40iw_debug(qp->dev, I40IW_DEBUG_WQE,
2506                     "%s: hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\n",
2507                     __func__, qp->hw_rq_size, qp->qp_uk.rq_size, wqe_size);
2508         qp->sq_tph_val = info->sq_tph_val;
2509         qp->rq_tph_val = info->rq_tph_val;
2510         qp->sq_tph_en = info->sq_tph_en;
2511         qp->rq_tph_en = info->rq_tph_en;
2512         qp->rcv_tph_en = info->rcv_tph_en;
2513         qp->xmit_tph_en = info->xmit_tph_en;
2514         qp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle;
2515
2516         return 0;
2517 }
2518
2519 /**
2520  * i40iw_sc_qp_create - create qp
2521  * @qp: sc qp
2522  * @info: qp create info
2523  * @scratch: u64 saved to be used during cqp completion
2524  * @post_sq: flag for cqp db to ring
2525  */
2526 static enum i40iw_status_code i40iw_sc_qp_create(
2527                                 struct i40iw_sc_qp *qp,
2528                                 struct i40iw_create_qp_info *info,
2529                                 u64 scratch,
2530                                 bool post_sq)
2531 {
2532         struct i40iw_sc_cqp *cqp;
2533         u64 *wqe;
2534         u64 header;
2535
2536         if ((qp->qp_uk.qp_id < I40IW_MIN_IW_QP_ID) ||
2537             (qp->qp_uk.qp_id > I40IW_MAX_IW_QP_ID))
2538                 return I40IW_ERR_INVALID_QP_ID;
2539
2540         cqp = qp->pd->dev->cqp;
2541         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2542         if (!wqe)
2543                 return I40IW_ERR_RING_FULL;
2544
2545         set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
2546
2547         set_64bit_val(wqe, 40, qp->shadow_area_pa);
2548
2549         header = qp->qp_uk.qp_id |
2550                  LS_64(I40IW_CQP_OP_CREATE_QP, I40IW_CQPSQ_OPCODE) |
2551                  LS_64((info->ord_valid ? 1 : 0), I40IW_CQPSQ_QP_ORDVALID) |
2552                  LS_64(info->tcp_ctx_valid, I40IW_CQPSQ_QP_TOECTXVALID) |
2553                  LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) |
2554                  LS_64(qp->virtual_map, I40IW_CQPSQ_QP_VQ) |
2555                  LS_64(info->cq_num_valid, I40IW_CQPSQ_QP_CQNUMVALID) |
2556                  LS_64(info->arp_cache_idx_valid, I40IW_CQPSQ_QP_ARPTABIDXVALID) |
2557                  LS_64(info->next_iwarp_state, I40IW_CQPSQ_QP_NEXTIWSTATE) |
2558                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2559
2560         i40iw_insert_wqe_hdr(wqe, header);
2561         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_CREATE WQE",
2562                         wqe, I40IW_CQP_WQE_SIZE * 8);
2563
2564         if (post_sq)
2565                 i40iw_sc_cqp_post_sq(cqp);
2566         return 0;
2567 }
2568
2569 /**
2570  * i40iw_sc_qp_modify - modify qp cqp wqe
2571  * @qp: sc qp
2572  * @info: modify qp info
2573  * @scratch: u64 saved to be used during cqp completion
2574  * @post_sq: flag for cqp db to ring
2575  */
2576 static enum i40iw_status_code i40iw_sc_qp_modify(
2577                                 struct i40iw_sc_qp *qp,
2578                                 struct i40iw_modify_qp_info *info,
2579                                 u64 scratch,
2580                                 bool post_sq)
2581 {
2582         u64 *wqe;
2583         struct i40iw_sc_cqp *cqp;
2584         u64 header;
2585         u8 term_actions = 0;
2586         u8 term_len = 0;
2587
2588         cqp = qp->pd->dev->cqp;
2589         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2590         if (!wqe)
2591                 return I40IW_ERR_RING_FULL;
2592         if (info->next_iwarp_state == I40IW_QP_STATE_TERMINATE) {
2593                 if (info->dont_send_fin)
2594                         term_actions += I40IWQP_TERM_SEND_TERM_ONLY;
2595                 if (info->dont_send_term)
2596                         term_actions += I40IWQP_TERM_SEND_FIN_ONLY;
2597                 if ((term_actions == I40IWQP_TERM_SEND_TERM_AND_FIN) ||
2598                     (term_actions == I40IWQP_TERM_SEND_TERM_ONLY))
2599                         term_len = info->termlen;
2600         }
2601
2602         set_64bit_val(wqe,
2603                       8,
2604                       LS_64(term_len, I40IW_CQPSQ_QP_TERMLEN));
2605
2606         set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
2607         set_64bit_val(wqe, 40, qp->shadow_area_pa);
2608
2609         header = qp->qp_uk.qp_id |
2610                  LS_64(I40IW_CQP_OP_MODIFY_QP, I40IW_CQPSQ_OPCODE) |
2611                  LS_64(info->ord_valid, I40IW_CQPSQ_QP_ORDVALID) |
2612                  LS_64(info->tcp_ctx_valid, I40IW_CQPSQ_QP_TOECTXVALID) |
2613                  LS_64(info->cached_var_valid, I40IW_CQPSQ_QP_CACHEDVARVALID) |
2614                  LS_64(qp->virtual_map, I40IW_CQPSQ_QP_VQ) |
2615                  LS_64(info->cq_num_valid, I40IW_CQPSQ_QP_CQNUMVALID) |
2616                  LS_64(info->force_loopback, I40IW_CQPSQ_QP_FORCELOOPBACK) |
2617                  LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) |
2618                  LS_64(info->remove_hash_idx, I40IW_CQPSQ_QP_REMOVEHASHENTRY) |
2619                  LS_64(term_actions, I40IW_CQPSQ_QP_TERMACT) |
2620                  LS_64(info->reset_tcp_conn, I40IW_CQPSQ_QP_RESETCON) |
2621                  LS_64(info->arp_cache_idx_valid, I40IW_CQPSQ_QP_ARPTABIDXVALID) |
2622                  LS_64(info->next_iwarp_state, I40IW_CQPSQ_QP_NEXTIWSTATE) |
2623                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2624
2625         i40iw_insert_wqe_hdr(wqe, header);
2626
2627         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_MODIFY WQE",
2628                         wqe, I40IW_CQP_WQE_SIZE * 8);
2629
2630         if (post_sq)
2631                 i40iw_sc_cqp_post_sq(cqp);
2632         return 0;
2633 }
2634
2635 /**
2636  * i40iw_sc_qp_destroy - cqp destroy qp
2637  * @qp: sc qp
2638  * @scratch: u64 saved to be used during cqp completion
2639  * @remove_hash_idx: flag if to remove hash idx
2640  * @ignore_mw_bnd: memory window bind flag
2641  * @post_sq: flag for cqp db to ring
2642  */
2643 static enum i40iw_status_code i40iw_sc_qp_destroy(
2644                                         struct i40iw_sc_qp *qp,
2645                                         u64 scratch,
2646                                         bool remove_hash_idx,
2647                                         bool ignore_mw_bnd,
2648                                         bool post_sq)
2649 {
2650         u64 *wqe;
2651         struct i40iw_sc_cqp *cqp;
2652         u64 header;
2653
2654         i40iw_qp_rem_qos(qp);
2655         cqp = qp->pd->dev->cqp;
2656         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2657         if (!wqe)
2658                 return I40IW_ERR_RING_FULL;
2659         set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
2660         set_64bit_val(wqe, 40, qp->shadow_area_pa);
2661
2662         header = qp->qp_uk.qp_id |
2663                  LS_64(I40IW_CQP_OP_DESTROY_QP, I40IW_CQPSQ_OPCODE) |
2664                  LS_64(qp->qp_type, I40IW_CQPSQ_QP_QPTYPE) |
2665                  LS_64(ignore_mw_bnd, I40IW_CQPSQ_QP_IGNOREMWBOUND) |
2666                  LS_64(remove_hash_idx, I40IW_CQPSQ_QP_REMOVEHASHENTRY) |
2667                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2668
2669         i40iw_insert_wqe_hdr(wqe, header);
2670         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_DESTROY WQE",
2671                         wqe, I40IW_CQP_WQE_SIZE * 8);
2672
2673         if (post_sq)
2674                 i40iw_sc_cqp_post_sq(cqp);
2675         return 0;
2676 }
2677
2678 /**
2679  * i40iw_sc_qp_flush_wqes - flush qp's wqe
2680  * @qp: sc qp
2681  * @info: dlush information
2682  * @scratch: u64 saved to be used during cqp completion
2683  * @post_sq: flag for cqp db to ring
2684  */
2685 static enum i40iw_status_code i40iw_sc_qp_flush_wqes(
2686                                 struct i40iw_sc_qp *qp,
2687                                 struct i40iw_qp_flush_info *info,
2688                                 u64 scratch,
2689                                 bool post_sq)
2690 {
2691         u64 temp = 0;
2692         u64 *wqe;
2693         struct i40iw_sc_cqp *cqp;
2694         u64 header;
2695         bool flush_sq = false, flush_rq = false;
2696
2697         if (info->rq && !qp->flush_rq)
2698                 flush_rq = true;
2699
2700         if (info->sq && !qp->flush_sq)
2701                 flush_sq = true;
2702
2703         qp->flush_sq |= flush_sq;
2704         qp->flush_rq |= flush_rq;
2705         if (!flush_sq && !flush_rq)
2706                 return 0;
2707
2708         cqp = qp->pd->dev->cqp;
2709         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2710         if (!wqe)
2711                 return I40IW_ERR_RING_FULL;
2712         if (info->userflushcode) {
2713                 if (flush_rq) {
2714                         temp |= LS_64(info->rq_minor_code, I40IW_CQPSQ_FWQE_RQMNERR) |
2715                                 LS_64(info->rq_major_code, I40IW_CQPSQ_FWQE_RQMJERR);
2716                 }
2717                 if (flush_sq) {
2718                         temp |= LS_64(info->sq_minor_code, I40IW_CQPSQ_FWQE_SQMNERR) |
2719                                 LS_64(info->sq_major_code, I40IW_CQPSQ_FWQE_SQMJERR);
2720                 }
2721         }
2722         set_64bit_val(wqe, 16, temp);
2723
2724         temp = (info->generate_ae) ?
2725                 info->ae_code | LS_64(info->ae_source, I40IW_CQPSQ_FWQE_AESOURCE) : 0;
2726
2727         set_64bit_val(wqe, 8, temp);
2728
2729         header = qp->qp_uk.qp_id |
2730                  LS_64(I40IW_CQP_OP_FLUSH_WQES, I40IW_CQPSQ_OPCODE) |
2731                  LS_64(info->generate_ae, I40IW_CQPSQ_FWQE_GENERATE_AE) |
2732                  LS_64(info->userflushcode, I40IW_CQPSQ_FWQE_USERFLCODE) |
2733                  LS_64(flush_sq, I40IW_CQPSQ_FWQE_FLUSHSQ) |
2734                  LS_64(flush_rq, I40IW_CQPSQ_FWQE_FLUSHRQ) |
2735                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2736
2737         i40iw_insert_wqe_hdr(wqe, header);
2738
2739         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "QP_FLUSH WQE",
2740                         wqe, I40IW_CQP_WQE_SIZE * 8);
2741
2742         if (post_sq)
2743                 i40iw_sc_cqp_post_sq(cqp);
2744         return 0;
2745 }
2746
2747 /**
2748  * i40iw_sc_gen_ae - generate AE, currently uses flush WQE CQP OP
2749  * @qp: sc qp
2750  * @info: gen ae information
2751  * @scratch: u64 saved to be used during cqp completion
2752  * @post_sq: flag for cqp db to ring
2753  */
2754 static enum i40iw_status_code i40iw_sc_gen_ae(
2755                                 struct i40iw_sc_qp *qp,
2756                                 struct i40iw_gen_ae_info *info,
2757                                 u64 scratch,
2758                                 bool post_sq)
2759 {
2760         u64 temp;
2761         u64 *wqe;
2762         struct i40iw_sc_cqp *cqp;
2763         u64 header;
2764
2765         cqp = qp->pd->dev->cqp;
2766         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2767         if (!wqe)
2768                 return I40IW_ERR_RING_FULL;
2769
2770         temp = info->ae_code |
2771                LS_64(info->ae_source, I40IW_CQPSQ_FWQE_AESOURCE);
2772
2773         set_64bit_val(wqe, 8, temp);
2774
2775         header = qp->qp_uk.qp_id |
2776                  LS_64(I40IW_CQP_OP_GEN_AE, I40IW_CQPSQ_OPCODE) |
2777                  LS_64(1, I40IW_CQPSQ_FWQE_GENERATE_AE) |
2778                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2779
2780         i40iw_insert_wqe_hdr(wqe, header);
2781
2782         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "GEN_AE WQE",
2783                         wqe, I40IW_CQP_WQE_SIZE * 8);
2784
2785         if (post_sq)
2786                 i40iw_sc_cqp_post_sq(cqp);
2787         return 0;
2788 }
2789
2790 /**
2791  * i40iw_sc_qp_upload_context - upload qp's context
2792  * @dev: sc device struct
2793  * @info: upload context info ptr for return
2794  * @scratch: u64 saved to be used during cqp completion
2795  * @post_sq: flag for cqp db to ring
2796  */
2797 static enum i40iw_status_code i40iw_sc_qp_upload_context(
2798                                         struct i40iw_sc_dev *dev,
2799                                         struct i40iw_upload_context_info *info,
2800                                         u64 scratch,
2801                                         bool post_sq)
2802 {
2803         u64 *wqe;
2804         struct i40iw_sc_cqp *cqp;
2805         u64 header;
2806
2807         cqp = dev->cqp;
2808         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
2809         if (!wqe)
2810                 return I40IW_ERR_RING_FULL;
2811         set_64bit_val(wqe, 16, info->buf_pa);
2812
2813         header = LS_64(info->qp_id, I40IW_CQPSQ_UCTX_QPID) |
2814                  LS_64(I40IW_CQP_OP_UPLOAD_CONTEXT, I40IW_CQPSQ_OPCODE) |
2815                  LS_64(info->qp_type, I40IW_CQPSQ_UCTX_QPTYPE) |
2816                  LS_64(info->raw_format, I40IW_CQPSQ_UCTX_RAWFORMAT) |
2817                  LS_64(info->freeze_qp, I40IW_CQPSQ_UCTX_FREEZEQP) |
2818                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
2819
2820         i40iw_insert_wqe_hdr(wqe, header);
2821
2822         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "QP_UPLOAD_CTX WQE",
2823                         wqe, I40IW_CQP_WQE_SIZE * 8);
2824
2825         if (post_sq)
2826                 i40iw_sc_cqp_post_sq(cqp);
2827         return 0;
2828 }
2829
2830 /**
2831  * i40iw_sc_qp_setctx - set qp's context
2832  * @qp: sc qp
2833  * @qp_ctx: context ptr
2834  * @info: ctx info
2835  */
2836 static enum i40iw_status_code i40iw_sc_qp_setctx(
2837                                 struct i40iw_sc_qp *qp,
2838                                 u64 *qp_ctx,
2839                                 struct i40iw_qp_host_ctx_info *info)
2840 {
2841         struct i40iwarp_offload_info *iw;
2842         struct i40iw_tcp_offload_info *tcp;
2843         struct i40iw_sc_vsi *vsi;
2844         struct i40iw_sc_dev *dev;
2845         u64 qw0, qw3, qw7 = 0;
2846
2847         iw = info->iwarp_info;
2848         tcp = info->tcp_info;
2849         vsi = qp->vsi;
2850         dev = qp->dev;
2851         if (info->add_to_qoslist) {
2852                 qp->user_pri = info->user_pri;
2853                 i40iw_qp_add_qos(qp);
2854                 i40iw_debug(qp->dev, I40IW_DEBUG_DCB, "%s qp[%d] UP[%d] qset[%d]\n",
2855                             __func__, qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle);
2856         }
2857         qw0 = LS_64(qp->qp_uk.rq_wqe_size, I40IWQPC_RQWQESIZE) |
2858               LS_64(info->err_rq_idx_valid, I40IWQPC_ERR_RQ_IDX_VALID) |
2859               LS_64(qp->rcv_tph_en, I40IWQPC_RCVTPHEN) |
2860               LS_64(qp->xmit_tph_en, I40IWQPC_XMITTPHEN) |
2861               LS_64(qp->rq_tph_en, I40IWQPC_RQTPHEN) |
2862               LS_64(qp->sq_tph_en, I40IWQPC_SQTPHEN) |
2863               LS_64(info->push_idx, I40IWQPC_PPIDX) |
2864               LS_64(info->push_mode_en, I40IWQPC_PMENA);
2865
2866         set_64bit_val(qp_ctx, 8, qp->sq_pa);
2867         set_64bit_val(qp_ctx, 16, qp->rq_pa);
2868
2869         qw3 = LS_64(qp->src_mac_addr_idx, I40IWQPC_SRCMACADDRIDX) |
2870               LS_64(qp->hw_rq_size, I40IWQPC_RQSIZE) |
2871               LS_64(qp->hw_sq_size, I40IWQPC_SQSIZE);
2872
2873         set_64bit_val(qp_ctx,
2874                       128,
2875                       LS_64(info->err_rq_idx, I40IWQPC_ERR_RQ_IDX));
2876
2877         set_64bit_val(qp_ctx,
2878                       136,
2879                       LS_64(info->send_cq_num, I40IWQPC_TXCQNUM) |
2880                       LS_64(info->rcv_cq_num, I40IWQPC_RXCQNUM));
2881
2882         set_64bit_val(qp_ctx,
2883                       168,
2884                       LS_64(info->qp_compl_ctx, I40IWQPC_QPCOMPCTX));
2885         set_64bit_val(qp_ctx,
2886                       176,
2887                       LS_64(qp->sq_tph_val, I40IWQPC_SQTPHVAL) |
2888                       LS_64(qp->rq_tph_val, I40IWQPC_RQTPHVAL) |
2889                       LS_64(qp->qs_handle, I40IWQPC_QSHANDLE) |
2890                       LS_64(vsi->exception_lan_queue, I40IWQPC_EXCEPTION_LAN_QUEUE));
2891
2892         if (info->iwarp_info_valid) {
2893                 qw0 |= LS_64(iw->ddp_ver, I40IWQPC_DDP_VER) |
2894                        LS_64(iw->rdmap_ver, I40IWQPC_RDMAP_VER);
2895
2896                 qw7 |= LS_64(iw->pd_id, I40IWQPC_PDIDX);
2897                 set_64bit_val(qp_ctx,
2898                               144,
2899                               LS_64(qp->q2_pa, I40IWQPC_Q2ADDR) |
2900                               LS_64(vsi->fcn_id, I40IWQPC_STAT_INDEX));
2901                 set_64bit_val(qp_ctx,
2902                               152,
2903                               LS_64(iw->last_byte_sent, I40IWQPC_LASTBYTESENT));
2904
2905                 set_64bit_val(qp_ctx,
2906                               160,
2907                               LS_64(iw->ord_size, I40IWQPC_ORDSIZE) |
2908                               LS_64(iw->ird_size, I40IWQPC_IRDSIZE) |
2909                               LS_64(iw->wr_rdresp_en, I40IWQPC_WRRDRSPOK) |
2910                               LS_64(iw->rd_enable, I40IWQPC_RDOK) |
2911                               LS_64(iw->snd_mark_en, I40IWQPC_SNDMARKERS) |
2912                               LS_64(iw->bind_en, I40IWQPC_BINDEN) |
2913                               LS_64(iw->fast_reg_en, I40IWQPC_FASTREGEN) |
2914                               LS_64(iw->priv_mode_en, I40IWQPC_PRIVEN) |
2915                               LS_64((((vsi->stats_fcn_id_alloc) &&
2916                                       (dev->is_pf) && (vsi->fcn_id >= I40IW_FIRST_NON_PF_STAT)) ? 1 : 0),
2917                                     I40IWQPC_USESTATSINSTANCE) |
2918                               LS_64(1, I40IWQPC_IWARPMODE) |
2919                               LS_64(iw->rcv_mark_en, I40IWQPC_RCVMARKERS) |
2920                               LS_64(iw->align_hdrs, I40IWQPC_ALIGNHDRS) |
2921                               LS_64(iw->rcv_no_mpa_crc, I40IWQPC_RCVNOMPACRC) |
2922                               LS_64(iw->rcv_mark_offset, I40IWQPC_RCVMARKOFFSET) |
2923                               LS_64(iw->snd_mark_offset, I40IWQPC_SNDMARKOFFSET));
2924         }
2925         if (info->tcp_info_valid) {
2926                 qw0 |= LS_64(tcp->ipv4, I40IWQPC_IPV4) |
2927                        LS_64(tcp->no_nagle, I40IWQPC_NONAGLE) |
2928                        LS_64(tcp->insert_vlan_tag, I40IWQPC_INSERTVLANTAG) |
2929                        LS_64(tcp->time_stamp, I40IWQPC_TIMESTAMP) |
2930                        LS_64(tcp->cwnd_inc_limit, I40IWQPC_LIMIT) |
2931                        LS_64(tcp->drop_ooo_seg, I40IWQPC_DROPOOOSEG) |
2932                        LS_64(tcp->dup_ack_thresh, I40IWQPC_DUPACK_THRESH);
2933
2934                 qw3 |= LS_64(tcp->ttl, I40IWQPC_TTL) |
2935                        LS_64(tcp->src_mac_addr_idx, I40IWQPC_SRCMACADDRIDX) |
2936                        LS_64(tcp->avoid_stretch_ack, I40IWQPC_AVOIDSTRETCHACK) |
2937                        LS_64(tcp->tos, I40IWQPC_TOS) |
2938                        LS_64(tcp->src_port, I40IWQPC_SRCPORTNUM) |
2939                        LS_64(tcp->dst_port, I40IWQPC_DESTPORTNUM);
2940
2941                 qp->src_mac_addr_idx = tcp->src_mac_addr_idx;
2942                 set_64bit_val(qp_ctx,
2943                               32,
2944                               LS_64(tcp->dest_ip_addr2, I40IWQPC_DESTIPADDR2) |
2945                               LS_64(tcp->dest_ip_addr3, I40IWQPC_DESTIPADDR3));
2946
2947                 set_64bit_val(qp_ctx,
2948                               40,
2949                               LS_64(tcp->dest_ip_addr0, I40IWQPC_DESTIPADDR0) |
2950                               LS_64(tcp->dest_ip_addr1, I40IWQPC_DESTIPADDR1));
2951
2952                 set_64bit_val(qp_ctx,
2953                               48,
2954                               LS_64(tcp->snd_mss, I40IWQPC_SNDMSS) |
2955                                 LS_64(tcp->vlan_tag, I40IWQPC_VLANTAG) |
2956                                 LS_64(tcp->arp_idx, I40IWQPC_ARPIDX));
2957
2958                 qw7 |= LS_64(tcp->flow_label, I40IWQPC_FLOWLABEL) |
2959                        LS_64(tcp->wscale, I40IWQPC_WSCALE) |
2960                        LS_64(tcp->ignore_tcp_opt, I40IWQPC_IGNORE_TCP_OPT) |
2961                        LS_64(tcp->ignore_tcp_uns_opt, I40IWQPC_IGNORE_TCP_UNS_OPT) |
2962                        LS_64(tcp->tcp_state, I40IWQPC_TCPSTATE) |
2963                        LS_64(tcp->rcv_wscale, I40IWQPC_RCVSCALE) |
2964                        LS_64(tcp->snd_wscale, I40IWQPC_SNDSCALE);
2965
2966                 set_64bit_val(qp_ctx,
2967                               72,
2968                               LS_64(tcp->time_stamp_recent, I40IWQPC_TIMESTAMP_RECENT) |
2969                               LS_64(tcp->time_stamp_age, I40IWQPC_TIMESTAMP_AGE));
2970                 set_64bit_val(qp_ctx,
2971                               80,
2972                               LS_64(tcp->snd_nxt, I40IWQPC_SNDNXT) |
2973                               LS_64(tcp->snd_wnd, I40IWQPC_SNDWND));
2974
2975                 set_64bit_val(qp_ctx,
2976                               88,
2977                               LS_64(tcp->rcv_nxt, I40IWQPC_RCVNXT) |
2978                               LS_64(tcp->rcv_wnd, I40IWQPC_RCVWND));
2979                 set_64bit_val(qp_ctx,
2980                               96,
2981                               LS_64(tcp->snd_max, I40IWQPC_SNDMAX) |
2982                               LS_64(tcp->snd_una, I40IWQPC_SNDUNA));
2983                 set_64bit_val(qp_ctx,
2984                               104,
2985                               LS_64(tcp->srtt, I40IWQPC_SRTT) |
2986                               LS_64(tcp->rtt_var, I40IWQPC_RTTVAR));
2987                 set_64bit_val(qp_ctx,
2988                               112,
2989                               LS_64(tcp->ss_thresh, I40IWQPC_SSTHRESH) |
2990                               LS_64(tcp->cwnd, I40IWQPC_CWND));
2991                 set_64bit_val(qp_ctx,
2992                               120,
2993                               LS_64(tcp->snd_wl1, I40IWQPC_SNDWL1) |
2994                               LS_64(tcp->snd_wl2, I40IWQPC_SNDWL2));
2995                 set_64bit_val(qp_ctx,
2996                               128,
2997                               LS_64(tcp->max_snd_window, I40IWQPC_MAXSNDWND) |
2998                               LS_64(tcp->rexmit_thresh, I40IWQPC_REXMIT_THRESH));
2999                 set_64bit_val(qp_ctx,
3000                               184,
3001                               LS_64(tcp->local_ipaddr3, I40IWQPC_LOCAL_IPADDR3) |
3002                               LS_64(tcp->local_ipaddr2, I40IWQPC_LOCAL_IPADDR2));
3003                 set_64bit_val(qp_ctx,
3004                               192,
3005                               LS_64(tcp->local_ipaddr1, I40IWQPC_LOCAL_IPADDR1) |
3006                               LS_64(tcp->local_ipaddr0, I40IWQPC_LOCAL_IPADDR0));
3007         }
3008
3009         set_64bit_val(qp_ctx, 0, qw0);
3010         set_64bit_val(qp_ctx, 24, qw3);
3011         set_64bit_val(qp_ctx, 56, qw7);
3012
3013         i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "QP_HOST)CTX WQE",
3014                         qp_ctx, I40IW_QP_CTX_SIZE);
3015         return 0;
3016 }
3017
3018 /**
3019  * i40iw_sc_alloc_stag - mr stag alloc
3020  * @dev: sc device struct
3021  * @info: stag info
3022  * @scratch: u64 saved to be used during cqp completion
3023  * @post_sq: flag for cqp db to ring
3024  */
3025 static enum i40iw_status_code i40iw_sc_alloc_stag(
3026                                 struct i40iw_sc_dev *dev,
3027                                 struct i40iw_allocate_stag_info *info,
3028                                 u64 scratch,
3029                                 bool post_sq)
3030 {
3031         u64 *wqe;
3032         struct i40iw_sc_cqp *cqp;
3033         u64 header;
3034         enum i40iw_page_size page_size;
3035
3036         page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
3037         cqp = dev->cqp;
3038         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3039         if (!wqe)
3040                 return I40IW_ERR_RING_FULL;
3041         set_64bit_val(wqe,
3042                       8,
3043                       LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID) |
3044                       LS_64(info->total_len, I40IW_CQPSQ_STAG_STAGLEN));
3045         set_64bit_val(wqe,
3046                       16,
3047                       LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX));
3048         set_64bit_val(wqe,
3049                       40,
3050                       LS_64(info->hmc_fcn_index, I40IW_CQPSQ_STAG_HMCFNIDX));
3051
3052         header = LS_64(I40IW_CQP_OP_ALLOC_STAG, I40IW_CQPSQ_OPCODE) |
3053                  LS_64(1, I40IW_CQPSQ_STAG_MR) |
3054                  LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |
3055                  LS_64(info->chunk_size, I40IW_CQPSQ_STAG_LPBLSIZE) |
3056                  LS_64(page_size, I40IW_CQPSQ_STAG_HPAGESIZE) |
3057                  LS_64(info->remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |
3058                  LS_64(info->use_hmc_fcn_index, I40IW_CQPSQ_STAG_USEHMCFNIDX) |
3059                  LS_64(info->use_pf_rid, I40IW_CQPSQ_STAG_USEPFRID) |
3060                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3061
3062         i40iw_insert_wqe_hdr(wqe, header);
3063
3064         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "ALLOC_STAG WQE",
3065                         wqe, I40IW_CQP_WQE_SIZE * 8);
3066
3067         if (post_sq)
3068                 i40iw_sc_cqp_post_sq(cqp);
3069         return 0;
3070 }
3071
3072 /**
3073  * i40iw_sc_mr_reg_non_shared - non-shared mr registration
3074  * @dev: sc device struct
3075  * @info: mr info
3076  * @scratch: u64 saved to be used during cqp completion
3077  * @post_sq: flag for cqp db to ring
3078  */
3079 static enum i40iw_status_code i40iw_sc_mr_reg_non_shared(
3080                                 struct i40iw_sc_dev *dev,
3081                                 struct i40iw_reg_ns_stag_info *info,
3082                                 u64 scratch,
3083                                 bool post_sq)
3084 {
3085         u64 *wqe;
3086         u64 temp;
3087         struct i40iw_sc_cqp *cqp;
3088         u64 header;
3089         u32 pble_obj_cnt;
3090         bool remote_access;
3091         u8 addr_type;
3092         enum i40iw_page_size page_size;
3093
3094         page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
3095         if (info->access_rights & (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY |
3096                                    I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY))
3097                 remote_access = true;
3098         else
3099                 remote_access = false;
3100
3101         pble_obj_cnt = dev->hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt;
3102
3103         if (info->chunk_size && (info->first_pm_pbl_index >= pble_obj_cnt))
3104                 return I40IW_ERR_INVALID_PBLE_INDEX;
3105
3106         cqp = dev->cqp;
3107         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3108         if (!wqe)
3109                 return I40IW_ERR_RING_FULL;
3110
3111         temp = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? (uintptr_t)info->va : info->fbo;
3112         set_64bit_val(wqe, 0, temp);
3113
3114         set_64bit_val(wqe,
3115                       8,
3116                       LS_64(info->total_len, I40IW_CQPSQ_STAG_STAGLEN) |
3117                       LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID));
3118
3119         set_64bit_val(wqe,
3120                       16,
3121                       LS_64(info->stag_key, I40IW_CQPSQ_STAG_KEY) |
3122                       LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX));
3123         if (!info->chunk_size) {
3124                 set_64bit_val(wqe, 32, info->reg_addr_pa);
3125                 set_64bit_val(wqe, 48, 0);
3126         } else {
3127                 set_64bit_val(wqe, 32, 0);
3128                 set_64bit_val(wqe, 48, info->first_pm_pbl_index);
3129         }
3130         set_64bit_val(wqe, 40, info->hmc_fcn_index);
3131         set_64bit_val(wqe, 56, 0);
3132
3133         addr_type = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? 1 : 0;
3134         header = LS_64(I40IW_CQP_OP_REG_MR, I40IW_CQPSQ_OPCODE) |
3135                  LS_64(1, I40IW_CQPSQ_STAG_MR) |
3136                  LS_64(info->chunk_size, I40IW_CQPSQ_STAG_LPBLSIZE) |
3137                  LS_64(page_size, I40IW_CQPSQ_STAG_HPAGESIZE) |
3138                  LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |
3139                  LS_64(remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |
3140                  LS_64(addr_type, I40IW_CQPSQ_STAG_VABASEDTO) |
3141                  LS_64(info->use_hmc_fcn_index, I40IW_CQPSQ_STAG_USEHMCFNIDX) |
3142                  LS_64(info->use_pf_rid, I40IW_CQPSQ_STAG_USEPFRID) |
3143                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3144
3145         i40iw_insert_wqe_hdr(wqe, header);
3146
3147         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MR_REG_NS WQE",
3148                         wqe, I40IW_CQP_WQE_SIZE * 8);
3149
3150         if (post_sq)
3151                 i40iw_sc_cqp_post_sq(cqp);
3152         return 0;
3153 }
3154
3155 /**
3156  * i40iw_sc_mr_reg_shared - registered shared memory region
3157  * @dev: sc device struct
3158  * @info: info for shared memory registeration
3159  * @scratch: u64 saved to be used during cqp completion
3160  * @post_sq: flag for cqp db to ring
3161  */
3162 static enum i40iw_status_code i40iw_sc_mr_reg_shared(
3163                                         struct i40iw_sc_dev *dev,
3164                                         struct i40iw_register_shared_stag *info,
3165                                         u64 scratch,
3166                                         bool post_sq)
3167 {
3168         u64 *wqe;
3169         struct i40iw_sc_cqp *cqp;
3170         u64 temp, va64, fbo, header;
3171         u32 va32;
3172         bool remote_access;
3173         u8 addr_type;
3174
3175         if (info->access_rights & (I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY |
3176                                    I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY))
3177                 remote_access = true;
3178         else
3179                 remote_access = false;
3180         cqp = dev->cqp;
3181         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3182         if (!wqe)
3183                 return I40IW_ERR_RING_FULL;
3184         va64 = (uintptr_t)(info->va);
3185         va32 = (u32)(va64 & 0x00000000FFFFFFFF);
3186         fbo = (u64)(va32 & (4096 - 1));
3187
3188         set_64bit_val(wqe,
3189                       0,
3190                       (info->addr_type == I40IW_ADDR_TYPE_VA_BASED ? (uintptr_t)info->va : fbo));
3191
3192         set_64bit_val(wqe,
3193                       8,
3194                       LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID));
3195         temp = LS_64(info->new_stag_key, I40IW_CQPSQ_STAG_KEY) |
3196                LS_64(info->new_stag_idx, I40IW_CQPSQ_STAG_IDX) |
3197                LS_64(info->parent_stag_idx, I40IW_CQPSQ_STAG_PARENTSTAGIDX);
3198         set_64bit_val(wqe, 16, temp);
3199
3200         addr_type = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? 1 : 0;
3201         header = LS_64(I40IW_CQP_OP_REG_SMR, I40IW_CQPSQ_OPCODE) |
3202                  LS_64(1, I40IW_CQPSQ_STAG_MR) |
3203                  LS_64(info->access_rights, I40IW_CQPSQ_STAG_ARIGHTS) |
3204                  LS_64(remote_access, I40IW_CQPSQ_STAG_REMACCENABLED) |
3205                  LS_64(addr_type, I40IW_CQPSQ_STAG_VABASEDTO) |
3206                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3207
3208         i40iw_insert_wqe_hdr(wqe, header);
3209
3210         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MR_REG_SHARED WQE",
3211                         wqe, I40IW_CQP_WQE_SIZE * 8);
3212
3213         if (post_sq)
3214                 i40iw_sc_cqp_post_sq(cqp);
3215         return 0;
3216 }
3217
3218 /**
3219  * i40iw_sc_dealloc_stag - deallocate stag
3220  * @dev: sc device struct
3221  * @info: dealloc stag info
3222  * @scratch: u64 saved to be used during cqp completion
3223  * @post_sq: flag for cqp db to ring
3224  */
3225 static enum i40iw_status_code i40iw_sc_dealloc_stag(
3226                                         struct i40iw_sc_dev *dev,
3227                                         struct i40iw_dealloc_stag_info *info,
3228                                         u64 scratch,
3229                                         bool post_sq)
3230 {
3231         u64 header;
3232         u64 *wqe;
3233         struct i40iw_sc_cqp *cqp;
3234
3235         cqp = dev->cqp;
3236         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3237         if (!wqe)
3238                 return I40IW_ERR_RING_FULL;
3239         set_64bit_val(wqe,
3240                       8,
3241                       LS_64(info->pd_id, I40IW_CQPSQ_STAG_PDID));
3242         set_64bit_val(wqe,
3243                       16,
3244                       LS_64(info->stag_idx, I40IW_CQPSQ_STAG_IDX));
3245
3246         header = LS_64(I40IW_CQP_OP_DEALLOC_STAG, I40IW_CQPSQ_OPCODE) |
3247                  LS_64(info->mr, I40IW_CQPSQ_STAG_MR) |
3248                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3249
3250         i40iw_insert_wqe_hdr(wqe, header);
3251
3252         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "DEALLOC_STAG WQE",
3253                         wqe, I40IW_CQP_WQE_SIZE * 8);
3254
3255         if (post_sq)
3256                 i40iw_sc_cqp_post_sq(cqp);
3257         return 0;
3258 }
3259
3260 /**
3261  * i40iw_sc_query_stag - query hardware for stag
3262  * @dev: sc device struct
3263  * @scratch: u64 saved to be used during cqp completion
3264  * @stag_index: stag index for query
3265  * @post_sq: flag for cqp db to ring
3266  */
3267 static enum i40iw_status_code i40iw_sc_query_stag(struct i40iw_sc_dev *dev,
3268                                                   u64 scratch,
3269                                                   u32 stag_index,
3270                                                   bool post_sq)
3271 {
3272         u64 header;
3273         u64 *wqe;
3274         struct i40iw_sc_cqp *cqp;
3275
3276         cqp = dev->cqp;
3277         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3278         if (!wqe)
3279                 return I40IW_ERR_RING_FULL;
3280         set_64bit_val(wqe,
3281                       16,
3282                       LS_64(stag_index, I40IW_CQPSQ_QUERYSTAG_IDX));
3283
3284         header = LS_64(I40IW_CQP_OP_QUERY_STAG, I40IW_CQPSQ_OPCODE) |
3285                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3286
3287         i40iw_insert_wqe_hdr(wqe, header);
3288
3289         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "QUERY_STAG WQE",
3290                         wqe, I40IW_CQP_WQE_SIZE * 8);
3291
3292         if (post_sq)
3293                 i40iw_sc_cqp_post_sq(cqp);
3294         return 0;
3295 }
3296
3297 /**
3298  * i40iw_sc_mw_alloc - mw allocate
3299  * @dev: sc device struct
3300  * @scratch: u64 saved to be used during cqp completion
3301  * @mw_stag_index:stag index
3302  * @pd_id: pd is for this mw
3303  * @post_sq: flag for cqp db to ring
3304  */
3305 static enum i40iw_status_code i40iw_sc_mw_alloc(
3306                                         struct i40iw_sc_dev *dev,
3307                                         u64 scratch,
3308                                         u32 mw_stag_index,
3309                                         u16 pd_id,
3310                                         bool post_sq)
3311 {
3312         u64 header;
3313         struct i40iw_sc_cqp *cqp;
3314         u64 *wqe;
3315
3316         cqp = dev->cqp;
3317         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3318         if (!wqe)
3319                 return I40IW_ERR_RING_FULL;
3320         set_64bit_val(wqe, 8, LS_64(pd_id, I40IW_CQPSQ_STAG_PDID));
3321         set_64bit_val(wqe,
3322                       16,
3323                       LS_64(mw_stag_index, I40IW_CQPSQ_STAG_IDX));
3324
3325         header = LS_64(I40IW_CQP_OP_ALLOC_STAG, I40IW_CQPSQ_OPCODE) |
3326                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3327
3328         i40iw_insert_wqe_hdr(wqe, header);
3329
3330         i40iw_debug_buf(dev, I40IW_DEBUG_WQE, "MW_ALLOC WQE",
3331                         wqe, I40IW_CQP_WQE_SIZE * 8);
3332
3333         if (post_sq)
3334                 i40iw_sc_cqp_post_sq(cqp);
3335         return 0;
3336 }
3337
3338 /**
3339  * i40iw_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp
3340  * @qp: sc qp struct
3341  * @info: fast mr info
3342  * @post_sq: flag for cqp db to ring
3343  */
3344 enum i40iw_status_code i40iw_sc_mr_fast_register(
3345                                 struct i40iw_sc_qp *qp,
3346                                 struct i40iw_fast_reg_stag_info *info,
3347                                 bool post_sq)
3348 {
3349         u64 temp, header;
3350         u64 *wqe;
3351         u32 wqe_idx;
3352         enum i40iw_page_size page_size;
3353
3354         page_size = (info->page_size == 0x200000) ? I40IW_PAGE_SIZE_2M : I40IW_PAGE_SIZE_4K;
3355         wqe = i40iw_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx, I40IW_QP_WQE_MIN_SIZE,
3356                                          0, info->wr_id);
3357         if (!wqe)
3358                 return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
3359
3360         i40iw_debug(qp->dev, I40IW_DEBUG_MR, "%s: wr_id[%llxh] wqe_idx[%04d] location[%p]\n",
3361                     __func__, info->wr_id, wqe_idx,
3362                     &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid);
3363         temp = (info->addr_type == I40IW_ADDR_TYPE_VA_BASED) ? (uintptr_t)info->va : info->fbo;
3364         set_64bit_val(wqe, 0, temp);
3365
3366         temp = RS_64(info->first_pm_pbl_index >> 16, I40IWQPSQ_FIRSTPMPBLIDXHI);
3367         set_64bit_val(wqe,
3368                       8,
3369                       LS_64(temp, I40IWQPSQ_FIRSTPMPBLIDXHI) |
3370                       LS_64(info->reg_addr_pa >> I40IWQPSQ_PBLADDR_SHIFT, I40IWQPSQ_PBLADDR));
3371
3372         set_64bit_val(wqe,
3373                       16,
3374                       info->total_len |
3375                       LS_64(info->first_pm_pbl_index, I40IWQPSQ_FIRSTPMPBLIDXLO));
3376
3377         header = LS_64(info->stag_key, I40IWQPSQ_STAGKEY) |
3378                  LS_64(info->stag_idx, I40IWQPSQ_STAGINDEX) |
3379                  LS_64(I40IWQP_OP_FAST_REGISTER, I40IWQPSQ_OPCODE) |
3380                  LS_64(info->chunk_size, I40IWQPSQ_LPBLSIZE) |
3381                  LS_64(page_size, I40IWQPSQ_HPAGESIZE) |
3382                  LS_64(info->access_rights, I40IWQPSQ_STAGRIGHTS) |
3383                  LS_64(info->addr_type, I40IWQPSQ_VABASEDTO) |
3384                  LS_64(info->read_fence, I40IWQPSQ_READFENCE) |
3385                  LS_64(info->local_fence, I40IWQPSQ_LOCALFENCE) |
3386                  LS_64(info->signaled, I40IWQPSQ_SIGCOMPL) |
3387                  LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3388
3389         i40iw_insert_wqe_hdr(wqe, header);
3390
3391         i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "FAST_REG WQE",
3392                         wqe, I40IW_QP_WQE_MIN_SIZE);
3393
3394         if (post_sq)
3395                 i40iw_qp_post_wr(&qp->qp_uk);
3396         return 0;
3397 }
3398
3399 /**
3400  * i40iw_sc_send_lsmm - send last streaming mode message
3401  * @qp: sc qp struct
3402  * @lsmm_buf: buffer with lsmm message
3403  * @size: size of lsmm buffer
3404  * @stag: stag of lsmm buffer
3405  */
3406 static void i40iw_sc_send_lsmm(struct i40iw_sc_qp *qp,
3407                                void *lsmm_buf,
3408                                u32 size,
3409                                i40iw_stag stag)
3410 {
3411         u64 *wqe;
3412         u64 header;
3413         struct i40iw_qp_uk *qp_uk;
3414
3415         qp_uk = &qp->qp_uk;
3416         wqe = qp_uk->sq_base->elem;
3417
3418         set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);
3419
3420         set_64bit_val(wqe, 8, (size | LS_64(stag, I40IWQPSQ_FRAG_STAG)));
3421
3422         set_64bit_val(wqe, 16, 0);
3423
3424         header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) |
3425                  LS_64(1, I40IWQPSQ_STREAMMODE) |
3426                  LS_64(1, I40IWQPSQ_WAITFORRCVPDU) |
3427                  LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3428
3429         i40iw_insert_wqe_hdr(wqe, header);
3430
3431         i40iw_debug_buf(qp->dev, I40IW_DEBUG_QP, "SEND_LSMM WQE",
3432                         wqe, I40IW_QP_WQE_MIN_SIZE);
3433 }
3434
3435 /**
3436  * i40iw_sc_send_lsmm_nostag - for privilege qp
3437  * @qp: sc qp struct
3438  * @lsmm_buf: buffer with lsmm message
3439  * @size: size of lsmm buffer
3440  */
3441 static void i40iw_sc_send_lsmm_nostag(struct i40iw_sc_qp *qp,
3442                                       void *lsmm_buf,
3443                                       u32 size)
3444 {
3445         u64 *wqe;
3446         u64 header;
3447         struct i40iw_qp_uk *qp_uk;
3448
3449         qp_uk = &qp->qp_uk;
3450         wqe = qp_uk->sq_base->elem;
3451
3452         set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);
3453
3454         set_64bit_val(wqe, 8, size);
3455
3456         set_64bit_val(wqe, 16, 0);
3457
3458         header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) |
3459                  LS_64(1, I40IWQPSQ_STREAMMODE) |
3460                  LS_64(1, I40IWQPSQ_WAITFORRCVPDU) |
3461                  LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3462
3463         i40iw_insert_wqe_hdr(wqe, header);
3464
3465         i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "SEND_LSMM_NOSTAG WQE",
3466                         wqe, I40IW_QP_WQE_MIN_SIZE);
3467 }
3468
3469 /**
3470  * i40iw_sc_send_rtt - send last read0 or write0
3471  * @qp: sc qp struct
3472  * @read: Do read0 or write0
3473  */
3474 static void i40iw_sc_send_rtt(struct i40iw_sc_qp *qp, bool read)
3475 {
3476         u64 *wqe;
3477         u64 header;
3478         struct i40iw_qp_uk *qp_uk;
3479
3480         qp_uk = &qp->qp_uk;
3481         wqe = qp_uk->sq_base->elem;
3482
3483         set_64bit_val(wqe, 0, 0);
3484         set_64bit_val(wqe, 8, 0);
3485         set_64bit_val(wqe, 16, 0);
3486         if (read) {
3487                 header = LS_64(0x1234, I40IWQPSQ_REMSTAG) |
3488                          LS_64(I40IWQP_OP_RDMA_READ, I40IWQPSQ_OPCODE) |
3489                          LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3490                 set_64bit_val(wqe, 8, ((u64)0xabcd << 32));
3491         } else {
3492                 header = LS_64(I40IWQP_OP_RDMA_WRITE, I40IWQPSQ_OPCODE) |
3493                          LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3494         }
3495
3496         i40iw_insert_wqe_hdr(wqe, header);
3497
3498         i40iw_debug_buf(qp->dev, I40IW_DEBUG_WQE, "RTR WQE",
3499                         wqe, I40IW_QP_WQE_MIN_SIZE);
3500 }
3501
3502 /**
3503  * i40iw_sc_post_wqe0 - send wqe with opcode
3504  * @qp: sc qp struct
3505  * @opcode: opcode to use for wqe0
3506  */
3507 static enum i40iw_status_code i40iw_sc_post_wqe0(struct i40iw_sc_qp *qp, u8 opcode)
3508 {
3509         u64 *wqe;
3510         u64 header;
3511         struct i40iw_qp_uk *qp_uk;
3512
3513         qp_uk = &qp->qp_uk;
3514         wqe = qp_uk->sq_base->elem;
3515
3516         if (!wqe)
3517                 return I40IW_ERR_QP_TOOMANY_WRS_POSTED;
3518         switch (opcode) {
3519         case I40IWQP_OP_NOP:
3520                 set_64bit_val(wqe, 0, 0);
3521                 set_64bit_val(wqe, 8, 0);
3522                 set_64bit_val(wqe, 16, 0);
3523                 header = LS_64(I40IWQP_OP_NOP, I40IWQPSQ_OPCODE) |
3524                          LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID);
3525
3526                 i40iw_insert_wqe_hdr(wqe, header);
3527                 break;
3528         case I40IWQP_OP_RDMA_SEND:
3529                 set_64bit_val(wqe, 0, 0);
3530                 set_64bit_val(wqe, 8, 0);
3531                 set_64bit_val(wqe, 16, 0);
3532                 header = LS_64(I40IWQP_OP_RDMA_SEND, I40IWQPSQ_OPCODE) |
3533                          LS_64(qp->qp_uk.swqe_polarity, I40IWQPSQ_VALID) |
3534                          LS_64(1, I40IWQPSQ_STREAMMODE) |
3535                          LS_64(1, I40IWQPSQ_WAITFORRCVPDU);
3536
3537                 i40iw_insert_wqe_hdr(wqe, header);
3538                 break;
3539         default:
3540                 i40iw_debug(qp->dev, I40IW_DEBUG_QP, "%s: Invalid WQE zero opcode\n",
3541                             __func__);
3542                 break;
3543         }
3544         return 0;
3545 }
3546
3547 /**
3548  * i40iw_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info
3549  * @dev : ptr to i40iw_dev struct
3550  * @hmc_fn_id: hmc function id
3551  */
3552 enum i40iw_status_code i40iw_sc_init_iw_hmc(struct i40iw_sc_dev *dev, u8 hmc_fn_id)
3553 {
3554         struct i40iw_hmc_info *hmc_info;
3555         struct i40iw_dma_mem query_fpm_mem;
3556         struct i40iw_virt_mem virt_mem;
3557         struct i40iw_vfdev *vf_dev = NULL;
3558         u32 mem_size;
3559         enum i40iw_status_code ret_code = 0;
3560         bool poll_registers = true;
3561         u16 iw_vf_idx;
3562         u8 wait_type;
3563
3564         if (hmc_fn_id >= I40IW_MAX_VF_FPM_ID ||
3565             (dev->hmc_fn_id != hmc_fn_id && hmc_fn_id < I40IW_FIRST_VF_FPM_ID))
3566                 return I40IW_ERR_INVALID_HMCFN_ID;
3567
3568         i40iw_debug(dev, I40IW_DEBUG_HMC, "hmc_fn_id %u, dev->hmc_fn_id %u\n", hmc_fn_id,
3569                     dev->hmc_fn_id);
3570         if (hmc_fn_id == dev->hmc_fn_id) {
3571                 hmc_info = dev->hmc_info;
3572                 query_fpm_mem.pa = dev->fpm_query_buf_pa;
3573                 query_fpm_mem.va = dev->fpm_query_buf;
3574         } else {
3575                 vf_dev = i40iw_vfdev_from_fpm(dev, hmc_fn_id);
3576                 if (!vf_dev)
3577                         return I40IW_ERR_INVALID_VF_ID;
3578
3579                 hmc_info = &vf_dev->hmc_info;
3580                 iw_vf_idx = vf_dev->iw_vf_idx;
3581                 i40iw_debug(dev, I40IW_DEBUG_HMC, "vf_dev %p, hmc_info %p, hmc_obj %p\n", vf_dev,
3582                             hmc_info, hmc_info->hmc_obj);
3583                 if (!vf_dev->fpm_query_buf) {
3584                         if (!dev->vf_fpm_query_buf[iw_vf_idx].va) {
3585                                 ret_code = i40iw_alloc_query_fpm_buf(dev,
3586                                                                      &dev->vf_fpm_query_buf[iw_vf_idx]);
3587                                 if (ret_code)
3588                                         return ret_code;
3589                         }
3590                         vf_dev->fpm_query_buf = dev->vf_fpm_query_buf[iw_vf_idx].va;
3591                         vf_dev->fpm_query_buf_pa = dev->vf_fpm_query_buf[iw_vf_idx].pa;
3592                 }
3593                 query_fpm_mem.pa = vf_dev->fpm_query_buf_pa;
3594                 query_fpm_mem.va = vf_dev->fpm_query_buf;
3595                 /**
3596                  * It is HARDWARE specific:
3597                  * this call is done by PF for VF and
3598                  * i40iw_sc_query_fpm_values needs ccq poll
3599                  * because PF ccq is already created.
3600                  */
3601                 poll_registers = false;
3602         }
3603
3604         hmc_info->hmc_fn_id = hmc_fn_id;
3605
3606         if (hmc_fn_id != dev->hmc_fn_id) {
3607                 ret_code =
3608                         i40iw_cqp_query_fpm_values_cmd(dev, &query_fpm_mem, hmc_fn_id);
3609         } else {
3610                 wait_type = poll_registers ? (u8)I40IW_CQP_WAIT_POLL_REGS :
3611                             (u8)I40IW_CQP_WAIT_POLL_CQ;
3612
3613                 ret_code = i40iw_sc_query_fpm_values(
3614                                         dev->cqp,
3615                                         0,
3616                                         hmc_info->hmc_fn_id,
3617                                         &query_fpm_mem,
3618                                         true,
3619                                         wait_type);
3620         }
3621         if (ret_code)
3622                 return ret_code;
3623
3624         /* parse the fpm_query_buf and fill hmc obj info */
3625         ret_code =
3626                 i40iw_sc_parse_fpm_query_buf((u64 *)query_fpm_mem.va,
3627                                              hmc_info,
3628                                              &dev->hmc_fpm_misc);
3629         if (ret_code)
3630                 return ret_code;
3631         i40iw_debug_buf(dev, I40IW_DEBUG_HMC, "QUERY FPM BUFFER",
3632                         query_fpm_mem.va, I40IW_QUERY_FPM_BUF_SIZE);
3633
3634         if (hmc_fn_id != dev->hmc_fn_id) {
3635                 i40iw_cqp_commit_fpm_values_cmd(dev, &query_fpm_mem, hmc_fn_id);
3636
3637                 /* parse the fpm_commit_buf and fill hmc obj info */
3638                 i40iw_sc_parse_fpm_commit_buf((u64 *)query_fpm_mem.va, hmc_info->hmc_obj, &hmc_info->sd_table.sd_cnt);
3639                 mem_size = sizeof(struct i40iw_hmc_sd_entry) *
3640                            (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index);
3641                 ret_code = i40iw_allocate_virt_mem(dev->hw, &virt_mem, mem_size);
3642                 if (ret_code)
3643                         return ret_code;
3644                 hmc_info->sd_table.sd_entry = virt_mem.va;
3645         }
3646
3647         return ret_code;
3648 }
3649
3650 /**
3651  * i40iw_sc_configure_iw_fpm() - commits hmc obj cnt values using cqp command and
3652  * populates fpm base address in hmc_info
3653  * @dev : ptr to i40iw_dev struct
3654  * @hmc_fn_id: hmc function id
3655  */
3656 static enum i40iw_status_code i40iw_sc_configure_iw_fpm(struct i40iw_sc_dev *dev,
3657                                                         u8 hmc_fn_id)
3658 {
3659         struct i40iw_hmc_info *hmc_info;
3660         struct i40iw_hmc_obj_info *obj_info;
3661         u64 *buf;
3662         struct i40iw_dma_mem commit_fpm_mem;
3663         u32 i, j;
3664         enum i40iw_status_code ret_code = 0;
3665         bool poll_registers = true;
3666         u8 wait_type;
3667
3668         if (hmc_fn_id >= I40IW_MAX_VF_FPM_ID ||
3669             (dev->hmc_fn_id != hmc_fn_id && hmc_fn_id < I40IW_FIRST_VF_FPM_ID))
3670                 return I40IW_ERR_INVALID_HMCFN_ID;
3671
3672         if (hmc_fn_id == dev->hmc_fn_id) {
3673                 hmc_info = dev->hmc_info;
3674         } else {
3675                 hmc_info = i40iw_vf_hmcinfo_from_fpm(dev, hmc_fn_id);
3676                 poll_registers = false;
3677         }
3678         if (!hmc_info)
3679                 return I40IW_ERR_BAD_PTR;
3680
3681         obj_info = hmc_info->hmc_obj;
3682         buf = dev->fpm_commit_buf;
3683
3684         /* copy cnt values in commit buf */
3685         for (i = I40IW_HMC_IW_QP, j = 0; i <= I40IW_HMC_IW_PBLE;
3686              i++, j += 8)
3687                 set_64bit_val(buf, j, (u64)obj_info[i].cnt);
3688
3689         set_64bit_val(buf, 40, 0);   /* APBVT rsvd */
3690
3691         commit_fpm_mem.pa = dev->fpm_commit_buf_pa;
3692         commit_fpm_mem.va = dev->fpm_commit_buf;
3693         wait_type = poll_registers ? (u8)I40IW_CQP_WAIT_POLL_REGS :
3694                         (u8)I40IW_CQP_WAIT_POLL_CQ;
3695         ret_code = i40iw_sc_commit_fpm_values(
3696                                         dev->cqp,
3697                                         0,
3698                                         hmc_info->hmc_fn_id,
3699                                         &commit_fpm_mem,
3700                                         true,
3701                                         wait_type);
3702
3703         /* parse the fpm_commit_buf and fill hmc obj info */
3704         if (!ret_code)
3705                 ret_code = i40iw_sc_parse_fpm_commit_buf(dev->fpm_commit_buf,
3706                                                          hmc_info->hmc_obj,
3707                                                          &hmc_info->sd_table.sd_cnt);
3708
3709         i40iw_debug_buf(dev, I40IW_DEBUG_HMC, "COMMIT FPM BUFFER",
3710                         commit_fpm_mem.va, I40IW_COMMIT_FPM_BUF_SIZE);
3711
3712         return ret_code;
3713 }
3714
3715 /**
3716  * cqp_sds_wqe_fill - fill cqp wqe doe sd
3717  * @cqp: struct for cqp hw
3718  * @info; sd info for wqe
3719  * @scratch: u64 saved to be used during cqp completion
3720  */
3721 static enum i40iw_status_code cqp_sds_wqe_fill(struct i40iw_sc_cqp *cqp,
3722                                                struct i40iw_update_sds_info *info,
3723                                                u64 scratch)
3724 {
3725         u64 data;
3726         u64 header;
3727         u64 *wqe;
3728         int mem_entries, wqe_entries;
3729         struct i40iw_dma_mem *sdbuf = &cqp->sdbuf;
3730         u64 offset;
3731         u32 wqe_idx;
3732
3733         wqe = i40iw_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
3734         if (!wqe)
3735                 return I40IW_ERR_RING_FULL;
3736
3737         I40IW_CQP_INIT_WQE(wqe);
3738         wqe_entries = (info->cnt > 3) ? 3 : info->cnt;
3739         mem_entries = info->cnt - wqe_entries;
3740
3741         header = LS_64(I40IW_CQP_OP_UPDATE_PE_SDS, I40IW_CQPSQ_OPCODE) |
3742                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID) |
3743                  LS_64(mem_entries, I40IW_CQPSQ_UPESD_ENTRY_COUNT);
3744
3745         if (mem_entries) {
3746                 offset = wqe_idx * I40IW_UPDATE_SD_BUF_SIZE;
3747                 memcpy((char *)sdbuf->va + offset, &info->entry[3],
3748                        mem_entries << 4);
3749                 data = (u64)sdbuf->pa + offset;
3750         } else {
3751                 data = 0;
3752         }
3753         data |= LS_64(info->hmc_fn_id, I40IW_CQPSQ_UPESD_HMCFNID);
3754
3755         set_64bit_val(wqe, 16, data);
3756
3757         switch (wqe_entries) {
3758         case 3:
3759                 set_64bit_val(wqe, 48,
3760                               (LS_64(info->entry[2].cmd, I40IW_CQPSQ_UPESD_SDCMD) |
3761                                         LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID)));
3762
3763                 set_64bit_val(wqe, 56, info->entry[2].data);
3764                 fallthrough;
3765         case 2:
3766                 set_64bit_val(wqe, 32,
3767                               (LS_64(info->entry[1].cmd, I40IW_CQPSQ_UPESD_SDCMD) |
3768                                         LS_64(1, I40IW_CQPSQ_UPESD_ENTRY_VALID)));
3769
3770                 set_64bit_val(wqe, 40, info->entry[1].data);
3771                 fallthrough;
3772         case 1:
3773                 set_64bit_val(wqe, 0,
3774                               LS_64(info->entry[0].cmd, I40IW_CQPSQ_UPESD_SDCMD));
3775
3776                 set_64bit_val(wqe, 8, info->entry[0].data);
3777                 break;
3778         default:
3779                 break;
3780         }
3781
3782         i40iw_insert_wqe_hdr(wqe, header);
3783
3784         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "UPDATE_PE_SDS WQE",
3785                         wqe, I40IW_CQP_WQE_SIZE * 8);
3786         return 0;
3787 }
3788
3789 /**
3790  * i40iw_update_pe_sds - cqp wqe for sd
3791  * @dev: ptr to i40iw_dev struct
3792  * @info: sd info for sd's
3793  * @scratch: u64 saved to be used during cqp completion
3794  */
3795 static enum i40iw_status_code i40iw_update_pe_sds(struct i40iw_sc_dev *dev,
3796                                                   struct i40iw_update_sds_info *info,
3797                                                   u64 scratch)
3798 {
3799         struct i40iw_sc_cqp *cqp = dev->cqp;
3800         enum i40iw_status_code ret_code;
3801
3802         ret_code = cqp_sds_wqe_fill(cqp, info, scratch);
3803         if (!ret_code)
3804                 i40iw_sc_cqp_post_sq(cqp);
3805
3806         return ret_code;
3807 }
3808
3809 /**
3810  * i40iw_update_sds_noccq - update sd before ccq created
3811  * @dev: sc device struct
3812  * @info: sd info for sd's
3813  */
3814 enum i40iw_status_code i40iw_update_sds_noccq(struct i40iw_sc_dev *dev,
3815                                               struct i40iw_update_sds_info *info)
3816 {
3817         u32 error, val, tail;
3818         struct i40iw_sc_cqp *cqp = dev->cqp;
3819         enum i40iw_status_code ret_code;
3820
3821         ret_code = cqp_sds_wqe_fill(cqp, info, 0);
3822         if (ret_code)
3823                 return ret_code;
3824         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
3825         if (error)
3826                 return I40IW_ERR_CQP_COMPL_ERROR;
3827
3828         i40iw_sc_cqp_post_sq(cqp);
3829         ret_code = i40iw_cqp_poll_registers(cqp, tail, I40IW_DONE_COUNT);
3830
3831         return ret_code;
3832 }
3833
3834 /**
3835  * i40iw_sc_suspend_qp - suspend qp for param change
3836  * @cqp: struct for cqp hw
3837  * @qp: sc qp struct
3838  * @scratch: u64 saved to be used during cqp completion
3839  */
3840 enum i40iw_status_code i40iw_sc_suspend_qp(struct i40iw_sc_cqp *cqp,
3841                                            struct i40iw_sc_qp *qp,
3842                                            u64 scratch)
3843 {
3844         u64 header;
3845         u64 *wqe;
3846
3847         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3848         if (!wqe)
3849                 return I40IW_ERR_RING_FULL;
3850         header = LS_64(qp->qp_uk.qp_id, I40IW_CQPSQ_SUSPENDQP_QPID) |
3851                  LS_64(I40IW_CQP_OP_SUSPEND_QP, I40IW_CQPSQ_OPCODE) |
3852                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3853
3854         i40iw_insert_wqe_hdr(wqe, header);
3855
3856         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "SUSPEND_QP WQE",
3857                         wqe, I40IW_CQP_WQE_SIZE * 8);
3858
3859         i40iw_sc_cqp_post_sq(cqp);
3860         return 0;
3861 }
3862
3863 /**
3864  * i40iw_sc_resume_qp - resume qp after suspend
3865  * @cqp: struct for cqp hw
3866  * @qp: sc qp struct
3867  * @scratch: u64 saved to be used during cqp completion
3868  */
3869 enum i40iw_status_code i40iw_sc_resume_qp(struct i40iw_sc_cqp *cqp,
3870                                           struct i40iw_sc_qp *qp,
3871                                           u64 scratch)
3872 {
3873         u64 header;
3874         u64 *wqe;
3875
3876         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3877         if (!wqe)
3878                 return I40IW_ERR_RING_FULL;
3879         set_64bit_val(wqe,
3880                       16,
3881                         LS_64(qp->qs_handle, I40IW_CQPSQ_RESUMEQP_QSHANDLE));
3882
3883         header = LS_64(qp->qp_uk.qp_id, I40IW_CQPSQ_RESUMEQP_QPID) |
3884                  LS_64(I40IW_CQP_OP_RESUME_QP, I40IW_CQPSQ_OPCODE) |
3885                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3886
3887         i40iw_insert_wqe_hdr(wqe, header);
3888
3889         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "RESUME_QP WQE",
3890                         wqe, I40IW_CQP_WQE_SIZE * 8);
3891
3892         i40iw_sc_cqp_post_sq(cqp);
3893         return 0;
3894 }
3895
3896 /**
3897  * i40iw_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages
3898  * @cqp: struct for cqp hw
3899  * @scratch: u64 saved to be used during cqp completion
3900  * @hmc_fn_id: hmc function id
3901  * @post_sq: flag for cqp db to ring
3902  * @poll_registers: flag to poll register for cqp completion
3903  */
3904 enum i40iw_status_code i40iw_sc_static_hmc_pages_allocated(
3905                                         struct i40iw_sc_cqp *cqp,
3906                                         u64 scratch,
3907                                         u8 hmc_fn_id,
3908                                         bool post_sq,
3909                                         bool poll_registers)
3910 {
3911         u64 header;
3912         u64 *wqe;
3913         u32 tail, val, error;
3914         enum i40iw_status_code ret_code = 0;
3915
3916         wqe = i40iw_sc_cqp_get_next_send_wqe(cqp, scratch);
3917         if (!wqe)
3918                 return I40IW_ERR_RING_FULL;
3919         set_64bit_val(wqe,
3920                       16,
3921                       LS_64(hmc_fn_id, I40IW_SHMC_PAGE_ALLOCATED_HMC_FN_ID));
3922
3923         header = LS_64(I40IW_CQP_OP_SHMC_PAGES_ALLOCATED, I40IW_CQPSQ_OPCODE) |
3924                  LS_64(cqp->polarity, I40IW_CQPSQ_WQEVALID);
3925
3926         i40iw_insert_wqe_hdr(wqe, header);
3927
3928         i40iw_debug_buf(cqp->dev, I40IW_DEBUG_WQE, "SHMC_PAGES_ALLOCATED WQE",
3929                         wqe, I40IW_CQP_WQE_SIZE * 8);
3930         i40iw_get_cqp_reg_info(cqp, &val, &tail, &error);
3931         if (error) {
3932                 ret_code = I40IW_ERR_CQP_COMPL_ERROR;
3933                 return ret_code;
3934         }
3935         if (post_sq) {
3936                 i40iw_sc_cqp_post_sq(cqp);
3937                 if (poll_registers)
3938                         /* check for cqp sq tail update */
3939                         ret_code = i40iw_cqp_poll_registers(cqp, tail, 1000);
3940                 else
3941                         ret_code = i40iw_sc_poll_for_cqp_op_done(cqp,
3942                                                                  I40IW_CQP_OP_SHMC_PAGES_ALLOCATED,
3943                                                                  NULL);
3944         }
3945
3946         return ret_code;
3947 }
3948
3949 /**
3950  * i40iw_ring_full - check if cqp ring is full
3951  * @cqp: struct for cqp hw
3952  */
3953 static bool i40iw_ring_full(struct i40iw_sc_cqp *cqp)
3954 {
3955         return I40IW_RING_FULL_ERR(cqp->sq_ring);
3956 }
3957
3958 /**
3959  * i40iw_est_sd - returns approximate number of SDs for HMC
3960  * @dev: sc device struct
3961  * @hmc_info: hmc structure, size and count for HMC objects
3962  */
3963 static u64 i40iw_est_sd(struct i40iw_sc_dev *dev, struct i40iw_hmc_info *hmc_info)
3964 {
3965         int i;
3966         u64 size = 0;
3967         u64 sd;
3968
3969         for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_PBLE; i++)
3970                 size += hmc_info->hmc_obj[i].cnt * hmc_info->hmc_obj[i].size;
3971
3972         if (dev->is_pf)
3973                 size += hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt * hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].size;
3974
3975         if (size & 0x1FFFFF)
3976                 sd = (size >> 21) + 1; /* add 1 for remainder */
3977         else
3978                 sd = size >> 21;
3979
3980         if (!dev->is_pf) {
3981                 /* 2MB alignment for VF PBLE HMC */
3982                 size = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt * hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].size;
3983                 if (size & 0x1FFFFF)
3984                         sd += (size >> 21) + 1; /* add 1 for remainder */
3985                 else
3986                         sd += size >> 21;
3987         }
3988
3989         return sd;
3990 }
3991
3992 /**
3993  * i40iw_config_fpm_values - configure HMC objects
3994  * @dev: sc device struct
3995  * @qp_count: desired qp count
3996  */
3997 enum i40iw_status_code i40iw_config_fpm_values(struct i40iw_sc_dev *dev, u32 qp_count)
3998 {
3999         struct i40iw_virt_mem virt_mem;
4000         u32 i, mem_size;
4001         u32 qpwantedoriginal, qpwanted, mrwanted, pblewanted;
4002         u64 sd_needed;
4003         u32 loop_count = 0;
4004
4005         struct i40iw_hmc_info *hmc_info;
4006         struct i40iw_hmc_fpm_misc *hmc_fpm_misc;
4007         enum i40iw_status_code ret_code = 0;
4008
4009         hmc_info = dev->hmc_info;
4010         hmc_fpm_misc = &dev->hmc_fpm_misc;
4011
4012         ret_code = i40iw_sc_init_iw_hmc(dev, dev->hmc_fn_id);
4013         if (ret_code) {
4014                 i40iw_debug(dev, I40IW_DEBUG_HMC,
4015                             "i40iw_sc_init_iw_hmc returned error_code = %d\n",
4016                             ret_code);
4017                 return ret_code;
4018         }
4019
4020         for (i = I40IW_HMC_IW_QP; i < I40IW_HMC_IW_MAX; i++)
4021                 hmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt;
4022         sd_needed = i40iw_est_sd(dev, hmc_info);
4023         i40iw_debug(dev, I40IW_DEBUG_HMC,
4024                     "%s: FW initial max sd_count[%08lld] first_sd_index[%04d]\n",
4025                     __func__, sd_needed, hmc_info->first_sd_index);
4026         i40iw_debug(dev, I40IW_DEBUG_HMC,
4027                     "%s: sd count %d where max sd is %d\n",
4028                     __func__, hmc_info->sd_table.sd_cnt,
4029                     hmc_fpm_misc->max_sds);
4030
4031         qpwanted = min(qp_count, hmc_info->hmc_obj[I40IW_HMC_IW_QP].max_cnt);
4032         qpwantedoriginal = qpwanted;
4033         mrwanted = hmc_info->hmc_obj[I40IW_HMC_IW_MR].max_cnt;
4034         pblewanted = hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].max_cnt;
4035
4036         i40iw_debug(dev, I40IW_DEBUG_HMC,
4037                     "req_qp=%d max_sd=%d, max_qp = %d, max_cq=%d, max_mr=%d, max_pble=%d\n",
4038                     qp_count, hmc_fpm_misc->max_sds,
4039                     hmc_info->hmc_obj[I40IW_HMC_IW_QP].max_cnt,
4040                     hmc_info->hmc_obj[I40IW_HMC_IW_CQ].max_cnt,
4041                     hmc_info->hmc_obj[I40IW_HMC_IW_MR].max_cnt,
4042                     hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].max_cnt);
4043
4044         do {
4045                 ++loop_count;
4046                 hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt = qpwanted;
4047                 hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt =
4048                         min(2 * qpwanted, hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt);
4049                 hmc_info->hmc_obj[I40IW_HMC_IW_SRQ].cnt = 0x00; /* Reserved */
4050                 hmc_info->hmc_obj[I40IW_HMC_IW_HTE].cnt =
4051                                         qpwanted * hmc_fpm_misc->ht_multiplier;
4052                 hmc_info->hmc_obj[I40IW_HMC_IW_ARP].cnt =
4053                         hmc_info->hmc_obj[I40IW_HMC_IW_ARP].max_cnt;
4054                 hmc_info->hmc_obj[I40IW_HMC_IW_APBVT_ENTRY].cnt = 1;
4055                 hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt = mrwanted;
4056
4057                 hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt =
4058                         roundup_pow_of_two(I40IW_MAX_WQ_ENTRIES * qpwanted);
4059                 hmc_info->hmc_obj[I40IW_HMC_IW_Q1].cnt =
4060                         roundup_pow_of_two(2 * I40IW_MAX_IRD_SIZE * qpwanted);
4061                 hmc_info->hmc_obj[I40IW_HMC_IW_XFFL].cnt =
4062                         hmc_info->hmc_obj[I40IW_HMC_IW_XF].cnt / hmc_fpm_misc->xf_block_size;
4063                 hmc_info->hmc_obj[I40IW_HMC_IW_Q1FL].cnt =
4064                         hmc_info->hmc_obj[I40IW_HMC_IW_Q1].cnt / hmc_fpm_misc->q1_block_size;
4065                 hmc_info->hmc_obj[I40IW_HMC_IW_TIMER].cnt =
4066                         ((qpwanted) / 512 + 1) * hmc_fpm_misc->timer_bucket;
4067                 hmc_info->hmc_obj[I40IW_HMC_IW_FSIMC].cnt = 0x00;
4068                 hmc_info->hmc_obj[I40IW_HMC_IW_FSIAV].cnt = 0x00;
4069                 hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt = pblewanted;
4070
4071                 /* How much memory is needed for all the objects. */
4072                 sd_needed = i40iw_est_sd(dev, hmc_info);
4073                 if ((loop_count > 1000) ||
4074                     ((!(loop_count % 10)) &&
4075                     (qpwanted > qpwantedoriginal * 2 / 3))) {
4076                         if (qpwanted > FPM_MULTIPLIER)
4077                                 qpwanted = roundup_pow_of_two(qpwanted -
4078                                                               FPM_MULTIPLIER);
4079                         qpwanted >>= 1;
4080                 }
4081                 if (mrwanted > FPM_MULTIPLIER * 10)
4082                         mrwanted -= FPM_MULTIPLIER * 10;
4083                 if (pblewanted > FPM_MULTIPLIER * 1000)
4084                         pblewanted -= FPM_MULTIPLIER * 1000;
4085         } while (sd_needed > hmc_fpm_misc->max_sds && loop_count < 2000);
4086
4087         i40iw_debug(dev, I40IW_DEBUG_HMC,
4088                     "loop_cnt=%d, sd_needed=%lld, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d\n",
4089                     loop_count, sd_needed,
4090                     hmc_info->hmc_obj[I40IW_HMC_IW_QP].cnt,
4091                     hmc_info->hmc_obj[I40IW_HMC_IW_CQ].cnt,
4092                     hmc_info->hmc_obj[I40IW_HMC_IW_MR].cnt,
4093                     hmc_info->hmc_obj[I40IW_HMC_IW_PBLE].cnt);
4094
4095         ret_code = i40iw_sc_configure_iw_fpm(dev, dev->hmc_fn_id);
4096         if (ret_code) {
4097                 i40iw_debug(dev, I40IW_DEBUG_HMC,
4098                             "configure_iw_fpm returned error_code[x%08X]\n",
4099                             i40iw_rd32(dev->hw, dev->is_pf ? I40E_PFPE_CQPERRCODES : I40E_VFPE_CQPERRCODES1));
4100                 return ret_code;
4101         }
4102
4103         mem_size = sizeof(struct i40iw_hmc_sd_entry) *
4104                    (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index + 1);
4105         ret_code = i40iw_allocate_virt_mem(dev->hw, &virt_mem, mem_size);
4106         if (ret_code) {
4107                 i40iw_debug(dev, I40IW_DEBUG_HMC,
4108                             "%s: failed to allocate memory for sd_entry buffer\n",
4109                             __func__);
4110                 return ret_code;
4111         }
4112         hmc_info->sd_table.sd_entry = virt_mem.va;
4113
4114         return ret_code;
4115 }
4116
4117 /**
4118  * i40iw_exec_cqp_cmd - execute cqp cmd when wqe are available
4119  * @dev: rdma device
4120  * @pcmdinfo: cqp command info
4121  */
4122 static enum i40iw_status_code i40iw_exec_cqp_cmd(struct i40iw_sc_dev *dev,
4123                                                  struct cqp_commands_info *pcmdinfo)
4124 {
4125         enum i40iw_status_code status;
4126         struct i40iw_dma_mem values_mem;
4127
4128         dev->cqp_cmd_stats[pcmdinfo->cqp_cmd]++;
4129         switch (pcmdinfo->cqp_cmd) {
4130         case OP_DELETE_LOCAL_MAC_IPADDR_ENTRY:
4131                 status = i40iw_sc_del_local_mac_ipaddr_entry(
4132                                 pcmdinfo->in.u.del_local_mac_ipaddr_entry.cqp,
4133                                 pcmdinfo->in.u.del_local_mac_ipaddr_entry.scratch,
4134                                 pcmdinfo->in.u.del_local_mac_ipaddr_entry.entry_idx,
4135                                 pcmdinfo->in.u.del_local_mac_ipaddr_entry.ignore_ref_count,
4136                                 pcmdinfo->post_sq);
4137                 break;
4138         case OP_CEQ_DESTROY:
4139                 status = i40iw_sc_ceq_destroy(pcmdinfo->in.u.ceq_destroy.ceq,
4140                                               pcmdinfo->in.u.ceq_destroy.scratch,
4141                                               pcmdinfo->post_sq);
4142                 break;
4143         case OP_AEQ_DESTROY:
4144                 status = i40iw_sc_aeq_destroy(pcmdinfo->in.u.aeq_destroy.aeq,
4145                                               pcmdinfo->in.u.aeq_destroy.scratch,
4146                                               pcmdinfo->post_sq);
4147
4148                 break;
4149         case OP_DELETE_ARP_CACHE_ENTRY:
4150                 status = i40iw_sc_del_arp_cache_entry(
4151                                 pcmdinfo->in.u.del_arp_cache_entry.cqp,
4152                                 pcmdinfo->in.u.del_arp_cache_entry.scratch,
4153                                 pcmdinfo->in.u.del_arp_cache_entry.arp_index,
4154                                 pcmdinfo->post_sq);
4155                 break;
4156         case OP_MANAGE_APBVT_ENTRY:
4157                 status = i40iw_sc_manage_apbvt_entry(
4158                                 pcmdinfo->in.u.manage_apbvt_entry.cqp,
4159                                 &pcmdinfo->in.u.manage_apbvt_entry.info,
4160                                 pcmdinfo->in.u.manage_apbvt_entry.scratch,
4161                                 pcmdinfo->post_sq);
4162                 break;
4163         case OP_CEQ_CREATE:
4164                 status = i40iw_sc_ceq_create(pcmdinfo->in.u.ceq_create.ceq,
4165                                              pcmdinfo->in.u.ceq_create.scratch,
4166                                              pcmdinfo->post_sq);
4167                 break;
4168         case OP_AEQ_CREATE:
4169                 status = i40iw_sc_aeq_create(pcmdinfo->in.u.aeq_create.aeq,
4170                                              pcmdinfo->in.u.aeq_create.scratch,
4171                                              pcmdinfo->post_sq);
4172                 break;
4173         case OP_ALLOC_LOCAL_MAC_IPADDR_ENTRY:
4174                 status = i40iw_sc_alloc_local_mac_ipaddr_entry(
4175                                 pcmdinfo->in.u.alloc_local_mac_ipaddr_entry.cqp,
4176                                 pcmdinfo->in.u.alloc_local_mac_ipaddr_entry.scratch,
4177                                 pcmdinfo->post_sq);
4178                 break;
4179         case OP_ADD_LOCAL_MAC_IPADDR_ENTRY:
4180                 status = i40iw_sc_add_local_mac_ipaddr_entry(
4181                                 pcmdinfo->in.u.add_local_mac_ipaddr_entry.cqp,
4182                                 &pcmdinfo->in.u.add_local_mac_ipaddr_entry.info,
4183                                 pcmdinfo->in.u.add_local_mac_ipaddr_entry.scratch,
4184                                 pcmdinfo->post_sq);
4185                 break;
4186         case OP_MANAGE_QHASH_TABLE_ENTRY:
4187                 status = i40iw_sc_manage_qhash_table_entry(
4188                                 pcmdinfo->in.u.manage_qhash_table_entry.cqp,
4189                                 &pcmdinfo->in.u.manage_qhash_table_entry.info,
4190                                 pcmdinfo->in.u.manage_qhash_table_entry.scratch,
4191                                 pcmdinfo->post_sq);
4192
4193                 break;
4194         case OP_QP_MODIFY:
4195                 status = i40iw_sc_qp_modify(
4196                                 pcmdinfo->in.u.qp_modify.qp,
4197                                 &pcmdinfo->in.u.qp_modify.info,
4198                                 pcmdinfo->in.u.qp_modify.scratch,
4199                                 pcmdinfo->post_sq);
4200
4201                 break;
4202         case OP_QP_UPLOAD_CONTEXT:
4203                 status = i40iw_sc_qp_upload_context(
4204                                 pcmdinfo->in.u.qp_upload_context.dev,
4205                                 &pcmdinfo->in.u.qp_upload_context.info,
4206                                 pcmdinfo->in.u.qp_upload_context.scratch,
4207                                 pcmdinfo->post_sq);
4208
4209                 break;
4210         case OP_CQ_CREATE:
4211                 status = i40iw_sc_cq_create(
4212                                 pcmdinfo->in.u.cq_create.cq,
4213                                 pcmdinfo->in.u.cq_create.scratch,
4214                                 pcmdinfo->in.u.cq_create.check_overflow,
4215                                 pcmdinfo->post_sq);
4216                 break;
4217         case OP_CQ_DESTROY:
4218                 status = i40iw_sc_cq_destroy(
4219                                 pcmdinfo->in.u.cq_destroy.cq,
4220                                 pcmdinfo->in.u.cq_destroy.scratch,
4221                                 pcmdinfo->post_sq);
4222
4223                 break;
4224         case OP_QP_CREATE:
4225                 status = i40iw_sc_qp_create(
4226                                 pcmdinfo->in.u.qp_create.qp,
4227                                 &pcmdinfo->in.u.qp_create.info,
4228                                 pcmdinfo->in.u.qp_create.scratch,
4229                                 pcmdinfo->post_sq);
4230                 break;
4231         case OP_QP_DESTROY:
4232                 status = i40iw_sc_qp_destroy(
4233                                 pcmdinfo->in.u.qp_destroy.qp,
4234                                 pcmdinfo->in.u.qp_destroy.scratch,
4235                                 pcmdinfo->in.u.qp_destroy.remove_hash_idx,
4236                                 pcmdinfo->in.u.qp_destroy.
4237                                 ignore_mw_bnd,
4238                                 pcmdinfo->post_sq);
4239
4240                 break;
4241         case OP_ALLOC_STAG:
4242                 status = i40iw_sc_alloc_stag(
4243                                 pcmdinfo->in.u.alloc_stag.dev,
4244                                 &pcmdinfo->in.u.alloc_stag.info,
4245                                 pcmdinfo->in.u.alloc_stag.scratch,
4246                                 pcmdinfo->post_sq);
4247                 break;
4248         case OP_MR_REG_NON_SHARED:
4249                 status = i40iw_sc_mr_reg_non_shared(
4250                                 pcmdinfo->in.u.mr_reg_non_shared.dev,
4251                                 &pcmdinfo->in.u.mr_reg_non_shared.info,
4252                                 pcmdinfo->in.u.mr_reg_non_shared.scratch,
4253                                 pcmdinfo->post_sq);
4254
4255                 break;
4256         case OP_DEALLOC_STAG:
4257                 status = i40iw_sc_dealloc_stag(
4258                                 pcmdinfo->in.u.dealloc_stag.dev,
4259                                 &pcmdinfo->in.u.dealloc_stag.info,
4260                                 pcmdinfo->in.u.dealloc_stag.scratch,
4261                                 pcmdinfo->post_sq);
4262
4263                 break;
4264         case OP_MW_ALLOC:
4265                 status = i40iw_sc_mw_alloc(
4266                                 pcmdinfo->in.u.mw_alloc.dev,
4267                                 pcmdinfo->in.u.mw_alloc.scratch,
4268                                 pcmdinfo->in.u.mw_alloc.mw_stag_index,
4269                                 pcmdinfo->in.u.mw_alloc.pd_id,
4270                                 pcmdinfo->post_sq);
4271
4272                 break;
4273         case OP_QP_FLUSH_WQES:
4274                 status = i40iw_sc_qp_flush_wqes(
4275                                 pcmdinfo->in.u.qp_flush_wqes.qp,
4276                                 &pcmdinfo->in.u.qp_flush_wqes.info,
4277                                 pcmdinfo->in.u.qp_flush_wqes.
4278                                 scratch, pcmdinfo->post_sq);
4279                 break;
4280         case OP_GEN_AE:
4281                 status = i40iw_sc_gen_ae(
4282                                 pcmdinfo->in.u.gen_ae.qp,
4283                                 &pcmdinfo->in.u.gen_ae.info,
4284                                 pcmdinfo->in.u.gen_ae.scratch,
4285                                 pcmdinfo->post_sq);
4286                 break;
4287         case OP_ADD_ARP_CACHE_ENTRY:
4288                 status = i40iw_sc_add_arp_cache_entry(
4289                                 pcmdinfo->in.u.add_arp_cache_entry.cqp,
4290                                 &pcmdinfo->in.u.add_arp_cache_entry.info,
4291                                 pcmdinfo->in.u.add_arp_cache_entry.scratch,
4292                                 pcmdinfo->post_sq);
4293                 break;
4294         case OP_MANAGE_PUSH_PAGE:
4295                 status = i40iw_sc_manage_push_page(
4296                                 pcmdinfo->in.u.manage_push_page.cqp,
4297                                 &pcmdinfo->in.u.manage_push_page.info,
4298                                 pcmdinfo->in.u.manage_push_page.scratch,
4299                                 pcmdinfo->post_sq);
4300                 break;
4301         case OP_UPDATE_PE_SDS:
4302                 /* case I40IW_CQP_OP_UPDATE_PE_SDS */
4303                 status = i40iw_update_pe_sds(
4304                                 pcmdinfo->in.u.update_pe_sds.dev,
4305                                 &pcmdinfo->in.u.update_pe_sds.info,
4306                                 pcmdinfo->in.u.update_pe_sds.
4307                                 scratch);
4308
4309                 break;
4310         case OP_MANAGE_HMC_PM_FUNC_TABLE:
4311                 status = i40iw_sc_manage_hmc_pm_func_table(
4312                                 pcmdinfo->in.u.manage_hmc_pm.dev->cqp,
4313                                 pcmdinfo->in.u.manage_hmc_pm.scratch,
4314                                 (u8)pcmdinfo->in.u.manage_hmc_pm.info.vf_id,
4315                                 pcmdinfo->in.u.manage_hmc_pm.info.free_fcn,
4316                                 true);
4317                 break;
4318         case OP_SUSPEND:
4319                 status = i40iw_sc_suspend_qp(
4320                                 pcmdinfo->in.u.suspend_resume.cqp,
4321                                 pcmdinfo->in.u.suspend_resume.qp,
4322                                 pcmdinfo->in.u.suspend_resume.scratch);
4323                 break;
4324         case OP_RESUME:
4325                 status = i40iw_sc_resume_qp(
4326                                 pcmdinfo->in.u.suspend_resume.cqp,
4327                                 pcmdinfo->in.u.suspend_resume.qp,
4328                                 pcmdinfo->in.u.suspend_resume.scratch);
4329                 break;
4330         case OP_MANAGE_VF_PBLE_BP:
4331                 status = i40iw_manage_vf_pble_bp(
4332                                 pcmdinfo->in.u.manage_vf_pble_bp.cqp,
4333                                 &pcmdinfo->in.u.manage_vf_pble_bp.info,
4334                                 pcmdinfo->in.u.manage_vf_pble_bp.scratch, true);
4335                 break;
4336         case OP_QUERY_FPM_VALUES:
4337                 values_mem.pa = pcmdinfo->in.u.query_fpm_values.fpm_values_pa;
4338                 values_mem.va = pcmdinfo->in.u.query_fpm_values.fpm_values_va;
4339                 status = i40iw_sc_query_fpm_values(
4340                                 pcmdinfo->in.u.query_fpm_values.cqp,
4341                                 pcmdinfo->in.u.query_fpm_values.scratch,
4342                                 pcmdinfo->in.u.query_fpm_values.hmc_fn_id,
4343                                 &values_mem, true, I40IW_CQP_WAIT_EVENT);
4344                 break;
4345         case OP_COMMIT_FPM_VALUES:
4346                 values_mem.pa = pcmdinfo->in.u.commit_fpm_values.fpm_values_pa;
4347                 values_mem.va = pcmdinfo->in.u.commit_fpm_values.fpm_values_va;
4348                 status = i40iw_sc_commit_fpm_values(
4349                                 pcmdinfo->in.u.commit_fpm_values.cqp,
4350                                 pcmdinfo->in.u.commit_fpm_values.scratch,
4351                                 pcmdinfo->in.u.commit_fpm_values.hmc_fn_id,
4352                                 &values_mem,
4353                                 true,
4354                                 I40IW_CQP_WAIT_EVENT);
4355                 break;
4356         case OP_QUERY_RDMA_FEATURES:
4357                 values_mem.pa = pcmdinfo->in.u.query_rdma_features.cap_pa;
4358                 values_mem.va = pcmdinfo->in.u.query_rdma_features.cap_va;
4359                 status = i40iw_sc_query_rdma_features(
4360                         pcmdinfo->in.u.query_rdma_features.cqp, &values_mem,
4361                         pcmdinfo->in.u.query_rdma_features.scratch);
4362                 break;
4363         default:
4364                 status = I40IW_NOT_SUPPORTED;
4365                 break;
4366         }
4367
4368         return status;
4369 }
4370
4371 /**
4372  * i40iw_process_cqp_cmd - process all cqp commands
4373  * @dev: sc device struct
4374  * @pcmdinfo: cqp command info
4375  */
4376 enum i40iw_status_code i40iw_process_cqp_cmd(struct i40iw_sc_dev *dev,
4377                                              struct cqp_commands_info *pcmdinfo)
4378 {
4379         enum i40iw_status_code status = 0;
4380         unsigned long flags;
4381
4382         spin_lock_irqsave(&dev->cqp_lock, flags);
4383         if (list_empty(&dev->cqp_cmd_head) && !i40iw_ring_full(dev->cqp))
4384                 status = i40iw_exec_cqp_cmd(dev, pcmdinfo);
4385         else
4386                 list_add_tail(&pcmdinfo->cqp_cmd_entry, &dev->cqp_cmd_head);
4387         spin_unlock_irqrestore(&dev->cqp_lock, flags);
4388         return status;
4389 }
4390
4391 /**
4392  * i40iw_process_bh - called from tasklet for cqp list
4393  * @dev: sc device struct
4394  */
4395 enum i40iw_status_code i40iw_process_bh(struct i40iw_sc_dev *dev)
4396 {
4397         enum i40iw_status_code status = 0;
4398         struct cqp_commands_info *pcmdinfo;
4399         unsigned long flags;
4400
4401         spin_lock_irqsave(&dev->cqp_lock, flags);
4402         while (!list_empty(&dev->cqp_cmd_head) && !i40iw_ring_full(dev->cqp)) {
4403                 pcmdinfo = (struct cqp_commands_info *)i40iw_remove_head(&dev->cqp_cmd_head);
4404
4405                 status = i40iw_exec_cqp_cmd(dev, pcmdinfo);
4406                 if (status)
4407                         break;
4408         }
4409         spin_unlock_irqrestore(&dev->cqp_lock, flags);
4410         return status;
4411 }
4412
4413 /**
4414  * i40iw_iwarp_opcode - determine if incoming is rdma layer
4415  * @info: aeq info for the packet
4416  * @pkt: packet for error
4417  */
4418 static u32 i40iw_iwarp_opcode(struct i40iw_aeqe_info *info, u8 *pkt)
4419 {
4420         __be16 *mpa;
4421         u32 opcode = 0xffffffff;
4422
4423         if (info->q2_data_written) {
4424                 mpa = (__be16 *)pkt;
4425                 opcode = ntohs(mpa[1]) & 0xf;
4426         }
4427         return opcode;
4428 }
4429
4430 /**
4431  * i40iw_locate_mpa - return pointer to mpa in the pkt
4432  * @pkt: packet with data
4433  */
4434 static u8 *i40iw_locate_mpa(u8 *pkt)
4435 {
4436         /* skip over ethernet header */
4437         pkt += I40IW_MAC_HLEN;
4438
4439         /* Skip over IP and TCP headers */
4440         pkt += 4 * (pkt[0] & 0x0f);
4441         pkt += 4 * ((pkt[12] >> 4) & 0x0f);
4442         return pkt;
4443 }
4444
4445 /**
4446  * i40iw_setup_termhdr - termhdr for terminate pkt
4447  * @qp: sc qp ptr for pkt
4448  * @hdr: term hdr
4449  * @opcode: flush opcode for termhdr
4450  * @layer_etype: error layer + error type
4451  * @err: error cod ein the header
4452  */
4453 static void i40iw_setup_termhdr(struct i40iw_sc_qp *qp,
4454                                 struct i40iw_terminate_hdr *hdr,
4455                                 enum i40iw_flush_opcode opcode,
4456                                 u8 layer_etype,
4457                                 u8 err)
4458 {
4459         qp->flush_code = opcode;
4460         hdr->layer_etype = layer_etype;
4461         hdr->error_code = err;
4462 }
4463
4464 /**
4465  * i40iw_bld_terminate_hdr - build terminate message header
4466  * @qp: qp associated with received terminate AE
4467  * @info: the struct contiaing AE information
4468  */
4469 static int i40iw_bld_terminate_hdr(struct i40iw_sc_qp *qp,
4470                                    struct i40iw_aeqe_info *info)
4471 {
4472         u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
4473         u16 ddp_seg_len;
4474         int copy_len = 0;
4475         u8 is_tagged = 0;
4476         u32 opcode;
4477         struct i40iw_terminate_hdr *termhdr;
4478
4479         termhdr = (struct i40iw_terminate_hdr *)qp->q2_buf;
4480         memset(termhdr, 0, Q2_BAD_FRAME_OFFSET);
4481
4482         if (info->q2_data_written) {
4483                 /* Use data from offending packet to fill in ddp & rdma hdrs */
4484                 pkt = i40iw_locate_mpa(pkt);
4485                 ddp_seg_len = ntohs(*(__be16 *)pkt);
4486                 if (ddp_seg_len) {
4487                         copy_len = 2;
4488                         termhdr->hdrct = DDP_LEN_FLAG;
4489                         if (pkt[2] & 0x80) {
4490                                 is_tagged = 1;
4491                                 if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
4492                                         copy_len += TERM_DDP_LEN_TAGGED;
4493                                         termhdr->hdrct |= DDP_HDR_FLAG;
4494                                 }
4495                         } else {
4496                                 if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
4497                                         copy_len += TERM_DDP_LEN_UNTAGGED;
4498                                         termhdr->hdrct |= DDP_HDR_FLAG;
4499                                 }
4500
4501                                 if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) {
4502                                         if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) {
4503                                                 copy_len += TERM_RDMA_LEN;
4504                                                 termhdr->hdrct |= RDMA_HDR_FLAG;
4505                                         }
4506                                 }
4507                         }
4508                 }
4509         }
4510
4511         opcode = i40iw_iwarp_opcode(info, pkt);
4512
4513         switch (info->ae_id) {
4514         case I40IW_AE_AMP_UNALLOCATED_STAG:
4515                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4516                 if (opcode == I40IW_OP_TYPE_RDMA_WRITE)
4517                         i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,
4518                                             (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_INV_STAG);
4519                 else
4520                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4521                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_INV_STAG);
4522                 break;
4523         case I40IW_AE_AMP_BOUNDS_VIOLATION:
4524                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4525                 if (info->q2_data_written)
4526                         i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,
4527                                             (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_BOUNDS);
4528                 else
4529                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4530                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_INV_BOUNDS);
4531                 break;
4532         case I40IW_AE_AMP_BAD_PD:
4533                 switch (opcode) {
4534                 case I40IW_OP_TYPE_RDMA_WRITE:
4535                         i40iw_setup_termhdr(qp, termhdr, FLUSH_PROT_ERR,
4536                                             (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_UNASSOC_STAG);
4537                         break;
4538                 case I40IW_OP_TYPE_SEND_INV:
4539                 case I40IW_OP_TYPE_SEND_SOL_INV:
4540                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4541                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_CANT_INV_STAG);
4542                         break;
4543                 default:
4544                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4545                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_UNASSOC_STAG);
4546                 }
4547                 break;
4548         case I40IW_AE_AMP_INVALID_STAG:
4549                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4550                 i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4551                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_INV_STAG);
4552                 break;
4553         case I40IW_AE_AMP_BAD_QP:
4554                 i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_QP_OP_ERR,
4555                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_QN);
4556                 break;
4557         case I40IW_AE_AMP_BAD_STAG_KEY:
4558         case I40IW_AE_AMP_BAD_STAG_INDEX:
4559                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4560                 switch (opcode) {
4561                 case I40IW_OP_TYPE_SEND_INV:
4562                 case I40IW_OP_TYPE_SEND_SOL_INV:
4563                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_OP_ERR,
4564                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_CANT_INV_STAG);
4565                         break;
4566                 default:
4567                         i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4568                                             (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_INV_STAG);
4569                 }
4570                 break;
4571         case I40IW_AE_AMP_RIGHTS_VIOLATION:
4572         case I40IW_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
4573         case I40IW_AE_PRIV_OPERATION_DENIED:
4574                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4575                 i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4576                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_ACCESS);
4577                 break;
4578         case I40IW_AE_AMP_TO_WRAP:
4579                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4580                 i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_ACCESS_ERR,
4581                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT, RDMAP_TO_WRAP);
4582                 break;
4583         case I40IW_AE_LLP_RECEIVED_MPA_CRC_ERROR:
4584                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4585                                     (LAYER_MPA << 4) | DDP_LLP, MPA_CRC);
4586                 break;
4587         case I40IW_AE_LLP_SEGMENT_TOO_LARGE:
4588         case I40IW_AE_LLP_SEGMENT_TOO_SMALL:
4589                 i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_LEN_ERR,
4590                                     (LAYER_DDP << 4) | DDP_CATASTROPHIC, DDP_CATASTROPHIC_LOCAL);
4591                 break;
4592         case I40IW_AE_LCE_QP_CATASTROPHIC:
4593         case I40IW_AE_DDP_NO_L_BIT:
4594                 i40iw_setup_termhdr(qp, termhdr, FLUSH_FATAL_ERR,
4595                                     (LAYER_DDP << 4) | DDP_CATASTROPHIC, DDP_CATASTROPHIC_LOCAL);
4596                 break;
4597         case I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN:
4598                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4599                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MSN_RANGE);
4600                 break;
4601         case I40IW_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
4602                 qp->eventtype = TERM_EVENT_QP_ACCESS_ERR;
4603                 i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_LEN_ERR,
4604                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_TOO_LONG);
4605                 break;
4606         case I40IW_AE_DDP_UBE_INVALID_DDP_VERSION:
4607                 if (is_tagged)
4608                         i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4609                                             (LAYER_DDP << 4) | DDP_TAGGED_BUFFER, DDP_TAGGED_INV_DDP_VER);
4610                 else
4611                         i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4612                                             (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_DDP_VER);
4613                 break;
4614         case I40IW_AE_DDP_UBE_INVALID_MO:
4615                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4616                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MO);
4617                 break;
4618         case I40IW_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
4619                 i40iw_setup_termhdr(qp, termhdr, FLUSH_REM_OP_ERR,
4620                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_MSN_NO_BUF);
4621                 break;
4622         case I40IW_AE_DDP_UBE_INVALID_QN:
4623                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4624                                     (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER, DDP_UNTAGGED_INV_QN);
4625                 break;
4626         case I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
4627                 i40iw_setup_termhdr(qp, termhdr, FLUSH_GENERAL_ERR,
4628                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_INV_RDMAP_VER);
4629                 break;
4630         case I40IW_AE_RDMAP_ROE_UNEXPECTED_OPCODE:
4631                 i40iw_setup_termhdr(qp, termhdr, FLUSH_LOC_QP_OP_ERR,
4632                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_UNEXPECTED_OP);
4633                 break;
4634         default:
4635                 i40iw_setup_termhdr(qp, termhdr, FLUSH_FATAL_ERR,
4636                                     (LAYER_RDMA << 4) | RDMAP_REMOTE_OP, RDMAP_UNSPECIFIED);
4637                 break;
4638         }
4639
4640         if (copy_len)
4641                 memcpy(termhdr + 1, pkt, copy_len);
4642
4643         return sizeof(struct i40iw_terminate_hdr) + copy_len;
4644 }
4645
4646 /**
4647  * i40iw_terminate_send_fin() - Send fin for terminate message
4648  * @qp: qp associated with received terminate AE
4649  */
4650 void i40iw_terminate_send_fin(struct i40iw_sc_qp *qp)
4651 {
4652         /* Send the fin only */
4653         i40iw_term_modify_qp(qp,
4654                              I40IW_QP_STATE_TERMINATE,
4655                              I40IWQP_TERM_SEND_FIN_ONLY,
4656                              0);
4657 }
4658
4659 /**
4660  * i40iw_terminate_connection() - Bad AE and send terminate to remote QP
4661  * @qp: qp associated with received terminate AE
4662  * @info: the struct contiaing AE information
4663  */
4664 void i40iw_terminate_connection(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info)
4665 {
4666         u8 termlen = 0;
4667
4668         if (qp->term_flags & I40IW_TERM_SENT)
4669                 return;         /* Sanity check */
4670
4671         /* Eventtype can change from bld_terminate_hdr */
4672         qp->eventtype = TERM_EVENT_QP_FATAL;
4673         termlen = i40iw_bld_terminate_hdr(qp, info);
4674         i40iw_terminate_start_timer(qp);
4675         qp->term_flags |= I40IW_TERM_SENT;
4676         i40iw_term_modify_qp(qp, I40IW_QP_STATE_TERMINATE,
4677                              I40IWQP_TERM_SEND_TERM_ONLY, termlen);
4678 }
4679
4680 /**
4681  * i40iw_terminate_received - handle terminate received AE
4682  * @qp: qp associated with received terminate AE
4683  * @info: the struct contiaing AE information
4684  */
4685 void i40iw_terminate_received(struct i40iw_sc_qp *qp, struct i40iw_aeqe_info *info)
4686 {
4687         u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
4688         __be32 *mpa;
4689         u8 ddp_ctl;
4690         u8 rdma_ctl;
4691         u16 aeq_id = 0;
4692         struct i40iw_terminate_hdr *termhdr;
4693
4694         mpa = (__be32 *)i40iw_locate_mpa(pkt);
4695         if (info->q2_data_written) {
4696                 /* did not validate the frame - do it now */
4697                 ddp_ctl = (ntohl(mpa[0]) >> 8) & 0xff;
4698                 rdma_ctl = ntohl(mpa[0]) & 0xff;
4699                 if ((ddp_ctl & 0xc0) != 0x40)
4700                         aeq_id = I40IW_AE_LCE_QP_CATASTROPHIC;
4701                 else if ((ddp_ctl & 0x03) != 1)
4702                         aeq_id = I40IW_AE_DDP_UBE_INVALID_DDP_VERSION;
4703                 else if (ntohl(mpa[2]) != 2)
4704                         aeq_id = I40IW_AE_DDP_UBE_INVALID_QN;
4705                 else if (ntohl(mpa[3]) != 1)
4706                         aeq_id = I40IW_AE_DDP_INVALID_MSN_GAP_IN_MSN;
4707                 else if (ntohl(mpa[4]) != 0)
4708                         aeq_id = I40IW_AE_DDP_UBE_INVALID_MO;
4709                 else if ((rdma_ctl & 0xc0) != 0x40)
4710                         aeq_id = I40IW_AE_RDMAP_ROE_INVALID_RDMAP_VERSION;
4711
4712                 info->ae_id = aeq_id;
4713                 if (info->ae_id) {
4714                         /* Bad terminate recvd - send back a terminate */
4715                         i40iw_terminate_connection(qp, info);
4716                         return;
4717                 }
4718         }
4719
4720         qp->term_flags |= I40IW_TERM_RCVD;
4721         qp->eventtype = TERM_EVENT_QP_FATAL;
4722         termhdr = (struct i40iw_terminate_hdr *)&mpa[5];
4723         if (termhdr->layer_etype == RDMAP_REMOTE_PROT ||
4724             termhdr->layer_etype == RDMAP_REMOTE_OP) {
4725                 i40iw_terminate_done(qp, 0);
4726         } else {
4727                 i40iw_terminate_start_timer(qp);
4728                 i40iw_terminate_send_fin(qp);
4729         }
4730 }
4731
4732 /**
4733  * i40iw_sc_vsi_init - Initialize virtual device
4734  * @vsi: pointer to the vsi structure
4735  * @info: parameters to initialize vsi
4736  **/
4737 void i40iw_sc_vsi_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_init_info *info)
4738 {
4739         int i;
4740
4741         vsi->dev = info->dev;
4742         vsi->back_vsi = info->back_vsi;
4743         vsi->mtu = info->params->mtu;
4744         vsi->exception_lan_queue = info->exception_lan_queue;
4745         i40iw_fill_qos_list(info->params->qs_handle_list);
4746
4747         for (i = 0; i < I40IW_MAX_USER_PRIORITY; i++) {
4748                 vsi->qos[i].qs_handle = info->params->qs_handle_list[i];
4749                 i40iw_debug(vsi->dev, I40IW_DEBUG_DCB, "qset[%d]: %d\n", i,
4750                             vsi->qos[i].qs_handle);
4751                 spin_lock_init(&vsi->qos[i].lock);
4752                 INIT_LIST_HEAD(&vsi->qos[i].qplist);
4753         }
4754 }
4755
4756 /**
4757  * i40iw_hw_stats_init - Initiliaze HW stats table
4758  * @stats: pestat struct
4759  * @fcn_idx: PCI fn id
4760  * @is_pf: Is it a PF?
4761  *
4762  * Populate the HW stats table with register offset addr for each
4763  * stats. And start the perioidic stats timer.
4764  */
4765 void i40iw_hw_stats_init(struct i40iw_vsi_pestat *stats, u8 fcn_idx, bool is_pf)
4766 {
4767         u32 stats_reg_offset;
4768         u32 stats_index;
4769         struct i40iw_dev_hw_stats_offsets *stats_table =
4770                 &stats->hw_stats_offsets;
4771         struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
4772
4773         if (is_pf) {
4774                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] =
4775                                 I40E_GLPES_PFIP4RXDISCARD(fcn_idx);
4776                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] =
4777                                 I40E_GLPES_PFIP4RXTRUNC(fcn_idx);
4778                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] =
4779                                 I40E_GLPES_PFIP4TXNOROUTE(fcn_idx);
4780                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] =
4781                                 I40E_GLPES_PFIP6RXDISCARD(fcn_idx);
4782                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] =
4783                                 I40E_GLPES_PFIP6RXTRUNC(fcn_idx);
4784                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] =
4785                                 I40E_GLPES_PFIP6TXNOROUTE(fcn_idx);
4786                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] =
4787                                 I40E_GLPES_PFTCPRTXSEG(fcn_idx);
4788                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] =
4789                                 I40E_GLPES_PFTCPRXOPTERR(fcn_idx);
4790                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] =
4791                                 I40E_GLPES_PFTCPRXPROTOERR(fcn_idx);
4792
4793                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] =
4794                                 I40E_GLPES_PFIP4RXOCTSLO(fcn_idx);
4795                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] =
4796                                 I40E_GLPES_PFIP4RXPKTSLO(fcn_idx);
4797                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] =
4798                                 I40E_GLPES_PFIP4RXFRAGSLO(fcn_idx);
4799                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] =
4800                                 I40E_GLPES_PFIP4RXMCPKTSLO(fcn_idx);
4801                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] =
4802                                 I40E_GLPES_PFIP4TXOCTSLO(fcn_idx);
4803                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] =
4804                                 I40E_GLPES_PFIP4TXPKTSLO(fcn_idx);
4805                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] =
4806                                 I40E_GLPES_PFIP4TXFRAGSLO(fcn_idx);
4807                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] =
4808                                 I40E_GLPES_PFIP4TXMCPKTSLO(fcn_idx);
4809                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] =
4810                                 I40E_GLPES_PFIP6RXOCTSLO(fcn_idx);
4811                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] =
4812                                 I40E_GLPES_PFIP6RXPKTSLO(fcn_idx);
4813                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] =
4814                                 I40E_GLPES_PFIP6RXFRAGSLO(fcn_idx);
4815                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] =
4816                                 I40E_GLPES_PFIP6RXMCPKTSLO(fcn_idx);
4817                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] =
4818                                 I40E_GLPES_PFIP6TXOCTSLO(fcn_idx);
4819                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
4820                                 I40E_GLPES_PFIP6TXPKTSLO(fcn_idx);
4821                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
4822                                 I40E_GLPES_PFIP6TXPKTSLO(fcn_idx);
4823                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] =
4824                                 I40E_GLPES_PFIP6TXFRAGSLO(fcn_idx);
4825                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] =
4826                                 I40E_GLPES_PFTCPRXSEGSLO(fcn_idx);
4827                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] =
4828                                 I40E_GLPES_PFTCPTXSEGLO(fcn_idx);
4829                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] =
4830                                 I40E_GLPES_PFRDMARXRDSLO(fcn_idx);
4831                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] =
4832                                 I40E_GLPES_PFRDMARXSNDSLO(fcn_idx);
4833                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] =
4834                                 I40E_GLPES_PFRDMARXWRSLO(fcn_idx);
4835                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] =
4836                                 I40E_GLPES_PFRDMATXRDSLO(fcn_idx);
4837                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] =
4838                                 I40E_GLPES_PFRDMATXSNDSLO(fcn_idx);
4839                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] =
4840                                 I40E_GLPES_PFRDMATXWRSLO(fcn_idx);
4841                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] =
4842                                 I40E_GLPES_PFRDMAVBNDLO(fcn_idx);
4843                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] =
4844                                 I40E_GLPES_PFRDMAVINVLO(fcn_idx);
4845         } else {
4846                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXDISCARD] =
4847                                 I40E_GLPES_VFIP4RXDISCARD(fcn_idx);
4848                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4RXTRUNC] =
4849                                 I40E_GLPES_VFIP4RXTRUNC(fcn_idx);
4850                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP4TXNOROUTE] =
4851                                 I40E_GLPES_VFIP4TXNOROUTE(fcn_idx);
4852                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXDISCARD] =
4853                                 I40E_GLPES_VFIP6RXDISCARD(fcn_idx);
4854                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6RXTRUNC] =
4855                                 I40E_GLPES_VFIP6RXTRUNC(fcn_idx);
4856                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_IP6TXNOROUTE] =
4857                                 I40E_GLPES_VFIP6TXNOROUTE(fcn_idx);
4858                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRTXSEG] =
4859                                 I40E_GLPES_VFTCPRTXSEG(fcn_idx);
4860                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXOPTERR] =
4861                                 I40E_GLPES_VFTCPRXOPTERR(fcn_idx);
4862                 stats_table->stats_offset_32[I40IW_HW_STAT_INDEX_TCPRXPROTOERR] =
4863                                 I40E_GLPES_VFTCPRXPROTOERR(fcn_idx);
4864
4865                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXOCTS] =
4866                                 I40E_GLPES_VFIP4RXOCTSLO(fcn_idx);
4867                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXPKTS] =
4868                                 I40E_GLPES_VFIP4RXPKTSLO(fcn_idx);
4869                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXFRAGS] =
4870                                 I40E_GLPES_VFIP4RXFRAGSLO(fcn_idx);
4871                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4RXMCPKTS] =
4872                                 I40E_GLPES_VFIP4RXMCPKTSLO(fcn_idx);
4873                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXOCTS] =
4874                                 I40E_GLPES_VFIP4TXOCTSLO(fcn_idx);
4875                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXPKTS] =
4876                                 I40E_GLPES_VFIP4TXPKTSLO(fcn_idx);
4877                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXFRAGS] =
4878                                 I40E_GLPES_VFIP4TXFRAGSLO(fcn_idx);
4879                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP4TXMCPKTS] =
4880                                 I40E_GLPES_VFIP4TXMCPKTSLO(fcn_idx);
4881                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXOCTS] =
4882                                 I40E_GLPES_VFIP6RXOCTSLO(fcn_idx);
4883                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXPKTS] =
4884                                 I40E_GLPES_VFIP6RXPKTSLO(fcn_idx);
4885                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXFRAGS] =
4886                                 I40E_GLPES_VFIP6RXFRAGSLO(fcn_idx);
4887                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6RXMCPKTS] =
4888                                 I40E_GLPES_VFIP6RXMCPKTSLO(fcn_idx);
4889                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXOCTS] =
4890                                 I40E_GLPES_VFIP6TXOCTSLO(fcn_idx);
4891                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
4892                                 I40E_GLPES_VFIP6TXPKTSLO(fcn_idx);
4893                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXPKTS] =
4894                                 I40E_GLPES_VFIP6TXPKTSLO(fcn_idx);
4895                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_IP6TXFRAGS] =
4896                                 I40E_GLPES_VFIP6TXFRAGSLO(fcn_idx);
4897                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPRXSEGS] =
4898                                 I40E_GLPES_VFTCPRXSEGSLO(fcn_idx);
4899                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_TCPTXSEG] =
4900                                 I40E_GLPES_VFTCPTXSEGLO(fcn_idx);
4901                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXRDS] =
4902                                 I40E_GLPES_VFRDMARXRDSLO(fcn_idx);
4903                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXSNDS] =
4904                                 I40E_GLPES_VFRDMARXSNDSLO(fcn_idx);
4905                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMARXWRS] =
4906                                 I40E_GLPES_VFRDMARXWRSLO(fcn_idx);
4907                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXRDS] =
4908                                 I40E_GLPES_VFRDMATXRDSLO(fcn_idx);
4909                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXSNDS] =
4910                                 I40E_GLPES_VFRDMATXSNDSLO(fcn_idx);
4911                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMATXWRS] =
4912                                 I40E_GLPES_VFRDMATXWRSLO(fcn_idx);
4913                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVBND] =
4914                                 I40E_GLPES_VFRDMAVBNDLO(fcn_idx);
4915                 stats_table->stats_offset_64[I40IW_HW_STAT_INDEX_RDMAVINV] =
4916                                 I40E_GLPES_VFRDMAVINVLO(fcn_idx);
4917         }
4918
4919         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
4920              stats_index++) {
4921                 stats_reg_offset = stats_table->stats_offset_64[stats_index];
4922                 last_rd_stats->stats_value_64[stats_index] =
4923                         readq(stats->hw->hw_addr + stats_reg_offset);
4924         }
4925
4926         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
4927              stats_index++) {
4928                 stats_reg_offset = stats_table->stats_offset_32[stats_index];
4929                 last_rd_stats->stats_value_32[stats_index] =
4930                         i40iw_rd32(stats->hw, stats_reg_offset);
4931         }
4932 }
4933
4934 /**
4935  * i40iw_hw_stats_read_32 - Read 32-bit HW stats counters and accommodates for roll-overs.
4936  * @stat: pestat struct
4937  * @index: index in HW stats table which contains offset reg-addr
4938  * @value: hw stats value
4939  */
4940 void i40iw_hw_stats_read_32(struct i40iw_vsi_pestat *stats,
4941                             enum i40iw_hw_stats_index_32b index,
4942                             u64 *value)
4943 {
4944         struct i40iw_dev_hw_stats_offsets *stats_table =
4945                 &stats->hw_stats_offsets;
4946         struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
4947         struct i40iw_dev_hw_stats *hw_stats = &stats->hw_stats;
4948         u64 new_stats_value = 0;
4949         u32 stats_reg_offset = stats_table->stats_offset_32[index];
4950
4951         new_stats_value = i40iw_rd32(stats->hw, stats_reg_offset);
4952         /*roll-over case */
4953         if (new_stats_value < last_rd_stats->stats_value_32[index])
4954                 hw_stats->stats_value_32[index] += new_stats_value;
4955         else
4956                 hw_stats->stats_value_32[index] +=
4957                         new_stats_value - last_rd_stats->stats_value_32[index];
4958         last_rd_stats->stats_value_32[index] = new_stats_value;
4959         *value = hw_stats->stats_value_32[index];
4960 }
4961
4962 /**
4963  * i40iw_hw_stats_read_64 - Read HW stats counters (greater than 32-bit) and accommodates for roll-overs.
4964  * @stats: pestat struct
4965  * @index: index in HW stats table which contains offset reg-addr
4966  * @value: hw stats value
4967  */
4968 void i40iw_hw_stats_read_64(struct i40iw_vsi_pestat *stats,
4969                             enum i40iw_hw_stats_index_64b index,
4970                             u64 *value)
4971 {
4972         struct i40iw_dev_hw_stats_offsets *stats_table =
4973                 &stats->hw_stats_offsets;
4974         struct i40iw_dev_hw_stats *last_rd_stats = &stats->last_read_hw_stats;
4975         struct i40iw_dev_hw_stats *hw_stats = &stats->hw_stats;
4976         u64 new_stats_value = 0;
4977         u32 stats_reg_offset = stats_table->stats_offset_64[index];
4978
4979         new_stats_value = readq(stats->hw->hw_addr + stats_reg_offset);
4980         /*roll-over case */
4981         if (new_stats_value < last_rd_stats->stats_value_64[index])
4982                 hw_stats->stats_value_64[index] += new_stats_value;
4983         else
4984                 hw_stats->stats_value_64[index] +=
4985                         new_stats_value - last_rd_stats->stats_value_64[index];
4986         last_rd_stats->stats_value_64[index] = new_stats_value;
4987         *value = hw_stats->stats_value_64[index];
4988 }
4989
4990 /**
4991  * i40iw_hw_stats_read_all - read all HW stat counters
4992  * @stats: pestat struct
4993  * @stats_values: hw stats structure
4994  *
4995  * Read all the HW stat counters and populates hw_stats structure
4996  * of passed-in vsi's pestat as well as copy created in stat_values.
4997  */
4998 void i40iw_hw_stats_read_all(struct i40iw_vsi_pestat *stats,
4999                              struct i40iw_dev_hw_stats *stats_values)
5000 {
5001         u32 stats_index;
5002         unsigned long flags;
5003
5004         spin_lock_irqsave(&stats->lock, flags);
5005
5006         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
5007              stats_index++)
5008                 i40iw_hw_stats_read_32(stats, stats_index,
5009                                        &stats_values->stats_value_32[stats_index]);
5010         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
5011              stats_index++)
5012                 i40iw_hw_stats_read_64(stats, stats_index,
5013                                        &stats_values->stats_value_64[stats_index]);
5014         spin_unlock_irqrestore(&stats->lock, flags);
5015 }
5016
5017 /**
5018  * i40iw_hw_stats_refresh_all - Update all HW stats structs
5019  * @stats: pestat struct
5020  *
5021  * Read all the HW stats counters to refresh values in hw_stats structure
5022  * of passed-in dev's pestat
5023  */
5024 void i40iw_hw_stats_refresh_all(struct i40iw_vsi_pestat *stats)
5025 {
5026         u64 stats_value;
5027         u32 stats_index;
5028         unsigned long flags;
5029
5030         spin_lock_irqsave(&stats->lock, flags);
5031
5032         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_32;
5033              stats_index++)
5034                 i40iw_hw_stats_read_32(stats, stats_index, &stats_value);
5035         for (stats_index = 0; stats_index < I40IW_HW_STAT_INDEX_MAX_64;
5036              stats_index++)
5037                 i40iw_hw_stats_read_64(stats, stats_index, &stats_value);
5038         spin_unlock_irqrestore(&stats->lock, flags);
5039 }
5040
5041 /**
5042  * i40iw_get_fcn_id - Return the function id
5043  * @dev: pointer to the device
5044  */
5045 static u8 i40iw_get_fcn_id(struct i40iw_sc_dev *dev)
5046 {
5047         u8 fcn_id = I40IW_INVALID_FCN_ID;
5048         u8 i;
5049
5050         for (i = I40IW_FIRST_NON_PF_STAT; i < I40IW_MAX_STATS_COUNT; i++)
5051                 if (!dev->fcn_id_array[i]) {
5052                         fcn_id = i;
5053                         dev->fcn_id_array[i] = true;
5054                         break;
5055                 }
5056         return fcn_id;
5057 }
5058
5059 /**
5060  * i40iw_vsi_stats_init - Initialize the vsi statistics
5061  * @vsi: pointer to the vsi structure
5062  * @info: The info structure used for initialization
5063  */
5064 enum i40iw_status_code i40iw_vsi_stats_init(struct i40iw_sc_vsi *vsi, struct i40iw_vsi_stats_info *info)
5065 {
5066         u8 fcn_id = info->fcn_id;
5067
5068         if (info->alloc_fcn_id)
5069                 fcn_id = i40iw_get_fcn_id(vsi->dev);
5070
5071         if (fcn_id == I40IW_INVALID_FCN_ID)
5072                 return I40IW_ERR_NOT_READY;
5073
5074         vsi->pestat = info->pestat;
5075         vsi->pestat->hw = vsi->dev->hw;
5076         vsi->pestat->vsi = vsi;
5077
5078         if (info->stats_initialize) {
5079                 i40iw_hw_stats_init(vsi->pestat, fcn_id, true);
5080                 spin_lock_init(&vsi->pestat->lock);
5081                 i40iw_hw_stats_start_timer(vsi);
5082         }
5083         vsi->stats_fcn_id_alloc = info->alloc_fcn_id;
5084         vsi->fcn_id = fcn_id;
5085         return I40IW_SUCCESS;
5086 }
5087
5088 /**
5089  * i40iw_vsi_stats_free - Free the vsi stats
5090  * @vsi: pointer to the vsi structure
5091  */
5092 void i40iw_vsi_stats_free(struct i40iw_sc_vsi *vsi)
5093 {
5094         u8 fcn_id = vsi->fcn_id;
5095
5096         if (vsi->stats_fcn_id_alloc && fcn_id < I40IW_MAX_STATS_COUNT)
5097                 vsi->dev->fcn_id_array[fcn_id] = false;
5098         i40iw_hw_stats_stop_timer(vsi);
5099 }
5100
5101 static struct i40iw_cqp_ops iw_cqp_ops = {
5102         .cqp_init = i40iw_sc_cqp_init,
5103         .cqp_create = i40iw_sc_cqp_create,
5104         .cqp_post_sq = i40iw_sc_cqp_post_sq,
5105         .cqp_get_next_send_wqe = i40iw_sc_cqp_get_next_send_wqe,
5106         .cqp_destroy = i40iw_sc_cqp_destroy,
5107         .poll_for_cqp_op_done = i40iw_sc_poll_for_cqp_op_done
5108 };
5109
5110 static struct i40iw_ccq_ops iw_ccq_ops = {
5111         .ccq_init = i40iw_sc_ccq_init,
5112         .ccq_create = i40iw_sc_ccq_create,
5113         .ccq_destroy = i40iw_sc_ccq_destroy,
5114         .ccq_create_done = i40iw_sc_ccq_create_done,
5115         .ccq_get_cqe_info = i40iw_sc_ccq_get_cqe_info,
5116         .ccq_arm = i40iw_sc_ccq_arm
5117 };
5118
5119 static struct i40iw_ceq_ops iw_ceq_ops = {
5120         .ceq_init = i40iw_sc_ceq_init,
5121         .ceq_create = i40iw_sc_ceq_create,
5122         .cceq_create_done = i40iw_sc_cceq_create_done,
5123         .cceq_destroy_done = i40iw_sc_cceq_destroy_done,
5124         .cceq_create = i40iw_sc_cceq_create,
5125         .ceq_destroy = i40iw_sc_ceq_destroy,
5126         .process_ceq = i40iw_sc_process_ceq
5127 };
5128
5129 static struct i40iw_aeq_ops iw_aeq_ops = {
5130         .aeq_init = i40iw_sc_aeq_init,
5131         .aeq_create = i40iw_sc_aeq_create,
5132         .aeq_destroy = i40iw_sc_aeq_destroy,
5133         .get_next_aeqe = i40iw_sc_get_next_aeqe,
5134         .repost_aeq_entries = i40iw_sc_repost_aeq_entries,
5135         .aeq_create_done = i40iw_sc_aeq_create_done,
5136         .aeq_destroy_done = i40iw_sc_aeq_destroy_done
5137 };
5138
5139 /* iwarp pd ops */
5140 static struct i40iw_pd_ops iw_pd_ops = {
5141         .pd_init = i40iw_sc_pd_init,
5142 };
5143
5144 static struct i40iw_priv_qp_ops iw_priv_qp_ops = {
5145         .qp_init = i40iw_sc_qp_init,
5146         .qp_create = i40iw_sc_qp_create,
5147         .qp_modify = i40iw_sc_qp_modify,
5148         .qp_destroy = i40iw_sc_qp_destroy,
5149         .qp_flush_wqes = i40iw_sc_qp_flush_wqes,
5150         .qp_upload_context = i40iw_sc_qp_upload_context,
5151         .qp_setctx = i40iw_sc_qp_setctx,
5152         .qp_send_lsmm = i40iw_sc_send_lsmm,
5153         .qp_send_lsmm_nostag = i40iw_sc_send_lsmm_nostag,
5154         .qp_send_rtt = i40iw_sc_send_rtt,
5155         .qp_post_wqe0 = i40iw_sc_post_wqe0,
5156         .iw_mr_fast_register = i40iw_sc_mr_fast_register
5157 };
5158
5159 static struct i40iw_priv_cq_ops iw_priv_cq_ops = {
5160         .cq_init = i40iw_sc_cq_init,
5161         .cq_create = i40iw_sc_cq_create,
5162         .cq_destroy = i40iw_sc_cq_destroy,
5163         .cq_modify = i40iw_sc_cq_modify,
5164 };
5165
5166 static struct i40iw_mr_ops iw_mr_ops = {
5167         .alloc_stag = i40iw_sc_alloc_stag,
5168         .mr_reg_non_shared = i40iw_sc_mr_reg_non_shared,
5169         .mr_reg_shared = i40iw_sc_mr_reg_shared,
5170         .dealloc_stag = i40iw_sc_dealloc_stag,
5171         .query_stag = i40iw_sc_query_stag,
5172         .mw_alloc = i40iw_sc_mw_alloc
5173 };
5174
5175 static struct i40iw_cqp_misc_ops iw_cqp_misc_ops = {
5176         .manage_push_page = i40iw_sc_manage_push_page,
5177         .manage_hmc_pm_func_table = i40iw_sc_manage_hmc_pm_func_table,
5178         .set_hmc_resource_profile = i40iw_sc_set_hmc_resource_profile,
5179         .commit_fpm_values = i40iw_sc_commit_fpm_values,
5180         .query_fpm_values = i40iw_sc_query_fpm_values,
5181         .static_hmc_pages_allocated = i40iw_sc_static_hmc_pages_allocated,
5182         .add_arp_cache_entry = i40iw_sc_add_arp_cache_entry,
5183         .del_arp_cache_entry = i40iw_sc_del_arp_cache_entry,
5184         .query_arp_cache_entry = i40iw_sc_query_arp_cache_entry,
5185         .manage_apbvt_entry = i40iw_sc_manage_apbvt_entry,
5186         .manage_qhash_table_entry = i40iw_sc_manage_qhash_table_entry,
5187         .alloc_local_mac_ipaddr_table_entry = i40iw_sc_alloc_local_mac_ipaddr_entry,
5188         .add_local_mac_ipaddr_entry = i40iw_sc_add_local_mac_ipaddr_entry,
5189         .del_local_mac_ipaddr_entry = i40iw_sc_del_local_mac_ipaddr_entry,
5190         .cqp_nop = i40iw_sc_cqp_nop,
5191         .commit_fpm_values_done = i40iw_sc_commit_fpm_values_done,
5192         .query_fpm_values_done = i40iw_sc_query_fpm_values_done,
5193         .manage_hmc_pm_func_table_done = i40iw_sc_manage_hmc_pm_func_table_done,
5194         .update_suspend_qp = i40iw_sc_suspend_qp,
5195         .update_resume_qp = i40iw_sc_resume_qp
5196 };
5197
5198 static struct i40iw_hmc_ops iw_hmc_ops = {
5199         .init_iw_hmc = i40iw_sc_init_iw_hmc,
5200         .parse_fpm_query_buf = i40iw_sc_parse_fpm_query_buf,
5201         .configure_iw_fpm = i40iw_sc_configure_iw_fpm,
5202         .parse_fpm_commit_buf = i40iw_sc_parse_fpm_commit_buf,
5203         .create_hmc_object = i40iw_sc_create_hmc_obj,
5204         .del_hmc_object = i40iw_sc_del_hmc_obj
5205 };
5206
5207 /**
5208  * i40iw_device_init - Initialize IWARP device
5209  * @dev: IWARP device pointer
5210  * @info: IWARP init info
5211  */
5212 enum i40iw_status_code i40iw_device_init(struct i40iw_sc_dev *dev,
5213                                          struct i40iw_device_init_info *info)
5214 {
5215         u32 val;
5216         u32 vchnl_ver = 0;
5217         u16 hmc_fcn = 0;
5218         enum i40iw_status_code ret_code = 0;
5219         u8 db_size;
5220
5221         spin_lock_init(&dev->cqp_lock);
5222
5223         i40iw_device_init_uk(&dev->dev_uk);
5224
5225         dev->debug_mask = info->debug_mask;
5226
5227         dev->hmc_fn_id = info->hmc_fn_id;
5228         dev->is_pf = info->is_pf;
5229
5230         dev->fpm_query_buf_pa = info->fpm_query_buf_pa;
5231         dev->fpm_query_buf = info->fpm_query_buf;
5232
5233         dev->fpm_commit_buf_pa = info->fpm_commit_buf_pa;
5234         dev->fpm_commit_buf = info->fpm_commit_buf;
5235
5236         dev->hw = info->hw;
5237         dev->hw->hw_addr = info->bar0;
5238
5239         if (dev->is_pf) {
5240                 val = i40iw_rd32(dev->hw, I40E_GLPCI_DREVID);
5241                 dev->hw_rev = (u8)RS_32(val, I40E_GLPCI_DREVID_DEFAULT_REVID);
5242
5243                 val = i40iw_rd32(dev->hw, I40E_GLPCI_LBARCTRL);
5244                 db_size = (u8)RS_32(val, I40E_GLPCI_LBARCTRL_PE_DB_SIZE);
5245                 if ((db_size != I40IW_PE_DB_SIZE_4M) &&
5246                     (db_size != I40IW_PE_DB_SIZE_8M)) {
5247                         i40iw_debug(dev, I40IW_DEBUG_DEV,
5248                                     "%s: PE doorbell is not enabled in CSR val 0x%x\n",
5249                                     __func__, val);
5250                         ret_code = I40IW_ERR_PE_DOORBELL_NOT_ENABLED;
5251                         return ret_code;
5252                 }
5253                 dev->db_addr = dev->hw->hw_addr + I40IW_DB_ADDR_OFFSET;
5254                 dev->vchnl_if.vchnl_recv = i40iw_vchnl_recv_pf;
5255         } else {
5256                 dev->db_addr = dev->hw->hw_addr + I40IW_VF_DB_ADDR_OFFSET;
5257         }
5258
5259         dev->cqp_ops = &iw_cqp_ops;
5260         dev->ccq_ops = &iw_ccq_ops;
5261         dev->ceq_ops = &iw_ceq_ops;
5262         dev->aeq_ops = &iw_aeq_ops;
5263         dev->cqp_misc_ops = &iw_cqp_misc_ops;
5264         dev->iw_pd_ops = &iw_pd_ops;
5265         dev->iw_priv_qp_ops = &iw_priv_qp_ops;
5266         dev->iw_priv_cq_ops = &iw_priv_cq_ops;
5267         dev->mr_ops = &iw_mr_ops;
5268         dev->hmc_ops = &iw_hmc_ops;
5269         dev->vchnl_if.vchnl_send = info->vchnl_send;
5270         if (dev->vchnl_if.vchnl_send)
5271                 dev->vchnl_up = true;
5272         else
5273                 dev->vchnl_up = false;
5274         if (!dev->is_pf) {
5275                 dev->vchnl_if.vchnl_recv = i40iw_vchnl_recv_vf;
5276                 ret_code = i40iw_vchnl_vf_get_ver(dev, &vchnl_ver);
5277                 if (!ret_code) {
5278                         i40iw_debug(dev, I40IW_DEBUG_DEV,
5279                                     "%s: Get Channel version rc = 0x%0x, version is %u\n",
5280                                 __func__, ret_code, vchnl_ver);
5281                         ret_code = i40iw_vchnl_vf_get_hmc_fcn(dev, &hmc_fcn);
5282                         if (!ret_code) {
5283                                 i40iw_debug(dev, I40IW_DEBUG_DEV,
5284                                             "%s Get HMC function rc = 0x%0x, hmc fcn is %u\n",
5285                                             __func__, ret_code, hmc_fcn);
5286                                 dev->hmc_fn_id = (u8)hmc_fcn;
5287                         }
5288                 }
5289         }
5290         dev->iw_vf_cqp_ops = &iw_vf_cqp_ops;
5291
5292         return ret_code;
5293 }