hw/irdma/uk.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /* Copyright (c) 2015 - 2021 Intel Corporation */
   3 #include "osdep.h"
   4 #include "defs.h"
   5 #include "user.h"
   6 #include "irdma.h"
   7
   8 /**
   9  * irdma_set_fragment - set fragment in wqe
  10  * @wqe: wqe for setting fragment
  11  * @offset: offset value
  12  * @sge: sge length and stag
  13  * @valid: The wqe valid
  14  */
  15 static void irdma_set_fragment(__le64 *wqe, u32 offset, struct ib_sge *sge,
  16                                u8 valid)
  17 {
  18         if (sge) {
  19                 set_64bit_val(wqe, offset,
  20                               FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr));
  21                 set_64bit_val(wqe, offset + 8,
  22                               FIELD_PREP(IRDMAQPSQ_VALID, valid) |
  23                               FIELD_PREP(IRDMAQPSQ_FRAG_LEN, sge->length) |
  24                               FIELD_PREP(IRDMAQPSQ_FRAG_STAG, sge->lkey));
  25         } else {
  26                 set_64bit_val(wqe, offset, 0);
  27                 set_64bit_val(wqe, offset + 8,
  28                               FIELD_PREP(IRDMAQPSQ_VALID, valid));
  29         }
  30 }
  31
  32 /**
  33  * irdma_set_fragment_gen_1 - set fragment in wqe
  34  * @wqe: wqe for setting fragment
  35  * @offset: offset value
  36  * @sge: sge length and stag
  37  * @valid: wqe valid flag
  38  */
  39 static void irdma_set_fragment_gen_1(__le64 *wqe, u32 offset,
  40                                      struct ib_sge *sge, u8 valid)
  41 {
  42         if (sge) {
  43                 set_64bit_val(wqe, offset,
  44                               FIELD_PREP(IRDMAQPSQ_FRAG_TO, sge->addr));
  45                 set_64bit_val(wqe, offset + 8,
  46                               FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, sge->length) |
  47                               FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, sge->lkey));
  48         } else {
  49                 set_64bit_val(wqe, offset, 0);
  50                 set_64bit_val(wqe, offset + 8, 0);
  51         }
  52 }
  53
  54 /**
  55  * irdma_nop_1 - insert a NOP wqe
  56  * @qp: hw qp ptr
  57  */
  58 static int irdma_nop_1(struct irdma_qp_uk *qp)
  59 {
  60         u64 hdr;
  61         __le64 *wqe;
  62         u32 wqe_idx;
  63         bool signaled = false;
  64
  65         if (!qp->sq_ring.head)
  66                 return -EINVAL;
  67
  68         wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
  69         wqe = qp->sq_base[wqe_idx].elem;
  70
  71         qp->sq_wrtrk_array[wqe_idx].quanta = IRDMA_QP_WQE_MIN_QUANTA;
  72
  73         set_64bit_val(wqe, 0, 0);
  74         set_64bit_val(wqe, 8, 0);
  75         set_64bit_val(wqe, 16, 0);
  76
  77         hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
  78               FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) |
  79               FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
  80
  81         /* make sure WQE is written before valid bit is set */
  82         dma_wmb();
  83
  84         set_64bit_val(wqe, 24, hdr);
  85
  86         return 0;
  87 }
  88
  89 /**
  90  * irdma_clr_wqes - clear next 128 sq entries
  91  * @qp: hw qp ptr
  92  * @qp_wqe_idx: wqe_idx
  93  */
  94 void irdma_clr_wqes(struct irdma_qp_uk *qp, u32 qp_wqe_idx)
  95 {
  96         struct irdma_qp_quanta *sq;
  97         u32 wqe_idx;
  98
  99         if (!(qp_wqe_idx & 0x7F)) {
 100                 wqe_idx = (qp_wqe_idx + 128) % qp->sq_ring.size;
 101                 sq = qp->sq_base + wqe_idx;
 102                 if (wqe_idx)
 103                         memset(sq, qp->swqe_polarity ? 0 : 0xFF,
 104                                128 * sizeof(*sq));
 105                 else
 106                         memset(sq, qp->swqe_polarity ? 0xFF : 0,
 107                                128 * sizeof(*sq));
 108         }
 109 }
 110
 111 /**
 112  * irdma_uk_qp_post_wr - ring doorbell
 113  * @qp: hw qp ptr
 114  */
 115 void irdma_uk_qp_post_wr(struct irdma_qp_uk *qp)
 116 {
 117         u64 temp;
 118         u32 hw_sq_tail;
 119         u32 sw_sq_head;
 120
 121         /* valid bit is written and loads completed before reading shadow */
 122         mb();
 123
 124         /* read the doorbell shadow area */
 125         get_64bit_val(qp->shadow_area, 0, &temp);
 126
 127         hw_sq_tail = (u32)FIELD_GET(IRDMA_QP_DBSA_HW_SQ_TAIL, temp);
 128         sw_sq_head = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
 129         if (sw_sq_head != qp->initial_ring.head) {
 130                 if (sw_sq_head != hw_sq_tail) {
 131                         if (sw_sq_head > qp->initial_ring.head) {
 132                                 if (hw_sq_tail >= qp->initial_ring.head &&
 133                                     hw_sq_tail < sw_sq_head)
 134                                         writel(qp->qp_id, qp->wqe_alloc_db);
 135                         } else {
 136                                 if (hw_sq_tail >= qp->initial_ring.head ||
 137                                     hw_sq_tail < sw_sq_head)
 138                                         writel(qp->qp_id, qp->wqe_alloc_db);
 139                         }
 140                 }
 141         }
 142
 143         qp->initial_ring.head = qp->sq_ring.head;
 144 }
 145
 146 /**
 147  * irdma_qp_get_next_send_wqe - pad with NOP if needed, return where next WR should go
 148  * @qp: hw qp ptr
 149  * @wqe_idx: return wqe index
 150  * @quanta: size of WR in quanta
 151  * @total_size: size of WR in bytes
 152  * @info: info on WR
 153  */
 154 __le64 *irdma_qp_get_next_send_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx,
 155                                    u16 quanta, u32 total_size,
 156                                    struct irdma_post_sq_info *info)
 157 {
 158         __le64 *wqe;
 159         __le64 *wqe_0 = NULL;
 160         u16 avail_quanta;
 161         u16 i;
 162
 163         avail_quanta = qp->uk_attrs->max_hw_sq_chunk -
 164                        (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) %
 165                        qp->uk_attrs->max_hw_sq_chunk);
 166         if (quanta <= avail_quanta) {
 167                 /* WR fits in current chunk */
 168                 if (quanta > IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring))
 169                         return NULL;
 170         } else {
 171                 /* Need to pad with NOP */
 172                 if (quanta + avail_quanta >
 173                         IRDMA_SQ_RING_FREE_QUANTA(qp->sq_ring))
 174                         return NULL;
 175
 176                 for (i = 0; i < avail_quanta; i++) {
 177                         irdma_nop_1(qp);
 178                         IRDMA_RING_MOVE_HEAD_NOCHECK(qp->sq_ring);
 179                 }
 180         }
 181
 182         *wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
 183         if (!*wqe_idx)
 184                 qp->swqe_polarity = !qp->swqe_polarity;
 185
 186         IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, quanta);
 187
 188         wqe = qp->sq_base[*wqe_idx].elem;
 189         if (qp->uk_attrs->hw_rev == IRDMA_GEN_1 && quanta == 1 &&
 190             (IRDMA_RING_CURRENT_HEAD(qp->sq_ring) & 1)) {
 191                 wqe_0 = qp->sq_base[IRDMA_RING_CURRENT_HEAD(qp->sq_ring)].elem;
 192                 wqe_0[3] = cpu_to_le64(FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity ? 0 : 1));
 193         }
 194         qp->sq_wrtrk_array[*wqe_idx].wrid = info->wr_id;
 195         qp->sq_wrtrk_array[*wqe_idx].wr_len = total_size;
 196         qp->sq_wrtrk_array[*wqe_idx].quanta = quanta;
 197
 198         return wqe;
 199 }
 200
 201 /**
 202  * irdma_qp_get_next_recv_wqe - get next qp's rcv wqe
 203  * @qp: hw qp ptr
 204  * @wqe_idx: return wqe index
 205  */
 206 __le64 *irdma_qp_get_next_recv_wqe(struct irdma_qp_uk *qp, u32 *wqe_idx)
 207 {
 208         __le64 *wqe;
 209         int ret_code;
 210
 211         if (IRDMA_RING_FULL_ERR(qp->rq_ring))
 212                 return NULL;
 213
 214         IRDMA_ATOMIC_RING_MOVE_HEAD(qp->rq_ring, *wqe_idx, ret_code);
 215         if (ret_code)
 216                 return NULL;
 217
 218         if (!*wqe_idx)
 219                 qp->rwqe_polarity = !qp->rwqe_polarity;
 220         /* rq_wqe_size_multiplier is no of 32 byte quanta in one rq wqe */
 221         wqe = qp->rq_base[*wqe_idx * qp->rq_wqe_size_multiplier].elem;
 222
 223         return wqe;
 224 }
 225
 226 /**
 227  * irdma_uk_rdma_write - rdma write operation
 228  * @qp: hw qp ptr
 229  * @info: post sq information
 230  * @post_sq: flag to post sq
 231  */
 232 int irdma_uk_rdma_write(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
 233                         bool post_sq)
 234 {
 235         u64 hdr;
 236         __le64 *wqe;
 237         struct irdma_rdma_write *op_info;
 238         u32 i, wqe_idx;
 239         u32 total_size = 0, byte_off;
 240         int ret_code;
 241         u32 frag_cnt, addl_frag_cnt;
 242         bool read_fence = false;
 243         u16 quanta;
 244
 245         op_info = &info->op.rdma_write;
 246         if (op_info->num_lo_sges > qp->max_sq_frag_cnt)
 247                 return -EINVAL;
 248
 249         for (i = 0; i < op_info->num_lo_sges; i++)
 250                 total_size += op_info->lo_sg_list[i].length;
 251
 252         read_fence |= info->read_fence;
 253
 254         if (info->imm_data_valid)
 255                 frag_cnt = op_info->num_lo_sges + 1;
 256         else
 257                 frag_cnt = op_info->num_lo_sges;
 258         addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0;
 259         ret_code = irdma_fragcnt_to_quanta_sq(frag_cnt, &quanta);
 260         if (ret_code)
 261                 return ret_code;
 262
 263         wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
 264                                          info);
 265         if (!wqe)
 266                 return -ENOMEM;
 267
 268         irdma_clr_wqes(qp, wqe_idx);
 269
 270         set_64bit_val(wqe, 16,
 271                       FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
 272
 273         if (info->imm_data_valid) {
 274                 set_64bit_val(wqe, 0,
 275                               FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
 276                 i = 0;
 277         } else {
 278                 qp->wqe_ops.iw_set_fragment(wqe, 0,
 279                                             op_info->lo_sg_list,
 280                                             qp->swqe_polarity);
 281                 i = 1;
 282         }
 283
 284         for (byte_off = 32; i < op_info->num_lo_sges; i++) {
 285                 qp->wqe_ops.iw_set_fragment(wqe, byte_off,
 286                                             &op_info->lo_sg_list[i],
 287                                             qp->swqe_polarity);
 288                 byte_off += 16;
 289         }
 290
 291         /* if not an odd number set valid bit in next fragment */
 292         if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(frag_cnt & 0x01) &&
 293             frag_cnt) {
 294                 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
 295                                             qp->swqe_polarity);
 296                 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
 297                         ++addl_frag_cnt;
 298         }
 299
 300         hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
 301               FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
 302               FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid) |
 303               FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt) |
 304               FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
 305               FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
 306               FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
 307               FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
 308               FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
 309
 310         dma_wmb(); /* make sure WQE is populated before valid bit is set */
 311
 312         set_64bit_val(wqe, 24, hdr);
 313
 314         if (post_sq)
 315                 irdma_uk_qp_post_wr(qp);
 316
 317         return 0;
 318 }
 319
 320 /**
 321  * irdma_uk_rdma_read - rdma read command
 322  * @qp: hw qp ptr
 323  * @info: post sq information
 324  * @inv_stag: flag for inv_stag
 325  * @post_sq: flag to post sq
 326  */
 327 int irdma_uk_rdma_read(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
 328                        bool inv_stag, bool post_sq)
 329 {
 330         struct irdma_rdma_read *op_info;
 331         int ret_code;
 332         u32 i, byte_off, total_size = 0;
 333         bool local_fence = false;
 334         u32 addl_frag_cnt;
 335         __le64 *wqe;
 336         u32 wqe_idx;
 337         u16 quanta;
 338         u64 hdr;
 339
 340         op_info = &info->op.rdma_read;
 341         if (qp->max_sq_frag_cnt < op_info->num_lo_sges)
 342                 return -EINVAL;
 343
 344         for (i = 0; i < op_info->num_lo_sges; i++)
 345                 total_size += op_info->lo_sg_list[i].length;
 346
 347         ret_code = irdma_fragcnt_to_quanta_sq(op_info->num_lo_sges, &quanta);
 348         if (ret_code)
 349                 return ret_code;
 350
 351         wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
 352                                          info);
 353         if (!wqe)
 354                 return -ENOMEM;
 355
 356         irdma_clr_wqes(qp, wqe_idx);
 357
 358         addl_frag_cnt = op_info->num_lo_sges > 1 ?
 359                         (op_info->num_lo_sges - 1) : 0;
 360         local_fence |= info->local_fence;
 361
 362         qp->wqe_ops.iw_set_fragment(wqe, 0, op_info->lo_sg_list,
 363                                     qp->swqe_polarity);
 364         for (i = 1, byte_off = 32; i < op_info->num_lo_sges; ++i) {
 365                 qp->wqe_ops.iw_set_fragment(wqe, byte_off,
 366                                             &op_info->lo_sg_list[i],
 367                                             qp->swqe_polarity);
 368                 byte_off += 16;
 369         }
 370
 371         /* if not an odd number set valid bit in next fragment */
 372         if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 &&
 373             !(op_info->num_lo_sges & 0x01) && op_info->num_lo_sges) {
 374                 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
 375                                             qp->swqe_polarity);
 376                 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
 377                         ++addl_frag_cnt;
 378         }
 379         set_64bit_val(wqe, 16,
 380                       FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
 381         hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
 382               FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
 383               FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
 384               FIELD_PREP(IRDMAQPSQ_OPCODE,
 385                          (inv_stag ? IRDMAQP_OP_RDMA_READ_LOC_INV : IRDMAQP_OP_RDMA_READ)) |
 386               FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
 387               FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) |
 388               FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
 389               FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
 390
 391         dma_wmb(); /* make sure WQE is populated before valid bit is set */
 392
 393         set_64bit_val(wqe, 24, hdr);
 394
 395         if (post_sq)
 396                 irdma_uk_qp_post_wr(qp);
 397
 398         return 0;
 399 }
 400
 401 /**
 402  * irdma_uk_send - rdma send command
 403  * @qp: hw qp ptr
 404  * @info: post sq information
 405  * @post_sq: flag to post sq
 406  */
 407 int irdma_uk_send(struct irdma_qp_uk *qp, struct irdma_post_sq_info *info,
 408                   bool post_sq)
 409 {
 410         __le64 *wqe;
 411         struct irdma_post_send *op_info;
 412         u64 hdr;
 413         u32 i, wqe_idx, total_size = 0, byte_off;
 414         int ret_code;
 415         u32 frag_cnt, addl_frag_cnt;
 416         bool read_fence = false;
 417         u16 quanta;
 418
 419         op_info = &info->op.send;
 420         if (qp->max_sq_frag_cnt < op_info->num_sges)
 421                 return -EINVAL;
 422
 423         for (i = 0; i < op_info->num_sges; i++)
 424                 total_size += op_info->sg_list[i].length;
 425
 426         if (info->imm_data_valid)
 427                 frag_cnt = op_info->num_sges + 1;
 428         else
 429                 frag_cnt = op_info->num_sges;
 430         ret_code = irdma_fragcnt_to_quanta_sq(frag_cnt, &quanta);
 431         if (ret_code)
 432                 return ret_code;
 433
 434         wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
 435                                          info);
 436         if (!wqe)
 437                 return -ENOMEM;
 438
 439         irdma_clr_wqes(qp, wqe_idx);
 440
 441         read_fence |= info->read_fence;
 442         addl_frag_cnt = frag_cnt > 1 ? (frag_cnt - 1) : 0;
 443         if (info->imm_data_valid) {
 444                 set_64bit_val(wqe, 0,
 445                               FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
 446                 i = 0;
 447         } else {
 448                 qp->wqe_ops.iw_set_fragment(wqe, 0,
 449                                             frag_cnt ? op_info->sg_list : NULL,
 450                                             qp->swqe_polarity);
 451                 i = 1;
 452         }
 453
 454         for (byte_off = 32; i < op_info->num_sges; i++) {
 455                 qp->wqe_ops.iw_set_fragment(wqe, byte_off, &op_info->sg_list[i],
 456                                             qp->swqe_polarity);
 457                 byte_off += 16;
 458         }
 459
 460         /* if not an odd number set valid bit in next fragment */
 461         if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(frag_cnt & 0x01) &&
 462             frag_cnt) {
 463                 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
 464                                             qp->swqe_polarity);
 465                 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
 466                         ++addl_frag_cnt;
 467         }
 468
 469         set_64bit_val(wqe, 16,
 470                       FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info->qkey) |
 471                       FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info->dest_qp));
 472         hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) |
 473               FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) |
 474               FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG,
 475                          (info->imm_data_valid ? 1 : 0)) |
 476               FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
 477               FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
 478               FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
 479               FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
 480               FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
 481               FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
 482               FIELD_PREP(IRDMAQPSQ_UDPHEADER, info->udp_hdr) |
 483               FIELD_PREP(IRDMAQPSQ_L4LEN, info->l4len) |
 484               FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
 485
 486         dma_wmb(); /* make sure WQE is populated before valid bit is set */
 487
 488         set_64bit_val(wqe, 24, hdr);
 489
 490         if (post_sq)
 491                 irdma_uk_qp_post_wr(qp);
 492
 493         return 0;
 494 }
 495
 496 /**
 497  * irdma_set_mw_bind_wqe_gen_1 - set mw bind wqe
 498  * @wqe: wqe for setting fragment
 499  * @op_info: info for setting bind wqe values
 500  */
 501 static void irdma_set_mw_bind_wqe_gen_1(__le64 *wqe,
 502                                         struct irdma_bind_window *op_info)
 503 {
 504         set_64bit_val(wqe, 0, (uintptr_t)op_info->va);
 505         set_64bit_val(wqe, 8,
 506                       FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mw_stag) |
 507                       FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mr_stag));
 508         set_64bit_val(wqe, 16, op_info->bind_len);
 509 }
 510
 511 /**
 512  * irdma_copy_inline_data_gen_1 - Copy inline data to wqe
 513  * @wqe: pointer to wqe
 514  * @sge_list: table of pointers to inline data
 515  * @num_sges: Total inline data length
 516  * @polarity: compatibility parameter
 517  */
 518 static void irdma_copy_inline_data_gen_1(u8 *wqe, struct ib_sge *sge_list,
 519                                          u32 num_sges, u8 polarity)
 520 {
 521         u32 quanta_bytes_remaining = 16;
 522         int i;
 523
 524         for (i = 0; i < num_sges; i++) {
 525                 u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;
 526                 u32 sge_len = sge_list[i].length;
 527
 528                 while (sge_len) {
 529                         u32 bytes_copied;
 530
 531                         bytes_copied = min(sge_len, quanta_bytes_remaining);
 532                         memcpy(wqe, cur_sge, bytes_copied);
 533                         wqe += bytes_copied;
 534                         cur_sge += bytes_copied;
 535                         quanta_bytes_remaining -= bytes_copied;
 536                         sge_len -= bytes_copied;
 537
 538                         if (!quanta_bytes_remaining) {
 539                                 /* Remaining inline bytes reside after hdr */
 540                                 wqe += 16;
 541                                 quanta_bytes_remaining = 32;
 542                         }
 543                 }
 544         }
 545 }
 546
 547 /**
 548  * irdma_inline_data_size_to_quanta_gen_1 - based on inline data, quanta
 549  * @data_size: data size for inline
 550  *
 551  * Gets the quanta based on inline and immediate data.
 552  */
 553 static inline u16 irdma_inline_data_size_to_quanta_gen_1(u32 data_size)
 554 {
 555         return data_size <= 16 ? IRDMA_QP_WQE_MIN_QUANTA : 2;
 556 }
 557
 558 /**
 559  * irdma_set_mw_bind_wqe - set mw bind in wqe
 560  * @wqe: wqe for setting mw bind
 561  * @op_info: info for setting wqe values
 562  */
 563 static void irdma_set_mw_bind_wqe(__le64 *wqe,
 564                                   struct irdma_bind_window *op_info)
 565 {
 566         set_64bit_val(wqe, 0, (uintptr_t)op_info->va);
 567         set_64bit_val(wqe, 8,
 568                       FIELD_PREP(IRDMAQPSQ_PARENTMRSTAG, op_info->mr_stag) |
 569                       FIELD_PREP(IRDMAQPSQ_MWSTAG, op_info->mw_stag));
 570         set_64bit_val(wqe, 16, op_info->bind_len);
 571 }
 572
 573 /**
 574  * irdma_copy_inline_data - Copy inline data to wqe
 575  * @wqe: pointer to wqe
 576  * @sge_list: table of pointers to inline data
 577  * @num_sges: number of SGE's
 578  * @polarity: polarity of wqe valid bit
 579  */
 580 static void irdma_copy_inline_data(u8 *wqe, struct ib_sge *sge_list,
 581                                    u32 num_sges, u8 polarity)
 582 {
 583         u8 inline_valid = polarity << IRDMA_INLINE_VALID_S;
 584         u32 quanta_bytes_remaining = 8;
 585         bool first_quanta = true;
 586         int i;
 587
 588         wqe += 8;
 589
 590         for (i = 0; i < num_sges; i++) {
 591                 u8 *cur_sge = (u8 *)(uintptr_t)sge_list[i].addr;
 592                 u32 sge_len = sge_list[i].length;
 593
 594                 while (sge_len) {
 595                         u32 bytes_copied;
 596
 597                         bytes_copied = min(sge_len, quanta_bytes_remaining);
 598                         memcpy(wqe, cur_sge, bytes_copied);
 599                         wqe += bytes_copied;
 600                         cur_sge += bytes_copied;
 601                         quanta_bytes_remaining -= bytes_copied;
 602                         sge_len -= bytes_copied;
 603
 604                         if (!quanta_bytes_remaining) {
 605                                 quanta_bytes_remaining = 31;
 606
 607                                 /* Remaining inline bytes reside after hdr */
 608                                 if (first_quanta) {
 609                                         first_quanta = false;
 610                                         wqe += 16;
 611                                 } else {
 612                                         *wqe = inline_valid;
 613                                         wqe++;
 614                                 }
 615                         }
 616                 }
 617         }
 618         if (!first_quanta && quanta_bytes_remaining < 31)
 619                 *(wqe + quanta_bytes_remaining) = inline_valid;
 620 }
 621
 622 /**
 623  * irdma_inline_data_size_to_quanta - based on inline data, quanta
 624  * @data_size: data size for inline
 625  *
 626  * Gets the quanta based on inline and immediate data.
 627  */
 628 static u16 irdma_inline_data_size_to_quanta(u32 data_size)
 629 {
 630         if (data_size <= 8)
 631                 return IRDMA_QP_WQE_MIN_QUANTA;
 632         else if (data_size <= 39)
 633                 return 2;
 634         else if (data_size <= 70)
 635                 return 3;
 636         else if (data_size <= 101)
 637                 return 4;
 638         else if (data_size <= 132)
 639                 return 5;
 640         else if (data_size <= 163)
 641                 return 6;
 642         else if (data_size <= 194)
 643                 return 7;
 644         else
 645                 return 8;
 646 }
 647
 648 /**
 649  * irdma_uk_inline_rdma_write - inline rdma write operation
 650  * @qp: hw qp ptr
 651  * @info: post sq information
 652  * @post_sq: flag to post sq
 653  */
 654 int irdma_uk_inline_rdma_write(struct irdma_qp_uk *qp,
 655                                struct irdma_post_sq_info *info, bool post_sq)
 656 {
 657         __le64 *wqe;
 658         struct irdma_rdma_write *op_info;
 659         u64 hdr = 0;
 660         u32 wqe_idx;
 661         bool read_fence = false;
 662         u32 i, total_size = 0;
 663         u16 quanta;
 664
 665         op_info = &info->op.rdma_write;
 666
 667         if (unlikely(qp->max_sq_frag_cnt < op_info->num_lo_sges))
 668                 return -EINVAL;
 669
 670         for (i = 0; i < op_info->num_lo_sges; i++)
 671                 total_size += op_info->lo_sg_list[i].length;
 672
 673         if (unlikely(total_size > qp->max_inline_data))
 674                 return -EINVAL;
 675
 676         quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size);
 677         wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
 678                                          info);
 679         if (!wqe)
 680                 return -ENOMEM;
 681
 682         irdma_clr_wqes(qp, wqe_idx);
 683
 684         read_fence |= info->read_fence;
 685         set_64bit_val(wqe, 16,
 686                       FIELD_PREP(IRDMAQPSQ_FRAG_TO, op_info->rem_addr.addr));
 687
 688         hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, op_info->rem_addr.lkey) |
 689               FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
 690               FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) |
 691               FIELD_PREP(IRDMAQPSQ_REPORTRTT, info->report_rtt ? 1 : 0) |
 692               FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) |
 693               FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG, info->imm_data_valid ? 1 : 0) |
 694               FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
 695               FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
 696               FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
 697               FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
 698
 699         if (info->imm_data_valid)
 700                 set_64bit_val(wqe, 0,
 701                               FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
 702
 703         qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->lo_sg_list,
 704                                         op_info->num_lo_sges,
 705                                         qp->swqe_polarity);
 706         dma_wmb(); /* make sure WQE is populated before valid bit is set */
 707
 708         set_64bit_val(wqe, 24, hdr);
 709
 710         if (post_sq)
 711                 irdma_uk_qp_post_wr(qp);
 712
 713         return 0;
 714 }
 715
 716 /**
 717  * irdma_uk_inline_send - inline send operation
 718  * @qp: hw qp ptr
 719  * @info: post sq information
 720  * @post_sq: flag to post sq
 721  */
 722 int irdma_uk_inline_send(struct irdma_qp_uk *qp,
 723                          struct irdma_post_sq_info *info, bool post_sq)
 724 {
 725         __le64 *wqe;
 726         struct irdma_post_send *op_info;
 727         u64 hdr;
 728         u32 wqe_idx;
 729         bool read_fence = false;
 730         u32 i, total_size = 0;
 731         u16 quanta;
 732
 733         op_info = &info->op.send;
 734
 735         if (unlikely(qp->max_sq_frag_cnt < op_info->num_sges))
 736                 return -EINVAL;
 737
 738         for (i = 0; i < op_info->num_sges; i++)
 739                 total_size += op_info->sg_list[i].length;
 740
 741         if (unlikely(total_size > qp->max_inline_data))
 742                 return -EINVAL;
 743
 744         quanta = qp->wqe_ops.iw_inline_data_size_to_quanta(total_size);
 745         wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, quanta, total_size,
 746                                          info);
 747         if (!wqe)
 748                 return -ENOMEM;
 749
 750         irdma_clr_wqes(qp, wqe_idx);
 751
 752         set_64bit_val(wqe, 16,
 753                       FIELD_PREP(IRDMAQPSQ_DESTQKEY, op_info->qkey) |
 754                       FIELD_PREP(IRDMAQPSQ_DESTQPN, op_info->dest_qp));
 755
 756         read_fence |= info->read_fence;
 757         hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, info->stag_to_inv) |
 758               FIELD_PREP(IRDMAQPSQ_AHID, op_info->ah_id) |
 759               FIELD_PREP(IRDMAQPSQ_OPCODE, info->op_type) |
 760               FIELD_PREP(IRDMAQPSQ_INLINEDATALEN, total_size) |
 761               FIELD_PREP(IRDMAQPSQ_IMMDATAFLAG,
 762                          (info->imm_data_valid ? 1 : 0)) |
 763               FIELD_PREP(IRDMAQPSQ_REPORTRTT, (info->report_rtt ? 1 : 0)) |
 764               FIELD_PREP(IRDMAQPSQ_INLINEDATAFLAG, 1) |
 765               FIELD_PREP(IRDMAQPSQ_READFENCE, read_fence) |
 766               FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
 767               FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
 768               FIELD_PREP(IRDMAQPSQ_UDPHEADER, info->udp_hdr) |
 769               FIELD_PREP(IRDMAQPSQ_L4LEN, info->l4len) |
 770               FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
 771
 772         if (info->imm_data_valid)
 773                 set_64bit_val(wqe, 0,
 774                               FIELD_PREP(IRDMAQPSQ_IMMDATA, info->imm_data));
 775         qp->wqe_ops.iw_copy_inline_data((u8 *)wqe, op_info->sg_list,
 776                                         op_info->num_sges, qp->swqe_polarity);
 777
 778         dma_wmb(); /* make sure WQE is populated before valid bit is set */
 779
 780         set_64bit_val(wqe, 24, hdr);
 781
 782         if (post_sq)
 783                 irdma_uk_qp_post_wr(qp);
 784
 785         return 0;
 786 }
 787
 788 /**
 789  * irdma_uk_stag_local_invalidate - stag invalidate operation
 790  * @qp: hw qp ptr
 791  * @info: post sq information
 792  * @post_sq: flag to post sq
 793  */
 794 int irdma_uk_stag_local_invalidate(struct irdma_qp_uk *qp,
 795                                    struct irdma_post_sq_info *info,
 796                                    bool post_sq)
 797 {
 798         __le64 *wqe;
 799         struct irdma_inv_local_stag *op_info;
 800         u64 hdr;
 801         u32 wqe_idx;
 802         bool local_fence = false;
 803         struct ib_sge sge = {};
 804
 805         op_info = &info->op.inv_local_stag;
 806         local_fence = info->local_fence;
 807
 808         wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,
 809                                          0, info);
 810         if (!wqe)
 811                 return -ENOMEM;
 812
 813         irdma_clr_wqes(qp, wqe_idx);
 814
 815         sge.lkey = op_info->target_stag;
 816         qp->wqe_ops.iw_set_fragment(wqe, 0, &sge, 0);
 817
 818         set_64bit_val(wqe, 16, 0);
 819
 820         hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMA_OP_TYPE_INV_STAG) |
 821               FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
 822               FIELD_PREP(IRDMAQPSQ_LOCALFENCE, local_fence) |
 823               FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
 824               FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
 825
 826         dma_wmb(); /* make sure WQE is populated before valid bit is set */
 827
 828         set_64bit_val(wqe, 24, hdr);
 829
 830         if (post_sq)
 831                 irdma_uk_qp_post_wr(qp);
 832
 833         return 0;
 834 }
 835
 836 /**
 837  * irdma_uk_post_receive - post receive wqe
 838  * @qp: hw qp ptr
 839  * @info: post rq information
 840  */
 841 int irdma_uk_post_receive(struct irdma_qp_uk *qp,
 842                           struct irdma_post_rq_info *info)
 843 {
 844         u32 wqe_idx, i, byte_off;
 845         u32 addl_frag_cnt;
 846         __le64 *wqe;
 847         u64 hdr;
 848
 849         if (qp->max_rq_frag_cnt < info->num_sges)
 850                 return -EINVAL;
 851
 852         wqe = irdma_qp_get_next_recv_wqe(qp, &wqe_idx);
 853         if (!wqe)
 854                 return -ENOMEM;
 855
 856         qp->rq_wrid_array[wqe_idx] = info->wr_id;
 857         addl_frag_cnt = info->num_sges > 1 ? (info->num_sges - 1) : 0;
 858         qp->wqe_ops.iw_set_fragment(wqe, 0, info->sg_list,
 859                                     qp->rwqe_polarity);
 860
 861         for (i = 1, byte_off = 32; i < info->num_sges; i++) {
 862                 qp->wqe_ops.iw_set_fragment(wqe, byte_off, &info->sg_list[i],
 863                                             qp->rwqe_polarity);
 864                 byte_off += 16;
 865         }
 866
 867         /* if not an odd number set valid bit in next fragment */
 868         if (qp->uk_attrs->hw_rev >= IRDMA_GEN_2 && !(info->num_sges & 0x01) &&
 869             info->num_sges) {
 870                 qp->wqe_ops.iw_set_fragment(wqe, byte_off, NULL,
 871                                             qp->rwqe_polarity);
 872                 if (qp->uk_attrs->hw_rev == IRDMA_GEN_2)
 873                         ++addl_frag_cnt;
 874         }
 875
 876         set_64bit_val(wqe, 16, 0);
 877         hdr = FIELD_PREP(IRDMAQPSQ_ADDFRAGCNT, addl_frag_cnt) |
 878               FIELD_PREP(IRDMAQPSQ_VALID, qp->rwqe_polarity);
 879
 880         dma_wmb(); /* make sure WQE is populated before valid bit is set */
 881
 882         set_64bit_val(wqe, 24, hdr);
 883
 884         return 0;
 885 }
 886
 887 /**
 888  * irdma_uk_cq_resize - reset the cq buffer info
 889  * @cq: cq to resize
 890  * @cq_base: new cq buffer addr
 891  * @cq_size: number of cqes
 892  */
 893 void irdma_uk_cq_resize(struct irdma_cq_uk *cq, void *cq_base, int cq_size)
 894 {
 895         cq->cq_base = cq_base;
 896         cq->cq_size = cq_size;
 897         IRDMA_RING_INIT(cq->cq_ring, cq->cq_size);
 898         cq->polarity = 1;
 899 }
 900
 901 /**
 902  * irdma_uk_cq_set_resized_cnt - record the count of the resized buffers
 903  * @cq: cq to resize
 904  * @cq_cnt: the count of the resized cq buffers
 905  */
 906 void irdma_uk_cq_set_resized_cnt(struct irdma_cq_uk *cq, u16 cq_cnt)
 907 {
 908         u64 temp_val;
 909         u16 sw_cq_sel;
 910         u8 arm_next_se;
 911         u8 arm_next;
 912         u8 arm_seq_num;
 913
 914         get_64bit_val(cq->shadow_area, 32, &temp_val);
 915
 916         sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);
 917         sw_cq_sel += cq_cnt;
 918
 919         arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val);
 920         arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);
 921         arm_next = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT, temp_val);
 922
 923         temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) |
 924                    FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) |
 925                    FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |
 926                    FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, arm_next);
 927
 928         set_64bit_val(cq->shadow_area, 32, temp_val);
 929 }
 930
 931 /**
 932  * irdma_uk_cq_request_notification - cq notification request (door bell)
 933  * @cq: hw cq
 934  * @cq_notify: notification type
 935  */
 936 void irdma_uk_cq_request_notification(struct irdma_cq_uk *cq,
 937                                       enum irdma_cmpl_notify cq_notify)
 938 {
 939         u64 temp_val;
 940         u16 sw_cq_sel;
 941         u8 arm_next_se = 0;
 942         u8 arm_next = 0;
 943         u8 arm_seq_num;
 944
 945         get_64bit_val(cq->shadow_area, 32, &temp_val);
 946         arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val);
 947         arm_seq_num++;
 948         sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);
 949         arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);
 950         arm_next_se |= 1;
 951         if (cq_notify == IRDMA_CQ_COMPL_EVENT)
 952                 arm_next = 1;
 953         temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) |
 954                    FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) |
 955                    FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |
 956                    FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, arm_next);
 957
 958         set_64bit_val(cq->shadow_area, 32, temp_val);
 959
 960         dma_wmb(); /* make sure WQE is populated before valid bit is set */
 961
 962         writel(cq->cq_id, cq->cqe_alloc_db);
 963 }
 964
 965 /**
 966  * irdma_uk_cq_poll_cmpl - get cq completion info
 967  * @cq: hw cq
 968  * @info: cq poll information returned
 969  */
 970 int irdma_uk_cq_poll_cmpl(struct irdma_cq_uk *cq,
 971                           struct irdma_cq_poll_info *info)
 972 {
 973         u64 comp_ctx, qword0, qword2, qword3;
 974         __le64 *cqe;
 975         struct irdma_qp_uk *qp;
 976         struct irdma_ring *pring = NULL;
 977         u32 wqe_idx;
 978         int ret_code;
 979         bool move_cq_head = true;
 980         u8 polarity;
 981         bool ext_valid;
 982         __le64 *ext_cqe;
 983
 984         if (cq->avoid_mem_cflct)
 985                 cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(cq);
 986         else
 987                 cqe = IRDMA_GET_CURRENT_CQ_ELEM(cq);
 988
 989         get_64bit_val(cqe, 24, &qword3);
 990         polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
 991         if (polarity != cq->polarity)
 992                 return -ENOENT;
 993
 994         /* Ensure CQE contents are read after valid bit is checked */
 995         dma_rmb();
 996
 997         ext_valid = (bool)FIELD_GET(IRDMA_CQ_EXTCQE, qword3);
 998         if (ext_valid) {
 999                 u64 qword6, qword7;
1000                 u32 peek_head;
1001
1002                 if (cq->avoid_mem_cflct) {
1003                         ext_cqe = (__le64 *)((u8 *)cqe + 32);
1004                         get_64bit_val(ext_cqe, 24, &qword7);
1005                         polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7);
1006                 } else {
1007                         peek_head = (cq->cq_ring.head + 1) % cq->cq_ring.size;
1008                         ext_cqe = cq->cq_base[peek_head].buf;
1009                         get_64bit_val(ext_cqe, 24, &qword7);
1010                         polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7);
1011                         if (!peek_head)
1012                                 polarity ^= 1;
1013                 }
1014                 if (polarity != cq->polarity)
1015                         return -ENOENT;
1016
1017                 /* Ensure ext CQE contents are read after ext valid bit is checked */
1018                 dma_rmb();
1019
1020                 info->imm_valid = (bool)FIELD_GET(IRDMA_CQ_IMMVALID, qword7);
1021                 if (info->imm_valid) {
1022                         u64 qword4;
1023
1024                         get_64bit_val(ext_cqe, 0, &qword4);
1025                         info->imm_data = (u32)FIELD_GET(IRDMA_CQ_IMMDATALOW32, qword4);
1026                 }
1027                 info->ud_smac_valid = (bool)FIELD_GET(IRDMA_CQ_UDSMACVALID, qword7);
1028                 info->ud_vlan_valid = (bool)FIELD_GET(IRDMA_CQ_UDVLANVALID, qword7);
1029                 if (info->ud_smac_valid || info->ud_vlan_valid) {
1030                         get_64bit_val(ext_cqe, 16, &qword6);
1031                         if (info->ud_vlan_valid)
1032                                 info->ud_vlan = (u16)FIELD_GET(IRDMA_CQ_UDVLAN, qword6);
1033                         if (info->ud_smac_valid) {
1034                                 info->ud_smac[5] = qword6 & 0xFF;
1035                                 info->ud_smac[4] = (qword6 >> 8) & 0xFF;
1036                                 info->ud_smac[3] = (qword6 >> 16) & 0xFF;
1037                                 info->ud_smac[2] = (qword6 >> 24) & 0xFF;
1038                                 info->ud_smac[1] = (qword6 >> 32) & 0xFF;
1039                                 info->ud_smac[0] = (qword6 >> 40) & 0xFF;
1040                         }
1041                 }
1042         } else {
1043                 info->imm_valid = false;
1044                 info->ud_smac_valid = false;
1045                 info->ud_vlan_valid = false;
1046         }
1047
1048         info->q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);
1049         info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3);
1050         info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3);
1051         if (info->error) {
1052                 info->major_err = FIELD_GET(IRDMA_CQ_MAJERR, qword3);
1053                 info->minor_err = FIELD_GET(IRDMA_CQ_MINERR, qword3);
1054                 if (info->major_err == IRDMA_FLUSH_MAJOR_ERR) {
1055                         info->comp_status = IRDMA_COMPL_STATUS_FLUSHED;
1056                         /* Set the min error to standard flush error code for remaining cqes */
1057                         if (info->minor_err != FLUSH_GENERAL_ERR) {
1058                                 qword3 &= ~IRDMA_CQ_MINERR;
1059                                 qword3 |= FIELD_PREP(IRDMA_CQ_MINERR, FLUSH_GENERAL_ERR);
1060                                 set_64bit_val(cqe, 24, qword3);
1061                         }
1062                 } else {
1063                         info->comp_status = IRDMA_COMPL_STATUS_UNKNOWN;
1064                 }
1065         } else {
1066                 info->comp_status = IRDMA_COMPL_STATUS_SUCCESS;
1067         }
1068
1069         get_64bit_val(cqe, 0, &qword0);
1070         get_64bit_val(cqe, 16, &qword2);
1071
1072         info->tcp_seq_num_rtt = (u32)FIELD_GET(IRDMACQ_TCPSEQNUMRTT, qword0);
1073         info->qp_id = (u32)FIELD_GET(IRDMACQ_QPID, qword2);
1074         info->ud_src_qpn = (u32)FIELD_GET(IRDMACQ_UDSRCQPN, qword2);
1075
1076         get_64bit_val(cqe, 8, &comp_ctx);
1077
1078         info->solicited_event = (bool)FIELD_GET(IRDMACQ_SOEVENT, qword3);
1079         qp = (struct irdma_qp_uk *)(unsigned long)comp_ctx;
1080         if (!qp || qp->destroy_pending) {
1081                 ret_code = -EFAULT;
1082                 goto exit;
1083         }
1084         wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3);
1085         info->qp_handle = (irdma_qp_handle)(unsigned long)qp;
1086         info->op_type = (u8)FIELD_GET(IRDMACQ_OP, qword3);
1087
1088         if (info->q_type == IRDMA_CQE_QTYPE_RQ) {
1089                 u32 array_idx;
1090
1091                 array_idx = wqe_idx / qp->rq_wqe_size_multiplier;
1092
1093                 if (info->comp_status == IRDMA_COMPL_STATUS_FLUSHED ||
1094                     info->comp_status == IRDMA_COMPL_STATUS_UNKNOWN) {
1095                         if (!IRDMA_RING_MORE_WORK(qp->rq_ring)) {
1096                                 ret_code = -ENOENT;
1097                                 goto exit;
1098                         }
1099
1100                         info->wr_id = qp->rq_wrid_array[qp->rq_ring.tail];
1101                         array_idx = qp->rq_ring.tail;
1102                 } else {
1103                         info->wr_id = qp->rq_wrid_array[array_idx];
1104                 }
1105
1106                 info->bytes_xfered = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0);
1107
1108                 if (qword3 & IRDMACQ_STAG) {
1109                         info->stag_invalid_set = true;
1110                         info->inv_stag = (u32)FIELD_GET(IRDMACQ_INVSTAG, qword2);
1111                 } else {
1112                         info->stag_invalid_set = false;
1113                 }
1114                 IRDMA_RING_SET_TAIL(qp->rq_ring, array_idx + 1);
1115                 if (info->comp_status == IRDMA_COMPL_STATUS_FLUSHED) {
1116                         qp->rq_flush_seen = true;
1117                         if (!IRDMA_RING_MORE_WORK(qp->rq_ring))
1118                                 qp->rq_flush_complete = true;
1119                         else
1120                                 move_cq_head = false;
1121                 }
1122                 pring = &qp->rq_ring;
1123         } else { /* q_type is IRDMA_CQE_QTYPE_SQ */
1124                 if (qp->first_sq_wq) {
1125                         if (wqe_idx + 1 >= qp->conn_wqes)
1126                                 qp->first_sq_wq = false;
1127
1128                         if (wqe_idx < qp->conn_wqes && qp->sq_ring.head == qp->sq_ring.tail) {
1129                                 IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
1130                                 IRDMA_RING_MOVE_TAIL(cq->cq_ring);
1131                                 set_64bit_val(cq->shadow_area, 0,
1132                                               IRDMA_RING_CURRENT_HEAD(cq->cq_ring));
1133                                 memset(info, 0,
1134                                        sizeof(struct irdma_cq_poll_info));
1135                                 return irdma_uk_cq_poll_cmpl(cq, info);
1136                         }
1137                 }
1138                 if (info->comp_status != IRDMA_COMPL_STATUS_FLUSHED) {
1139                         info->wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
1140                         if (!info->comp_status)
1141                                 info->bytes_xfered = qp->sq_wrtrk_array[wqe_idx].wr_len;
1142                         info->op_type = (u8)FIELD_GET(IRDMACQ_OP, qword3);
1143                         IRDMA_RING_SET_TAIL(qp->sq_ring,
1144                                             wqe_idx + qp->sq_wrtrk_array[wqe_idx].quanta);
1145                 } else {
1146                         if (!IRDMA_RING_MORE_WORK(qp->sq_ring)) {
1147                                 ret_code = -ENOENT;
1148                                 goto exit;
1149                         }
1150
1151                         do {
1152                                 __le64 *sw_wqe;
1153                                 u64 wqe_qword;
1154                                 u32 tail;
1155
1156                                 tail = qp->sq_ring.tail;
1157                                 sw_wqe = qp->sq_base[tail].elem;
1158                                 get_64bit_val(sw_wqe, 24,
1159                                               &wqe_qword);
1160                                 info->op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE,
1161                                                               wqe_qword);
1162                                 IRDMA_RING_SET_TAIL(qp->sq_ring,
1163                                                     tail + qp->sq_wrtrk_array[tail].quanta);
1164                                 if (info->op_type != IRDMAQP_OP_NOP) {
1165                                         info->wr_id = qp->sq_wrtrk_array[tail].wrid;
1166                                         info->bytes_xfered = qp->sq_wrtrk_array[tail].wr_len;
1167                                         break;
1168                                 }
1169                         } while (1);
1170                         if (info->op_type == IRDMA_OP_TYPE_BIND_MW &&
1171                             info->minor_err == FLUSH_PROT_ERR)
1172                                 info->minor_err = FLUSH_MW_BIND_ERR;
1173                         qp->sq_flush_seen = true;
1174                         if (!IRDMA_RING_MORE_WORK(qp->sq_ring))
1175                                 qp->sq_flush_complete = true;
1176                 }
1177                 pring = &qp->sq_ring;
1178         }
1179
1180         ret_code = 0;
1181
1182 exit:
1183         if (!ret_code && info->comp_status == IRDMA_COMPL_STATUS_FLUSHED)
1184                 if (pring && IRDMA_RING_MORE_WORK(*pring))
1185                         move_cq_head = false;
1186
1187         if (move_cq_head) {
1188                 IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
1189                 if (!IRDMA_RING_CURRENT_HEAD(cq->cq_ring))
1190                         cq->polarity ^= 1;
1191
1192                 if (ext_valid && !cq->avoid_mem_cflct) {
1193                         IRDMA_RING_MOVE_HEAD_NOCHECK(cq->cq_ring);
1194                         if (!IRDMA_RING_CURRENT_HEAD(cq->cq_ring))
1195                                 cq->polarity ^= 1;
1196                 }
1197
1198                 IRDMA_RING_MOVE_TAIL(cq->cq_ring);
1199                 if (!cq->avoid_mem_cflct && ext_valid)
1200                         IRDMA_RING_MOVE_TAIL(cq->cq_ring);
1201                 set_64bit_val(cq->shadow_area, 0,
1202                               IRDMA_RING_CURRENT_HEAD(cq->cq_ring));
1203         } else {
1204                 qword3 &= ~IRDMA_CQ_WQEIDX;
1205                 qword3 |= FIELD_PREP(IRDMA_CQ_WQEIDX, pring->tail);
1206                 set_64bit_val(cqe, 24, qword3);
1207         }
1208
1209         return ret_code;
1210 }
1211
1212 /**
1213  * irdma_qp_round_up - return round up qp wq depth
1214  * @wqdepth: wq depth in quanta to round up
1215  */
1216 static int irdma_qp_round_up(u32 wqdepth)
1217 {
1218         int scount = 1;
1219
1220         for (wqdepth--; scount <= 16; scount *= 2)
1221                 wqdepth |= wqdepth >> scount;
1222
1223         return ++wqdepth;
1224 }
1225
1226 /**
1227  * irdma_get_wqe_shift - get shift count for maximum wqe size
1228  * @uk_attrs: qp HW attributes
1229  * @sge: Maximum Scatter Gather Elements wqe
1230  * @inline_data: Maximum inline data size
1231  * @shift: Returns the shift needed based on sge
1232  *
1233  * Shift can be used to left shift the wqe size based on number of SGEs and inlind data size.
1234  * For 1 SGE or inline data <= 8, shift = 0 (wqe size of 32
1235  * bytes). For 2 or 3 SGEs or inline data <= 39, shift = 1 (wqe
1236  * size of 64 bytes).
1237  * For 4-7 SGE's and inline <= 101 Shift of 2 otherwise (wqe
1238  * size of 256 bytes).
1239  */
1240 void irdma_get_wqe_shift(struct irdma_uk_attrs *uk_attrs, u32 sge,
1241                          u32 inline_data, u8 *shift)
1242 {
1243         *shift = 0;
1244         if (uk_attrs->hw_rev >= IRDMA_GEN_2) {
1245                 if (sge > 1 || inline_data > 8) {
1246                         if (sge < 4 && inline_data <= 39)
1247                                 *shift = 1;
1248                         else if (sge < 8 && inline_data <= 101)
1249                                 *shift = 2;
1250                         else
1251                                 *shift = 3;
1252                 }
1253         } else if (sge > 1 || inline_data > 16) {
1254                 *shift = (sge < 4 && inline_data <= 48) ? 1 : 2;
1255         }
1256 }
1257
1258 /*
1259  * irdma_get_sqdepth - get SQ depth (quanta)
1260  * @uk_attrs: qp HW attributes
1261  * @sq_size: SQ size
1262  * @shift: shift which determines size of WQE
1263  * @sqdepth: depth of SQ
1264  *
1265  */
1266 int irdma_get_sqdepth(struct irdma_uk_attrs *uk_attrs, u32 sq_size, u8 shift,
1267                       u32 *sqdepth)
1268 {
1269         u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift;
1270
1271         *sqdepth = irdma_qp_round_up((sq_size << shift) + IRDMA_SQ_RSVD);
1272
1273         if (*sqdepth < min_size)
1274                 *sqdepth = min_size;
1275         else if (*sqdepth > uk_attrs->max_hw_wq_quanta)
1276                 return -EINVAL;
1277
1278         return 0;
1279 }
1280
1281 /*
1282  * irdma_get_rqdepth - get RQ depth (quanta)
1283  * @uk_attrs: qp HW attributes
1284  * @rq_size: RQ size
1285  * @shift: shift which determines size of WQE
1286  * @rqdepth: depth of RQ
1287  */
1288 int irdma_get_rqdepth(struct irdma_uk_attrs *uk_attrs, u32 rq_size, u8 shift,
1289                       u32 *rqdepth)
1290 {
1291         u32 min_size = (u32)uk_attrs->min_hw_wq_size << shift;
1292
1293         *rqdepth = irdma_qp_round_up((rq_size << shift) + IRDMA_RQ_RSVD);
1294
1295         if (*rqdepth < min_size)
1296                 *rqdepth = min_size;
1297         else if (*rqdepth > uk_attrs->max_hw_rq_quanta)
1298                 return -EINVAL;
1299
1300         return 0;
1301 }
1302
1303 static const struct irdma_wqe_uk_ops iw_wqe_uk_ops = {
1304         .iw_copy_inline_data = irdma_copy_inline_data,
1305         .iw_inline_data_size_to_quanta = irdma_inline_data_size_to_quanta,
1306         .iw_set_fragment = irdma_set_fragment,
1307         .iw_set_mw_bind_wqe = irdma_set_mw_bind_wqe,
1308 };
1309
1310 static const struct irdma_wqe_uk_ops iw_wqe_uk_ops_gen_1 = {
1311         .iw_copy_inline_data = irdma_copy_inline_data_gen_1,
1312         .iw_inline_data_size_to_quanta = irdma_inline_data_size_to_quanta_gen_1,
1313         .iw_set_fragment = irdma_set_fragment_gen_1,
1314         .iw_set_mw_bind_wqe = irdma_set_mw_bind_wqe_gen_1,
1315 };
1316
1317 /**
1318  * irdma_setup_connection_wqes - setup WQEs necessary to complete
1319  * connection.
1320  * @qp: hw qp (user and kernel)
1321  * @info: qp initialization info
1322  */
1323 static void irdma_setup_connection_wqes(struct irdma_qp_uk *qp,
1324                                         struct irdma_qp_uk_init_info *info)
1325 {
1326         u16 move_cnt = 1;
1327
1328         if (!info->legacy_mode &&
1329             (qp->uk_attrs->feature_flags & IRDMA_FEATURE_RTS_AE))
1330                 move_cnt = 3;
1331
1332         qp->conn_wqes = move_cnt;
1333         IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->sq_ring, move_cnt);
1334         IRDMA_RING_MOVE_TAIL_BY_COUNT(qp->sq_ring, move_cnt);
1335         IRDMA_RING_MOVE_HEAD_BY_COUNT_NOCHECK(qp->initial_ring, move_cnt);
1336 }
1337
1338 /**
1339  * irdma_uk_calc_shift_wq - calculate WQE shift for both SQ and RQ
1340  * @ukinfo: qp initialization info
1341  * @sq_shift: Returns shift of SQ
1342  * @rq_shift: Returns shift of RQ
1343  */
1344 void irdma_uk_calc_shift_wq(struct irdma_qp_uk_init_info *ukinfo, u8 *sq_shift,
1345                             u8 *rq_shift)
1346 {
1347         bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2;
1348
1349         irdma_get_wqe_shift(ukinfo->uk_attrs,
1350                             imm_support ? ukinfo->max_sq_frag_cnt + 1 :
1351                                           ukinfo->max_sq_frag_cnt,
1352                             ukinfo->max_inline_data, sq_shift);
1353
1354         irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_rq_frag_cnt, 0,
1355                             rq_shift);
1356
1357         if (ukinfo->uk_attrs->hw_rev == IRDMA_GEN_1) {
1358                 if (ukinfo->abi_ver > 4)
1359                         *rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;
1360         }
1361 }
1362
1363 /**
1364  * irdma_uk_calc_depth_shift_sq - calculate depth and shift for SQ size.
1365  * @ukinfo: qp initialization info
1366  * @sq_depth: Returns depth of SQ
1367  * @sq_shift: Returns shift of SQ
1368  */
1369 int irdma_uk_calc_depth_shift_sq(struct irdma_qp_uk_init_info *ukinfo,
1370                                  u32 *sq_depth, u8 *sq_shift)
1371 {
1372         bool imm_support = ukinfo->uk_attrs->hw_rev >= IRDMA_GEN_2;
1373         int status;
1374
1375         irdma_get_wqe_shift(ukinfo->uk_attrs,
1376                             imm_support ? ukinfo->max_sq_frag_cnt + 1 :
1377                             ukinfo->max_sq_frag_cnt,
1378                             ukinfo->max_inline_data, sq_shift);
1379         status = irdma_get_sqdepth(ukinfo->uk_attrs, ukinfo->sq_size,
1380                                    *sq_shift, sq_depth);
1381
1382         return status;
1383 }
1384
1385 /**
1386  * irdma_uk_calc_depth_shift_rq - calculate depth and shift for RQ size.
1387  * @ukinfo: qp initialization info
1388  * @rq_depth: Returns depth of RQ
1389  * @rq_shift: Returns shift of RQ
1390  */
1391 int irdma_uk_calc_depth_shift_rq(struct irdma_qp_uk_init_info *ukinfo,
1392                                  u32 *rq_depth, u8 *rq_shift)
1393 {
1394         int status;
1395
1396         irdma_get_wqe_shift(ukinfo->uk_attrs, ukinfo->max_rq_frag_cnt, 0,
1397                             rq_shift);
1398
1399         if (ukinfo->uk_attrs->hw_rev == IRDMA_GEN_1) {
1400                 if (ukinfo->abi_ver > 4)
1401                         *rq_shift = IRDMA_MAX_RQ_WQE_SHIFT_GEN1;
1402         }
1403
1404         status = irdma_get_rqdepth(ukinfo->uk_attrs, ukinfo->rq_size,
1405                                    *rq_shift, rq_depth);
1406
1407         return status;
1408 }
1409
1410 /**
1411  * irdma_uk_qp_init - initialize shared qp
1412  * @qp: hw qp (user and kernel)
1413  * @info: qp initialization info
1414  *
1415  * initializes the vars used in both user and kernel mode.
1416  * size of the wqe depends on numbers of max. fragements
1417  * allowed. Then size of wqe * the number of wqes should be the
1418  * amount of memory allocated for sq and rq.
1419  */
1420 int irdma_uk_qp_init(struct irdma_qp_uk *qp, struct irdma_qp_uk_init_info *info)
1421 {
1422         int ret_code = 0;
1423         u32 sq_ring_size;
1424
1425         qp->uk_attrs = info->uk_attrs;
1426         if (info->max_sq_frag_cnt > qp->uk_attrs->max_hw_wq_frags ||
1427             info->max_rq_frag_cnt > qp->uk_attrs->max_hw_wq_frags)
1428                 return -EINVAL;
1429
1430         qp->qp_caps = info->qp_caps;
1431         qp->sq_base = info->sq;
1432         qp->rq_base = info->rq;
1433         qp->qp_type = info->type ? info->type : IRDMA_QP_TYPE_IWARP;
1434         qp->shadow_area = info->shadow_area;
1435         qp->sq_wrtrk_array = info->sq_wrtrk_array;
1436
1437         qp->rq_wrid_array = info->rq_wrid_array;
1438         qp->wqe_alloc_db = info->wqe_alloc_db;
1439         qp->qp_id = info->qp_id;
1440         qp->sq_size = info->sq_size;
1441         qp->max_sq_frag_cnt = info->max_sq_frag_cnt;
1442         sq_ring_size = qp->sq_size << info->sq_shift;
1443         IRDMA_RING_INIT(qp->sq_ring, sq_ring_size);
1444         IRDMA_RING_INIT(qp->initial_ring, sq_ring_size);
1445         if (info->first_sq_wq) {
1446                 irdma_setup_connection_wqes(qp, info);
1447                 qp->swqe_polarity = 1;
1448                 qp->first_sq_wq = true;
1449         } else {
1450                 qp->swqe_polarity = 0;
1451         }
1452         qp->swqe_polarity_deferred = 1;
1453         qp->rwqe_polarity = 0;
1454         qp->rq_size = info->rq_size;
1455         qp->max_rq_frag_cnt = info->max_rq_frag_cnt;
1456         qp->max_inline_data = info->max_inline_data;
1457         qp->rq_wqe_size = info->rq_shift;
1458         IRDMA_RING_INIT(qp->rq_ring, qp->rq_size);
1459         qp->rq_wqe_size_multiplier = 1 << info->rq_shift;
1460         if (qp->uk_attrs->hw_rev == IRDMA_GEN_1)
1461                 qp->wqe_ops = iw_wqe_uk_ops_gen_1;
1462         else
1463                 qp->wqe_ops = iw_wqe_uk_ops;
1464         return ret_code;
1465 }
1466
1467 /**
1468  * irdma_uk_cq_init - initialize shared cq (user and kernel)
1469  * @cq: hw cq
1470  * @info: hw cq initialization info
1471  */
1472 void irdma_uk_cq_init(struct irdma_cq_uk *cq,
1473                       struct irdma_cq_uk_init_info *info)
1474 {
1475         cq->cq_base = info->cq_base;
1476         cq->cq_id = info->cq_id;
1477         cq->cq_size = info->cq_size;
1478         cq->cqe_alloc_db = info->cqe_alloc_db;
1479         cq->cq_ack_db = info->cq_ack_db;
1480         cq->shadow_area = info->shadow_area;
1481         cq->avoid_mem_cflct = info->avoid_mem_cflct;
1482         IRDMA_RING_INIT(cq->cq_ring, cq->cq_size);
1483         cq->polarity = 1;
1484 }
1485
1486 /**
1487  * irdma_uk_clean_cq - clean cq entries
1488  * @q: completion context
1489  * @cq: cq to clean
1490  */
1491 void irdma_uk_clean_cq(void *q, struct irdma_cq_uk *cq)
1492 {
1493         __le64 *cqe;
1494         u64 qword3, comp_ctx;
1495         u32 cq_head;
1496         u8 polarity, temp;
1497
1498         cq_head = cq->cq_ring.head;
1499         temp = cq->polarity;
1500         do {
1501                 if (cq->avoid_mem_cflct)
1502                         cqe = ((struct irdma_extended_cqe *)(cq->cq_base))[cq_head].buf;
1503                 else
1504                         cqe = cq->cq_base[cq_head].buf;
1505                 get_64bit_val(cqe, 24, &qword3);
1506                 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
1507
1508                 if (polarity != temp)
1509                         break;
1510
1511                 /* Ensure CQE contents are read after valid bit is checked */
1512                 dma_rmb();
1513
1514                 get_64bit_val(cqe, 8, &comp_ctx);
1515                 if ((void *)(unsigned long)comp_ctx == q)
1516                         set_64bit_val(cqe, 8, 0);
1517
1518                 cq_head = (cq_head + 1) % cq->cq_ring.size;
1519                 if (!cq_head)
1520                         temp ^= 1;
1521         } while (true);
1522 }
1523
1524 /**
1525  * irdma_nop - post a nop
1526  * @qp: hw qp ptr
1527  * @wr_id: work request id
1528  * @signaled: signaled for completion
1529  * @post_sq: ring doorbell
1530  */
1531 int irdma_nop(struct irdma_qp_uk *qp, u64 wr_id, bool signaled, bool post_sq)
1532 {
1533         __le64 *wqe;
1534         u64 hdr;
1535         u32 wqe_idx;
1536         struct irdma_post_sq_info info = {};
1537
1538         info.wr_id = wr_id;
1539         wqe = irdma_qp_get_next_send_wqe(qp, &wqe_idx, IRDMA_QP_WQE_MIN_QUANTA,
1540                                          0, &info);
1541         if (!wqe)
1542                 return -ENOMEM;
1543
1544         irdma_clr_wqes(qp, wqe_idx);
1545
1546         set_64bit_val(wqe, 0, 0);
1547         set_64bit_val(wqe, 8, 0);
1548         set_64bit_val(wqe, 16, 0);
1549
1550         hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
1551               FIELD_PREP(IRDMAQPSQ_SIGCOMPL, signaled) |
1552               FIELD_PREP(IRDMAQPSQ_VALID, qp->swqe_polarity);
1553
1554         dma_wmb(); /* make sure WQE is populated before valid bit is set */
1555
1556         set_64bit_val(wqe, 24, hdr);
1557         if (post_sq)
1558                 irdma_uk_qp_post_wr(qp);
1559
1560         return 0;
1561 }
1562
1563 /**
1564  * irdma_fragcnt_to_quanta_sq - calculate quanta based on fragment count for SQ
1565  * @frag_cnt: number of fragments
1566  * @quanta: quanta for frag_cnt
1567  */
1568 int irdma_fragcnt_to_quanta_sq(u32 frag_cnt, u16 *quanta)
1569 {
1570         switch (frag_cnt) {
1571         case 0:
1572         case 1:
1573                 *quanta = IRDMA_QP_WQE_MIN_QUANTA;
1574                 break;
1575         case 2:
1576         case 3:
1577                 *quanta = 2;
1578                 break;
1579         case 4:
1580         case 5:
1581                 *quanta = 3;
1582                 break;
1583         case 6:
1584         case 7:
1585                 *quanta = 4;
1586                 break;
1587         case 8:
1588         case 9:
1589                 *quanta = 5;
1590                 break;
1591         case 10:
1592         case 11:
1593                 *quanta = 6;
1594                 break;
1595         case 12:
1596         case 13:
1597                 *quanta = 7;
1598                 break;
1599         case 14:
1600         case 15: /* when immediate data is present */
1601                 *quanta = 8;
1602                 break;
1603         default:
1604                 return -EINVAL;
1605         }
1606
1607         return 0;
1608 }
1609
1610 /**
1611  * irdma_fragcnt_to_wqesize_rq - calculate wqe size based on fragment count for RQ
1612  * @frag_cnt: number of fragments
1613  * @wqe_size: size in bytes given frag_cnt
1614  */
1615 int irdma_fragcnt_to_wqesize_rq(u32 frag_cnt, u16 *wqe_size)
1616 {
1617         switch (frag_cnt) {
1618         case 0:
1619         case 1:
1620                 *wqe_size = 32;
1621                 break;
1622         case 2:
1623         case 3:
1624                 *wqe_size = 64;
1625                 break;
1626         case 4:
1627         case 5:
1628         case 6:
1629         case 7:
1630                 *wqe_size = 128;
1631                 break;
1632         case 8:
1633         case 9:
1634         case 10:
1635         case 11:
1636         case 12:
1637         case 13:
1638         case 14:
1639                 *wqe_size = 256;
1640                 break;
1641         default:
1642                 return -EINVAL;
1643         }
1644
1645         return 0;
1646 }