drivers/infiniband/sw/rxe/rxe_req.c

   1 /*
   2  * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
   3  * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #include <linux/skbuff.h>
  35
  36 #include "rxe.h"
  37 #include "rxe_loc.h"
  38 #include "rxe_queue.h"
  39
  40 static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
  41                        u32 opcode);
  42
  43 static inline void retry_first_write_send(struct rxe_qp *qp,
  44                                           struct rxe_send_wqe *wqe,
  45                                           unsigned mask, int npsn)
  46 {
  47         int i;
  48
  49         for (i = 0; i < npsn; i++) {
  50                 int to_send = (wqe->dma.resid > qp->mtu) ?
  51                                 qp->mtu : wqe->dma.resid;
  52
  53                 qp->req.opcode = next_opcode(qp, wqe,
  54                                              wqe->wr.opcode);
  55
  56                 if (wqe->wr.send_flags & IB_SEND_INLINE) {
  57                         wqe->dma.resid -= to_send;
  58                         wqe->dma.sge_offset += to_send;
  59                 } else {
  60                         advance_dma_data(&wqe->dma, to_send);
  61                 }
  62                 if (mask & WR_WRITE_MASK)
  63                         wqe->iova += qp->mtu;
  64         }
  65 }
  66
  67 static void req_retry(struct rxe_qp *qp)
  68 {
  69         struct rxe_send_wqe *wqe;
  70         unsigned int wqe_index;
  71         unsigned int mask;
  72         int npsn;
  73         int first = 1;
  74
  75         qp->req.wqe_index       = consumer_index(qp->sq.queue);
  76         qp->req.psn             = qp->comp.psn;
  77         qp->req.opcode          = -1;
  78
  79         for (wqe_index = consumer_index(qp->sq.queue);
  80                 wqe_index != producer_index(qp->sq.queue);
  81                 wqe_index = next_index(qp->sq.queue, wqe_index)) {
  82                 wqe = addr_from_index(qp->sq.queue, wqe_index);
  83                 mask = wr_opcode_mask(wqe->wr.opcode, qp);
  84
  85                 if (wqe->state == wqe_state_posted)
  86                         break;
  87
  88                 if (wqe->state == wqe_state_done)
  89                         continue;
  90
  91                 wqe->iova = (mask & WR_ATOMIC_MASK) ?
  92                              wqe->wr.wr.atomic.remote_addr :
  93                              (mask & WR_READ_OR_WRITE_MASK) ?
  94                              wqe->wr.wr.rdma.remote_addr :
  95                              0;
  96
  97                 if (!first || (mask & WR_READ_MASK) == 0) {
  98                         wqe->dma.resid = wqe->dma.length;
  99                         wqe->dma.cur_sge = 0;
 100                         wqe->dma.sge_offset = 0;
 101                 }
 102
 103                 if (first) {
 104                         first = 0;
 105
 106                         if (mask & WR_WRITE_OR_SEND_MASK) {
 107                                 npsn = (qp->comp.psn - wqe->first_psn) &
 108                                         BTH_PSN_MASK;
 109                                 retry_first_write_send(qp, wqe, mask, npsn);
 110                         }
 111
 112                         if (mask & WR_READ_MASK) {
 113                                 npsn = (wqe->dma.length - wqe->dma.resid) /
 114                                         qp->mtu;
 115                                 wqe->iova += npsn * qp->mtu;
 116                         }
 117                 }
 118
 119                 wqe->state = wqe_state_posted;
 120         }
 121 }
 122
 123 void rnr_nak_timer(unsigned long data)
 124 {
 125         struct rxe_qp *qp = (struct rxe_qp *)data;
 126
 127         pr_debug("qp#%d rnr nak timer fired\n", qp_num(qp));
 128         rxe_run_task(&qp->req.task, 1);
 129 }
 130
 131 static struct rxe_send_wqe *req_next_wqe(struct rxe_qp *qp)
 132 {
 133         struct rxe_send_wqe *wqe = queue_head(qp->sq.queue);
 134         unsigned long flags;
 135
 136         if (unlikely(qp->req.state == QP_STATE_DRAIN)) {
 137                 /* check to see if we are drained;
 138                  * state_lock used by requester and completer
 139                  */
 140                 spin_lock_irqsave(&qp->state_lock, flags);
 141                 do {
 142                         if (qp->req.state != QP_STATE_DRAIN) {
 143                                 /* comp just finished */
 144                                 spin_unlock_irqrestore(&qp->state_lock,
 145                                                        flags);
 146                                 break;
 147                         }
 148
 149                         if (wqe && ((qp->req.wqe_index !=
 150                                 consumer_index(qp->sq.queue)) ||
 151                                 (wqe->state != wqe_state_posted))) {
 152                                 /* comp not done yet */
 153                                 spin_unlock_irqrestore(&qp->state_lock,
 154                                                        flags);
 155                                 break;
 156                         }
 157
 158                         qp->req.state = QP_STATE_DRAINED;
 159                         spin_unlock_irqrestore(&qp->state_lock, flags);
 160
 161                         if (qp->ibqp.event_handler) {
 162                                 struct ib_event ev;
 163
 164                                 ev.device = qp->ibqp.device;
 165                                 ev.element.qp = &qp->ibqp;
 166                                 ev.event = IB_EVENT_SQ_DRAINED;
 167                                 qp->ibqp.event_handler(&ev,
 168                                         qp->ibqp.qp_context);
 169                         }
 170                 } while (0);
 171         }
 172
 173         if (qp->req.wqe_index == producer_index(qp->sq.queue))
 174                 return NULL;
 175
 176         wqe = addr_from_index(qp->sq.queue, qp->req.wqe_index);
 177
 178         if (unlikely((qp->req.state == QP_STATE_DRAIN ||
 179                       qp->req.state == QP_STATE_DRAINED) &&
 180                      (wqe->state != wqe_state_processing)))
 181                 return NULL;
 182
 183         if (unlikely((wqe->wr.send_flags & IB_SEND_FENCE) &&
 184                      (qp->req.wqe_index != consumer_index(qp->sq.queue)))) {
 185                 qp->req.wait_fence = 1;
 186                 return NULL;
 187         }
 188
 189         wqe->mask = wr_opcode_mask(wqe->wr.opcode, qp);
 190         return wqe;
 191 }
 192
 193 static int next_opcode_rc(struct rxe_qp *qp, u32 opcode, int fits)
 194 {
 195         switch (opcode) {
 196         case IB_WR_RDMA_WRITE:
 197                 if (qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_FIRST ||
 198                     qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE)
 199                         return fits ?
 200                                 IB_OPCODE_RC_RDMA_WRITE_LAST :
 201                                 IB_OPCODE_RC_RDMA_WRITE_MIDDLE;
 202                 else
 203                         return fits ?
 204                                 IB_OPCODE_RC_RDMA_WRITE_ONLY :
 205                                 IB_OPCODE_RC_RDMA_WRITE_FIRST;
 206
 207         case IB_WR_RDMA_WRITE_WITH_IMM:
 208                 if (qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_FIRST ||
 209                     qp->req.opcode == IB_OPCODE_RC_RDMA_WRITE_MIDDLE)
 210                         return fits ?
 211                                 IB_OPCODE_RC_RDMA_WRITE_LAST_WITH_IMMEDIATE :
 212                                 IB_OPCODE_RC_RDMA_WRITE_MIDDLE;
 213                 else
 214                         return fits ?
 215                                 IB_OPCODE_RC_RDMA_WRITE_ONLY_WITH_IMMEDIATE :
 216                                 IB_OPCODE_RC_RDMA_WRITE_FIRST;
 217
 218         case IB_WR_SEND:
 219                 if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST ||
 220                     qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE)
 221                         return fits ?
 222                                 IB_OPCODE_RC_SEND_LAST :
 223                                 IB_OPCODE_RC_SEND_MIDDLE;
 224                 else
 225                         return fits ?
 226                                 IB_OPCODE_RC_SEND_ONLY :
 227                                 IB_OPCODE_RC_SEND_FIRST;
 228
 229         case IB_WR_SEND_WITH_IMM:
 230                 if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST ||
 231                     qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE)
 232                         return fits ?
 233                                 IB_OPCODE_RC_SEND_LAST_WITH_IMMEDIATE :
 234                                 IB_OPCODE_RC_SEND_MIDDLE;
 235                 else
 236                         return fits ?
 237                                 IB_OPCODE_RC_SEND_ONLY_WITH_IMMEDIATE :
 238                                 IB_OPCODE_RC_SEND_FIRST;
 239
 240         case IB_WR_RDMA_READ:
 241                 return IB_OPCODE_RC_RDMA_READ_REQUEST;
 242
 243         case IB_WR_ATOMIC_CMP_AND_SWP:
 244                 return IB_OPCODE_RC_COMPARE_SWAP;
 245
 246         case IB_WR_ATOMIC_FETCH_AND_ADD:
 247                 return IB_OPCODE_RC_FETCH_ADD;
 248
 249         case IB_WR_SEND_WITH_INV:
 250                 if (qp->req.opcode == IB_OPCODE_RC_SEND_FIRST ||
 251                     qp->req.opcode == IB_OPCODE_RC_SEND_MIDDLE)
 252                         return fits ? IB_OPCODE_RC_SEND_LAST_WITH_INVALIDATE :
 253                                 IB_OPCODE_RC_SEND_MIDDLE;
 254                 else
 255                         return fits ? IB_OPCODE_RC_SEND_ONLY_WITH_INVALIDATE :
 256                                 IB_OPCODE_RC_SEND_FIRST;
 257         case IB_WR_REG_MR:
 258         case IB_WR_LOCAL_INV:
 259                 return opcode;
 260         }
 261
 262         return -EINVAL;
 263 }
 264
 265 static int next_opcode_uc(struct rxe_qp *qp, u32 opcode, int fits)
 266 {
 267         switch (opcode) {
 268         case IB_WR_RDMA_WRITE:
 269                 if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST ||
 270                     qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE)
 271                         return fits ?
 272                                 IB_OPCODE_UC_RDMA_WRITE_LAST :
 273                                 IB_OPCODE_UC_RDMA_WRITE_MIDDLE;
 274                 else
 275                         return fits ?
 276                                 IB_OPCODE_UC_RDMA_WRITE_ONLY :
 277                                 IB_OPCODE_UC_RDMA_WRITE_FIRST;
 278
 279         case IB_WR_RDMA_WRITE_WITH_IMM:
 280                 if (qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_FIRST ||
 281                     qp->req.opcode == IB_OPCODE_UC_RDMA_WRITE_MIDDLE)
 282                         return fits ?
 283                                 IB_OPCODE_UC_RDMA_WRITE_LAST_WITH_IMMEDIATE :
 284                                 IB_OPCODE_UC_RDMA_WRITE_MIDDLE;
 285                 else
 286                         return fits ?
 287                                 IB_OPCODE_UC_RDMA_WRITE_ONLY_WITH_IMMEDIATE :
 288                                 IB_OPCODE_UC_RDMA_WRITE_FIRST;
 289
 290         case IB_WR_SEND:
 291                 if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST ||
 292                     qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE)
 293                         return fits ?
 294                                 IB_OPCODE_UC_SEND_LAST :
 295                                 IB_OPCODE_UC_SEND_MIDDLE;
 296                 else
 297                         return fits ?
 298                                 IB_OPCODE_UC_SEND_ONLY :
 299                                 IB_OPCODE_UC_SEND_FIRST;
 300
 301         case IB_WR_SEND_WITH_IMM:
 302                 if (qp->req.opcode == IB_OPCODE_UC_SEND_FIRST ||
 303                     qp->req.opcode == IB_OPCODE_UC_SEND_MIDDLE)
 304                         return fits ?
 305                                 IB_OPCODE_UC_SEND_LAST_WITH_IMMEDIATE :
 306                                 IB_OPCODE_UC_SEND_MIDDLE;
 307                 else
 308                         return fits ?
 309                                 IB_OPCODE_UC_SEND_ONLY_WITH_IMMEDIATE :
 310                                 IB_OPCODE_UC_SEND_FIRST;
 311         }
 312
 313         return -EINVAL;
 314 }
 315
 316 static int next_opcode(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
 317                        u32 opcode)
 318 {
 319         int fits = (wqe->dma.resid <= qp->mtu);
 320
 321         switch (qp_type(qp)) {
 322         case IB_QPT_RC:
 323                 return next_opcode_rc(qp, opcode, fits);
 324
 325         case IB_QPT_UC:
 326                 return next_opcode_uc(qp, opcode, fits);
 327
 328         case IB_QPT_SMI:
 329         case IB_QPT_UD:
 330         case IB_QPT_GSI:
 331                 switch (opcode) {
 332                 case IB_WR_SEND:
 333                         return IB_OPCODE_UD_SEND_ONLY;
 334
 335                 case IB_WR_SEND_WITH_IMM:
 336                         return IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
 337                 }
 338                 break;
 339
 340         default:
 341                 break;
 342         }
 343
 344         return -EINVAL;
 345 }
 346
 347 static inline int check_init_depth(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
 348 {
 349         int depth;
 350
 351         if (wqe->has_rd_atomic)
 352                 return 0;
 353
 354         qp->req.need_rd_atomic = 1;
 355         depth = atomic_dec_return(&qp->req.rd_atomic);
 356
 357         if (depth >= 0) {
 358                 qp->req.need_rd_atomic = 0;
 359                 wqe->has_rd_atomic = 1;
 360                 return 0;
 361         }
 362
 363         atomic_inc(&qp->req.rd_atomic);
 364         return -EAGAIN;
 365 }
 366
 367 static inline int get_mtu(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
 368 {
 369         struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
 370         struct rxe_port *port;
 371         struct rxe_av *av;
 372
 373         if ((qp_type(qp) == IB_QPT_RC) || (qp_type(qp) == IB_QPT_UC))
 374                 return qp->mtu;
 375
 376         av = &wqe->av;
 377         port = &rxe->port;
 378
 379         return port->mtu_cap;
 380 }
 381
 382 static struct sk_buff *init_req_packet(struct rxe_qp *qp,
 383                                        struct rxe_send_wqe *wqe,
 384                                        int opcode, int payload,
 385                                        struct rxe_pkt_info *pkt)
 386 {
 387         struct rxe_dev          *rxe = to_rdev(qp->ibqp.device);
 388         struct rxe_port         *port = &rxe->port;
 389         struct sk_buff          *skb;
 390         struct rxe_send_wr      *ibwr = &wqe->wr;
 391         struct rxe_av           *av;
 392         int                     pad = (-payload) & 0x3;
 393         int                     paylen;
 394         int                     solicited;
 395         u16                     pkey;
 396         u32                     qp_num;
 397         int                     ack_req;
 398
 399         /* length from start of bth to end of icrc */
 400         paylen = rxe_opcode[opcode].length + payload + pad + RXE_ICRC_SIZE;
 401
 402         /* pkt->hdr, rxe, port_num and mask are initialized in ifc
 403          * layer
 404          */
 405         pkt->opcode     = opcode;
 406         pkt->qp         = qp;
 407         pkt->psn        = qp->req.psn;
 408         pkt->mask       = rxe_opcode[opcode].mask;
 409         pkt->paylen     = paylen;
 410         pkt->offset     = 0;
 411         pkt->wqe        = wqe;
 412
 413         /* init skb */
 414         av = rxe_get_av(pkt);
 415         skb = rxe_init_packet(rxe, av, paylen, pkt);
 416         if (unlikely(!skb))
 417                 return NULL;
 418
 419         /* init bth */
 420         solicited = (ibwr->send_flags & IB_SEND_SOLICITED) &&
 421                         (pkt->mask & RXE_END_MASK) &&
 422                         ((pkt->mask & (RXE_SEND_MASK)) ||
 423                         (pkt->mask & (RXE_WRITE_MASK | RXE_IMMDT_MASK)) ==
 424                         (RXE_WRITE_MASK | RXE_IMMDT_MASK));
 425
 426         pkey = (qp_type(qp) == IB_QPT_GSI) ?
 427                  port->pkey_tbl[ibwr->wr.ud.pkey_index] :
 428                  port->pkey_tbl[qp->attr.pkey_index];
 429
 430         qp_num = (pkt->mask & RXE_DETH_MASK) ? ibwr->wr.ud.remote_qpn :
 431                                          qp->attr.dest_qp_num;
 432
 433         ack_req = ((pkt->mask & RXE_END_MASK) ||
 434                 (qp->req.noack_pkts++ > RXE_MAX_PKT_PER_ACK));
 435         if (ack_req)
 436                 qp->req.noack_pkts = 0;
 437
 438         bth_init(pkt, pkt->opcode, solicited, 0, pad, pkey, qp_num,
 439                  ack_req, pkt->psn);
 440
 441         /* init optional headers */
 442         if (pkt->mask & RXE_RETH_MASK) {
 443                 reth_set_rkey(pkt, ibwr->wr.rdma.rkey);
 444                 reth_set_va(pkt, wqe->iova);
 445                 reth_set_len(pkt, wqe->dma.resid);
 446         }
 447
 448         if (pkt->mask & RXE_IMMDT_MASK)
 449                 immdt_set_imm(pkt, ibwr->ex.imm_data);
 450
 451         if (pkt->mask & RXE_IETH_MASK)
 452                 ieth_set_rkey(pkt, ibwr->ex.invalidate_rkey);
 453
 454         if (pkt->mask & RXE_ATMETH_MASK) {
 455                 atmeth_set_va(pkt, wqe->iova);
 456                 if (opcode == IB_OPCODE_RC_COMPARE_SWAP ||
 457                     opcode == IB_OPCODE_RD_COMPARE_SWAP) {
 458                         atmeth_set_swap_add(pkt, ibwr->wr.atomic.swap);
 459                         atmeth_set_comp(pkt, ibwr->wr.atomic.compare_add);
 460                 } else {
 461                         atmeth_set_swap_add(pkt, ibwr->wr.atomic.compare_add);
 462                 }
 463                 atmeth_set_rkey(pkt, ibwr->wr.atomic.rkey);
 464         }
 465
 466         if (pkt->mask & RXE_DETH_MASK) {
 467                 if (qp->ibqp.qp_num == 1)
 468                         deth_set_qkey(pkt, GSI_QKEY);
 469                 else
 470                         deth_set_qkey(pkt, ibwr->wr.ud.remote_qkey);
 471                 deth_set_sqp(pkt, qp->ibqp.qp_num);
 472         }
 473
 474         return skb;
 475 }
 476
 477 static int fill_packet(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
 478                        struct rxe_pkt_info *pkt, struct sk_buff *skb,
 479                        int paylen)
 480 {
 481         struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
 482         u32 crc = 0;
 483         u32 *p;
 484         int err;
 485
 486         err = rxe_prepare(rxe, pkt, skb, &crc);
 487         if (err)
 488                 return err;
 489
 490         if (pkt->mask & RXE_WRITE_OR_SEND) {
 491                 if (wqe->wr.send_flags & IB_SEND_INLINE) {
 492                         u8 *tmp = &wqe->dma.inline_data[wqe->dma.sge_offset];
 493
 494                         crc = crc32_le(crc, tmp, paylen);
 495
 496                         memcpy(payload_addr(pkt), tmp, paylen);
 497
 498                         wqe->dma.resid -= paylen;
 499                         wqe->dma.sge_offset += paylen;
 500                 } else {
 501                         err = copy_data(rxe, qp->pd, 0, &wqe->dma,
 502                                         payload_addr(pkt), paylen,
 503                                         from_mem_obj,
 504                                         &crc);
 505                         if (err)
 506                                 return err;
 507                 }
 508         }
 509         p = payload_addr(pkt) + paylen + bth_pad(pkt);
 510
 511         *p = ~crc;
 512
 513         return 0;
 514 }
 515
 516 static void update_wqe_state(struct rxe_qp *qp,
 517                 struct rxe_send_wqe *wqe,
 518                 struct rxe_pkt_info *pkt)
 519 {
 520         if (pkt->mask & RXE_END_MASK) {
 521                 if (qp_type(qp) == IB_QPT_RC)
 522                         wqe->state = wqe_state_pending;
 523         } else {
 524                 wqe->state = wqe_state_processing;
 525         }
 526 }
 527
 528 static void update_wqe_psn(struct rxe_qp *qp,
 529                            struct rxe_send_wqe *wqe,
 530                            struct rxe_pkt_info *pkt,
 531                            int payload)
 532 {
 533         /* number of packets left to send including current one */
 534         int num_pkt = (wqe->dma.resid + payload + qp->mtu - 1) / qp->mtu;
 535
 536         /* handle zero length packet case */
 537         if (num_pkt == 0)
 538                 num_pkt = 1;
 539
 540         if (pkt->mask & RXE_START_MASK) {
 541                 wqe->first_psn = qp->req.psn;
 542                 wqe->last_psn = (qp->req.psn + num_pkt - 1) & BTH_PSN_MASK;
 543         }
 544
 545         if (pkt->mask & RXE_READ_MASK)
 546                 qp->req.psn = (wqe->first_psn + num_pkt) & BTH_PSN_MASK;
 547         else
 548                 qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK;
 549 }
 550
 551 static void save_state(struct rxe_send_wqe *wqe,
 552                        struct rxe_qp *qp,
 553                        struct rxe_send_wqe *rollback_wqe,
 554                        u32 *rollback_psn)
 555 {
 556         rollback_wqe->state     = wqe->state;
 557         rollback_wqe->first_psn = wqe->first_psn;
 558         rollback_wqe->last_psn  = wqe->last_psn;
 559         *rollback_psn           = qp->req.psn;
 560 }
 561
 562 static void rollback_state(struct rxe_send_wqe *wqe,
 563                            struct rxe_qp *qp,
 564                            struct rxe_send_wqe *rollback_wqe,
 565                            u32 rollback_psn)
 566 {
 567         wqe->state     = rollback_wqe->state;
 568         wqe->first_psn = rollback_wqe->first_psn;
 569         wqe->last_psn  = rollback_wqe->last_psn;
 570         qp->req.psn    = rollback_psn;
 571 }
 572
 573 static void update_state(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
 574                          struct rxe_pkt_info *pkt, int payload)
 575 {
 576         qp->req.opcode = pkt->opcode;
 577
 578         if (pkt->mask & RXE_END_MASK)
 579                 qp->req.wqe_index = next_index(qp->sq.queue, qp->req.wqe_index);
 580
 581         qp->need_req_skb = 0;
 582
 583         if (qp->qp_timeout_jiffies && !timer_pending(&qp->retrans_timer))
 584                 mod_timer(&qp->retrans_timer,
 585                           jiffies + qp->qp_timeout_jiffies);
 586 }
 587
 588 int rxe_requester(void *arg)
 589 {
 590         struct rxe_qp *qp = (struct rxe_qp *)arg;
 591         struct rxe_pkt_info pkt;
 592         struct sk_buff *skb;
 593         struct rxe_send_wqe *wqe;
 594         enum rxe_hdr_mask mask;
 595         int payload;
 596         int mtu;
 597         int opcode;
 598         int ret;
 599         struct rxe_send_wqe rollback_wqe;
 600         u32 rollback_psn;
 601
 602 next_wqe:
 603         if (unlikely(!qp->valid || qp->req.state == QP_STATE_ERROR))
 604                 goto exit;
 605
 606         if (unlikely(qp->req.state == QP_STATE_RESET)) {
 607                 qp->req.wqe_index = consumer_index(qp->sq.queue);
 608                 qp->req.opcode = -1;
 609                 qp->req.need_rd_atomic = 0;
 610                 qp->req.wait_psn = 0;
 611                 qp->req.need_retry = 0;
 612                 goto exit;
 613         }
 614
 615         if (unlikely(qp->req.need_retry)) {
 616                 req_retry(qp);
 617                 qp->req.need_retry = 0;
 618         }
 619
 620         wqe = req_next_wqe(qp);
 621         if (unlikely(!wqe))
 622                 goto exit;
 623
 624         if (wqe->mask & WR_REG_MASK) {
 625                 if (wqe->wr.opcode == IB_WR_LOCAL_INV) {
 626                         struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
 627                         struct rxe_mem *rmr;
 628
 629                         rmr = rxe_pool_get_index(&rxe->mr_pool,
 630                                                  wqe->wr.ex.invalidate_rkey >> 8);
 631                         if (!rmr) {
 632                                 pr_err("No mr for key %#x\n",
 633                                        wqe->wr.ex.invalidate_rkey);
 634                                 wqe->state = wqe_state_error;
 635                                 wqe->status = IB_WC_MW_BIND_ERR;
 636                                 goto exit;
 637                         }
 638                         rmr->state = RXE_MEM_STATE_FREE;
 639                         rxe_drop_ref(rmr);
 640                         wqe->state = wqe_state_done;
 641                         wqe->status = IB_WC_SUCCESS;
 642                 } else if (wqe->wr.opcode == IB_WR_REG_MR) {
 643                         struct rxe_mem *rmr = to_rmr(wqe->wr.wr.reg.mr);
 644
 645                         rmr->state = RXE_MEM_STATE_VALID;
 646                         rmr->access = wqe->wr.wr.reg.access;
 647                         rmr->lkey = wqe->wr.wr.reg.key;
 648                         rmr->rkey = wqe->wr.wr.reg.key;
 649                         rmr->iova = wqe->wr.wr.reg.mr->iova;
 650                         wqe->state = wqe_state_done;
 651                         wqe->status = IB_WC_SUCCESS;
 652                 } else {
 653                         goto exit;
 654                 }
 655                 if ((wqe->wr.send_flags & IB_SEND_SIGNALED) ||
 656                     qp->sq_sig_type == IB_SIGNAL_ALL_WR)
 657                         rxe_run_task(&qp->comp.task, 1);
 658                 qp->req.wqe_index = next_index(qp->sq.queue,
 659                                                 qp->req.wqe_index);
 660                 goto next_wqe;
 661         }
 662
 663         if (unlikely(qp_type(qp) == IB_QPT_RC &&
 664                 psn_compare(qp->req.psn, (qp->comp.psn +
 665                                 RXE_MAX_UNACKED_PSNS)) > 0)) {
 666                 qp->req.wait_psn = 1;
 667                 goto exit;
 668         }
 669
 670         /* Limit the number of inflight SKBs per QP */
 671         if (unlikely(atomic_read(&qp->skb_out) >
 672                      RXE_INFLIGHT_SKBS_PER_QP_HIGH)) {
 673                 qp->need_req_skb = 1;
 674                 goto exit;
 675         }
 676
 677         opcode = next_opcode(qp, wqe, wqe->wr.opcode);
 678         if (unlikely(opcode < 0)) {
 679                 wqe->status = IB_WC_LOC_QP_OP_ERR;
 680                 goto err;
 681         }
 682
 683         mask = rxe_opcode[opcode].mask;
 684         if (unlikely(mask & RXE_READ_OR_ATOMIC)) {
 685                 if (check_init_depth(qp, wqe))
 686                         goto exit;
 687         }
 688
 689         mtu = get_mtu(qp, wqe);
 690         payload = (mask & RXE_WRITE_OR_SEND) ? wqe->dma.resid : 0;
 691         if (payload > mtu) {
 692                 if (qp_type(qp) == IB_QPT_UD) {
 693                         /* C10-93.1.1: If the total sum of all the buffer lengths specified for a
 694                          * UD message exceeds the MTU of the port as returned by QueryHCA, the CI
 695                          * shall not emit any packets for this message. Further, the CI shall not
 696                          * generate an error due to this condition.
 697                          */
 698
 699                         /* fake a successful UD send */
 700                         wqe->first_psn = qp->req.psn;
 701                         wqe->last_psn = qp->req.psn;
 702                         qp->req.psn = (qp->req.psn + 1) & BTH_PSN_MASK;
 703                         qp->req.opcode = IB_OPCODE_UD_SEND_ONLY;
 704                         qp->req.wqe_index = next_index(qp->sq.queue,
 705                                                        qp->req.wqe_index);
 706                         wqe->state = wqe_state_done;
 707                         wqe->status = IB_WC_SUCCESS;
 708                         __rxe_do_task(&qp->comp.task);
 709                         return 0;
 710                 }
 711                 payload = mtu;
 712         }
 713
 714         skb = init_req_packet(qp, wqe, opcode, payload, &pkt);
 715         if (unlikely(!skb)) {
 716                 pr_err("qp#%d Failed allocating skb\n", qp_num(qp));
 717                 goto err;
 718         }
 719
 720         if (fill_packet(qp, wqe, &pkt, skb, payload)) {
 721                 pr_debug("qp#%d Error during fill packet\n", qp_num(qp));
 722                 goto err;
 723         }
 724
 725         /*
 726          * To prevent a race on wqe access between requester and completer,
 727          * wqe members state and psn need to be set before calling
 728          * rxe_xmit_packet().
 729          * Otherwise, completer might initiate an unjustified retry flow.
 730          */
 731         save_state(wqe, qp, &rollback_wqe, &rollback_psn);
 732         update_wqe_state(qp, wqe, &pkt);
 733         update_wqe_psn(qp, wqe, &pkt, payload);
 734         ret = rxe_xmit_packet(to_rdev(qp->ibqp.device), qp, &pkt, skb);
 735         if (ret) {
 736                 qp->need_req_skb = 1;
 737
 738                 rollback_state(wqe, qp, &rollback_wqe, rollback_psn);
 739
 740                 if (ret == -EAGAIN) {
 741                         kfree_skb(skb);
 742                         rxe_run_task(&qp->req.task, 1);
 743                         goto exit;
 744                 }
 745
 746                 goto err;
 747         }
 748
 749         update_state(qp, wqe, &pkt, payload);
 750
 751         goto next_wqe;
 752
 753 err:
 754         kfree_skb(skb);
 755         wqe->status = IB_WC_LOC_PROT_ERR;
 756         wqe->state = wqe_state_error;
 757
 758         /*
 759          * IBA Spec. Section 10.7.3.1 SIGNALED COMPLETIONS
 760          * ---------8<---------8<-------------
 761          * ...Note that if a completion error occurs, a Work Completion
 762          * will always be generated, even if the signaling
 763          * indicator requests an Unsignaled Completion.
 764          * ---------8<---------8<-------------
 765          */
 766         wqe->wr.send_flags |= IB_SEND_SIGNALED;
 767         __rxe_do_task(&qp->comp.task);
 768         return -EAGAIN;
 769
 770 exit:
 771         return -EAGAIN;
 772 }