drivers/infiniband/hw/qib/qib_rc.c

   1 /*
   2  * Copyright (c) 2006, 2007, 2008, 2009 QLogic Corporation. All rights reserved.
   3  * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #include <linux/io.h>
  35
  36 #include "qib.h"
  37
  38 /* cut down ridiculously long IB macro names */
  39 #define OP(x) IB_OPCODE_RC_##x
  40
  41
  42 static u32 restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe,
  43                        u32 psn, u32 pmtu)
  44 {
  45         u32 len;
  46
  47         len = ((psn - wqe->psn) & QIB_PSN_MASK) * pmtu;
  48         ss->sge = wqe->sg_list[0];
  49         ss->sg_list = wqe->sg_list + 1;
  50         ss->num_sge = wqe->wr.num_sge;
  51         ss->total_len = wqe->length;
  52         rvt_skip_sge(ss, len, false);
  53         return wqe->length - len;
  54 }
  55
  56 /**
  57  * qib_make_rc_ack - construct a response packet (ACK, NAK, or RDMA read)
  58  * @dev: the device for this QP
  59  * @qp: a pointer to the QP
  60  * @ohdr: a pointer to the IB header being constructed
  61  * @pmtu: the path MTU
  62  *
  63  * Return 1 if constructed; otherwise, return 0.
  64  * Note that we are in the responder's side of the QP context.
  65  * Note the QP s_lock must be held.
  66  */
  67 static int qib_make_rc_ack(struct qib_ibdev *dev, struct rvt_qp *qp,
  68                            struct ib_other_headers *ohdr, u32 pmtu)
  69 {
  70         struct rvt_ack_entry *e;
  71         u32 hwords;
  72         u32 len;
  73         u32 bth0;
  74         u32 bth2;
  75
  76         /* Don't send an ACK if we aren't supposed to. */
  77         if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
  78                 goto bail;
  79
  80         /* header size in 32-bit words LRH+BTH = (8+12)/4. */
  81         hwords = 5;
  82
  83         switch (qp->s_ack_state) {
  84         case OP(RDMA_READ_RESPONSE_LAST):
  85         case OP(RDMA_READ_RESPONSE_ONLY):
  86                 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
  87                 if (e->rdma_sge.mr) {
  88                         rvt_put_mr(e->rdma_sge.mr);
  89                         e->rdma_sge.mr = NULL;
  90                 }
  91                 /* FALLTHROUGH */
  92         case OP(ATOMIC_ACKNOWLEDGE):
  93                 /*
  94                  * We can increment the tail pointer now that the last
  95                  * response has been sent instead of only being
  96                  * constructed.
  97                  */
  98                 if (++qp->s_tail_ack_queue > QIB_MAX_RDMA_ATOMIC)
  99                         qp->s_tail_ack_queue = 0;
 100                 /* FALLTHROUGH */
 101         case OP(SEND_ONLY):
 102         case OP(ACKNOWLEDGE):
 103                 /* Check for no next entry in the queue. */
 104                 if (qp->r_head_ack_queue == qp->s_tail_ack_queue) {
 105                         if (qp->s_flags & RVT_S_ACK_PENDING)
 106                                 goto normal;
 107                         goto bail;
 108                 }
 109
 110                 e = &qp->s_ack_queue[qp->s_tail_ack_queue];
 111                 if (e->opcode == OP(RDMA_READ_REQUEST)) {
 112                         /*
 113                          * If a RDMA read response is being resent and
 114                          * we haven't seen the duplicate request yet,
 115                          * then stop sending the remaining responses the
 116                          * responder has seen until the requester resends it.
 117                          */
 118                         len = e->rdma_sge.sge_length;
 119                         if (len && !e->rdma_sge.mr) {
 120                                 qp->s_tail_ack_queue = qp->r_head_ack_queue;
 121                                 goto bail;
 122                         }
 123                         /* Copy SGE state in case we need to resend */
 124                         qp->s_rdma_mr = e->rdma_sge.mr;
 125                         if (qp->s_rdma_mr)
 126                                 rvt_get_mr(qp->s_rdma_mr);
 127                         qp->s_ack_rdma_sge.sge = e->rdma_sge;
 128                         qp->s_ack_rdma_sge.num_sge = 1;
 129                         qp->s_cur_sge = &qp->s_ack_rdma_sge;
 130                         if (len > pmtu) {
 131                                 len = pmtu;
 132                                 qp->s_ack_state = OP(RDMA_READ_RESPONSE_FIRST);
 133                         } else {
 134                                 qp->s_ack_state = OP(RDMA_READ_RESPONSE_ONLY);
 135                                 e->sent = 1;
 136                         }
 137                         ohdr->u.aeth = rvt_compute_aeth(qp);
 138                         hwords++;
 139                         qp->s_ack_rdma_psn = e->psn;
 140                         bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK;
 141                 } else {
 142                         /* COMPARE_SWAP or FETCH_ADD */
 143                         qp->s_cur_sge = NULL;
 144                         len = 0;
 145                         qp->s_ack_state = OP(ATOMIC_ACKNOWLEDGE);
 146                         ohdr->u.at.aeth = rvt_compute_aeth(qp);
 147                         ib_u64_put(e->atomic_data, &ohdr->u.at.atomic_ack_eth);
 148                         hwords += sizeof(ohdr->u.at) / sizeof(u32);
 149                         bth2 = e->psn & QIB_PSN_MASK;
 150                         e->sent = 1;
 151                 }
 152                 bth0 = qp->s_ack_state << 24;
 153                 break;
 154
 155         case OP(RDMA_READ_RESPONSE_FIRST):
 156                 qp->s_ack_state = OP(RDMA_READ_RESPONSE_MIDDLE);
 157                 /* FALLTHROUGH */
 158         case OP(RDMA_READ_RESPONSE_MIDDLE):
 159                 qp->s_cur_sge = &qp->s_ack_rdma_sge;
 160                 qp->s_rdma_mr = qp->s_ack_rdma_sge.sge.mr;
 161                 if (qp->s_rdma_mr)
 162                         rvt_get_mr(qp->s_rdma_mr);
 163                 len = qp->s_ack_rdma_sge.sge.sge_length;
 164                 if (len > pmtu)
 165                         len = pmtu;
 166                 else {
 167                         ohdr->u.aeth = rvt_compute_aeth(qp);
 168                         hwords++;
 169                         qp->s_ack_state = OP(RDMA_READ_RESPONSE_LAST);
 170                         e = &qp->s_ack_queue[qp->s_tail_ack_queue];
 171                         e->sent = 1;
 172                 }
 173                 bth0 = qp->s_ack_state << 24;
 174                 bth2 = qp->s_ack_rdma_psn++ & QIB_PSN_MASK;
 175                 break;
 176
 177         default:
 178 normal:
 179                 /*
 180                  * Send a regular ACK.
 181                  * Set the s_ack_state so we wait until after sending
 182                  * the ACK before setting s_ack_state to ACKNOWLEDGE
 183                  * (see above).
 184                  */
 185                 qp->s_ack_state = OP(SEND_ONLY);
 186                 qp->s_flags &= ~RVT_S_ACK_PENDING;
 187                 qp->s_cur_sge = NULL;
 188                 if (qp->s_nak_state)
 189                         ohdr->u.aeth =
 190                                 cpu_to_be32((qp->r_msn & IB_MSN_MASK) |
 191                                             (qp->s_nak_state <<
 192                                              IB_AETH_CREDIT_SHIFT));
 193                 else
 194                         ohdr->u.aeth = rvt_compute_aeth(qp);
 195                 hwords++;
 196                 len = 0;
 197                 bth0 = OP(ACKNOWLEDGE) << 24;
 198                 bth2 = qp->s_ack_psn & QIB_PSN_MASK;
 199         }
 200         qp->s_rdma_ack_cnt++;
 201         qp->s_hdrwords = hwords;
 202         qp->s_cur_size = len;
 203         qib_make_ruc_header(qp, ohdr, bth0, bth2);
 204         return 1;
 205
 206 bail:
 207         qp->s_ack_state = OP(ACKNOWLEDGE);
 208         qp->s_flags &= ~(RVT_S_RESP_PENDING | RVT_S_ACK_PENDING);
 209         return 0;
 210 }
 211
 212 /**
 213  * qib_make_rc_req - construct a request packet (SEND, RDMA r/w, ATOMIC)
 214  * @qp: a pointer to the QP
 215  *
 216  * Assumes the s_lock is held.
 217  *
 218  * Return 1 if constructed; otherwise, return 0.
 219  */
 220 int qib_make_rc_req(struct rvt_qp *qp, unsigned long *flags)
 221 {
 222         struct qib_qp_priv *priv = qp->priv;
 223         struct qib_ibdev *dev = to_idev(qp->ibqp.device);
 224         struct ib_other_headers *ohdr;
 225         struct rvt_sge_state *ss;
 226         struct rvt_swqe *wqe;
 227         u32 hwords;
 228         u32 len;
 229         u32 bth0;
 230         u32 bth2;
 231         u32 pmtu = qp->pmtu;
 232         char newreq;
 233         int ret = 0;
 234         int delta;
 235
 236         ohdr = &priv->s_hdr->u.oth;
 237         if (rdma_ah_get_ah_flags(&qp->remote_ah_attr) & IB_AH_GRH)
 238                 ohdr = &priv->s_hdr->u.l.oth;
 239
 240         /* Sending responses has higher priority over sending requests. */
 241         if ((qp->s_flags & RVT_S_RESP_PENDING) &&
 242             qib_make_rc_ack(dev, qp, ohdr, pmtu))
 243                 goto done;
 244
 245         if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_SEND_OK)) {
 246                 if (!(ib_rvt_state_ops[qp->state] & RVT_FLUSH_SEND))
 247                         goto bail;
 248                 /* We are in the error state, flush the work request. */
 249                 if (qp->s_last == READ_ONCE(qp->s_head))
 250                         goto bail;
 251                 /* If DMAs are in progress, we can't flush immediately. */
 252                 if (atomic_read(&priv->s_dma_busy)) {
 253                         qp->s_flags |= RVT_S_WAIT_DMA;
 254                         goto bail;
 255                 }
 256                 wqe = rvt_get_swqe_ptr(qp, qp->s_last);
 257                 qib_send_complete(qp, wqe, qp->s_last != qp->s_acked ?
 258                         IB_WC_SUCCESS : IB_WC_WR_FLUSH_ERR);
 259                 /* will get called again */
 260                 goto done;
 261         }
 262
 263         if (qp->s_flags & (RVT_S_WAIT_RNR | RVT_S_WAIT_ACK))
 264                 goto bail;
 265
 266         if (qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) {
 267                 if (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) {
 268                         qp->s_flags |= RVT_S_WAIT_PSN;
 269                         goto bail;
 270                 }
 271                 qp->s_sending_psn = qp->s_psn;
 272                 qp->s_sending_hpsn = qp->s_psn - 1;
 273         }
 274
 275         /* header size in 32-bit words LRH+BTH = (8+12)/4. */
 276         hwords = 5;
 277         bth0 = 0;
 278
 279         /* Send a request. */
 280         wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
 281         switch (qp->s_state) {
 282         default:
 283                 if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_NEXT_SEND_OK))
 284                         goto bail;
 285                 /*
 286                  * Resend an old request or start a new one.
 287                  *
 288                  * We keep track of the current SWQE so that
 289                  * we don't reset the "furthest progress" state
 290                  * if we need to back up.
 291                  */
 292                 newreq = 0;
 293                 if (qp->s_cur == qp->s_tail) {
 294                         /* Check if send work queue is empty. */
 295                         if (qp->s_tail == READ_ONCE(qp->s_head))
 296                                 goto bail;
 297                         /*
 298                          * If a fence is requested, wait for previous
 299                          * RDMA read and atomic operations to finish.
 300                          */
 301                         if ((wqe->wr.send_flags & IB_SEND_FENCE) &&
 302                             qp->s_num_rd_atomic) {
 303                                 qp->s_flags |= RVT_S_WAIT_FENCE;
 304                                 goto bail;
 305                         }
 306                         newreq = 1;
 307                         qp->s_psn = wqe->psn;
 308                 }
 309                 /*
 310                  * Note that we have to be careful not to modify the
 311                  * original work request since we may need to resend
 312                  * it.
 313                  */
 314                 len = wqe->length;
 315                 ss = &qp->s_sge;
 316                 bth2 = qp->s_psn & QIB_PSN_MASK;
 317                 switch (wqe->wr.opcode) {
 318                 case IB_WR_SEND:
 319                 case IB_WR_SEND_WITH_IMM:
 320                         /* If no credit, return. */
 321                         if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT) &&
 322                             rvt_cmp_msn(wqe->ssn, qp->s_lsn + 1) > 0) {
 323                                 qp->s_flags |= RVT_S_WAIT_SSN_CREDIT;
 324                                 goto bail;
 325                         }
 326                         if (len > pmtu) {
 327                                 qp->s_state = OP(SEND_FIRST);
 328                                 len = pmtu;
 329                                 break;
 330                         }
 331                         if (wqe->wr.opcode == IB_WR_SEND)
 332                                 qp->s_state = OP(SEND_ONLY);
 333                         else {
 334                                 qp->s_state = OP(SEND_ONLY_WITH_IMMEDIATE);
 335                                 /* Immediate data comes after the BTH */
 336                                 ohdr->u.imm_data = wqe->wr.ex.imm_data;
 337                                 hwords += 1;
 338                         }
 339                         if (wqe->wr.send_flags & IB_SEND_SOLICITED)
 340                                 bth0 |= IB_BTH_SOLICITED;
 341                         bth2 |= IB_BTH_REQ_ACK;
 342                         if (++qp->s_cur == qp->s_size)
 343                                 qp->s_cur = 0;
 344                         break;
 345
 346                 case IB_WR_RDMA_WRITE:
 347                         if (newreq && !(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
 348                                 qp->s_lsn++;
 349                         goto no_flow_control;
 350                 case IB_WR_RDMA_WRITE_WITH_IMM:
 351                         /* If no credit, return. */
 352                         if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT) &&
 353                             rvt_cmp_msn(wqe->ssn, qp->s_lsn + 1) > 0) {
 354                                 qp->s_flags |= RVT_S_WAIT_SSN_CREDIT;
 355                                 goto bail;
 356                         }
 357 no_flow_control:
 358                         ohdr->u.rc.reth.vaddr =
 359                                 cpu_to_be64(wqe->rdma_wr.remote_addr);
 360                         ohdr->u.rc.reth.rkey =
 361                                 cpu_to_be32(wqe->rdma_wr.rkey);
 362                         ohdr->u.rc.reth.length = cpu_to_be32(len);
 363                         hwords += sizeof(struct ib_reth) / sizeof(u32);
 364                         if (len > pmtu) {
 365                                 qp->s_state = OP(RDMA_WRITE_FIRST);
 366                                 len = pmtu;
 367                                 break;
 368                         }
 369                         if (wqe->rdma_wr.wr.opcode == IB_WR_RDMA_WRITE)
 370                                 qp->s_state = OP(RDMA_WRITE_ONLY);
 371                         else {
 372                                 qp->s_state = OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE);
 373                                 /* Immediate data comes after RETH */
 374                                 ohdr->u.rc.imm_data =
 375                                         wqe->rdma_wr.wr.ex.imm_data;
 376                                 hwords += 1;
 377                                 if (wqe->rdma_wr.wr.send_flags & IB_SEND_SOLICITED)
 378                                         bth0 |= IB_BTH_SOLICITED;
 379                         }
 380                         bth2 |= IB_BTH_REQ_ACK;
 381                         if (++qp->s_cur == qp->s_size)
 382                                 qp->s_cur = 0;
 383                         break;
 384
 385                 case IB_WR_RDMA_READ:
 386                         /*
 387                          * Don't allow more operations to be started
 388                          * than the QP limits allow.
 389                          */
 390                         if (newreq) {
 391                                 if (qp->s_num_rd_atomic >=
 392                                     qp->s_max_rd_atomic) {
 393                                         qp->s_flags |= RVT_S_WAIT_RDMAR;
 394                                         goto bail;
 395                                 }
 396                                 qp->s_num_rd_atomic++;
 397                                 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
 398                                         qp->s_lsn++;
 399                         }
 400
 401                         ohdr->u.rc.reth.vaddr =
 402                                 cpu_to_be64(wqe->rdma_wr.remote_addr);
 403                         ohdr->u.rc.reth.rkey =
 404                                 cpu_to_be32(wqe->rdma_wr.rkey);
 405                         ohdr->u.rc.reth.length = cpu_to_be32(len);
 406                         qp->s_state = OP(RDMA_READ_REQUEST);
 407                         hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32);
 408                         ss = NULL;
 409                         len = 0;
 410                         bth2 |= IB_BTH_REQ_ACK;
 411                         if (++qp->s_cur == qp->s_size)
 412                                 qp->s_cur = 0;
 413                         break;
 414
 415                 case IB_WR_ATOMIC_CMP_AND_SWP:
 416                 case IB_WR_ATOMIC_FETCH_AND_ADD:
 417                         /*
 418                          * Don't allow more operations to be started
 419                          * than the QP limits allow.
 420                          */
 421                         if (newreq) {
 422                                 if (qp->s_num_rd_atomic >=
 423                                     qp->s_max_rd_atomic) {
 424                                         qp->s_flags |= RVT_S_WAIT_RDMAR;
 425                                         goto bail;
 426                                 }
 427                                 qp->s_num_rd_atomic++;
 428                                 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT))
 429                                         qp->s_lsn++;
 430                         }
 431                         if (wqe->atomic_wr.wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
 432                                 qp->s_state = OP(COMPARE_SWAP);
 433                                 put_ib_ateth_swap(wqe->atomic_wr.swap,
 434                                                   &ohdr->u.atomic_eth);
 435                                 put_ib_ateth_compare(wqe->atomic_wr.compare_add,
 436                                                      &ohdr->u.atomic_eth);
 437                         } else {
 438                                 qp->s_state = OP(FETCH_ADD);
 439                                 put_ib_ateth_swap(wqe->atomic_wr.compare_add,
 440                                                   &ohdr->u.atomic_eth);
 441                                 put_ib_ateth_compare(0, &ohdr->u.atomic_eth);
 442                         }
 443                         put_ib_ateth_vaddr(wqe->atomic_wr.remote_addr,
 444                                            &ohdr->u.atomic_eth);
 445                         ohdr->u.atomic_eth.rkey = cpu_to_be32(
 446                                 wqe->atomic_wr.rkey);
 447                         hwords += sizeof(struct ib_atomic_eth) / sizeof(u32);
 448                         ss = NULL;
 449                         len = 0;
 450                         bth2 |= IB_BTH_REQ_ACK;
 451                         if (++qp->s_cur == qp->s_size)
 452                                 qp->s_cur = 0;
 453                         break;
 454
 455                 default:
 456                         goto bail;
 457                 }
 458                 qp->s_sge.sge = wqe->sg_list[0];
 459                 qp->s_sge.sg_list = wqe->sg_list + 1;
 460                 qp->s_sge.num_sge = wqe->wr.num_sge;
 461                 qp->s_sge.total_len = wqe->length;
 462                 qp->s_len = wqe->length;
 463                 if (newreq) {
 464                         qp->s_tail++;
 465                         if (qp->s_tail >= qp->s_size)
 466                                 qp->s_tail = 0;
 467                 }
 468                 if (wqe->wr.opcode == IB_WR_RDMA_READ)
 469                         qp->s_psn = wqe->lpsn + 1;
 470                 else
 471                         qp->s_psn++;
 472                 break;
 473
 474         case OP(RDMA_READ_RESPONSE_FIRST):
 475                 /*
 476                  * qp->s_state is normally set to the opcode of the
 477                  * last packet constructed for new requests and therefore
 478                  * is never set to RDMA read response.
 479                  * RDMA_READ_RESPONSE_FIRST is used by the ACK processing
 480                  * thread to indicate a SEND needs to be restarted from an
 481                  * earlier PSN without interferring with the sending thread.
 482                  * See qib_restart_rc().
 483                  */
 484                 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
 485                 /* FALLTHROUGH */
 486         case OP(SEND_FIRST):
 487                 qp->s_state = OP(SEND_MIDDLE);
 488                 /* FALLTHROUGH */
 489         case OP(SEND_MIDDLE):
 490                 bth2 = qp->s_psn++ & QIB_PSN_MASK;
 491                 ss = &qp->s_sge;
 492                 len = qp->s_len;
 493                 if (len > pmtu) {
 494                         len = pmtu;
 495                         break;
 496                 }
 497                 if (wqe->wr.opcode == IB_WR_SEND)
 498                         qp->s_state = OP(SEND_LAST);
 499                 else {
 500                         qp->s_state = OP(SEND_LAST_WITH_IMMEDIATE);
 501                         /* Immediate data comes after the BTH */
 502                         ohdr->u.imm_data = wqe->wr.ex.imm_data;
 503                         hwords += 1;
 504                 }
 505                 if (wqe->wr.send_flags & IB_SEND_SOLICITED)
 506                         bth0 |= IB_BTH_SOLICITED;
 507                 bth2 |= IB_BTH_REQ_ACK;
 508                 qp->s_cur++;
 509                 if (qp->s_cur >= qp->s_size)
 510                         qp->s_cur = 0;
 511                 break;
 512
 513         case OP(RDMA_READ_RESPONSE_LAST):
 514                 /*
 515                  * qp->s_state is normally set to the opcode of the
 516                  * last packet constructed for new requests and therefore
 517                  * is never set to RDMA read response.
 518                  * RDMA_READ_RESPONSE_LAST is used by the ACK processing
 519                  * thread to indicate a RDMA write needs to be restarted from
 520                  * an earlier PSN without interferring with the sending thread.
 521                  * See qib_restart_rc().
 522                  */
 523                 qp->s_len = restart_sge(&qp->s_sge, wqe, qp->s_psn, pmtu);
 524                 /* FALLTHROUGH */
 525         case OP(RDMA_WRITE_FIRST):
 526                 qp->s_state = OP(RDMA_WRITE_MIDDLE);
 527                 /* FALLTHROUGH */
 528         case OP(RDMA_WRITE_MIDDLE):
 529                 bth2 = qp->s_psn++ & QIB_PSN_MASK;
 530                 ss = &qp->s_sge;
 531                 len = qp->s_len;
 532                 if (len > pmtu) {
 533                         len = pmtu;
 534                         break;
 535                 }
 536                 if (wqe->wr.opcode == IB_WR_RDMA_WRITE)
 537                         qp->s_state = OP(RDMA_WRITE_LAST);
 538                 else {
 539                         qp->s_state = OP(RDMA_WRITE_LAST_WITH_IMMEDIATE);
 540                         /* Immediate data comes after the BTH */
 541                         ohdr->u.imm_data = wqe->wr.ex.imm_data;
 542                         hwords += 1;
 543                         if (wqe->wr.send_flags & IB_SEND_SOLICITED)
 544                                 bth0 |= IB_BTH_SOLICITED;
 545                 }
 546                 bth2 |= IB_BTH_REQ_ACK;
 547                 qp->s_cur++;
 548                 if (qp->s_cur >= qp->s_size)
 549                         qp->s_cur = 0;
 550                 break;
 551
 552         case OP(RDMA_READ_RESPONSE_MIDDLE):
 553                 /*
 554                  * qp->s_state is normally set to the opcode of the
 555                  * last packet constructed for new requests and therefore
 556                  * is never set to RDMA read response.
 557                  * RDMA_READ_RESPONSE_MIDDLE is used by the ACK processing
 558                  * thread to indicate a RDMA read needs to be restarted from
 559                  * an earlier PSN without interferring with the sending thread.
 560                  * See qib_restart_rc().
 561                  */
 562                 len = ((qp->s_psn - wqe->psn) & QIB_PSN_MASK) * pmtu;
 563                 ohdr->u.rc.reth.vaddr =
 564                         cpu_to_be64(wqe->rdma_wr.remote_addr + len);
 565                 ohdr->u.rc.reth.rkey =
 566                         cpu_to_be32(wqe->rdma_wr.rkey);
 567                 ohdr->u.rc.reth.length = cpu_to_be32(wqe->length - len);
 568                 qp->s_state = OP(RDMA_READ_REQUEST);
 569                 hwords += sizeof(ohdr->u.rc.reth) / sizeof(u32);
 570                 bth2 = (qp->s_psn & QIB_PSN_MASK) | IB_BTH_REQ_ACK;
 571                 qp->s_psn = wqe->lpsn + 1;
 572                 ss = NULL;
 573                 len = 0;
 574                 qp->s_cur++;
 575                 if (qp->s_cur == qp->s_size)
 576                         qp->s_cur = 0;
 577                 break;
 578         }
 579         qp->s_sending_hpsn = bth2;
 580         delta = (((int) bth2 - (int) wqe->psn) << 8) >> 8;
 581         if (delta && delta % QIB_PSN_CREDIT == 0)
 582                 bth2 |= IB_BTH_REQ_ACK;
 583         if (qp->s_flags & RVT_S_SEND_ONE) {
 584                 qp->s_flags &= ~RVT_S_SEND_ONE;
 585                 qp->s_flags |= RVT_S_WAIT_ACK;
 586                 bth2 |= IB_BTH_REQ_ACK;
 587         }
 588         qp->s_len -= len;
 589         qp->s_hdrwords = hwords;
 590         qp->s_cur_sge = ss;
 591         qp->s_cur_size = len;
 592         qib_make_ruc_header(qp, ohdr, bth0 | (qp->s_state << 24), bth2);
 593 done:
 594         return 1;
 595 bail:
 596         qp->s_flags &= ~RVT_S_BUSY;
 597         return ret;
 598 }
 599
 600 /**
 601  * qib_send_rc_ack - Construct an ACK packet and send it
 602  * @qp: a pointer to the QP
 603  *
 604  * This is called from qib_rc_rcv() and qib_kreceive().
 605  * Note that RDMA reads and atomics are handled in the
 606  * send side QP state and tasklet.
 607  */
 608 void qib_send_rc_ack(struct rvt_qp *qp)
 609 {
 610         struct qib_devdata *dd = dd_from_ibdev(qp->ibqp.device);
 611         struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
 612         struct qib_pportdata *ppd = ppd_from_ibp(ibp);
 613         u64 pbc;
 614         u16 lrh0;
 615         u32 bth0;
 616         u32 hwords;
 617         u32 pbufn;
 618         u32 __iomem *piobuf;
 619         struct ib_header hdr;
 620         struct ib_other_headers *ohdr;
 621         u32 control;
 622         unsigned long flags;
 623
 624         spin_lock_irqsave(&qp->s_lock, flags);
 625
 626         if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
 627                 goto unlock;
 628
 629         /* Don't send ACK or NAK if a RDMA read or atomic is pending. */
 630         if ((qp->s_flags & RVT_S_RESP_PENDING) || qp->s_rdma_ack_cnt)
 631                 goto queue_ack;
 632
 633         /* Construct the header with s_lock held so APM doesn't change it. */
 634         ohdr = &hdr.u.oth;
 635         lrh0 = QIB_LRH_BTH;
 636         /* header size in 32-bit words LRH+BTH+AETH = (8+12+4)/4. */
 637         hwords = 6;
 638         if (unlikely(rdma_ah_get_ah_flags(&qp->remote_ah_attr) &
 639                      IB_AH_GRH)) {
 640                 hwords += qib_make_grh(ibp, &hdr.u.l.grh,
 641                                        rdma_ah_read_grh(&qp->remote_ah_attr),
 642                                        hwords, 0);
 643                 ohdr = &hdr.u.l.oth;
 644                 lrh0 = QIB_LRH_GRH;
 645         }
 646         /* read pkey_index w/o lock (its atomic) */
 647         bth0 = qib_get_pkey(ibp, qp->s_pkey_index) | (OP(ACKNOWLEDGE) << 24);
 648         if (qp->s_mig_state == IB_MIG_MIGRATED)
 649                 bth0 |= IB_BTH_MIG_REQ;
 650         if (qp->r_nak_state)
 651                 ohdr->u.aeth = cpu_to_be32((qp->r_msn & IB_MSN_MASK) |
 652                                             (qp->r_nak_state <<
 653                                              IB_AETH_CREDIT_SHIFT));
 654         else
 655                 ohdr->u.aeth = rvt_compute_aeth(qp);
 656         lrh0 |= ibp->sl_to_vl[rdma_ah_get_sl(&qp->remote_ah_attr)] << 12 |
 657                 rdma_ah_get_sl(&qp->remote_ah_attr) << 4;
 658         hdr.lrh[0] = cpu_to_be16(lrh0);
 659         hdr.lrh[1] = cpu_to_be16(rdma_ah_get_dlid(&qp->remote_ah_attr));
 660         hdr.lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC);
 661         hdr.lrh[3] = cpu_to_be16(ppd->lid |
 662                                  rdma_ah_get_path_bits(&qp->remote_ah_attr));
 663         ohdr->bth[0] = cpu_to_be32(bth0);
 664         ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
 665         ohdr->bth[2] = cpu_to_be32(qp->r_ack_psn & QIB_PSN_MASK);
 666
 667         spin_unlock_irqrestore(&qp->s_lock, flags);
 668
 669         /* Don't try to send ACKs if the link isn't ACTIVE */
 670         if (!(ppd->lflags & QIBL_LINKACTIVE))
 671                 goto done;
 672
 673         control = dd->f_setpbc_control(ppd, hwords + SIZE_OF_CRC,
 674                                        qp->s_srate, lrh0 >> 12);
 675         /* length is + 1 for the control dword */
 676         pbc = ((u64) control << 32) | (hwords + 1);
 677
 678         piobuf = dd->f_getsendbuf(ppd, pbc, &pbufn);
 679         if (!piobuf) {
 680                 /*
 681                  * We are out of PIO buffers at the moment.
 682                  * Pass responsibility for sending the ACK to the
 683                  * send tasklet so that when a PIO buffer becomes
 684                  * available, the ACK is sent ahead of other outgoing
 685                  * packets.
 686                  */
 687                 spin_lock_irqsave(&qp->s_lock, flags);
 688                 goto queue_ack;
 689         }
 690
 691         /*
 692          * Write the pbc.
 693          * We have to flush after the PBC for correctness
 694          * on some cpus or WC buffer can be written out of order.
 695          */
 696         writeq(pbc, piobuf);
 697
 698         if (dd->flags & QIB_PIO_FLUSH_WC) {
 699                 u32 *hdrp = (u32 *) &hdr;
 700
 701                 qib_flush_wc();
 702                 qib_pio_copy(piobuf + 2, hdrp, hwords - 1);
 703                 qib_flush_wc();
 704                 __raw_writel(hdrp[hwords - 1], piobuf + hwords + 1);
 705         } else
 706                 qib_pio_copy(piobuf + 2, (u32 *) &hdr, hwords);
 707
 708         if (dd->flags & QIB_USE_SPCL_TRIG) {
 709                 u32 spcl_off = (pbufn >= dd->piobcnt2k) ? 2047 : 1023;
 710
 711                 qib_flush_wc();
 712                 __raw_writel(0xaebecede, piobuf + spcl_off);
 713         }
 714
 715         qib_flush_wc();
 716         qib_sendbuf_done(dd, pbufn);
 717
 718         this_cpu_inc(ibp->pmastats->n_unicast_xmit);
 719         goto done;
 720
 721 queue_ack:
 722         if (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK) {
 723                 this_cpu_inc(*ibp->rvp.rc_qacks);
 724                 qp->s_flags |= RVT_S_ACK_PENDING | RVT_S_RESP_PENDING;
 725                 qp->s_nak_state = qp->r_nak_state;
 726                 qp->s_ack_psn = qp->r_ack_psn;
 727
 728                 /* Schedule the send tasklet. */
 729                 qib_schedule_send(qp);
 730         }
 731 unlock:
 732         spin_unlock_irqrestore(&qp->s_lock, flags);
 733 done:
 734         return;
 735 }
 736
 737 /**
 738  * reset_psn - reset the QP state to send starting from PSN
 739  * @qp: the QP
 740  * @psn: the packet sequence number to restart at
 741  *
 742  * This is called from qib_rc_rcv() to process an incoming RC ACK
 743  * for the given QP.
 744  * Called at interrupt level with the QP s_lock held.
 745  */
 746 static void reset_psn(struct rvt_qp *qp, u32 psn)
 747 {
 748         u32 n = qp->s_acked;
 749         struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, n);
 750         u32 opcode;
 751
 752         qp->s_cur = n;
 753
 754         /*
 755          * If we are starting the request from the beginning,
 756          * let the normal send code handle initialization.
 757          */
 758         if (qib_cmp24(psn, wqe->psn) <= 0) {
 759                 qp->s_state = OP(SEND_LAST);
 760                 goto done;
 761         }
 762
 763         /* Find the work request opcode corresponding to the given PSN. */
 764         opcode = wqe->wr.opcode;
 765         for (;;) {
 766                 int diff;
 767
 768                 if (++n == qp->s_size)
 769                         n = 0;
 770                 if (n == qp->s_tail)
 771                         break;
 772                 wqe = rvt_get_swqe_ptr(qp, n);
 773                 diff = qib_cmp24(psn, wqe->psn);
 774                 if (diff < 0)
 775                         break;
 776                 qp->s_cur = n;
 777                 /*
 778                  * If we are starting the request from the beginning,
 779                  * let the normal send code handle initialization.
 780                  */
 781                 if (diff == 0) {
 782                         qp->s_state = OP(SEND_LAST);
 783                         goto done;
 784                 }
 785                 opcode = wqe->wr.opcode;
 786         }
 787
 788         /*
 789          * Set the state to restart in the middle of a request.
 790          * Don't change the s_sge, s_cur_sge, or s_cur_size.
 791          * See qib_make_rc_req().
 792          */
 793         switch (opcode) {
 794         case IB_WR_SEND:
 795         case IB_WR_SEND_WITH_IMM:
 796                 qp->s_state = OP(RDMA_READ_RESPONSE_FIRST);
 797                 break;
 798
 799         case IB_WR_RDMA_WRITE:
 800         case IB_WR_RDMA_WRITE_WITH_IMM:
 801                 qp->s_state = OP(RDMA_READ_RESPONSE_LAST);
 802                 break;
 803
 804         case IB_WR_RDMA_READ:
 805                 qp->s_state = OP(RDMA_READ_RESPONSE_MIDDLE);
 806                 break;
 807
 808         default:
 809                 /*
 810                  * This case shouldn't happen since its only
 811                  * one PSN per req.
 812                  */
 813                 qp->s_state = OP(SEND_LAST);
 814         }
 815 done:
 816         qp->s_psn = psn;
 817         /*
 818          * Set RVT_S_WAIT_PSN as qib_rc_complete() may start the timer
 819          * asynchronously before the send tasklet can get scheduled.
 820          * Doing it in qib_make_rc_req() is too late.
 821          */
 822         if ((qib_cmp24(qp->s_psn, qp->s_sending_hpsn) <= 0) &&
 823             (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0))
 824                 qp->s_flags |= RVT_S_WAIT_PSN;
 825 }
 826
 827 /*
 828  * Back up requester to resend the last un-ACKed request.
 829  * The QP r_lock and s_lock should be held and interrupts disabled.
 830  */
 831 void qib_restart_rc(struct rvt_qp *qp, u32 psn, int wait)
 832 {
 833         struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
 834         struct qib_ibport *ibp;
 835
 836         if (qp->s_retry == 0) {
 837                 if (qp->s_mig_state == IB_MIG_ARMED) {
 838                         qib_migrate_qp(qp);
 839                         qp->s_retry = qp->s_retry_cnt;
 840                 } else if (qp->s_last == qp->s_acked) {
 841                         qib_send_complete(qp, wqe, IB_WC_RETRY_EXC_ERR);
 842                         rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
 843                         return;
 844                 } else /* XXX need to handle delayed completion */
 845                         return;
 846         } else
 847                 qp->s_retry--;
 848
 849         ibp = to_iport(qp->ibqp.device, qp->port_num);
 850         if (wqe->wr.opcode == IB_WR_RDMA_READ)
 851                 ibp->rvp.n_rc_resends++;
 852         else
 853                 ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK;
 854
 855         qp->s_flags &= ~(RVT_S_WAIT_FENCE | RVT_S_WAIT_RDMAR |
 856                          RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_PSN |
 857                          RVT_S_WAIT_ACK);
 858         if (wait)
 859                 qp->s_flags |= RVT_S_SEND_ONE;
 860         reset_psn(qp, psn);
 861 }
 862
 863 /*
 864  * Set qp->s_sending_psn to the next PSN after the given one.
 865  * This would be psn+1 except when RDMA reads are present.
 866  */
 867 static void reset_sending_psn(struct rvt_qp *qp, u32 psn)
 868 {
 869         struct rvt_swqe *wqe;
 870         u32 n = qp->s_last;
 871
 872         /* Find the work request corresponding to the given PSN. */
 873         for (;;) {
 874                 wqe = rvt_get_swqe_ptr(qp, n);
 875                 if (qib_cmp24(psn, wqe->lpsn) <= 0) {
 876                         if (wqe->wr.opcode == IB_WR_RDMA_READ)
 877                                 qp->s_sending_psn = wqe->lpsn + 1;
 878                         else
 879                                 qp->s_sending_psn = psn + 1;
 880                         break;
 881                 }
 882                 if (++n == qp->s_size)
 883                         n = 0;
 884                 if (n == qp->s_tail)
 885                         break;
 886         }
 887 }
 888
 889 /*
 890  * This should be called with the QP s_lock held and interrupts disabled.
 891  */
 892 void qib_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr)
 893 {
 894         struct ib_other_headers *ohdr;
 895         struct rvt_swqe *wqe;
 896         u32 opcode;
 897         u32 psn;
 898
 899         if (!(ib_rvt_state_ops[qp->state] & RVT_SEND_OR_FLUSH_OR_RECV_OK))
 900                 return;
 901
 902         /* Find out where the BTH is */
 903         if ((be16_to_cpu(hdr->lrh[0]) & 3) == QIB_LRH_BTH)
 904                 ohdr = &hdr->u.oth;
 905         else
 906                 ohdr = &hdr->u.l.oth;
 907
 908         opcode = be32_to_cpu(ohdr->bth[0]) >> 24;
 909         if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) &&
 910             opcode <= OP(ATOMIC_ACKNOWLEDGE)) {
 911                 WARN_ON(!qp->s_rdma_ack_cnt);
 912                 qp->s_rdma_ack_cnt--;
 913                 return;
 914         }
 915
 916         psn = be32_to_cpu(ohdr->bth[2]);
 917         reset_sending_psn(qp, psn);
 918
 919         /*
 920          * Start timer after a packet requesting an ACK has been sent and
 921          * there are still requests that haven't been acked.
 922          */
 923         if ((psn & IB_BTH_REQ_ACK) && qp->s_acked != qp->s_tail &&
 924             !(qp->s_flags & (RVT_S_TIMER | RVT_S_WAIT_RNR | RVT_S_WAIT_PSN)) &&
 925             (ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
 926                 rvt_add_retry_timer(qp);
 927
 928         while (qp->s_last != qp->s_acked) {
 929                 u32 s_last;
 930
 931                 wqe = rvt_get_swqe_ptr(qp, qp->s_last);
 932                 if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 &&
 933                     qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)
 934                         break;
 935                 s_last = qp->s_last;
 936                 if (++s_last >= qp->s_size)
 937                         s_last = 0;
 938                 qp->s_last = s_last;
 939                 /* see post_send() */
 940                 barrier();
 941                 rvt_put_swqe(wqe);
 942                 rvt_qp_swqe_complete(qp,
 943                                      wqe,
 944                                      ib_qib_wc_opcode[wqe->wr.opcode],
 945                                      IB_WC_SUCCESS);
 946         }
 947         /*
 948          * If we were waiting for sends to complete before resending,
 949          * and they are now complete, restart sending.
 950          */
 951         if (qp->s_flags & RVT_S_WAIT_PSN &&
 952             qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
 953                 qp->s_flags &= ~RVT_S_WAIT_PSN;
 954                 qp->s_sending_psn = qp->s_psn;
 955                 qp->s_sending_hpsn = qp->s_psn - 1;
 956                 qib_schedule_send(qp);
 957         }
 958 }
 959
 960 static inline void update_last_psn(struct rvt_qp *qp, u32 psn)
 961 {
 962         qp->s_last_psn = psn;
 963 }
 964
 965 /*
 966  * Generate a SWQE completion.
 967  * This is similar to qib_send_complete but has to check to be sure
 968  * that the SGEs are not being referenced if the SWQE is being resent.
 969  */
 970 static struct rvt_swqe *do_rc_completion(struct rvt_qp *qp,
 971                                          struct rvt_swqe *wqe,
 972                                          struct qib_ibport *ibp)
 973 {
 974         /*
 975          * Don't decrement refcount and don't generate a
 976          * completion if the SWQE is being resent until the send
 977          * is finished.
 978          */
 979         if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 ||
 980             qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) {
 981                 u32 s_last;
 982
 983                 rvt_put_swqe(wqe);
 984                 s_last = qp->s_last;
 985                 if (++s_last >= qp->s_size)
 986                         s_last = 0;
 987                 qp->s_last = s_last;
 988                 /* see post_send() */
 989                 barrier();
 990                 rvt_qp_swqe_complete(qp,
 991                                      wqe,
 992                                      ib_qib_wc_opcode[wqe->wr.opcode],
 993                                      IB_WC_SUCCESS);
 994         } else
 995                 this_cpu_inc(*ibp->rvp.rc_delayed_comp);
 996
 997         qp->s_retry = qp->s_retry_cnt;
 998         update_last_psn(qp, wqe->lpsn);
 999
1000         /*
1001          * If we are completing a request which is in the process of
1002          * being resent, we can stop resending it since we know the
1003          * responder has already seen it.
1004          */
1005         if (qp->s_acked == qp->s_cur) {
1006                 if (++qp->s_cur >= qp->s_size)
1007                         qp->s_cur = 0;
1008                 qp->s_acked = qp->s_cur;
1009                 wqe = rvt_get_swqe_ptr(qp, qp->s_cur);
1010                 if (qp->s_acked != qp->s_tail) {
1011                         qp->s_state = OP(SEND_LAST);
1012                         qp->s_psn = wqe->psn;
1013                 }
1014         } else {
1015                 if (++qp->s_acked >= qp->s_size)
1016                         qp->s_acked = 0;
1017                 if (qp->state == IB_QPS_SQD && qp->s_acked == qp->s_cur)
1018                         qp->s_draining = 0;
1019                 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1020         }
1021         return wqe;
1022 }
1023
1024 /**
1025  * do_rc_ack - process an incoming RC ACK
1026  * @qp: the QP the ACK came in on
1027  * @psn: the packet sequence number of the ACK
1028  * @opcode: the opcode of the request that resulted in the ACK
1029  *
1030  * This is called from qib_rc_rcv_resp() to process an incoming RC ACK
1031  * for the given QP.
1032  * Called at interrupt level with the QP s_lock held.
1033  * Returns 1 if OK, 0 if current operation should be aborted (NAK).
1034  */
1035 static int do_rc_ack(struct rvt_qp *qp, u32 aeth, u32 psn, int opcode,
1036                      u64 val, struct qib_ctxtdata *rcd)
1037 {
1038         struct qib_ibport *ibp;
1039         enum ib_wc_status status;
1040         struct rvt_swqe *wqe;
1041         int ret = 0;
1042         u32 ack_psn;
1043         int diff;
1044
1045         /*
1046          * Note that NAKs implicitly ACK outstanding SEND and RDMA write
1047          * requests and implicitly NAK RDMA read and atomic requests issued
1048          * before the NAK'ed request.  The MSN won't include the NAK'ed
1049          * request but will include an ACK'ed request(s).
1050          */
1051         ack_psn = psn;
1052         if (aeth >> IB_AETH_NAK_SHIFT)
1053                 ack_psn--;
1054         wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1055         ibp = to_iport(qp->ibqp.device, qp->port_num);
1056
1057         /*
1058          * The MSN might be for a later WQE than the PSN indicates so
1059          * only complete WQEs that the PSN finishes.
1060          */
1061         while ((diff = qib_cmp24(ack_psn, wqe->lpsn)) >= 0) {
1062                 /*
1063                  * RDMA_READ_RESPONSE_ONLY is a special case since
1064                  * we want to generate completion events for everything
1065                  * before the RDMA read, copy the data, then generate
1066                  * the completion for the read.
1067                  */
1068                 if (wqe->wr.opcode == IB_WR_RDMA_READ &&
1069                     opcode == OP(RDMA_READ_RESPONSE_ONLY) &&
1070                     diff == 0) {
1071                         ret = 1;
1072                         goto bail;
1073                 }
1074                 /*
1075                  * If this request is a RDMA read or atomic, and the ACK is
1076                  * for a later operation, this ACK NAKs the RDMA read or
1077                  * atomic.  In other words, only a RDMA_READ_LAST or ONLY
1078                  * can ACK a RDMA read and likewise for atomic ops.  Note
1079                  * that the NAK case can only happen if relaxed ordering is
1080                  * used and requests are sent after an RDMA read or atomic
1081                  * is sent but before the response is received.
1082                  */
1083                 if ((wqe->wr.opcode == IB_WR_RDMA_READ &&
1084                      (opcode != OP(RDMA_READ_RESPONSE_LAST) || diff != 0)) ||
1085                     ((wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1086                       wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) &&
1087                      (opcode != OP(ATOMIC_ACKNOWLEDGE) || diff != 0))) {
1088                         /* Retry this request. */
1089                         if (!(qp->r_flags & RVT_R_RDMAR_SEQ)) {
1090                                 qp->r_flags |= RVT_R_RDMAR_SEQ;
1091                                 qib_restart_rc(qp, qp->s_last_psn + 1, 0);
1092                                 if (list_empty(&qp->rspwait)) {
1093                                         qp->r_flags |= RVT_R_RSP_SEND;
1094                                         rvt_get_qp(qp);
1095                                         list_add_tail(&qp->rspwait,
1096                                                       &rcd->qp_wait_list);
1097                                 }
1098                         }
1099                         /*
1100                          * No need to process the ACK/NAK since we are
1101                          * restarting an earlier request.
1102                          */
1103                         goto bail;
1104                 }
1105                 if (wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1106                     wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
1107                         u64 *vaddr = wqe->sg_list[0].vaddr;
1108                         *vaddr = val;
1109                 }
1110                 if (qp->s_num_rd_atomic &&
1111                     (wqe->wr.opcode == IB_WR_RDMA_READ ||
1112                      wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1113                      wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)) {
1114                         qp->s_num_rd_atomic--;
1115                         /* Restart sending task if fence is complete */
1116                         if ((qp->s_flags & RVT_S_WAIT_FENCE) &&
1117                             !qp->s_num_rd_atomic) {
1118                                 qp->s_flags &= ~(RVT_S_WAIT_FENCE |
1119                                                  RVT_S_WAIT_ACK);
1120                                 qib_schedule_send(qp);
1121                         } else if (qp->s_flags & RVT_S_WAIT_RDMAR) {
1122                                 qp->s_flags &= ~(RVT_S_WAIT_RDMAR |
1123                                                  RVT_S_WAIT_ACK);
1124                                 qib_schedule_send(qp);
1125                         }
1126                 }
1127                 wqe = do_rc_completion(qp, wqe, ibp);
1128                 if (qp->s_acked == qp->s_tail)
1129                         break;
1130         }
1131
1132         switch (aeth >> IB_AETH_NAK_SHIFT) {
1133         case 0:         /* ACK */
1134                 this_cpu_inc(*ibp->rvp.rc_acks);
1135                 if (qp->s_acked != qp->s_tail) {
1136                         /*
1137                          * We are expecting more ACKs so
1138                          * reset the retransmit timer.
1139                          */
1140                         rvt_mod_retry_timer(qp);
1141                         /*
1142                          * We can stop resending the earlier packets and
1143                          * continue with the next packet the receiver wants.
1144                          */
1145                         if (qib_cmp24(qp->s_psn, psn) <= 0)
1146                                 reset_psn(qp, psn + 1);
1147                 } else {
1148                         /* No more acks - kill all timers */
1149                         rvt_stop_rc_timers(qp);
1150                         if (qib_cmp24(qp->s_psn, psn) <= 0) {
1151                                 qp->s_state = OP(SEND_LAST);
1152                                 qp->s_psn = psn + 1;
1153                         }
1154                 }
1155                 if (qp->s_flags & RVT_S_WAIT_ACK) {
1156                         qp->s_flags &= ~RVT_S_WAIT_ACK;
1157                         qib_schedule_send(qp);
1158                 }
1159                 rvt_get_credit(qp, aeth);
1160                 qp->s_rnr_retry = qp->s_rnr_retry_cnt;
1161                 qp->s_retry = qp->s_retry_cnt;
1162                 update_last_psn(qp, psn);
1163                 return 1;
1164
1165         case 1:         /* RNR NAK */
1166                 ibp->rvp.n_rnr_naks++;
1167                 if (qp->s_acked == qp->s_tail)
1168                         goto bail;
1169                 if (qp->s_flags & RVT_S_WAIT_RNR)
1170                         goto bail;
1171                 if (qp->s_rnr_retry == 0) {
1172                         status = IB_WC_RNR_RETRY_EXC_ERR;
1173                         goto class_b;
1174                 }
1175                 if (qp->s_rnr_retry_cnt < 7)
1176                         qp->s_rnr_retry--;
1177
1178                 /* The last valid PSN is the previous PSN. */
1179                 update_last_psn(qp, psn - 1);
1180
1181                 ibp->rvp.n_rc_resends += (qp->s_psn - psn) & QIB_PSN_MASK;
1182
1183                 reset_psn(qp, psn);
1184
1185                 qp->s_flags &= ~(RVT_S_WAIT_SSN_CREDIT | RVT_S_WAIT_ACK);
1186                 rvt_stop_rc_timers(qp);
1187                 rvt_add_rnr_timer(qp, aeth);
1188                 return 0;
1189
1190         case 3:         /* NAK */
1191                 if (qp->s_acked == qp->s_tail)
1192                         goto bail;
1193                 /* The last valid PSN is the previous PSN. */
1194                 update_last_psn(qp, psn - 1);
1195                 switch ((aeth >> IB_AETH_CREDIT_SHIFT) &
1196                         IB_AETH_CREDIT_MASK) {
1197                 case 0: /* PSN sequence error */
1198                         ibp->rvp.n_seq_naks++;
1199                         /*
1200                          * Back up to the responder's expected PSN.
1201                          * Note that we might get a NAK in the middle of an
1202                          * RDMA READ response which terminates the RDMA
1203                          * READ.
1204                          */
1205                         qib_restart_rc(qp, psn, 0);
1206                         qib_schedule_send(qp);
1207                         break;
1208
1209                 case 1: /* Invalid Request */
1210                         status = IB_WC_REM_INV_REQ_ERR;
1211                         ibp->rvp.n_other_naks++;
1212                         goto class_b;
1213
1214                 case 2: /* Remote Access Error */
1215                         status = IB_WC_REM_ACCESS_ERR;
1216                         ibp->rvp.n_other_naks++;
1217                         goto class_b;
1218
1219                 case 3: /* Remote Operation Error */
1220                         status = IB_WC_REM_OP_ERR;
1221                         ibp->rvp.n_other_naks++;
1222 class_b:
1223                         if (qp->s_last == qp->s_acked) {
1224                                 qib_send_complete(qp, wqe, status);
1225                                 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
1226                         }
1227                         break;
1228
1229                 default:
1230                         /* Ignore other reserved NAK error codes */
1231                         goto reserved;
1232                 }
1233                 qp->s_retry = qp->s_retry_cnt;
1234                 qp->s_rnr_retry = qp->s_rnr_retry_cnt;
1235                 goto bail;
1236
1237         default:                /* 2: reserved */
1238 reserved:
1239                 /* Ignore reserved NAK codes. */
1240                 goto bail;
1241         }
1242
1243 bail:
1244         rvt_stop_rc_timers(qp);
1245         return ret;
1246 }
1247
1248 /*
1249  * We have seen an out of sequence RDMA read middle or last packet.
1250  * This ACKs SENDs and RDMA writes up to the first RDMA read or atomic SWQE.
1251  */
1252 static void rdma_seq_err(struct rvt_qp *qp, struct qib_ibport *ibp, u32 psn,
1253                          struct qib_ctxtdata *rcd)
1254 {
1255         struct rvt_swqe *wqe;
1256
1257         /* Remove QP from retry timer */
1258         rvt_stop_rc_timers(qp);
1259
1260         wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1261
1262         while (qib_cmp24(psn, wqe->lpsn) > 0) {
1263                 if (wqe->wr.opcode == IB_WR_RDMA_READ ||
1264                     wqe->wr.opcode == IB_WR_ATOMIC_CMP_AND_SWP ||
1265                     wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD)
1266                         break;
1267                 wqe = do_rc_completion(qp, wqe, ibp);
1268         }
1269
1270         ibp->rvp.n_rdma_seq++;
1271         qp->r_flags |= RVT_R_RDMAR_SEQ;
1272         qib_restart_rc(qp, qp->s_last_psn + 1, 0);
1273         if (list_empty(&qp->rspwait)) {
1274                 qp->r_flags |= RVT_R_RSP_SEND;
1275                 rvt_get_qp(qp);
1276                 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
1277         }
1278 }
1279
1280 /**
1281  * qib_rc_rcv_resp - process an incoming RC response packet
1282  * @ibp: the port this packet came in on
1283  * @ohdr: the other headers for this packet
1284  * @data: the packet data
1285  * @tlen: the packet length
1286  * @qp: the QP for this packet
1287  * @opcode: the opcode for this packet
1288  * @psn: the packet sequence number for this packet
1289  * @hdrsize: the header length
1290  * @pmtu: the path MTU
1291  *
1292  * This is called from qib_rc_rcv() to process an incoming RC response
1293  * packet for the given QP.
1294  * Called at interrupt level.
1295  */
1296 static void qib_rc_rcv_resp(struct qib_ibport *ibp,
1297                             struct ib_other_headers *ohdr,
1298                             void *data, u32 tlen,
1299                             struct rvt_qp *qp,
1300                             u32 opcode,
1301                             u32 psn, u32 hdrsize, u32 pmtu,
1302                             struct qib_ctxtdata *rcd)
1303 {
1304         struct rvt_swqe *wqe;
1305         struct qib_pportdata *ppd = ppd_from_ibp(ibp);
1306         enum ib_wc_status status;
1307         unsigned long flags;
1308         int diff;
1309         u32 pad;
1310         u32 aeth;
1311         u64 val;
1312
1313         if (opcode != OP(RDMA_READ_RESPONSE_MIDDLE)) {
1314                 /*
1315                  * If ACK'd PSN on SDMA busy list try to make progress to
1316                  * reclaim SDMA credits.
1317                  */
1318                 if ((qib_cmp24(psn, qp->s_sending_psn) >= 0) &&
1319                     (qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0)) {
1320
1321                         /*
1322                          * If send tasklet not running attempt to progress
1323                          * SDMA queue.
1324                          */
1325                         if (!(qp->s_flags & RVT_S_BUSY)) {
1326                                 /* Acquire SDMA Lock */
1327                                 spin_lock_irqsave(&ppd->sdma_lock, flags);
1328                                 /* Invoke sdma make progress */
1329                                 qib_sdma_make_progress(ppd);
1330                                 /* Release SDMA Lock */
1331                                 spin_unlock_irqrestore(&ppd->sdma_lock, flags);
1332                         }
1333                 }
1334         }
1335
1336         spin_lock_irqsave(&qp->s_lock, flags);
1337         if (!(ib_rvt_state_ops[qp->state] & RVT_PROCESS_RECV_OK))
1338                 goto ack_done;
1339
1340         /* Ignore invalid responses. */
1341         if (qib_cmp24(psn, READ_ONCE(qp->s_next_psn)) >= 0)
1342                 goto ack_done;
1343
1344         /* Ignore duplicate responses. */
1345         diff = qib_cmp24(psn, qp->s_last_psn);
1346         if (unlikely(diff <= 0)) {
1347                 /* Update credits for "ghost" ACKs */
1348                 if (diff == 0 && opcode == OP(ACKNOWLEDGE)) {
1349                         aeth = be32_to_cpu(ohdr->u.aeth);
1350                         if ((aeth >> IB_AETH_NAK_SHIFT) == 0)
1351                                 rvt_get_credit(qp, aeth);
1352                 }
1353                 goto ack_done;
1354         }
1355
1356         /*
1357          * Skip everything other than the PSN we expect, if we are waiting
1358          * for a reply to a restarted RDMA read or atomic op.
1359          */
1360         if (qp->r_flags & RVT_R_RDMAR_SEQ) {
1361                 if (qib_cmp24(psn, qp->s_last_psn + 1) != 0)
1362                         goto ack_done;
1363                 qp->r_flags &= ~RVT_R_RDMAR_SEQ;
1364         }
1365
1366         if (unlikely(qp->s_acked == qp->s_tail))
1367                 goto ack_done;
1368         wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1369         status = IB_WC_SUCCESS;
1370
1371         switch (opcode) {
1372         case OP(ACKNOWLEDGE):
1373         case OP(ATOMIC_ACKNOWLEDGE):
1374         case OP(RDMA_READ_RESPONSE_FIRST):
1375                 aeth = be32_to_cpu(ohdr->u.aeth);
1376                 if (opcode == OP(ATOMIC_ACKNOWLEDGE))
1377                         val = ib_u64_get(&ohdr->u.at.atomic_ack_eth);
1378                 else
1379                         val = 0;
1380                 if (!do_rc_ack(qp, aeth, psn, opcode, val, rcd) ||
1381                     opcode != OP(RDMA_READ_RESPONSE_FIRST))
1382                         goto ack_done;
1383                 hdrsize += 4;
1384                 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1385                 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
1386                         goto ack_op_err;
1387                 /*
1388                  * If this is a response to a resent RDMA read, we
1389                  * have to be careful to copy the data to the right
1390                  * location.
1391                  */
1392                 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge,
1393                                                   wqe, psn, pmtu);
1394                 goto read_middle;
1395
1396         case OP(RDMA_READ_RESPONSE_MIDDLE):
1397                 /* no AETH, no ACK */
1398                 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1)))
1399                         goto ack_seq_err;
1400                 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
1401                         goto ack_op_err;
1402 read_middle:
1403                 if (unlikely(tlen != (hdrsize + pmtu + 4)))
1404                         goto ack_len_err;
1405                 if (unlikely(pmtu >= qp->s_rdma_read_len))
1406                         goto ack_len_err;
1407
1408                 /*
1409                  * We got a response so update the timeout.
1410                  * 4.096 usec. * (1 << qp->timeout)
1411                  */
1412                 rvt_mod_retry_timer(qp);
1413                 if (qp->s_flags & RVT_S_WAIT_ACK) {
1414                         qp->s_flags &= ~RVT_S_WAIT_ACK;
1415                         qib_schedule_send(qp);
1416                 }
1417
1418                 if (opcode == OP(RDMA_READ_RESPONSE_MIDDLE))
1419                         qp->s_retry = qp->s_retry_cnt;
1420
1421                 /*
1422                  * Update the RDMA receive state but do the copy w/o
1423                  * holding the locks and blocking interrupts.
1424                  */
1425                 qp->s_rdma_read_len -= pmtu;
1426                 update_last_psn(qp, psn);
1427                 spin_unlock_irqrestore(&qp->s_lock, flags);
1428                 qib_copy_sge(&qp->s_rdma_read_sge, data, pmtu, 0);
1429                 goto bail;
1430
1431         case OP(RDMA_READ_RESPONSE_ONLY):
1432                 aeth = be32_to_cpu(ohdr->u.aeth);
1433                 if (!do_rc_ack(qp, aeth, psn, opcode, 0, rcd))
1434                         goto ack_done;
1435                 /* Get the number of bytes the message was padded by. */
1436                 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
1437                 /*
1438                  * Check that the data size is >= 0 && <= pmtu.
1439                  * Remember to account for the AETH header (4) and
1440                  * ICRC (4).
1441                  */
1442                 if (unlikely(tlen < (hdrsize + pad + 8)))
1443                         goto ack_len_err;
1444                 /*
1445                  * If this is a response to a resent RDMA read, we
1446                  * have to be careful to copy the data to the right
1447                  * location.
1448                  */
1449                 wqe = rvt_get_swqe_ptr(qp, qp->s_acked);
1450                 qp->s_rdma_read_len = restart_sge(&qp->s_rdma_read_sge,
1451                                                   wqe, psn, pmtu);
1452                 goto read_last;
1453
1454         case OP(RDMA_READ_RESPONSE_LAST):
1455                 /* ACKs READ req. */
1456                 if (unlikely(qib_cmp24(psn, qp->s_last_psn + 1)))
1457                         goto ack_seq_err;
1458                 if (unlikely(wqe->wr.opcode != IB_WR_RDMA_READ))
1459                         goto ack_op_err;
1460                 /* Get the number of bytes the message was padded by. */
1461                 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
1462                 /*
1463                  * Check that the data size is >= 1 && <= pmtu.
1464                  * Remember to account for the AETH header (4) and
1465                  * ICRC (4).
1466                  */
1467                 if (unlikely(tlen <= (hdrsize + pad + 8)))
1468                         goto ack_len_err;
1469 read_last:
1470                 tlen -= hdrsize + pad + 8;
1471                 if (unlikely(tlen != qp->s_rdma_read_len))
1472                         goto ack_len_err;
1473                 aeth = be32_to_cpu(ohdr->u.aeth);
1474                 qib_copy_sge(&qp->s_rdma_read_sge, data, tlen, 0);
1475                 WARN_ON(qp->s_rdma_read_sge.num_sge);
1476                 (void) do_rc_ack(qp, aeth, psn,
1477                                  OP(RDMA_READ_RESPONSE_LAST), 0, rcd);
1478                 goto ack_done;
1479         }
1480
1481 ack_op_err:
1482         status = IB_WC_LOC_QP_OP_ERR;
1483         goto ack_err;
1484
1485 ack_seq_err:
1486         rdma_seq_err(qp, ibp, psn, rcd);
1487         goto ack_done;
1488
1489 ack_len_err:
1490         status = IB_WC_LOC_LEN_ERR;
1491 ack_err:
1492         if (qp->s_last == qp->s_acked) {
1493                 qib_send_complete(qp, wqe, status);
1494                 rvt_error_qp(qp, IB_WC_WR_FLUSH_ERR);
1495         }
1496 ack_done:
1497         spin_unlock_irqrestore(&qp->s_lock, flags);
1498 bail:
1499         return;
1500 }
1501
1502 /**
1503  * qib_rc_rcv_error - process an incoming duplicate or error RC packet
1504  * @ohdr: the other headers for this packet
1505  * @data: the packet data
1506  * @qp: the QP for this packet
1507  * @opcode: the opcode for this packet
1508  * @psn: the packet sequence number for this packet
1509  * @diff: the difference between the PSN and the expected PSN
1510  *
1511  * This is called from qib_rc_rcv() to process an unexpected
1512  * incoming RC packet for the given QP.
1513  * Called at interrupt level.
1514  * Return 1 if no more processing is needed; otherwise return 0 to
1515  * schedule a response to be sent.
1516  */
1517 static int qib_rc_rcv_error(struct ib_other_headers *ohdr,
1518                             void *data,
1519                             struct rvt_qp *qp,
1520                             u32 opcode,
1521                             u32 psn,
1522                             int diff,
1523                             struct qib_ctxtdata *rcd)
1524 {
1525         struct qib_ibport *ibp = to_iport(qp->ibqp.device, qp->port_num);
1526         struct rvt_ack_entry *e;
1527         unsigned long flags;
1528         u8 i, prev;
1529         int old_req;
1530
1531         if (diff > 0) {
1532                 /*
1533                  * Packet sequence error.
1534                  * A NAK will ACK earlier sends and RDMA writes.
1535                  * Don't queue the NAK if we already sent one.
1536                  */
1537                 if (!qp->r_nak_state) {
1538                         ibp->rvp.n_rc_seqnak++;
1539                         qp->r_nak_state = IB_NAK_PSN_ERROR;
1540                         /* Use the expected PSN. */
1541                         qp->r_ack_psn = qp->r_psn;
1542                         /*
1543                          * Wait to send the sequence NAK until all packets
1544                          * in the receive queue have been processed.
1545                          * Otherwise, we end up propagating congestion.
1546                          */
1547                         if (list_empty(&qp->rspwait)) {
1548                                 qp->r_flags |= RVT_R_RSP_NAK;
1549                                 rvt_get_qp(qp);
1550                                 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
1551                         }
1552                 }
1553                 goto done;
1554         }
1555
1556         /*
1557          * Handle a duplicate request.  Don't re-execute SEND, RDMA
1558          * write or atomic op.  Don't NAK errors, just silently drop
1559          * the duplicate request.  Note that r_sge, r_len, and
1560          * r_rcv_len may be in use so don't modify them.
1561          *
1562          * We are supposed to ACK the earliest duplicate PSN but we
1563          * can coalesce an outstanding duplicate ACK.  We have to
1564          * send the earliest so that RDMA reads can be restarted at
1565          * the requester's expected PSN.
1566          *
1567          * First, find where this duplicate PSN falls within the
1568          * ACKs previously sent.
1569          * old_req is true if there is an older response that is scheduled
1570          * to be sent before sending this one.
1571          */
1572         e = NULL;
1573         old_req = 1;
1574         ibp->rvp.n_rc_dupreq++;
1575
1576         spin_lock_irqsave(&qp->s_lock, flags);
1577
1578         for (i = qp->r_head_ack_queue; ; i = prev) {
1579                 if (i == qp->s_tail_ack_queue)
1580                         old_req = 0;
1581                 if (i)
1582                         prev = i - 1;
1583                 else
1584                         prev = QIB_MAX_RDMA_ATOMIC;
1585                 if (prev == qp->r_head_ack_queue) {
1586                         e = NULL;
1587                         break;
1588                 }
1589                 e = &qp->s_ack_queue[prev];
1590                 if (!e->opcode) {
1591                         e = NULL;
1592                         break;
1593                 }
1594                 if (qib_cmp24(psn, e->psn) >= 0) {
1595                         if (prev == qp->s_tail_ack_queue &&
1596                             qib_cmp24(psn, e->lpsn) <= 0)
1597                                 old_req = 0;
1598                         break;
1599                 }
1600         }
1601         switch (opcode) {
1602         case OP(RDMA_READ_REQUEST): {
1603                 struct ib_reth *reth;
1604                 u32 offset;
1605                 u32 len;
1606
1607                 /*
1608                  * If we didn't find the RDMA read request in the ack queue,
1609                  * we can ignore this request.
1610                  */
1611                 if (!e || e->opcode != OP(RDMA_READ_REQUEST))
1612                         goto unlock_done;
1613                 /* RETH comes after BTH */
1614                 reth = &ohdr->u.rc.reth;
1615                 /*
1616                  * Address range must be a subset of the original
1617                  * request and start on pmtu boundaries.
1618                  * We reuse the old ack_queue slot since the requester
1619                  * should not back up and request an earlier PSN for the
1620                  * same request.
1621                  */
1622                 offset = ((psn - e->psn) & QIB_PSN_MASK) *
1623                         qp->pmtu;
1624                 len = be32_to_cpu(reth->length);
1625                 if (unlikely(offset + len != e->rdma_sge.sge_length))
1626                         goto unlock_done;
1627                 if (e->rdma_sge.mr) {
1628                         rvt_put_mr(e->rdma_sge.mr);
1629                         e->rdma_sge.mr = NULL;
1630                 }
1631                 if (len != 0) {
1632                         u32 rkey = be32_to_cpu(reth->rkey);
1633                         u64 vaddr = be64_to_cpu(reth->vaddr);
1634                         int ok;
1635
1636                         ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr, rkey,
1637                                          IB_ACCESS_REMOTE_READ);
1638                         if (unlikely(!ok))
1639                                 goto unlock_done;
1640                 } else {
1641                         e->rdma_sge.vaddr = NULL;
1642                         e->rdma_sge.length = 0;
1643                         e->rdma_sge.sge_length = 0;
1644                 }
1645                 e->psn = psn;
1646                 if (old_req)
1647                         goto unlock_done;
1648                 qp->s_tail_ack_queue = prev;
1649                 break;
1650         }
1651
1652         case OP(COMPARE_SWAP):
1653         case OP(FETCH_ADD): {
1654                 /*
1655                  * If we didn't find the atomic request in the ack queue
1656                  * or the send tasklet is already backed up to send an
1657                  * earlier entry, we can ignore this request.
1658                  */
1659                 if (!e || e->opcode != (u8) opcode || old_req)
1660                         goto unlock_done;
1661                 qp->s_tail_ack_queue = prev;
1662                 break;
1663         }
1664
1665         default:
1666                 /*
1667                  * Ignore this operation if it doesn't request an ACK
1668                  * or an earlier RDMA read or atomic is going to be resent.
1669                  */
1670                 if (!(psn & IB_BTH_REQ_ACK) || old_req)
1671                         goto unlock_done;
1672                 /*
1673                  * Resend the most recent ACK if this request is
1674                  * after all the previous RDMA reads and atomics.
1675                  */
1676                 if (i == qp->r_head_ack_queue) {
1677                         spin_unlock_irqrestore(&qp->s_lock, flags);
1678                         qp->r_nak_state = 0;
1679                         qp->r_ack_psn = qp->r_psn - 1;
1680                         goto send_ack;
1681                 }
1682                 /*
1683                  * Try to send a simple ACK to work around a Mellanox bug
1684                  * which doesn't accept a RDMA read response or atomic
1685                  * response as an ACK for earlier SENDs or RDMA writes.
1686                  */
1687                 if (!(qp->s_flags & RVT_S_RESP_PENDING)) {
1688                         spin_unlock_irqrestore(&qp->s_lock, flags);
1689                         qp->r_nak_state = 0;
1690                         qp->r_ack_psn = qp->s_ack_queue[i].psn - 1;
1691                         goto send_ack;
1692                 }
1693                 /*
1694                  * Resend the RDMA read or atomic op which
1695                  * ACKs this duplicate request.
1696                  */
1697                 qp->s_tail_ack_queue = i;
1698                 break;
1699         }
1700         qp->s_ack_state = OP(ACKNOWLEDGE);
1701         qp->s_flags |= RVT_S_RESP_PENDING;
1702         qp->r_nak_state = 0;
1703         qib_schedule_send(qp);
1704
1705 unlock_done:
1706         spin_unlock_irqrestore(&qp->s_lock, flags);
1707 done:
1708         return 1;
1709
1710 send_ack:
1711         return 0;
1712 }
1713
1714 static inline void qib_update_ack_queue(struct rvt_qp *qp, unsigned n)
1715 {
1716         unsigned next;
1717
1718         next = n + 1;
1719         if (next > QIB_MAX_RDMA_ATOMIC)
1720                 next = 0;
1721         qp->s_tail_ack_queue = next;
1722         qp->s_ack_state = OP(ACKNOWLEDGE);
1723 }
1724
1725 /**
1726  * qib_rc_rcv - process an incoming RC packet
1727  * @rcd: the context pointer
1728  * @hdr: the header of this packet
1729  * @has_grh: true if the header has a GRH
1730  * @data: the packet data
1731  * @tlen: the packet length
1732  * @qp: the QP for this packet
1733  *
1734  * This is called from qib_qp_rcv() to process an incoming RC packet
1735  * for the given QP.
1736  * Called at interrupt level.
1737  */
1738 void qib_rc_rcv(struct qib_ctxtdata *rcd, struct ib_header *hdr,
1739                 int has_grh, void *data, u32 tlen, struct rvt_qp *qp)
1740 {
1741         struct qib_ibport *ibp = &rcd->ppd->ibport_data;
1742         struct ib_other_headers *ohdr;
1743         u32 opcode;
1744         u32 hdrsize;
1745         u32 psn;
1746         u32 pad;
1747         struct ib_wc wc;
1748         u32 pmtu = qp->pmtu;
1749         int diff;
1750         struct ib_reth *reth;
1751         unsigned long flags;
1752         int ret;
1753
1754         /* Check for GRH */
1755         if (!has_grh) {
1756                 ohdr = &hdr->u.oth;
1757                 hdrsize = 8 + 12;       /* LRH + BTH */
1758         } else {
1759                 ohdr = &hdr->u.l.oth;
1760                 hdrsize = 8 + 40 + 12;  /* LRH + GRH + BTH */
1761         }
1762
1763         opcode = be32_to_cpu(ohdr->bth[0]);
1764         if (qib_ruc_check_hdr(ibp, hdr, has_grh, qp, opcode))
1765                 return;
1766
1767         psn = be32_to_cpu(ohdr->bth[2]);
1768         opcode >>= 24;
1769
1770         /*
1771          * Process responses (ACKs) before anything else.  Note that the
1772          * packet sequence number will be for something in the send work
1773          * queue rather than the expected receive packet sequence number.
1774          * In other words, this QP is the requester.
1775          */
1776         if (opcode >= OP(RDMA_READ_RESPONSE_FIRST) &&
1777             opcode <= OP(ATOMIC_ACKNOWLEDGE)) {
1778                 qib_rc_rcv_resp(ibp, ohdr, data, tlen, qp, opcode, psn,
1779                                 hdrsize, pmtu, rcd);
1780                 return;
1781         }
1782
1783         /* Compute 24 bits worth of difference. */
1784         diff = qib_cmp24(psn, qp->r_psn);
1785         if (unlikely(diff)) {
1786                 if (qib_rc_rcv_error(ohdr, data, qp, opcode, psn, diff, rcd))
1787                         return;
1788                 goto send_ack;
1789         }
1790
1791         /* Check for opcode sequence errors. */
1792         switch (qp->r_state) {
1793         case OP(SEND_FIRST):
1794         case OP(SEND_MIDDLE):
1795                 if (opcode == OP(SEND_MIDDLE) ||
1796                     opcode == OP(SEND_LAST) ||
1797                     opcode == OP(SEND_LAST_WITH_IMMEDIATE))
1798                         break;
1799                 goto nack_inv;
1800
1801         case OP(RDMA_WRITE_FIRST):
1802         case OP(RDMA_WRITE_MIDDLE):
1803                 if (opcode == OP(RDMA_WRITE_MIDDLE) ||
1804                     opcode == OP(RDMA_WRITE_LAST) ||
1805                     opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE))
1806                         break;
1807                 goto nack_inv;
1808
1809         default:
1810                 if (opcode == OP(SEND_MIDDLE) ||
1811                     opcode == OP(SEND_LAST) ||
1812                     opcode == OP(SEND_LAST_WITH_IMMEDIATE) ||
1813                     opcode == OP(RDMA_WRITE_MIDDLE) ||
1814                     opcode == OP(RDMA_WRITE_LAST) ||
1815                     opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE))
1816                         goto nack_inv;
1817                 /*
1818                  * Note that it is up to the requester to not send a new
1819                  * RDMA read or atomic operation before receiving an ACK
1820                  * for the previous operation.
1821                  */
1822                 break;
1823         }
1824
1825         if (qp->state == IB_QPS_RTR && !(qp->r_flags & RVT_R_COMM_EST))
1826                 rvt_comm_est(qp);
1827
1828         /* OK, process the packet. */
1829         switch (opcode) {
1830         case OP(SEND_FIRST):
1831                 ret = rvt_get_rwqe(qp, false);
1832                 if (ret < 0)
1833                         goto nack_op_err;
1834                 if (!ret)
1835                         goto rnr_nak;
1836                 qp->r_rcv_len = 0;
1837                 /* FALLTHROUGH */
1838         case OP(SEND_MIDDLE):
1839         case OP(RDMA_WRITE_MIDDLE):
1840 send_middle:
1841                 /* Check for invalid length PMTU or posted rwqe len. */
1842                 if (unlikely(tlen != (hdrsize + pmtu + 4)))
1843                         goto nack_inv;
1844                 qp->r_rcv_len += pmtu;
1845                 if (unlikely(qp->r_rcv_len > qp->r_len))
1846                         goto nack_inv;
1847                 qib_copy_sge(&qp->r_sge, data, pmtu, 1);
1848                 break;
1849
1850         case OP(RDMA_WRITE_LAST_WITH_IMMEDIATE):
1851                 /* consume RWQE */
1852                 ret = rvt_get_rwqe(qp, true);
1853                 if (ret < 0)
1854                         goto nack_op_err;
1855                 if (!ret)
1856                         goto rnr_nak;
1857                 goto send_last_imm;
1858
1859         case OP(SEND_ONLY):
1860         case OP(SEND_ONLY_WITH_IMMEDIATE):
1861                 ret = rvt_get_rwqe(qp, false);
1862                 if (ret < 0)
1863                         goto nack_op_err;
1864                 if (!ret)
1865                         goto rnr_nak;
1866                 qp->r_rcv_len = 0;
1867                 if (opcode == OP(SEND_ONLY))
1868                         goto no_immediate_data;
1869                 /* fall through -- for SEND_ONLY_WITH_IMMEDIATE */
1870         case OP(SEND_LAST_WITH_IMMEDIATE):
1871 send_last_imm:
1872                 wc.ex.imm_data = ohdr->u.imm_data;
1873                 hdrsize += 4;
1874                 wc.wc_flags = IB_WC_WITH_IMM;
1875                 goto send_last;
1876         case OP(SEND_LAST):
1877         case OP(RDMA_WRITE_LAST):
1878 no_immediate_data:
1879                 wc.wc_flags = 0;
1880                 wc.ex.imm_data = 0;
1881 send_last:
1882                 /* Get the number of bytes the message was padded by. */
1883                 pad = (be32_to_cpu(ohdr->bth[0]) >> 20) & 3;
1884                 /* Check for invalid length. */
1885                 /* XXX LAST len should be >= 1 */
1886                 if (unlikely(tlen < (hdrsize + pad + 4)))
1887                         goto nack_inv;
1888                 /* Don't count the CRC. */
1889                 tlen -= (hdrsize + pad + 4);
1890                 wc.byte_len = tlen + qp->r_rcv_len;
1891                 if (unlikely(wc.byte_len > qp->r_len))
1892                         goto nack_inv;
1893                 qib_copy_sge(&qp->r_sge, data, tlen, 1);
1894                 rvt_put_ss(&qp->r_sge);
1895                 qp->r_msn++;
1896                 if (!test_and_clear_bit(RVT_R_WRID_VALID, &qp->r_aflags))
1897                         break;
1898                 wc.wr_id = qp->r_wr_id;
1899                 wc.status = IB_WC_SUCCESS;
1900                 if (opcode == OP(RDMA_WRITE_LAST_WITH_IMMEDIATE) ||
1901                     opcode == OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE))
1902                         wc.opcode = IB_WC_RECV_RDMA_WITH_IMM;
1903                 else
1904                         wc.opcode = IB_WC_RECV;
1905                 wc.qp = &qp->ibqp;
1906                 wc.src_qp = qp->remote_qpn;
1907                 wc.slid = rdma_ah_get_dlid(&qp->remote_ah_attr);
1908                 wc.sl = rdma_ah_get_sl(&qp->remote_ah_attr);
1909                 /* zero fields that are N/A */
1910                 wc.vendor_err = 0;
1911                 wc.pkey_index = 0;
1912                 wc.dlid_path_bits = 0;
1913                 wc.port_num = 0;
1914                 /* Signal completion event if the solicited bit is set. */
1915                 rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc,
1916                              ib_bth_is_solicited(ohdr));
1917                 break;
1918
1919         case OP(RDMA_WRITE_FIRST):
1920         case OP(RDMA_WRITE_ONLY):
1921         case OP(RDMA_WRITE_ONLY_WITH_IMMEDIATE):
1922                 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_WRITE)))
1923                         goto nack_inv;
1924                 /* consume RWQE */
1925                 reth = &ohdr->u.rc.reth;
1926                 hdrsize += sizeof(*reth);
1927                 qp->r_len = be32_to_cpu(reth->length);
1928                 qp->r_rcv_len = 0;
1929                 qp->r_sge.sg_list = NULL;
1930                 if (qp->r_len != 0) {
1931                         u32 rkey = be32_to_cpu(reth->rkey);
1932                         u64 vaddr = be64_to_cpu(reth->vaddr);
1933                         int ok;
1934
1935                         /* Check rkey & NAK */
1936                         ok = rvt_rkey_ok(qp, &qp->r_sge.sge, qp->r_len, vaddr,
1937                                          rkey, IB_ACCESS_REMOTE_WRITE);
1938                         if (unlikely(!ok))
1939                                 goto nack_acc;
1940                         qp->r_sge.num_sge = 1;
1941                 } else {
1942                         qp->r_sge.num_sge = 0;
1943                         qp->r_sge.sge.mr = NULL;
1944                         qp->r_sge.sge.vaddr = NULL;
1945                         qp->r_sge.sge.length = 0;
1946                         qp->r_sge.sge.sge_length = 0;
1947                 }
1948                 if (opcode == OP(RDMA_WRITE_FIRST))
1949                         goto send_middle;
1950                 else if (opcode == OP(RDMA_WRITE_ONLY))
1951                         goto no_immediate_data;
1952                 ret = rvt_get_rwqe(qp, true);
1953                 if (ret < 0)
1954                         goto nack_op_err;
1955                 if (!ret) {
1956                         rvt_put_ss(&qp->r_sge);
1957                         goto rnr_nak;
1958                 }
1959                 wc.ex.imm_data = ohdr->u.rc.imm_data;
1960                 hdrsize += 4;
1961                 wc.wc_flags = IB_WC_WITH_IMM;
1962                 goto send_last;
1963
1964         case OP(RDMA_READ_REQUEST): {
1965                 struct rvt_ack_entry *e;
1966                 u32 len;
1967                 u8 next;
1968
1969                 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_READ)))
1970                         goto nack_inv;
1971                 next = qp->r_head_ack_queue + 1;
1972                 /* s_ack_queue is size QIB_MAX_RDMA_ATOMIC+1 so use > not >= */
1973                 if (next > QIB_MAX_RDMA_ATOMIC)
1974                         next = 0;
1975                 spin_lock_irqsave(&qp->s_lock, flags);
1976                 if (unlikely(next == qp->s_tail_ack_queue)) {
1977                         if (!qp->s_ack_queue[next].sent)
1978                                 goto nack_inv_unlck;
1979                         qib_update_ack_queue(qp, next);
1980                 }
1981                 e = &qp->s_ack_queue[qp->r_head_ack_queue];
1982                 if (e->opcode == OP(RDMA_READ_REQUEST) && e->rdma_sge.mr) {
1983                         rvt_put_mr(e->rdma_sge.mr);
1984                         e->rdma_sge.mr = NULL;
1985                 }
1986                 reth = &ohdr->u.rc.reth;
1987                 len = be32_to_cpu(reth->length);
1988                 if (len) {
1989                         u32 rkey = be32_to_cpu(reth->rkey);
1990                         u64 vaddr = be64_to_cpu(reth->vaddr);
1991                         int ok;
1992
1993                         /* Check rkey & NAK */
1994                         ok = rvt_rkey_ok(qp, &e->rdma_sge, len, vaddr,
1995                                          rkey, IB_ACCESS_REMOTE_READ);
1996                         if (unlikely(!ok))
1997                                 goto nack_acc_unlck;
1998                         /*
1999                          * Update the next expected PSN.  We add 1 later
2000                          * below, so only add the remainder here.
2001                          */
2002                         qp->r_psn += rvt_div_mtu(qp, len - 1);
2003                 } else {
2004                         e->rdma_sge.mr = NULL;
2005                         e->rdma_sge.vaddr = NULL;
2006                         e->rdma_sge.length = 0;
2007                         e->rdma_sge.sge_length = 0;
2008                 }
2009                 e->opcode = opcode;
2010                 e->sent = 0;
2011                 e->psn = psn;
2012                 e->lpsn = qp->r_psn;
2013                 /*
2014                  * We need to increment the MSN here instead of when we
2015                  * finish sending the result since a duplicate request would
2016                  * increment it more than once.
2017                  */
2018                 qp->r_msn++;
2019                 qp->r_psn++;
2020                 qp->r_state = opcode;
2021                 qp->r_nak_state = 0;
2022                 qp->r_head_ack_queue = next;
2023
2024                 /* Schedule the send tasklet. */
2025                 qp->s_flags |= RVT_S_RESP_PENDING;
2026                 qib_schedule_send(qp);
2027
2028                 goto sunlock;
2029         }
2030
2031         case OP(COMPARE_SWAP):
2032         case OP(FETCH_ADD): {
2033                 struct ib_atomic_eth *ateth;
2034                 struct rvt_ack_entry *e;
2035                 u64 vaddr;
2036                 atomic64_t *maddr;
2037                 u64 sdata;
2038                 u32 rkey;
2039                 u8 next;
2040
2041                 if (unlikely(!(qp->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC)))
2042                         goto nack_inv;
2043                 next = qp->r_head_ack_queue + 1;
2044                 if (next > QIB_MAX_RDMA_ATOMIC)
2045                         next = 0;
2046                 spin_lock_irqsave(&qp->s_lock, flags);
2047                 if (unlikely(next == qp->s_tail_ack_queue)) {
2048                         if (!qp->s_ack_queue[next].sent)
2049                                 goto nack_inv_unlck;
2050                         qib_update_ack_queue(qp, next);
2051                 }
2052                 e = &qp->s_ack_queue[qp->r_head_ack_queue];
2053                 if (e->opcode == OP(RDMA_READ_REQUEST) && e->rdma_sge.mr) {
2054                         rvt_put_mr(e->rdma_sge.mr);
2055                         e->rdma_sge.mr = NULL;
2056                 }
2057                 ateth = &ohdr->u.atomic_eth;
2058                 vaddr = get_ib_ateth_vaddr(ateth);
2059                 if (unlikely(vaddr & (sizeof(u64) - 1)))
2060                         goto nack_inv_unlck;
2061                 rkey = be32_to_cpu(ateth->rkey);
2062                 /* Check rkey & NAK */
2063                 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, sizeof(u64),
2064                                           vaddr, rkey,
2065                                           IB_ACCESS_REMOTE_ATOMIC)))
2066                         goto nack_acc_unlck;
2067                 /* Perform atomic OP and save result. */
2068                 maddr = (atomic64_t *) qp->r_sge.sge.vaddr;
2069                 sdata = get_ib_ateth_swap(ateth);
2070                 e->atomic_data = (opcode == OP(FETCH_ADD)) ?
2071                         (u64) atomic64_add_return(sdata, maddr) - sdata :
2072                         (u64) cmpxchg((u64 *) qp->r_sge.sge.vaddr,
2073                                       get_ib_ateth_compare(ateth),
2074                                       sdata);
2075                 rvt_put_mr(qp->r_sge.sge.mr);
2076                 qp->r_sge.num_sge = 0;
2077                 e->opcode = opcode;
2078                 e->sent = 0;
2079                 e->psn = psn;
2080                 e->lpsn = psn;
2081                 qp->r_msn++;
2082                 qp->r_psn++;
2083                 qp->r_state = opcode;
2084                 qp->r_nak_state = 0;
2085                 qp->r_head_ack_queue = next;
2086
2087                 /* Schedule the send tasklet. */
2088                 qp->s_flags |= RVT_S_RESP_PENDING;
2089                 qib_schedule_send(qp);
2090
2091                 goto sunlock;
2092         }
2093
2094         default:
2095                 /* NAK unknown opcodes. */
2096                 goto nack_inv;
2097         }
2098         qp->r_psn++;
2099         qp->r_state = opcode;
2100         qp->r_ack_psn = psn;
2101         qp->r_nak_state = 0;
2102         /* Send an ACK if requested or required. */
2103         if (psn & (1 << 31))
2104                 goto send_ack;
2105         return;
2106
2107 rnr_nak:
2108         qp->r_nak_state = IB_RNR_NAK | qp->r_min_rnr_timer;
2109         qp->r_ack_psn = qp->r_psn;
2110         /* Queue RNR NAK for later */
2111         if (list_empty(&qp->rspwait)) {
2112                 qp->r_flags |= RVT_R_RSP_NAK;
2113                 rvt_get_qp(qp);
2114                 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
2115         }
2116         return;
2117
2118 nack_op_err:
2119         rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
2120         qp->r_nak_state = IB_NAK_REMOTE_OPERATIONAL_ERROR;
2121         qp->r_ack_psn = qp->r_psn;
2122         /* Queue NAK for later */
2123         if (list_empty(&qp->rspwait)) {
2124                 qp->r_flags |= RVT_R_RSP_NAK;
2125                 rvt_get_qp(qp);
2126                 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
2127         }
2128         return;
2129
2130 nack_inv_unlck:
2131         spin_unlock_irqrestore(&qp->s_lock, flags);
2132 nack_inv:
2133         rvt_rc_error(qp, IB_WC_LOC_QP_OP_ERR);
2134         qp->r_nak_state = IB_NAK_INVALID_REQUEST;
2135         qp->r_ack_psn = qp->r_psn;
2136         /* Queue NAK for later */
2137         if (list_empty(&qp->rspwait)) {
2138                 qp->r_flags |= RVT_R_RSP_NAK;
2139                 rvt_get_qp(qp);
2140                 list_add_tail(&qp->rspwait, &rcd->qp_wait_list);
2141         }
2142         return;
2143
2144 nack_acc_unlck:
2145         spin_unlock_irqrestore(&qp->s_lock, flags);
2146 nack_acc:
2147         rvt_rc_error(qp, IB_WC_LOC_PROT_ERR);
2148         qp->r_nak_state = IB_NAK_REMOTE_ACCESS_ERROR;
2149         qp->r_ack_psn = qp->r_psn;
2150 send_ack:
2151         qib_send_rc_ack(qp);
2152         return;
2153
2154 sunlock:
2155         spin_unlock_irqrestore(&qp->s_lock, flags);
2156 }