drivers/infiniband/sw/rdmavt/rc.c

   1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
   2 /*
   3  * Copyright(c) 2016 Intel Corporation.
   4  */
   5
   6 #include <rdma/rdmavt_qp.h>
   7 #include <rdma/ib_hdrs.h>
   8
   9 /*
  10  * Convert the AETH credit code into the number of credits.
  11  */
  12 static const u16 credit_table[31] = {
  13         0,                      /* 0 */
  14         1,                      /* 1 */
  15         2,                      /* 2 */
  16         3,                      /* 3 */
  17         4,                      /* 4 */
  18         6,                      /* 5 */
  19         8,                      /* 6 */
  20         12,                     /* 7 */
  21         16,                     /* 8 */
  22         24,                     /* 9 */
  23         32,                     /* A */
  24         48,                     /* B */
  25         64,                     /* C */
  26         96,                     /* D */
  27         128,                    /* E */
  28         192,                    /* F */
  29         256,                    /* 10 */
  30         384,                    /* 11 */
  31         512,                    /* 12 */
  32         768,                    /* 13 */
  33         1024,                   /* 14 */
  34         1536,                   /* 15 */
  35         2048,                   /* 16 */
  36         3072,                   /* 17 */
  37         4096,                   /* 18 */
  38         6144,                   /* 19 */
  39         8192,                   /* 1A */
  40         12288,                  /* 1B */
  41         16384,                  /* 1C */
  42         24576,                  /* 1D */
  43         32768                   /* 1E */
  44 };
  45
  46 /**
  47  * rvt_compute_aeth - compute the AETH (syndrome + MSN)
  48  * @qp: the queue pair to compute the AETH for
  49  *
  50  * Returns the AETH.
  51  */
  52 __be32 rvt_compute_aeth(struct rvt_qp *qp)
  53 {
  54         u32 aeth = qp->r_msn & IB_MSN_MASK;
  55
  56         if (qp->ibqp.srq) {
  57                 /*
  58                  * Shared receive queues don't generate credits.
  59                  * Set the credit field to the invalid value.
  60                  */
  61                 aeth |= IB_AETH_CREDIT_INVAL << IB_AETH_CREDIT_SHIFT;
  62         } else {
  63                 u32 min, max, x;
  64                 u32 credits;
  65                 u32 head;
  66                 u32 tail;
  67
  68                 credits = READ_ONCE(qp->r_rq.kwq->count);
  69                 if (credits == 0) {
  70                         /* sanity check pointers before trusting them */
  71                         if (qp->ip) {
  72                                 head = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->head);
  73                                 tail = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->tail);
  74                         } else {
  75                                 head = READ_ONCE(qp->r_rq.kwq->head);
  76                                 tail = READ_ONCE(qp->r_rq.kwq->tail);
  77                         }
  78                         if (head >= qp->r_rq.size)
  79                                 head = 0;
  80                         if (tail >= qp->r_rq.size)
  81                                 tail = 0;
  82                         /*
  83                          * Compute the number of credits available (RWQEs).
  84                          * There is a small chance that the pair of reads are
  85                          * not atomic, which is OK, since the fuzziness is
  86                          * resolved as further ACKs go out.
  87                          */
  88                         credits = rvt_get_rq_count(&qp->r_rq, head, tail);
  89                 }
  90                 /*
  91                  * Binary search the credit table to find the code to
  92                  * use.
  93                  */
  94                 min = 0;
  95                 max = 31;
  96                 for (;;) {
  97                         x = (min + max) / 2;
  98                         if (credit_table[x] == credits)
  99                                 break;
 100                         if (credit_table[x] > credits) {
 101                                 max = x;
 102                         } else {
 103                                 if (min == x)
 104                                         break;
 105                                 min = x;
 106                         }
 107                 }
 108                 aeth |= x << IB_AETH_CREDIT_SHIFT;
 109         }
 110         return cpu_to_be32(aeth);
 111 }
 112 EXPORT_SYMBOL(rvt_compute_aeth);
 113
 114 /**
 115  * rvt_get_credit - flush the send work queue of a QP
 116  * @qp: the qp who's send work queue to flush
 117  * @aeth: the Acknowledge Extended Transport Header
 118  *
 119  * The QP s_lock should be held.
 120  */
 121 void rvt_get_credit(struct rvt_qp *qp, u32 aeth)
 122 {
 123         struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
 124         u32 credit = (aeth >> IB_AETH_CREDIT_SHIFT) & IB_AETH_CREDIT_MASK;
 125
 126         lockdep_assert_held(&qp->s_lock);
 127         /*
 128          * If the credit is invalid, we can send
 129          * as many packets as we like.  Otherwise, we have to
 130          * honor the credit field.
 131          */
 132         if (credit == IB_AETH_CREDIT_INVAL) {
 133                 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
 134                         qp->s_flags |= RVT_S_UNLIMITED_CREDIT;
 135                         if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
 136                                 qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
 137                                 rdi->driver_f.schedule_send(qp);
 138                         }
 139                 }
 140         } else if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
 141                 /* Compute new LSN (i.e., MSN + credit) */
 142                 credit = (aeth + credit_table[credit]) & IB_MSN_MASK;
 143                 if (rvt_cmp_msn(credit, qp->s_lsn) > 0) {
 144                         qp->s_lsn = credit;
 145                         if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
 146                                 qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
 147                                 rdi->driver_f.schedule_send(qp);
 148                         }
 149                 }
 150         }
 151 }
 152 EXPORT_SYMBOL(rvt_get_credit);
 153
 154 /**
 155  * rvt_restart_sge - rewind the sge state for a wqe
 156  * @ss: the sge state pointer
 157  * @wqe: the wqe to rewind
 158  * @len: the data length from the start of the wqe in bytes
 159  *
 160  * Returns the remaining data length.
 161  */
 162 u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len)
 163 {
 164         ss->sge = wqe->sg_list[0];
 165         ss->sg_list = wqe->sg_list + 1;
 166         ss->num_sge = wqe->wr.num_sge;
 167         ss->total_len = wqe->length;
 168         rvt_skip_sge(ss, len, false);
 169         return wqe->length - len;
 170 }
 171 EXPORT_SYMBOL(rvt_restart_sge);
 172