1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
3 * Copyright(c) 2016 Intel Corporation.
6 #include <rdma/rdmavt_qp.h>
7 #include <rdma/ib_hdrs.h>
10 * Convert the AETH credit code into the number of credits.
12 static const u16 credit_table[31] = {
47 * rvt_compute_aeth - compute the AETH (syndrome + MSN)
48 * @qp: the queue pair to compute the AETH for
52 __be32 rvt_compute_aeth(struct rvt_qp *qp)
54 u32 aeth = qp->r_msn & IB_MSN_MASK;
58 * Shared receive queues don't generate credits.
59 * Set the credit field to the invalid value.
61 aeth |= IB_AETH_CREDIT_INVAL << IB_AETH_CREDIT_SHIFT;
68 credits = READ_ONCE(qp->r_rq.kwq->count);
70 /* sanity check pointers before trusting them */
72 head = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->head);
73 tail = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->tail);
75 head = READ_ONCE(qp->r_rq.kwq->head);
76 tail = READ_ONCE(qp->r_rq.kwq->tail);
78 if (head >= qp->r_rq.size)
80 if (tail >= qp->r_rq.size)
83 * Compute the number of credits available (RWQEs).
84 * There is a small chance that the pair of reads are
85 * not atomic, which is OK, since the fuzziness is
86 * resolved as further ACKs go out.
88 credits = rvt_get_rq_count(&qp->r_rq, head, tail);
91 * Binary search the credit table to find the code to
98 if (credit_table[x] == credits)
100 if (credit_table[x] > credits) {
108 aeth |= x << IB_AETH_CREDIT_SHIFT;
110 return cpu_to_be32(aeth);
112 EXPORT_SYMBOL(rvt_compute_aeth);
115 * rvt_get_credit - flush the send work queue of a QP
116 * @qp: the qp who's send work queue to flush
117 * @aeth: the Acknowledge Extended Transport Header
119 * The QP s_lock should be held.
121 void rvt_get_credit(struct rvt_qp *qp, u32 aeth)
123 struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
124 u32 credit = (aeth >> IB_AETH_CREDIT_SHIFT) & IB_AETH_CREDIT_MASK;
126 lockdep_assert_held(&qp->s_lock);
128 * If the credit is invalid, we can send
129 * as many packets as we like. Otherwise, we have to
130 * honor the credit field.
132 if (credit == IB_AETH_CREDIT_INVAL) {
133 if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
134 qp->s_flags |= RVT_S_UNLIMITED_CREDIT;
135 if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
136 qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
137 rdi->driver_f.schedule_send(qp);
140 } else if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
141 /* Compute new LSN (i.e., MSN + credit) */
142 credit = (aeth + credit_table[credit]) & IB_MSN_MASK;
143 if (rvt_cmp_msn(credit, qp->s_lsn) > 0) {
145 if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
146 qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
147 rdi->driver_f.schedule_send(qp);
152 EXPORT_SYMBOL(rvt_get_credit);
155 * rvt_restart_sge - rewind the sge state for a wqe
156 * @ss: the sge state pointer
157 * @wqe: the wqe to rewind
158 * @len: the data length from the start of the wqe in bytes
160 * Returns the remaining data length.
162 u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len)
164 ss->sge = wqe->sg_list[0];
165 ss->sg_list = wqe->sg_list + 1;
166 ss->num_sge = wqe->wr.num_sge;
167 ss->total_len = wqe->length;
168 rvt_skip_sge(ss, len, false);
169 return wqe->length - len;
171 EXPORT_SYMBOL(rvt_restart_sge);