GNU Linux-libre 4.19.211-gnu1
[releases.git] / net / rds / ib_recv.c
1 /*
2  * Copyright (c) 2006, 2017 Oracle and/or its affiliates. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  *
32  */
33 #include <linux/kernel.h>
34 #include <linux/slab.h>
35 #include <linux/pci.h>
36 #include <linux/dma-mapping.h>
37 #include <rdma/rdma_cm.h>
38
39 #include "rds_single_path.h"
40 #include "rds.h"
41 #include "ib.h"
42
43 static struct kmem_cache *rds_ib_incoming_slab;
44 static struct kmem_cache *rds_ib_frag_slab;
45 static atomic_t rds_ib_allocation = ATOMIC_INIT(0);
46
47 void rds_ib_recv_init_ring(struct rds_ib_connection *ic)
48 {
49         struct rds_ib_recv_work *recv;
50         u32 i;
51
52         for (i = 0, recv = ic->i_recvs; i < ic->i_recv_ring.w_nr; i++, recv++) {
53                 struct ib_sge *sge;
54
55                 recv->r_ibinc = NULL;
56                 recv->r_frag = NULL;
57
58                 recv->r_wr.next = NULL;
59                 recv->r_wr.wr_id = i;
60                 recv->r_wr.sg_list = recv->r_sge;
61                 recv->r_wr.num_sge = RDS_IB_RECV_SGE;
62
63                 sge = &recv->r_sge[0];
64                 sge->addr = ic->i_recv_hdrs_dma + (i * sizeof(struct rds_header));
65                 sge->length = sizeof(struct rds_header);
66                 sge->lkey = ic->i_pd->local_dma_lkey;
67
68                 sge = &recv->r_sge[1];
69                 sge->addr = 0;
70                 sge->length = RDS_FRAG_SIZE;
71                 sge->lkey = ic->i_pd->local_dma_lkey;
72         }
73 }
74
75 /*
76  * The entire 'from' list, including the from element itself, is put on
77  * to the tail of the 'to' list.
78  */
79 static void list_splice_entire_tail(struct list_head *from,
80                                     struct list_head *to)
81 {
82         struct list_head *from_last = from->prev;
83
84         list_splice_tail(from_last, to);
85         list_add_tail(from_last, to);
86 }
87
88 static void rds_ib_cache_xfer_to_ready(struct rds_ib_refill_cache *cache)
89 {
90         struct list_head *tmp;
91
92         tmp = xchg(&cache->xfer, NULL);
93         if (tmp) {
94                 if (cache->ready)
95                         list_splice_entire_tail(tmp, cache->ready);
96                 else
97                         cache->ready = tmp;
98         }
99 }
100
101 static int rds_ib_recv_alloc_cache(struct rds_ib_refill_cache *cache, gfp_t gfp)
102 {
103         struct rds_ib_cache_head *head;
104         int cpu;
105
106         cache->percpu = alloc_percpu_gfp(struct rds_ib_cache_head, gfp);
107         if (!cache->percpu)
108                return -ENOMEM;
109
110         for_each_possible_cpu(cpu) {
111                 head = per_cpu_ptr(cache->percpu, cpu);
112                 head->first = NULL;
113                 head->count = 0;
114         }
115         cache->xfer = NULL;
116         cache->ready = NULL;
117
118         return 0;
119 }
120
121 int rds_ib_recv_alloc_caches(struct rds_ib_connection *ic, gfp_t gfp)
122 {
123         int ret;
124
125         ret = rds_ib_recv_alloc_cache(&ic->i_cache_incs, gfp);
126         if (!ret) {
127                 ret = rds_ib_recv_alloc_cache(&ic->i_cache_frags, gfp);
128                 if (ret)
129                         free_percpu(ic->i_cache_incs.percpu);
130         }
131
132         return ret;
133 }
134
135 static void rds_ib_cache_splice_all_lists(struct rds_ib_refill_cache *cache,
136                                           struct list_head *caller_list)
137 {
138         struct rds_ib_cache_head *head;
139         int cpu;
140
141         for_each_possible_cpu(cpu) {
142                 head = per_cpu_ptr(cache->percpu, cpu);
143                 if (head->first) {
144                         list_splice_entire_tail(head->first, caller_list);
145                         head->first = NULL;
146                 }
147         }
148
149         if (cache->ready) {
150                 list_splice_entire_tail(cache->ready, caller_list);
151                 cache->ready = NULL;
152         }
153 }
154
155 void rds_ib_recv_free_caches(struct rds_ib_connection *ic)
156 {
157         struct rds_ib_incoming *inc;
158         struct rds_ib_incoming *inc_tmp;
159         struct rds_page_frag *frag;
160         struct rds_page_frag *frag_tmp;
161         LIST_HEAD(list);
162
163         rds_ib_cache_xfer_to_ready(&ic->i_cache_incs);
164         rds_ib_cache_splice_all_lists(&ic->i_cache_incs, &list);
165         free_percpu(ic->i_cache_incs.percpu);
166
167         list_for_each_entry_safe(inc, inc_tmp, &list, ii_cache_entry) {
168                 list_del(&inc->ii_cache_entry);
169                 WARN_ON(!list_empty(&inc->ii_frags));
170                 kmem_cache_free(rds_ib_incoming_slab, inc);
171         }
172
173         rds_ib_cache_xfer_to_ready(&ic->i_cache_frags);
174         rds_ib_cache_splice_all_lists(&ic->i_cache_frags, &list);
175         free_percpu(ic->i_cache_frags.percpu);
176
177         list_for_each_entry_safe(frag, frag_tmp, &list, f_cache_entry) {
178                 list_del(&frag->f_cache_entry);
179                 WARN_ON(!list_empty(&frag->f_item));
180                 kmem_cache_free(rds_ib_frag_slab, frag);
181         }
182 }
183
184 /* fwd decl */
185 static void rds_ib_recv_cache_put(struct list_head *new_item,
186                                   struct rds_ib_refill_cache *cache);
187 static struct list_head *rds_ib_recv_cache_get(struct rds_ib_refill_cache *cache);
188
189
190 /* Recycle frag and attached recv buffer f_sg */
191 static void rds_ib_frag_free(struct rds_ib_connection *ic,
192                              struct rds_page_frag *frag)
193 {
194         rdsdebug("frag %p page %p\n", frag, sg_page(&frag->f_sg));
195
196         rds_ib_recv_cache_put(&frag->f_cache_entry, &ic->i_cache_frags);
197         atomic_add(RDS_FRAG_SIZE / SZ_1K, &ic->i_cache_allocs);
198         rds_ib_stats_add(s_ib_recv_added_to_cache, RDS_FRAG_SIZE);
199 }
200
201 /* Recycle inc after freeing attached frags */
202 void rds_ib_inc_free(struct rds_incoming *inc)
203 {
204         struct rds_ib_incoming *ibinc;
205         struct rds_page_frag *frag;
206         struct rds_page_frag *pos;
207         struct rds_ib_connection *ic = inc->i_conn->c_transport_data;
208
209         ibinc = container_of(inc, struct rds_ib_incoming, ii_inc);
210
211         /* Free attached frags */
212         list_for_each_entry_safe(frag, pos, &ibinc->ii_frags, f_item) {
213                 list_del_init(&frag->f_item);
214                 rds_ib_frag_free(ic, frag);
215         }
216         BUG_ON(!list_empty(&ibinc->ii_frags));
217
218         rdsdebug("freeing ibinc %p inc %p\n", ibinc, inc);
219         rds_ib_recv_cache_put(&ibinc->ii_cache_entry, &ic->i_cache_incs);
220 }
221
222 static void rds_ib_recv_clear_one(struct rds_ib_connection *ic,
223                                   struct rds_ib_recv_work *recv)
224 {
225         if (recv->r_ibinc) {
226                 rds_inc_put(&recv->r_ibinc->ii_inc);
227                 recv->r_ibinc = NULL;
228         }
229         if (recv->r_frag) {
230                 ib_dma_unmap_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, 1, DMA_FROM_DEVICE);
231                 rds_ib_frag_free(ic, recv->r_frag);
232                 recv->r_frag = NULL;
233         }
234 }
235
236 void rds_ib_recv_clear_ring(struct rds_ib_connection *ic)
237 {
238         u32 i;
239
240         for (i = 0; i < ic->i_recv_ring.w_nr; i++)
241                 rds_ib_recv_clear_one(ic, &ic->i_recvs[i]);
242 }
243
244 static struct rds_ib_incoming *rds_ib_refill_one_inc(struct rds_ib_connection *ic,
245                                                      gfp_t slab_mask)
246 {
247         struct rds_ib_incoming *ibinc;
248         struct list_head *cache_item;
249         int avail_allocs;
250
251         cache_item = rds_ib_recv_cache_get(&ic->i_cache_incs);
252         if (cache_item) {
253                 ibinc = container_of(cache_item, struct rds_ib_incoming, ii_cache_entry);
254         } else {
255                 avail_allocs = atomic_add_unless(&rds_ib_allocation,
256                                                  1, rds_ib_sysctl_max_recv_allocation);
257                 if (!avail_allocs) {
258                         rds_ib_stats_inc(s_ib_rx_alloc_limit);
259                         return NULL;
260                 }
261                 ibinc = kmem_cache_alloc(rds_ib_incoming_slab, slab_mask);
262                 if (!ibinc) {
263                         atomic_dec(&rds_ib_allocation);
264                         return NULL;
265                 }
266                 rds_ib_stats_inc(s_ib_rx_total_incs);
267         }
268         INIT_LIST_HEAD(&ibinc->ii_frags);
269         rds_inc_init(&ibinc->ii_inc, ic->conn, &ic->conn->c_faddr);
270
271         return ibinc;
272 }
273
274 static struct rds_page_frag *rds_ib_refill_one_frag(struct rds_ib_connection *ic,
275                                                     gfp_t slab_mask, gfp_t page_mask)
276 {
277         struct rds_page_frag *frag;
278         struct list_head *cache_item;
279         int ret;
280
281         cache_item = rds_ib_recv_cache_get(&ic->i_cache_frags);
282         if (cache_item) {
283                 frag = container_of(cache_item, struct rds_page_frag, f_cache_entry);
284                 atomic_sub(RDS_FRAG_SIZE / SZ_1K, &ic->i_cache_allocs);
285                 rds_ib_stats_add(s_ib_recv_added_to_cache, RDS_FRAG_SIZE);
286         } else {
287                 frag = kmem_cache_alloc(rds_ib_frag_slab, slab_mask);
288                 if (!frag)
289                         return NULL;
290
291                 sg_init_table(&frag->f_sg, 1);
292                 ret = rds_page_remainder_alloc(&frag->f_sg,
293                                                RDS_FRAG_SIZE, page_mask);
294                 if (ret) {
295                         kmem_cache_free(rds_ib_frag_slab, frag);
296                         return NULL;
297                 }
298                 rds_ib_stats_inc(s_ib_rx_total_frags);
299         }
300
301         INIT_LIST_HEAD(&frag->f_item);
302
303         return frag;
304 }
305
306 static int rds_ib_recv_refill_one(struct rds_connection *conn,
307                                   struct rds_ib_recv_work *recv, gfp_t gfp)
308 {
309         struct rds_ib_connection *ic = conn->c_transport_data;
310         struct ib_sge *sge;
311         int ret = -ENOMEM;
312         gfp_t slab_mask = GFP_NOWAIT;
313         gfp_t page_mask = GFP_NOWAIT;
314
315         if (gfp & __GFP_DIRECT_RECLAIM) {
316                 slab_mask = GFP_KERNEL;
317                 page_mask = GFP_HIGHUSER;
318         }
319
320         if (!ic->i_cache_incs.ready)
321                 rds_ib_cache_xfer_to_ready(&ic->i_cache_incs);
322         if (!ic->i_cache_frags.ready)
323                 rds_ib_cache_xfer_to_ready(&ic->i_cache_frags);
324
325         /*
326          * ibinc was taken from recv if recv contained the start of a message.
327          * recvs that were continuations will still have this allocated.
328          */
329         if (!recv->r_ibinc) {
330                 recv->r_ibinc = rds_ib_refill_one_inc(ic, slab_mask);
331                 if (!recv->r_ibinc)
332                         goto out;
333         }
334
335         WARN_ON(recv->r_frag); /* leak! */
336         recv->r_frag = rds_ib_refill_one_frag(ic, slab_mask, page_mask);
337         if (!recv->r_frag)
338                 goto out;
339
340         ret = ib_dma_map_sg(ic->i_cm_id->device, &recv->r_frag->f_sg,
341                             1, DMA_FROM_DEVICE);
342         WARN_ON(ret != 1);
343
344         sge = &recv->r_sge[0];
345         sge->addr = ic->i_recv_hdrs_dma + (recv - ic->i_recvs) * sizeof(struct rds_header);
346         sge->length = sizeof(struct rds_header);
347
348         sge = &recv->r_sge[1];
349         sge->addr = ib_sg_dma_address(ic->i_cm_id->device, &recv->r_frag->f_sg);
350         sge->length = ib_sg_dma_len(ic->i_cm_id->device, &recv->r_frag->f_sg);
351
352         ret = 0;
353 out:
354         return ret;
355 }
356
357 static int acquire_refill(struct rds_connection *conn)
358 {
359         return test_and_set_bit(RDS_RECV_REFILL, &conn->c_flags) == 0;
360 }
361
362 static void release_refill(struct rds_connection *conn)
363 {
364         clear_bit(RDS_RECV_REFILL, &conn->c_flags);
365
366         /* We don't use wait_on_bit()/wake_up_bit() because our waking is in a
367          * hot path and finding waiters is very rare.  We don't want to walk
368          * the system-wide hashed waitqueue buckets in the fast path only to
369          * almost never find waiters.
370          */
371         if (waitqueue_active(&conn->c_waitq))
372                 wake_up_all(&conn->c_waitq);
373 }
374
375 /*
376  * This tries to allocate and post unused work requests after making sure that
377  * they have all the allocations they need to queue received fragments into
378  * sockets.
379  */
380 void rds_ib_recv_refill(struct rds_connection *conn, int prefill, gfp_t gfp)
381 {
382         struct rds_ib_connection *ic = conn->c_transport_data;
383         struct rds_ib_recv_work *recv;
384         unsigned int posted = 0;
385         int ret = 0;
386         bool can_wait = !!(gfp & __GFP_DIRECT_RECLAIM);
387         u32 pos;
388
389         /* the goal here is to just make sure that someone, somewhere
390          * is posting buffers.  If we can't get the refill lock,
391          * let them do their thing
392          */
393         if (!acquire_refill(conn))
394                 return;
395
396         while ((prefill || rds_conn_up(conn)) &&
397                rds_ib_ring_alloc(&ic->i_recv_ring, 1, &pos)) {
398                 if (pos >= ic->i_recv_ring.w_nr) {
399                         printk(KERN_NOTICE "Argh - ring alloc returned pos=%u\n",
400                                         pos);
401                         break;
402                 }
403
404                 recv = &ic->i_recvs[pos];
405                 ret = rds_ib_recv_refill_one(conn, recv, gfp);
406                 if (ret) {
407                         break;
408                 }
409
410                 rdsdebug("recv %p ibinc %p page %p addr %lu\n", recv,
411                          recv->r_ibinc, sg_page(&recv->r_frag->f_sg),
412                          (long) ib_sg_dma_address(
413                                 ic->i_cm_id->device,
414                                 &recv->r_frag->f_sg));
415
416                 /* XXX when can this fail? */
417                 ret = ib_post_recv(ic->i_cm_id->qp, &recv->r_wr, NULL);
418                 if (ret) {
419                         rds_ib_conn_error(conn, "recv post on "
420                                "%pI6c returned %d, disconnecting and "
421                                "reconnecting\n", &conn->c_faddr,
422                                ret);
423                         break;
424                 }
425
426                 posted++;
427         }
428
429         /* We're doing flow control - update the window. */
430         if (ic->i_flowctl && posted)
431                 rds_ib_advertise_credits(conn, posted);
432
433         if (ret)
434                 rds_ib_ring_unalloc(&ic->i_recv_ring, 1);
435
436         release_refill(conn);
437
438         /* if we're called from the softirq handler, we'll be GFP_NOWAIT.
439          * in this case the ring being low is going to lead to more interrupts
440          * and we can safely let the softirq code take care of it unless the
441          * ring is completely empty.
442          *
443          * if we're called from krdsd, we'll be GFP_KERNEL.  In this case
444          * we might have raced with the softirq code while we had the refill
445          * lock held.  Use rds_ib_ring_low() instead of ring_empty to decide
446          * if we should requeue.
447          */
448         if (rds_conn_up(conn) &&
449             ((can_wait && rds_ib_ring_low(&ic->i_recv_ring)) ||
450             rds_ib_ring_empty(&ic->i_recv_ring))) {
451                 queue_delayed_work(rds_wq, &conn->c_recv_w, 1);
452         }
453 }
454
455 /*
456  * We want to recycle several types of recv allocations, like incs and frags.
457  * To use this, the *_free() function passes in the ptr to a list_head within
458  * the recyclee, as well as the cache to put it on.
459  *
460  * First, we put the memory on a percpu list. When this reaches a certain size,
461  * We move it to an intermediate non-percpu list in a lockless manner, with some
462  * xchg/compxchg wizardry.
463  *
464  * N.B. Instead of a list_head as the anchor, we use a single pointer, which can
465  * be NULL and xchg'd. The list is actually empty when the pointer is NULL, and
466  * list_empty() will return true with one element is actually present.
467  */
468 static void rds_ib_recv_cache_put(struct list_head *new_item,
469                                  struct rds_ib_refill_cache *cache)
470 {
471         unsigned long flags;
472         struct list_head *old, *chpfirst;
473
474         local_irq_save(flags);
475
476         chpfirst = __this_cpu_read(cache->percpu->first);
477         if (!chpfirst)
478                 INIT_LIST_HEAD(new_item);
479         else /* put on front */
480                 list_add_tail(new_item, chpfirst);
481
482         __this_cpu_write(cache->percpu->first, new_item);
483         __this_cpu_inc(cache->percpu->count);
484
485         if (__this_cpu_read(cache->percpu->count) < RDS_IB_RECYCLE_BATCH_COUNT)
486                 goto end;
487
488         /*
489          * Return our per-cpu first list to the cache's xfer by atomically
490          * grabbing the current xfer list, appending it to our per-cpu list,
491          * and then atomically returning that entire list back to the
492          * cache's xfer list as long as it's still empty.
493          */
494         do {
495                 old = xchg(&cache->xfer, NULL);
496                 if (old)
497                         list_splice_entire_tail(old, chpfirst);
498                 old = cmpxchg(&cache->xfer, NULL, chpfirst);
499         } while (old);
500
501
502         __this_cpu_write(cache->percpu->first, NULL);
503         __this_cpu_write(cache->percpu->count, 0);
504 end:
505         local_irq_restore(flags);
506 }
507
508 static struct list_head *rds_ib_recv_cache_get(struct rds_ib_refill_cache *cache)
509 {
510         struct list_head *head = cache->ready;
511
512         if (head) {
513                 if (!list_empty(head)) {
514                         cache->ready = head->next;
515                         list_del_init(head);
516                 } else
517                         cache->ready = NULL;
518         }
519
520         return head;
521 }
522
523 int rds_ib_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to)
524 {
525         struct rds_ib_incoming *ibinc;
526         struct rds_page_frag *frag;
527         unsigned long to_copy;
528         unsigned long frag_off = 0;
529         int copied = 0;
530         int ret;
531         u32 len;
532
533         ibinc = container_of(inc, struct rds_ib_incoming, ii_inc);
534         frag = list_entry(ibinc->ii_frags.next, struct rds_page_frag, f_item);
535         len = be32_to_cpu(inc->i_hdr.h_len);
536
537         while (iov_iter_count(to) && copied < len) {
538                 if (frag_off == RDS_FRAG_SIZE) {
539                         frag = list_entry(frag->f_item.next,
540                                           struct rds_page_frag, f_item);
541                         frag_off = 0;
542                 }
543                 to_copy = min_t(unsigned long, iov_iter_count(to),
544                                 RDS_FRAG_SIZE - frag_off);
545                 to_copy = min_t(unsigned long, to_copy, len - copied);
546
547                 /* XXX needs + offset for multiple recvs per page */
548                 rds_stats_add(s_copy_to_user, to_copy);
549                 ret = copy_page_to_iter(sg_page(&frag->f_sg),
550                                         frag->f_sg.offset + frag_off,
551                                         to_copy,
552                                         to);
553                 if (ret != to_copy)
554                         return -EFAULT;
555
556                 frag_off += to_copy;
557                 copied += to_copy;
558         }
559
560         return copied;
561 }
562
563 /* ic starts out kzalloc()ed */
564 void rds_ib_recv_init_ack(struct rds_ib_connection *ic)
565 {
566         struct ib_send_wr *wr = &ic->i_ack_wr;
567         struct ib_sge *sge = &ic->i_ack_sge;
568
569         sge->addr = ic->i_ack_dma;
570         sge->length = sizeof(struct rds_header);
571         sge->lkey = ic->i_pd->local_dma_lkey;
572
573         wr->sg_list = sge;
574         wr->num_sge = 1;
575         wr->opcode = IB_WR_SEND;
576         wr->wr_id = RDS_IB_ACK_WR_ID;
577         wr->send_flags = IB_SEND_SIGNALED | IB_SEND_SOLICITED;
578 }
579
580 /*
581  * You'd think that with reliable IB connections you wouldn't need to ack
582  * messages that have been received.  The problem is that IB hardware generates
583  * an ack message before it has DMAed the message into memory.  This creates a
584  * potential message loss if the HCA is disabled for any reason between when it
585  * sends the ack and before the message is DMAed and processed.  This is only a
586  * potential issue if another HCA is available for fail-over.
587  *
588  * When the remote host receives our ack they'll free the sent message from
589  * their send queue.  To decrease the latency of this we always send an ack
590  * immediately after we've received messages.
591  *
592  * For simplicity, we only have one ack in flight at a time.  This puts
593  * pressure on senders to have deep enough send queues to absorb the latency of
594  * a single ack frame being in flight.  This might not be good enough.
595  *
596  * This is implemented by have a long-lived send_wr and sge which point to a
597  * statically allocated ack frame.  This ack wr does not fall under the ring
598  * accounting that the tx and rx wrs do.  The QP attribute specifically makes
599  * room for it beyond the ring size.  Send completion notices its special
600  * wr_id and avoids working with the ring in that case.
601  */
602 #ifndef KERNEL_HAS_ATOMIC64
603 void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq, int ack_required)
604 {
605         unsigned long flags;
606
607         spin_lock_irqsave(&ic->i_ack_lock, flags);
608         ic->i_ack_next = seq;
609         if (ack_required)
610                 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
611         spin_unlock_irqrestore(&ic->i_ack_lock, flags);
612 }
613
614 static u64 rds_ib_get_ack(struct rds_ib_connection *ic)
615 {
616         unsigned long flags;
617         u64 seq;
618
619         clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
620
621         spin_lock_irqsave(&ic->i_ack_lock, flags);
622         seq = ic->i_ack_next;
623         spin_unlock_irqrestore(&ic->i_ack_lock, flags);
624
625         return seq;
626 }
627 #else
628 void rds_ib_set_ack(struct rds_ib_connection *ic, u64 seq, int ack_required)
629 {
630         atomic64_set(&ic->i_ack_next, seq);
631         if (ack_required) {
632                 smp_mb__before_atomic();
633                 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
634         }
635 }
636
637 static u64 rds_ib_get_ack(struct rds_ib_connection *ic)
638 {
639         clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
640         smp_mb__after_atomic();
641
642         return atomic64_read(&ic->i_ack_next);
643 }
644 #endif
645
646
647 static void rds_ib_send_ack(struct rds_ib_connection *ic, unsigned int adv_credits)
648 {
649         struct rds_header *hdr = ic->i_ack;
650         u64 seq;
651         int ret;
652
653         seq = rds_ib_get_ack(ic);
654
655         rdsdebug("send_ack: ic %p ack %llu\n", ic, (unsigned long long) seq);
656         rds_message_populate_header(hdr, 0, 0, 0);
657         hdr->h_ack = cpu_to_be64(seq);
658         hdr->h_credit = adv_credits;
659         rds_message_make_checksum(hdr);
660         ic->i_ack_queued = jiffies;
661
662         ret = ib_post_send(ic->i_cm_id->qp, &ic->i_ack_wr, NULL);
663         if (unlikely(ret)) {
664                 /* Failed to send. Release the WR, and
665                  * force another ACK.
666                  */
667                 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
668                 set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
669
670                 rds_ib_stats_inc(s_ib_ack_send_failure);
671
672                 rds_ib_conn_error(ic->conn, "sending ack failed\n");
673         } else
674                 rds_ib_stats_inc(s_ib_ack_sent);
675 }
676
677 /*
678  * There are 3 ways of getting acknowledgements to the peer:
679  *  1.  We call rds_ib_attempt_ack from the recv completion handler
680  *      to send an ACK-only frame.
681  *      However, there can be only one such frame in the send queue
682  *      at any time, so we may have to postpone it.
683  *  2.  When another (data) packet is transmitted while there's
684  *      an ACK in the queue, we piggyback the ACK sequence number
685  *      on the data packet.
686  *  3.  If the ACK WR is done sending, we get called from the
687  *      send queue completion handler, and check whether there's
688  *      another ACK pending (postponed because the WR was on the
689  *      queue). If so, we transmit it.
690  *
691  * We maintain 2 variables:
692  *  -   i_ack_flags, which keeps track of whether the ACK WR
693  *      is currently in the send queue or not (IB_ACK_IN_FLIGHT)
694  *  -   i_ack_next, which is the last sequence number we received
695  *
696  * Potentially, send queue and receive queue handlers can run concurrently.
697  * It would be nice to not have to use a spinlock to synchronize things,
698  * but the one problem that rules this out is that 64bit updates are
699  * not atomic on all platforms. Things would be a lot simpler if
700  * we had atomic64 or maybe cmpxchg64 everywhere.
701  *
702  * Reconnecting complicates this picture just slightly. When we
703  * reconnect, we may be seeing duplicate packets. The peer
704  * is retransmitting them, because it hasn't seen an ACK for
705  * them. It is important that we ACK these.
706  *
707  * ACK mitigation adds a header flag "ACK_REQUIRED"; any packet with
708  * this flag set *MUST* be acknowledged immediately.
709  */
710
711 /*
712  * When we get here, we're called from the recv queue handler.
713  * Check whether we ought to transmit an ACK.
714  */
715 void rds_ib_attempt_ack(struct rds_ib_connection *ic)
716 {
717         unsigned int adv_credits;
718
719         if (!test_bit(IB_ACK_REQUESTED, &ic->i_ack_flags))
720                 return;
721
722         if (test_and_set_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags)) {
723                 rds_ib_stats_inc(s_ib_ack_send_delayed);
724                 return;
725         }
726
727         /* Can we get a send credit? */
728         if (!rds_ib_send_grab_credits(ic, 1, &adv_credits, 0, RDS_MAX_ADV_CREDIT)) {
729                 rds_ib_stats_inc(s_ib_tx_throttle);
730                 clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
731                 return;
732         }
733
734         clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);
735         rds_ib_send_ack(ic, adv_credits);
736 }
737
738 /*
739  * We get here from the send completion handler, when the
740  * adapter tells us the ACK frame was sent.
741  */
742 void rds_ib_ack_send_complete(struct rds_ib_connection *ic)
743 {
744         clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
745         rds_ib_attempt_ack(ic);
746 }
747
748 /*
749  * This is called by the regular xmit code when it wants to piggyback
750  * an ACK on an outgoing frame.
751  */
752 u64 rds_ib_piggyb_ack(struct rds_ib_connection *ic)
753 {
754         if (test_and_clear_bit(IB_ACK_REQUESTED, &ic->i_ack_flags))
755                 rds_ib_stats_inc(s_ib_ack_send_piggybacked);
756         return rds_ib_get_ack(ic);
757 }
758
759 /*
760  * It's kind of lame that we're copying from the posted receive pages into
761  * long-lived bitmaps.  We could have posted the bitmaps and rdma written into
762  * them.  But receiving new congestion bitmaps should be a *rare* event, so
763  * hopefully we won't need to invest that complexity in making it more
764  * efficient.  By copying we can share a simpler core with TCP which has to
765  * copy.
766  */
767 static void rds_ib_cong_recv(struct rds_connection *conn,
768                               struct rds_ib_incoming *ibinc)
769 {
770         struct rds_cong_map *map;
771         unsigned int map_off;
772         unsigned int map_page;
773         struct rds_page_frag *frag;
774         unsigned long frag_off;
775         unsigned long to_copy;
776         unsigned long copied;
777         uint64_t uncongested = 0;
778         void *addr;
779
780         /* catch completely corrupt packets */
781         if (be32_to_cpu(ibinc->ii_inc.i_hdr.h_len) != RDS_CONG_MAP_BYTES)
782                 return;
783
784         map = conn->c_fcong;
785         map_page = 0;
786         map_off = 0;
787
788         frag = list_entry(ibinc->ii_frags.next, struct rds_page_frag, f_item);
789         frag_off = 0;
790
791         copied = 0;
792
793         while (copied < RDS_CONG_MAP_BYTES) {
794                 uint64_t *src, *dst;
795                 unsigned int k;
796
797                 to_copy = min(RDS_FRAG_SIZE - frag_off, PAGE_SIZE - map_off);
798                 BUG_ON(to_copy & 7); /* Must be 64bit aligned. */
799
800                 addr = kmap_atomic(sg_page(&frag->f_sg));
801
802                 src = addr + frag->f_sg.offset + frag_off;
803                 dst = (void *)map->m_page_addrs[map_page] + map_off;
804                 for (k = 0; k < to_copy; k += 8) {
805                         /* Record ports that became uncongested, ie
806                          * bits that changed from 0 to 1. */
807                         uncongested |= ~(*src) & *dst;
808                         *dst++ = *src++;
809                 }
810                 kunmap_atomic(addr);
811
812                 copied += to_copy;
813
814                 map_off += to_copy;
815                 if (map_off == PAGE_SIZE) {
816                         map_off = 0;
817                         map_page++;
818                 }
819
820                 frag_off += to_copy;
821                 if (frag_off == RDS_FRAG_SIZE) {
822                         frag = list_entry(frag->f_item.next,
823                                           struct rds_page_frag, f_item);
824                         frag_off = 0;
825                 }
826         }
827
828         /* the congestion map is in little endian order */
829         uncongested = le64_to_cpu(uncongested);
830
831         rds_cong_map_updated(map, uncongested);
832 }
833
834 static void rds_ib_process_recv(struct rds_connection *conn,
835                                 struct rds_ib_recv_work *recv, u32 data_len,
836                                 struct rds_ib_ack_state *state)
837 {
838         struct rds_ib_connection *ic = conn->c_transport_data;
839         struct rds_ib_incoming *ibinc = ic->i_ibinc;
840         struct rds_header *ihdr, *hdr;
841
842         /* XXX shut down the connection if port 0,0 are seen? */
843
844         rdsdebug("ic %p ibinc %p recv %p byte len %u\n", ic, ibinc, recv,
845                  data_len);
846
847         if (data_len < sizeof(struct rds_header)) {
848                 rds_ib_conn_error(conn, "incoming message "
849                        "from %pI6c didn't include a "
850                        "header, disconnecting and "
851                        "reconnecting\n",
852                        &conn->c_faddr);
853                 return;
854         }
855         data_len -= sizeof(struct rds_header);
856
857         ihdr = &ic->i_recv_hdrs[recv - ic->i_recvs];
858
859         /* Validate the checksum. */
860         if (!rds_message_verify_checksum(ihdr)) {
861                 rds_ib_conn_error(conn, "incoming message "
862                        "from %pI6c has corrupted header - "
863                        "forcing a reconnect\n",
864                        &conn->c_faddr);
865                 rds_stats_inc(s_recv_drop_bad_checksum);
866                 return;
867         }
868
869         /* Process the ACK sequence which comes with every packet */
870         state->ack_recv = be64_to_cpu(ihdr->h_ack);
871         state->ack_recv_valid = 1;
872
873         /* Process the credits update if there was one */
874         if (ihdr->h_credit)
875                 rds_ib_send_add_credits(conn, ihdr->h_credit);
876
877         if (ihdr->h_sport == 0 && ihdr->h_dport == 0 && data_len == 0) {
878                 /* This is an ACK-only packet. The fact that it gets
879                  * special treatment here is that historically, ACKs
880                  * were rather special beasts.
881                  */
882                 rds_ib_stats_inc(s_ib_ack_received);
883
884                 /*
885                  * Usually the frags make their way on to incs and are then freed as
886                  * the inc is freed.  We don't go that route, so we have to drop the
887                  * page ref ourselves.  We can't just leave the page on the recv
888                  * because that confuses the dma mapping of pages and each recv's use
889                  * of a partial page.
890                  *
891                  * FIXME: Fold this into the code path below.
892                  */
893                 rds_ib_frag_free(ic, recv->r_frag);
894                 recv->r_frag = NULL;
895                 return;
896         }
897
898         /*
899          * If we don't already have an inc on the connection then this
900          * fragment has a header and starts a message.. copy its header
901          * into the inc and save the inc so we can hang upcoming fragments
902          * off its list.
903          */
904         if (!ibinc) {
905                 ibinc = recv->r_ibinc;
906                 recv->r_ibinc = NULL;
907                 ic->i_ibinc = ibinc;
908
909                 hdr = &ibinc->ii_inc.i_hdr;
910                 ibinc->ii_inc.i_rx_lat_trace[RDS_MSG_RX_HDR] =
911                                 local_clock();
912                 memcpy(hdr, ihdr, sizeof(*hdr));
913                 ic->i_recv_data_rem = be32_to_cpu(hdr->h_len);
914                 ibinc->ii_inc.i_rx_lat_trace[RDS_MSG_RX_START] =
915                                 local_clock();
916
917                 rdsdebug("ic %p ibinc %p rem %u flag 0x%x\n", ic, ibinc,
918                          ic->i_recv_data_rem, hdr->h_flags);
919         } else {
920                 hdr = &ibinc->ii_inc.i_hdr;
921                 /* We can't just use memcmp here; fragments of a
922                  * single message may carry different ACKs */
923                 if (hdr->h_sequence != ihdr->h_sequence ||
924                     hdr->h_len != ihdr->h_len ||
925                     hdr->h_sport != ihdr->h_sport ||
926                     hdr->h_dport != ihdr->h_dport) {
927                         rds_ib_conn_error(conn,
928                                 "fragment header mismatch; forcing reconnect\n");
929                         return;
930                 }
931         }
932
933         list_add_tail(&recv->r_frag->f_item, &ibinc->ii_frags);
934         recv->r_frag = NULL;
935
936         if (ic->i_recv_data_rem > RDS_FRAG_SIZE)
937                 ic->i_recv_data_rem -= RDS_FRAG_SIZE;
938         else {
939                 ic->i_recv_data_rem = 0;
940                 ic->i_ibinc = NULL;
941
942                 if (ibinc->ii_inc.i_hdr.h_flags == RDS_FLAG_CONG_BITMAP) {
943                         rds_ib_cong_recv(conn, ibinc);
944                 } else {
945                         rds_recv_incoming(conn, &conn->c_faddr, &conn->c_laddr,
946                                           &ibinc->ii_inc, GFP_ATOMIC);
947                         state->ack_next = be64_to_cpu(hdr->h_sequence);
948                         state->ack_next_valid = 1;
949                 }
950
951                 /* Evaluate the ACK_REQUIRED flag *after* we received
952                  * the complete frame, and after bumping the next_rx
953                  * sequence. */
954                 if (hdr->h_flags & RDS_FLAG_ACK_REQUIRED) {
955                         rds_stats_inc(s_recv_ack_required);
956                         state->ack_required = 1;
957                 }
958
959                 rds_inc_put(&ibinc->ii_inc);
960         }
961 }
962
963 void rds_ib_recv_cqe_handler(struct rds_ib_connection *ic,
964                              struct ib_wc *wc,
965                              struct rds_ib_ack_state *state)
966 {
967         struct rds_connection *conn = ic->conn;
968         struct rds_ib_recv_work *recv;
969
970         rdsdebug("wc wr_id 0x%llx status %u (%s) byte_len %u imm_data %u\n",
971                  (unsigned long long)wc->wr_id, wc->status,
972                  ib_wc_status_msg(wc->status), wc->byte_len,
973                  be32_to_cpu(wc->ex.imm_data));
974
975         rds_ib_stats_inc(s_ib_rx_cq_event);
976         recv = &ic->i_recvs[rds_ib_ring_oldest(&ic->i_recv_ring)];
977         ib_dma_unmap_sg(ic->i_cm_id->device, &recv->r_frag->f_sg, 1,
978                         DMA_FROM_DEVICE);
979
980         /* Also process recvs in connecting state because it is possible
981          * to get a recv completion _before_ the rdmacm ESTABLISHED
982          * event is processed.
983          */
984         if (wc->status == IB_WC_SUCCESS) {
985                 rds_ib_process_recv(conn, recv, wc->byte_len, state);
986         } else {
987                 /* We expect errors as the qp is drained during shutdown */
988                 if (rds_conn_up(conn) || rds_conn_connecting(conn))
989                         rds_ib_conn_error(conn, "recv completion on <%pI6c,%pI6c> had status %u (%s), disconnecting and reconnecting\n",
990                                           &conn->c_laddr, &conn->c_faddr,
991                                           wc->status,
992                                           ib_wc_status_msg(wc->status));
993         }
994
995         /* rds_ib_process_recv() doesn't always consume the frag, and
996          * we might not have called it at all if the wc didn't indicate
997          * success. We already unmapped the frag's pages, though, and
998          * the following rds_ib_ring_free() call tells the refill path
999          * that it will not find an allocated frag here. Make sure we
1000          * keep that promise by freeing a frag that's still on the ring.
1001          */
1002         if (recv->r_frag) {
1003                 rds_ib_frag_free(ic, recv->r_frag);
1004                 recv->r_frag = NULL;
1005         }
1006         rds_ib_ring_free(&ic->i_recv_ring, 1);
1007
1008         /* If we ever end up with a really empty receive ring, we're
1009          * in deep trouble, as the sender will definitely see RNR
1010          * timeouts. */
1011         if (rds_ib_ring_empty(&ic->i_recv_ring))
1012                 rds_ib_stats_inc(s_ib_rx_ring_empty);
1013
1014         if (rds_ib_ring_low(&ic->i_recv_ring)) {
1015                 rds_ib_recv_refill(conn, 0, GFP_NOWAIT);
1016                 rds_ib_stats_inc(s_ib_rx_refill_from_cq);
1017         }
1018 }
1019
1020 int rds_ib_recv_path(struct rds_conn_path *cp)
1021 {
1022         struct rds_connection *conn = cp->cp_conn;
1023         struct rds_ib_connection *ic = conn->c_transport_data;
1024
1025         rdsdebug("conn %p\n", conn);
1026         if (rds_conn_up(conn)) {
1027                 rds_ib_attempt_ack(ic);
1028                 rds_ib_recv_refill(conn, 0, GFP_KERNEL);
1029                 rds_ib_stats_inc(s_ib_rx_refill_from_thread);
1030         }
1031
1032         return 0;
1033 }
1034
1035 int rds_ib_recv_init(void)
1036 {
1037         struct sysinfo si;
1038         int ret = -ENOMEM;
1039
1040         /* Default to 30% of all available RAM for recv memory */
1041         si_meminfo(&si);
1042         rds_ib_sysctl_max_recv_allocation = si.totalram / 3 * PAGE_SIZE / RDS_FRAG_SIZE;
1043
1044         rds_ib_incoming_slab = kmem_cache_create("rds_ib_incoming",
1045                                         sizeof(struct rds_ib_incoming),
1046                                         0, SLAB_HWCACHE_ALIGN, NULL);
1047         if (!rds_ib_incoming_slab)
1048                 goto out;
1049
1050         rds_ib_frag_slab = kmem_cache_create("rds_ib_frag",
1051                                         sizeof(struct rds_page_frag),
1052                                         0, SLAB_HWCACHE_ALIGN, NULL);
1053         if (!rds_ib_frag_slab) {
1054                 kmem_cache_destroy(rds_ib_incoming_slab);
1055                 rds_ib_incoming_slab = NULL;
1056         } else
1057                 ret = 0;
1058 out:
1059         return ret;
1060 }
1061
1062 void rds_ib_recv_exit(void)
1063 {
1064         kmem_cache_destroy(rds_ib_incoming_slab);
1065         kmem_cache_destroy(rds_ib_frag_slab);
1066 }