1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (c) 2016-2018 Oracle. All rights reserved.
5 * Use the core R/W API to move RPC-over-RDMA Read and Write chunks.
10 #include <linux/sunrpc/xdr.h>
11 #include <linux/sunrpc/rpc_rdma.h>
12 #include <linux/sunrpc/svc_rdma.h>
14 #include "xprt_rdma.h"
15 #include <trace/events/rpcrdma.h>
17 static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc);
18 static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc);
20 /* Each R/W context contains state for one chain of RDMA Read or
21 * Write Work Requests.
23 * Each WR chain handles a single contiguous server-side buffer,
24 * because scatterlist entries after the first have to start on
25 * page alignment. xdr_buf iovecs cannot guarantee alignment.
27 * Each WR chain handles only one R_key. Each RPC-over-RDMA segment
28 * from a client may contain a unique R_key, so each WR chain moves
29 * up to one segment at a time.
31 * The scatterlist makes this data structure over 4KB in size. To
32 * make it less likely to fail, and to handle the allocation for
33 * smaller I/O requests without disabling bottom-halves, these
34 * contexts are created on demand, but cached and reused until the
35 * controlling svcxprt_rdma is destroyed.
37 struct svc_rdma_rw_ctxt {
38 struct list_head rw_list;
39 struct rdma_rw_ctx rw_ctx;
40 unsigned int rw_nents;
41 struct sg_table rw_sg_table;
42 struct scatterlist rw_first_sgl[];
45 static inline struct svc_rdma_rw_ctxt *
46 svc_rdma_next_ctxt(struct list_head *list)
48 return list_first_entry_or_null(list, struct svc_rdma_rw_ctxt,
52 static struct svc_rdma_rw_ctxt *
53 svc_rdma_get_rw_ctxt(struct svcxprt_rdma *rdma, unsigned int sges)
55 struct svc_rdma_rw_ctxt *ctxt;
57 spin_lock(&rdma->sc_rw_ctxt_lock);
59 ctxt = svc_rdma_next_ctxt(&rdma->sc_rw_ctxts);
61 list_del(&ctxt->rw_list);
62 spin_unlock(&rdma->sc_rw_ctxt_lock);
64 spin_unlock(&rdma->sc_rw_ctxt_lock);
65 ctxt = kmalloc(struct_size(ctxt, rw_first_sgl, SG_CHUNK_SIZE),
69 INIT_LIST_HEAD(&ctxt->rw_list);
72 ctxt->rw_sg_table.sgl = ctxt->rw_first_sgl;
73 if (sg_alloc_table_chained(&ctxt->rw_sg_table, sges,
74 ctxt->rw_sg_table.sgl,
82 trace_svcrdma_no_rwctx_err(rdma, sges);
86 static void svc_rdma_put_rw_ctxt(struct svcxprt_rdma *rdma,
87 struct svc_rdma_rw_ctxt *ctxt)
89 sg_free_table_chained(&ctxt->rw_sg_table, SG_CHUNK_SIZE);
91 spin_lock(&rdma->sc_rw_ctxt_lock);
92 list_add(&ctxt->rw_list, &rdma->sc_rw_ctxts);
93 spin_unlock(&rdma->sc_rw_ctxt_lock);
97 * svc_rdma_destroy_rw_ctxts - Free accumulated R/W contexts
98 * @rdma: transport about to be destroyed
101 void svc_rdma_destroy_rw_ctxts(struct svcxprt_rdma *rdma)
103 struct svc_rdma_rw_ctxt *ctxt;
105 while ((ctxt = svc_rdma_next_ctxt(&rdma->sc_rw_ctxts)) != NULL) {
106 list_del(&ctxt->rw_list);
112 * svc_rdma_rw_ctx_init - Prepare a R/W context for I/O
113 * @rdma: controlling transport instance
114 * @ctxt: R/W context to prepare
115 * @offset: RDMA offset
116 * @handle: RDMA tag/handle
117 * @direction: I/O direction
119 * Returns on success, the number of WQEs that will be needed
120 * on the workqueue, or a negative errno.
122 static int svc_rdma_rw_ctx_init(struct svcxprt_rdma *rdma,
123 struct svc_rdma_rw_ctxt *ctxt,
124 u64 offset, u32 handle,
125 enum dma_data_direction direction)
129 ret = rdma_rw_ctx_init(&ctxt->rw_ctx, rdma->sc_qp, rdma->sc_port_num,
130 ctxt->rw_sg_table.sgl, ctxt->rw_nents,
131 0, offset, handle, direction);
132 if (unlikely(ret < 0)) {
133 svc_rdma_put_rw_ctxt(rdma, ctxt);
134 trace_svcrdma_dma_map_rw_err(rdma, ctxt->rw_nents, ret);
139 /* A chunk context tracks all I/O for moving one Read or Write
140 * chunk. This is a set of rdma_rw's that handle data movement
141 * for all segments of one chunk.
143 * These are small, acquired with a single allocator call, and
144 * no more than one is needed per chunk. They are allocated on
145 * demand, and not cached.
147 struct svc_rdma_chunk_ctxt {
148 struct rpc_rdma_cid cc_cid;
149 struct ib_cqe cc_cqe;
150 struct svcxprt_rdma *cc_rdma;
151 struct list_head cc_rwctxts;
155 static void svc_rdma_cc_cid_init(struct svcxprt_rdma *rdma,
156 struct rpc_rdma_cid *cid)
158 cid->ci_queue_id = rdma->sc_sq_cq->res.id;
159 cid->ci_completion_id = atomic_inc_return(&rdma->sc_completion_ids);
162 static void svc_rdma_cc_init(struct svcxprt_rdma *rdma,
163 struct svc_rdma_chunk_ctxt *cc)
165 svc_rdma_cc_cid_init(rdma, &cc->cc_cid);
168 INIT_LIST_HEAD(&cc->cc_rwctxts);
172 static void svc_rdma_cc_release(struct svc_rdma_chunk_ctxt *cc,
173 enum dma_data_direction dir)
175 struct svcxprt_rdma *rdma = cc->cc_rdma;
176 struct svc_rdma_rw_ctxt *ctxt;
178 while ((ctxt = svc_rdma_next_ctxt(&cc->cc_rwctxts)) != NULL) {
179 list_del(&ctxt->rw_list);
181 rdma_rw_ctx_destroy(&ctxt->rw_ctx, rdma->sc_qp,
182 rdma->sc_port_num, ctxt->rw_sg_table.sgl,
183 ctxt->rw_nents, dir);
184 svc_rdma_put_rw_ctxt(rdma, ctxt);
188 /* State for sending a Write or Reply chunk.
189 * - Tracks progress of writing one chunk over all its segments
190 * - Stores arguments for the SGL constructor functions
192 struct svc_rdma_write_info {
193 /* write state of this chunk */
194 unsigned int wi_seg_off;
195 unsigned int wi_seg_no;
196 unsigned int wi_nsegs;
199 /* SGL constructor arguments */
200 struct xdr_buf *wi_xdr;
201 unsigned char *wi_base;
202 unsigned int wi_next_off;
204 struct svc_rdma_chunk_ctxt wi_cc;
207 static struct svc_rdma_write_info *
208 svc_rdma_write_info_alloc(struct svcxprt_rdma *rdma, __be32 *chunk)
210 struct svc_rdma_write_info *info;
212 info = kmalloc(sizeof(*info), GFP_KERNEL);
216 info->wi_seg_off = 0;
218 info->wi_nsegs = be32_to_cpup(++chunk);
219 info->wi_segs = ++chunk;
220 svc_rdma_cc_init(rdma, &info->wi_cc);
221 info->wi_cc.cc_cqe.done = svc_rdma_write_done;
225 static void svc_rdma_write_info_free(struct svc_rdma_write_info *info)
227 svc_rdma_cc_release(&info->wi_cc, DMA_TO_DEVICE);
232 * svc_rdma_write_done - Write chunk completion
233 * @cq: controlling Completion Queue
234 * @wc: Work Completion
236 * Pages under I/O are freed by a subsequent Send completion.
238 static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc)
240 struct ib_cqe *cqe = wc->wr_cqe;
241 struct svc_rdma_chunk_ctxt *cc =
242 container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe);
243 struct svcxprt_rdma *rdma = cc->cc_rdma;
244 struct svc_rdma_write_info *info =
245 container_of(cc, struct svc_rdma_write_info, wi_cc);
247 trace_svcrdma_wc_write(wc, &cc->cc_cid);
249 atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail);
250 wake_up(&rdma->sc_send_wait);
252 if (unlikely(wc->status != IB_WC_SUCCESS))
253 set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags);
255 svc_rdma_write_info_free(info);
258 /* State for pulling a Read chunk.
260 struct svc_rdma_read_info {
261 struct svc_rdma_recv_ctxt *ri_readctxt;
262 unsigned int ri_position;
263 unsigned int ri_pageno;
264 unsigned int ri_pageoff;
265 unsigned int ri_chunklen;
267 struct svc_rdma_chunk_ctxt ri_cc;
270 static struct svc_rdma_read_info *
271 svc_rdma_read_info_alloc(struct svcxprt_rdma *rdma)
273 struct svc_rdma_read_info *info;
275 info = kmalloc(sizeof(*info), GFP_KERNEL);
279 svc_rdma_cc_init(rdma, &info->ri_cc);
280 info->ri_cc.cc_cqe.done = svc_rdma_wc_read_done;
284 static void svc_rdma_read_info_free(struct svc_rdma_read_info *info)
286 svc_rdma_cc_release(&info->ri_cc, DMA_FROM_DEVICE);
291 * svc_rdma_wc_read_done - Handle completion of an RDMA Read ctx
292 * @cq: controlling Completion Queue
293 * @wc: Work Completion
296 static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc)
298 struct ib_cqe *cqe = wc->wr_cqe;
299 struct svc_rdma_chunk_ctxt *cc =
300 container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe);
301 struct svcxprt_rdma *rdma = cc->cc_rdma;
302 struct svc_rdma_read_info *info =
303 container_of(cc, struct svc_rdma_read_info, ri_cc);
305 trace_svcrdma_wc_read(wc, &cc->cc_cid);
307 atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail);
308 wake_up(&rdma->sc_send_wait);
310 if (unlikely(wc->status != IB_WC_SUCCESS)) {
311 set_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags);
312 svc_rdma_recv_ctxt_put(rdma, info->ri_readctxt);
314 spin_lock(&rdma->sc_rq_dto_lock);
315 list_add_tail(&info->ri_readctxt->rc_list,
316 &rdma->sc_read_complete_q);
317 /* Note the unlock pairs with the smp_rmb in svc_xprt_ready: */
318 set_bit(XPT_DATA, &rdma->sc_xprt.xpt_flags);
319 spin_unlock(&rdma->sc_rq_dto_lock);
321 svc_xprt_enqueue(&rdma->sc_xprt);
324 svc_rdma_read_info_free(info);
327 /* This function sleeps when the transport's Send Queue is congested.
330 * - If ib_post_send() succeeds, only one completion is expected,
331 * even if one or more WRs are flushed. This is true when posting
332 * an rdma_rw_ctx or when posting a single signaled WR.
334 static int svc_rdma_post_chunk_ctxt(struct svc_rdma_chunk_ctxt *cc)
336 struct svcxprt_rdma *rdma = cc->cc_rdma;
337 struct svc_xprt *xprt = &rdma->sc_xprt;
338 struct ib_send_wr *first_wr;
339 const struct ib_send_wr *bad_wr;
340 struct list_head *tmp;
344 if (cc->cc_sqecount > rdma->sc_sq_depth)
349 list_for_each(tmp, &cc->cc_rwctxts) {
350 struct svc_rdma_rw_ctxt *ctxt;
352 ctxt = list_entry(tmp, struct svc_rdma_rw_ctxt, rw_list);
353 first_wr = rdma_rw_ctx_wrs(&ctxt->rw_ctx, rdma->sc_qp,
354 rdma->sc_port_num, cqe, first_wr);
359 if (atomic_sub_return(cc->cc_sqecount,
360 &rdma->sc_sq_avail) > 0) {
361 trace_svcrdma_post_chunk(&cc->cc_cid, cc->cc_sqecount);
362 ret = ib_post_send(rdma->sc_qp, first_wr, &bad_wr);
368 trace_svcrdma_sq_full(rdma);
369 atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail);
370 wait_event(rdma->sc_send_wait,
371 atomic_read(&rdma->sc_sq_avail) > cc->cc_sqecount);
372 trace_svcrdma_sq_retry(rdma);
375 trace_svcrdma_sq_post_err(rdma, ret);
376 set_bit(XPT_CLOSE, &xprt->xpt_flags);
378 /* If even one was posted, there will be a completion. */
379 if (bad_wr != first_wr)
382 atomic_add(cc->cc_sqecount, &rdma->sc_sq_avail);
383 wake_up(&rdma->sc_send_wait);
387 /* Build and DMA-map an SGL that covers one kvec in an xdr_buf
389 static void svc_rdma_vec_to_sg(struct svc_rdma_write_info *info,
391 struct svc_rdma_rw_ctxt *ctxt)
393 struct scatterlist *sg = ctxt->rw_sg_table.sgl;
395 sg_set_buf(&sg[0], info->wi_base, len);
396 info->wi_base += len;
401 /* Build and DMA-map an SGL that covers part of an xdr_buf's pagelist.
403 static void svc_rdma_pagelist_to_sg(struct svc_rdma_write_info *info,
404 unsigned int remaining,
405 struct svc_rdma_rw_ctxt *ctxt)
407 unsigned int sge_no, sge_bytes, page_off, page_no;
408 struct xdr_buf *xdr = info->wi_xdr;
409 struct scatterlist *sg;
412 page_off = info->wi_next_off + xdr->page_base;
413 page_no = page_off >> PAGE_SHIFT;
414 page_off = offset_in_page(page_off);
415 page = xdr->pages + page_no;
416 info->wi_next_off += remaining;
417 sg = ctxt->rw_sg_table.sgl;
420 sge_bytes = min_t(unsigned int, remaining,
421 PAGE_SIZE - page_off);
422 sg_set_page(sg, *page, sge_bytes, page_off);
424 remaining -= sge_bytes;
431 ctxt->rw_nents = sge_no;
434 /* Construct RDMA Write WRs to send a portion of an xdr_buf containing
438 svc_rdma_build_writes(struct svc_rdma_write_info *info,
439 void (*constructor)(struct svc_rdma_write_info *info,
441 struct svc_rdma_rw_ctxt *ctxt),
442 unsigned int remaining)
444 struct svc_rdma_chunk_ctxt *cc = &info->wi_cc;
445 struct svcxprt_rdma *rdma = cc->cc_rdma;
446 struct svc_rdma_rw_ctxt *ctxt;
450 seg = info->wi_segs + info->wi_seg_no * rpcrdma_segment_maxsz;
452 unsigned int write_len;
456 if (info->wi_seg_no >= info->wi_nsegs)
459 xdr_decode_rdma_segment(seg, &handle, &length, &offset);
460 offset += info->wi_seg_off;
462 write_len = min(remaining, length - info->wi_seg_off);
463 ctxt = svc_rdma_get_rw_ctxt(rdma,
464 (write_len >> PAGE_SHIFT) + 2);
468 constructor(info, write_len, ctxt);
469 ret = svc_rdma_rw_ctx_init(rdma, ctxt, offset, handle,
474 trace_svcrdma_send_wseg(handle, write_len, offset);
476 list_add(&ctxt->rw_list, &cc->cc_rwctxts);
477 cc->cc_sqecount += ret;
478 if (write_len == length - info->wi_seg_off) {
481 info->wi_seg_off = 0;
483 info->wi_seg_off += write_len;
485 remaining -= write_len;
491 trace_svcrdma_small_wrch_err(rdma, remaining, info->wi_seg_no,
496 /* Send one of an xdr_buf's kvecs by itself. To send a Reply
497 * chunk, the whole RPC Reply is written back to the client.
498 * This function writes either the head or tail of the xdr_buf
499 * containing the Reply.
501 static int svc_rdma_send_xdr_kvec(struct svc_rdma_write_info *info,
504 info->wi_base = vec->iov_base;
505 return svc_rdma_build_writes(info, svc_rdma_vec_to_sg,
509 /* Send an xdr_buf's page list by itself. A Write chunk is just
510 * the page list. A Reply chunk is @xdr's head, page list, and
511 * tail. This function is shared between the two types of chunk.
513 static int svc_rdma_send_xdr_pagelist(struct svc_rdma_write_info *info,
516 unsigned long length)
519 info->wi_next_off = offset - xdr->head[0].iov_len;
520 return svc_rdma_build_writes(info, svc_rdma_pagelist_to_sg,
525 * svc_rdma_send_write_chunk - Write all segments in a Write chunk
526 * @rdma: controlling RDMA transport
527 * @wr_ch: Write chunk provided by client
528 * @xdr: xdr_buf containing the data payload
529 * @offset: payload's byte offset in @xdr
530 * @length: size of payload, in bytes
532 * Returns a non-negative number of bytes the chunk consumed, or
533 * %-E2BIG if the payload was larger than the Write chunk,
534 * %-EINVAL if client provided too many segments,
535 * %-ENOMEM if rdma_rw context pool was exhausted,
536 * %-ENOTCONN if posting failed (connection is lost),
537 * %-EIO if rdma_rw initialization failed (DMA mapping, etc).
539 int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma, __be32 *wr_ch,
541 unsigned int offset, unsigned long length)
543 struct svc_rdma_write_info *info;
549 info = svc_rdma_write_info_alloc(rdma, wr_ch);
553 ret = svc_rdma_send_xdr_pagelist(info, xdr, offset, length);
557 ret = svc_rdma_post_chunk_ctxt(&info->wi_cc);
561 trace_svcrdma_send_write_chunk(xdr->page_len);
565 svc_rdma_write_info_free(info);
570 * svc_rdma_send_reply_chunk - Write all segments in the Reply chunk
571 * @rdma: controlling RDMA transport
572 * @rctxt: Write and Reply chunks from client
573 * @xdr: xdr_buf containing an RPC Reply
575 * Returns a non-negative number of bytes the chunk consumed, or
576 * %-E2BIG if the payload was larger than the Reply chunk,
577 * %-EINVAL if client provided too many segments,
578 * %-ENOMEM if rdma_rw context pool was exhausted,
579 * %-ENOTCONN if posting failed (connection is lost),
580 * %-EIO if rdma_rw initialization failed (DMA mapping, etc).
582 int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma,
583 const struct svc_rdma_recv_ctxt *rctxt,
586 struct svc_rdma_write_info *info;
589 info = svc_rdma_write_info_alloc(rdma, rctxt->rc_reply_chunk);
593 ret = svc_rdma_send_xdr_kvec(info, &xdr->head[0]);
596 consumed = xdr->head[0].iov_len;
598 /* Send the page list in the Reply chunk only if the
599 * client did not provide Write chunks.
601 if (!rctxt->rc_write_list && xdr->page_len) {
602 ret = svc_rdma_send_xdr_pagelist(info, xdr,
603 xdr->head[0].iov_len,
607 consumed += xdr->page_len;
610 if (xdr->tail[0].iov_len) {
611 ret = svc_rdma_send_xdr_kvec(info, &xdr->tail[0]);
614 consumed += xdr->tail[0].iov_len;
617 ret = svc_rdma_post_chunk_ctxt(&info->wi_cc);
621 trace_svcrdma_send_reply_chunk(consumed);
625 svc_rdma_write_info_free(info);
629 static int svc_rdma_build_read_segment(struct svc_rdma_read_info *info,
630 struct svc_rqst *rqstp,
631 u32 rkey, u32 len, u64 offset)
633 struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
634 struct svc_rdma_chunk_ctxt *cc = &info->ri_cc;
635 struct svc_rdma_rw_ctxt *ctxt;
636 unsigned int sge_no, seg_len;
637 struct scatterlist *sg;
640 sge_no = PAGE_ALIGN(info->ri_pageoff + len) >> PAGE_SHIFT;
641 ctxt = svc_rdma_get_rw_ctxt(cc->cc_rdma, sge_no);
644 ctxt->rw_nents = sge_no;
646 sg = ctxt->rw_sg_table.sgl;
647 for (sge_no = 0; sge_no < ctxt->rw_nents; sge_no++) {
648 seg_len = min_t(unsigned int, len,
649 PAGE_SIZE - info->ri_pageoff);
651 head->rc_arg.pages[info->ri_pageno] =
652 rqstp->rq_pages[info->ri_pageno];
653 if (!info->ri_pageoff)
654 head->rc_page_count++;
656 sg_set_page(sg, rqstp->rq_pages[info->ri_pageno],
657 seg_len, info->ri_pageoff);
660 info->ri_pageoff += seg_len;
661 if (info->ri_pageoff == PAGE_SIZE) {
663 info->ri_pageoff = 0;
669 &rqstp->rq_pages[info->ri_pageno + 1] > rqstp->rq_page_end)
673 ret = svc_rdma_rw_ctx_init(cc->cc_rdma, ctxt, offset, rkey,
678 list_add(&ctxt->rw_list, &cc->cc_rwctxts);
679 cc->cc_sqecount += ret;
683 trace_svcrdma_page_overrun_err(cc->cc_rdma, rqstp, info->ri_pageno);
687 /* Walk the segments in the Read chunk starting at @p and construct
688 * RDMA Read operations to pull the chunk to the server.
690 static int svc_rdma_build_read_chunk(struct svc_rqst *rqstp,
691 struct svc_rdma_read_info *info,
697 info->ri_chunklen = 0;
698 while (*p++ != xdr_zero && be32_to_cpup(p++) == info->ri_position) {
702 p = xdr_decode_rdma_segment(p, &handle, &length, &offset);
703 ret = svc_rdma_build_read_segment(info, rqstp, handle, length,
708 trace_svcrdma_send_rseg(handle, length, offset);
709 info->ri_chunklen += length;
715 /* Construct RDMA Reads to pull over a normal Read chunk. The chunk
716 * data lands in the page list of head->rc_arg.pages.
718 * Currently NFSD does not look at the head->rc_arg.tail[0] iovec.
719 * Therefore, XDR round-up of the Read chunk and trailing
720 * inline content must both be added at the end of the pagelist.
722 static int svc_rdma_build_normal_read_chunk(struct svc_rqst *rqstp,
723 struct svc_rdma_read_info *info,
726 struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
729 ret = svc_rdma_build_read_chunk(rqstp, info, p);
733 trace_svcrdma_send_read_chunk(info->ri_chunklen, info->ri_position);
735 head->rc_hdr_count = 0;
737 /* Split the Receive buffer between the head and tail
738 * buffers at Read chunk's position. XDR roundup of the
739 * chunk is not included in either the pagelist or in
742 head->rc_arg.tail[0].iov_base =
743 head->rc_arg.head[0].iov_base + info->ri_position;
744 head->rc_arg.tail[0].iov_len =
745 head->rc_arg.head[0].iov_len - info->ri_position;
746 head->rc_arg.head[0].iov_len = info->ri_position;
748 /* Read chunk may need XDR roundup (see RFC 8166, s. 3.4.5.2).
750 * If the client already rounded up the chunk length, the
751 * length does not change. Otherwise, the length of the page
752 * list is increased to include XDR round-up.
754 * Currently these chunks always start at page offset 0,
755 * thus the rounded-up length never crosses a page boundary.
757 info->ri_chunklen = XDR_QUADLEN(info->ri_chunklen) << 2;
759 head->rc_arg.page_len = info->ri_chunklen;
760 head->rc_arg.len += info->ri_chunklen;
761 head->rc_arg.buflen += info->ri_chunklen;
767 /* Construct RDMA Reads to pull over a Position Zero Read chunk.
768 * The start of the data lands in the first page just after
769 * the Transport header, and the rest lands in the page list of
770 * head->rc_arg.pages.
773 * - A PZRC has an XDR-aligned length (no implicit round-up).
774 * - There can be no trailing inline content (IOW, we assume
775 * a PZRC is never sent in an RDMA_MSG message, though it's
778 static int svc_rdma_build_pz_read_chunk(struct svc_rqst *rqstp,
779 struct svc_rdma_read_info *info,
782 struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
785 ret = svc_rdma_build_read_chunk(rqstp, info, p);
789 trace_svcrdma_send_pzr(info->ri_chunklen);
791 head->rc_arg.len += info->ri_chunklen;
792 head->rc_arg.buflen += info->ri_chunklen;
794 head->rc_hdr_count = 1;
795 head->rc_arg.head[0].iov_base = page_address(head->rc_pages[0]);
796 head->rc_arg.head[0].iov_len = min_t(size_t, PAGE_SIZE,
799 head->rc_arg.page_len = info->ri_chunklen -
800 head->rc_arg.head[0].iov_len;
806 /* Pages under I/O have been copied to head->rc_pages. Ensure they
807 * are not released by svc_xprt_release() until the I/O is complete.
809 * This has to be done after all Read WRs are constructed to properly
810 * handle a page that is part of I/O on behalf of two different RDMA
813 * Do this only if I/O has been posted. Otherwise, we do indeed want
814 * svc_xprt_release() to clean things up properly.
816 static void svc_rdma_save_io_pages(struct svc_rqst *rqstp,
817 const unsigned int start,
818 const unsigned int num_pages)
822 for (i = start; i < num_pages + start; i++)
823 rqstp->rq_pages[i] = NULL;
827 * svc_rdma_recv_read_chunk - Pull a Read chunk from the client
828 * @rdma: controlling RDMA transport
829 * @rqstp: set of pages to use as Read sink buffers
830 * @head: pages under I/O collect here
831 * @p: pointer to start of Read chunk
834 * %0 if all needed RDMA Reads were posted successfully,
835 * %-EINVAL if client provided too many segments,
836 * %-ENOMEM if rdma_rw context pool was exhausted,
837 * %-ENOTCONN if posting failed (connection is lost),
838 * %-EIO if rdma_rw initialization failed (DMA mapping, etc).
841 * - All Read segments in @p have the same Position value.
843 int svc_rdma_recv_read_chunk(struct svcxprt_rdma *rdma, struct svc_rqst *rqstp,
844 struct svc_rdma_recv_ctxt *head, __be32 *p)
846 struct svc_rdma_read_info *info;
849 /* The request (with page list) is constructed in
850 * head->rc_arg. Pages involved with RDMA Read I/O are
853 head->rc_arg.head[0] = rqstp->rq_arg.head[0];
854 head->rc_arg.tail[0] = rqstp->rq_arg.tail[0];
855 head->rc_arg.pages = head->rc_pages;
856 head->rc_arg.page_base = 0;
857 head->rc_arg.page_len = 0;
858 head->rc_arg.len = rqstp->rq_arg.len;
859 head->rc_arg.buflen = rqstp->rq_arg.buflen;
861 info = svc_rdma_read_info_alloc(rdma);
864 info->ri_readctxt = head;
866 info->ri_pageoff = 0;
868 info->ri_position = be32_to_cpup(p + 1);
869 if (info->ri_position)
870 ret = svc_rdma_build_normal_read_chunk(rqstp, info, p);
872 ret = svc_rdma_build_pz_read_chunk(rqstp, info, p);
876 ret = svc_rdma_post_chunk_ctxt(&info->ri_cc);
879 svc_rdma_save_io_pages(rqstp, 0, head->rc_page_count);
883 svc_rdma_read_info_free(info);