1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
7 #include <linux/libnvdimm.h>
12 /* Return a random 8 bit key value that is
13 * different than the last_key. Set last_key to -1
14 * if this is the first key for an MR or MW
16 u8 rxe_get_next_key(u32 last_key)
21 get_random_bytes(&key, 1);
22 } while (key == last_key);
27 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length)
29 switch (mr->ibmr.type) {
34 case IB_MR_TYPE_MEM_REG:
35 if (iova < mr->ibmr.iova ||
36 iova + length > mr->ibmr.iova + mr->ibmr.length) {
37 rxe_dbg_mr(mr, "iova/length out of range");
43 rxe_dbg_mr(mr, "mr type not supported\n");
48 static void rxe_mr_init(int access, struct rxe_mr *mr)
50 u32 key = mr->elem.index << 8 | rxe_get_next_key(-1);
52 /* set ibmr->l/rkey and also copy into private l/rkey
53 * for user MRs these will always be the same
54 * for cases where caller 'owns' the key portion
55 * they may be different until REG_MR WQE is executed.
57 mr->lkey = mr->ibmr.lkey = key;
58 mr->rkey = mr->ibmr.rkey = key;
61 mr->ibmr.page_size = PAGE_SIZE;
62 mr->page_mask = PAGE_MASK;
63 mr->page_shift = PAGE_SHIFT;
64 mr->state = RXE_MR_STATE_INVALID;
67 void rxe_mr_init_dma(int access, struct rxe_mr *mr)
69 rxe_mr_init(access, mr);
71 mr->state = RXE_MR_STATE_VALID;
72 mr->ibmr.type = IB_MR_TYPE_DMA;
75 static unsigned long rxe_mr_iova_to_index(struct rxe_mr *mr, u64 iova)
77 return (iova >> mr->page_shift) - (mr->ibmr.iova >> mr->page_shift);
80 static unsigned long rxe_mr_iova_to_page_offset(struct rxe_mr *mr, u64 iova)
82 return iova & (mr_page_size(mr) - 1);
85 static bool is_pmem_page(struct page *pg)
87 unsigned long paddr = page_to_phys(pg);
89 return REGION_INTERSECTS ==
90 region_intersects(paddr, PAGE_SIZE, IORESOURCE_MEM,
91 IORES_DESC_PERSISTENT_MEMORY);
94 static int rxe_mr_fill_pages_from_sgt(struct rxe_mr *mr, struct sg_table *sgt)
96 XA_STATE(xas, &mr->page_list, 0);
97 struct sg_page_iter sg_iter;
99 bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT);
101 __sg_page_iter_start(&sg_iter, sgt->sgl, sgt->orig_nents, 0);
102 if (!__sg_page_iter_next(&sg_iter))
108 page = sg_page_iter_page(&sg_iter);
110 if (persistent && !is_pmem_page(page)) {
111 rxe_dbg_mr(mr, "Page can't be persistent\n");
112 xas_set_err(&xas, -EINVAL);
116 xas_store(&xas, page);
120 if (!__sg_page_iter_next(&sg_iter))
124 } while (xas_nomem(&xas, GFP_KERNEL));
126 return xas_error(&xas);
129 int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
130 int access, struct rxe_mr *mr)
132 struct ib_umem *umem;
135 rxe_mr_init(access, mr);
137 xa_init(&mr->page_list);
139 umem = ib_umem_get(&rxe->ib_dev, start, length, access);
141 rxe_dbg_mr(mr, "Unable to pin memory region err = %d\n",
143 return PTR_ERR(umem);
146 err = rxe_mr_fill_pages_from_sgt(mr, &umem->sgt_append.sgt);
148 ib_umem_release(umem);
153 mr->ibmr.type = IB_MR_TYPE_USER;
154 mr->state = RXE_MR_STATE_VALID;
159 static int rxe_mr_alloc(struct rxe_mr *mr, int num_buf)
161 XA_STATE(xas, &mr->page_list, 0);
165 xa_init(&mr->page_list);
169 while (i != num_buf) {
170 xas_store(&xas, XA_ZERO_ENTRY);
177 } while (xas_nomem(&xas, GFP_KERNEL));
179 err = xas_error(&xas);
183 mr->num_buf = num_buf;
188 int rxe_mr_init_fast(int max_pages, struct rxe_mr *mr)
192 /* always allow remote access for FMRs */
193 rxe_mr_init(RXE_ACCESS_REMOTE, mr);
195 err = rxe_mr_alloc(mr, max_pages);
199 mr->state = RXE_MR_STATE_FREE;
200 mr->ibmr.type = IB_MR_TYPE_MEM_REG;
208 static int rxe_set_page(struct ib_mr *ibmr, u64 dma_addr)
210 struct rxe_mr *mr = to_rmr(ibmr);
211 struct page *page = ib_virt_dma_to_page(dma_addr);
212 bool persistent = !!(mr->access & IB_ACCESS_FLUSH_PERSISTENT);
215 if (persistent && !is_pmem_page(page)) {
216 rxe_dbg_mr(mr, "Page cannot be persistent\n");
220 if (unlikely(mr->nbuf == mr->num_buf))
223 err = xa_err(xa_store(&mr->page_list, mr->nbuf, page, GFP_KERNEL));
231 int rxe_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sgl,
232 int sg_nents, unsigned int *sg_offset)
234 struct rxe_mr *mr = to_rmr(ibmr);
235 unsigned int page_size = mr_page_size(mr);
238 mr->page_shift = ilog2(page_size);
239 mr->page_mask = ~((u64)page_size - 1);
240 mr->page_offset = mr->ibmr.iova & (page_size - 1);
242 return ib_sg_to_pages(ibmr, sgl, sg_nents, sg_offset, rxe_set_page);
245 static int rxe_mr_copy_xarray(struct rxe_mr *mr, u64 iova, void *addr,
246 unsigned int length, enum rxe_mr_copy_dir dir)
248 unsigned int page_offset = rxe_mr_iova_to_page_offset(mr, iova);
249 unsigned long index = rxe_mr_iova_to_index(mr, iova);
255 page = xa_load(&mr->page_list, index);
259 bytes = min_t(unsigned int, length,
260 mr_page_size(mr) - page_offset);
261 va = kmap_local_page(page);
262 if (dir == RXE_FROM_MR_OBJ)
263 memcpy(addr, va + page_offset, bytes);
265 memcpy(va + page_offset, addr, bytes);
277 static void rxe_mr_copy_dma(struct rxe_mr *mr, u64 dma_addr, void *addr,
278 unsigned int length, enum rxe_mr_copy_dir dir)
280 unsigned int page_offset = dma_addr & (PAGE_SIZE - 1);
286 page = ib_virt_dma_to_page(dma_addr);
287 bytes = min_t(unsigned int, length,
288 PAGE_SIZE - page_offset);
289 va = kmap_local_page(page);
291 if (dir == RXE_TO_MR_OBJ)
292 memcpy(va + page_offset, addr, bytes);
294 memcpy(addr, va + page_offset, bytes);
304 int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr,
305 unsigned int length, enum rxe_mr_copy_dir dir)
315 if (mr->ibmr.type == IB_MR_TYPE_DMA) {
316 rxe_mr_copy_dma(mr, iova, addr, length, dir);
320 err = mr_check_range(mr, iova, length);
322 rxe_dbg_mr(mr, "iova out of range");
326 return rxe_mr_copy_xarray(mr, iova, addr, length, dir);
329 /* copy data in or out of a wqe, i.e. sg list
330 * under the control of a dma descriptor
335 struct rxe_dma_info *dma,
338 enum rxe_mr_copy_dir dir)
341 struct rxe_sge *sge = &dma->sge[dma->cur_sge];
342 int offset = dma->sge_offset;
343 int resid = dma->resid;
344 struct rxe_mr *mr = NULL;
351 if (length > resid) {
356 if (sge->length && (offset < sge->length)) {
357 mr = lookup_mr(pd, access, sge->lkey, RXE_LOOKUP_LOCAL);
367 if (offset >= sge->length) {
376 if (dma->cur_sge >= dma->num_sge) {
382 mr = lookup_mr(pd, access, sge->lkey,
393 if (bytes > sge->length - offset)
394 bytes = sge->length - offset;
397 iova = sge->addr + offset;
398 err = rxe_mr_copy(mr, iova, addr, bytes, dir);
409 dma->sge_offset = offset;
424 int rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length)
426 unsigned int page_offset;
433 /* mr must be valid even if length is zero */
440 if (mr->ibmr.type == IB_MR_TYPE_DMA)
443 err = mr_check_range(mr, iova, length);
448 index = rxe_mr_iova_to_index(mr, iova);
449 page = xa_load(&mr->page_list, index);
450 page_offset = rxe_mr_iova_to_page_offset(mr, iova);
453 bytes = min_t(unsigned int, length,
454 mr_page_size(mr) - page_offset);
456 va = kmap_local_page(page);
457 arch_wb_cache_pmem(va + page_offset, bytes);
468 /* Guarantee atomicity of atomic operations at the machine level. */
469 static DEFINE_SPINLOCK(atomic_ops_lock);
471 int rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode,
472 u64 compare, u64 swap_add, u64 *orig_val)
474 unsigned int page_offset;
479 if (unlikely(mr->state != RXE_MR_STATE_VALID)) {
480 rxe_dbg_mr(mr, "mr not in valid state");
481 return RESPST_ERR_RKEY_VIOLATION;
484 if (mr->ibmr.type == IB_MR_TYPE_DMA) {
485 page_offset = iova & (PAGE_SIZE - 1);
486 page = ib_virt_dma_to_page(iova);
491 err = mr_check_range(mr, iova, sizeof(value));
493 rxe_dbg_mr(mr, "iova out of range");
494 return RESPST_ERR_RKEY_VIOLATION;
496 page_offset = rxe_mr_iova_to_page_offset(mr, iova);
497 index = rxe_mr_iova_to_index(mr, iova);
498 page = xa_load(&mr->page_list, index);
500 return RESPST_ERR_RKEY_VIOLATION;
503 if (unlikely(page_offset & 0x7)) {
504 rxe_dbg_mr(mr, "iova not aligned");
505 return RESPST_ERR_MISALIGNED_ATOMIC;
508 va = kmap_local_page(page);
510 spin_lock_bh(&atomic_ops_lock);
511 value = *orig_val = va[page_offset >> 3];
513 if (opcode == IB_OPCODE_RC_COMPARE_SWAP) {
514 if (value == compare)
515 va[page_offset >> 3] = swap_add;
518 va[page_offset >> 3] = value;
520 spin_unlock_bh(&atomic_ops_lock);
527 #if defined CONFIG_64BIT
528 /* only implemented or called for 64 bit architectures */
529 int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)
531 unsigned int page_offset;
535 /* See IBA oA19-28 */
536 if (unlikely(mr->state != RXE_MR_STATE_VALID)) {
537 rxe_dbg_mr(mr, "mr not in valid state");
538 return RESPST_ERR_RKEY_VIOLATION;
541 if (mr->ibmr.type == IB_MR_TYPE_DMA) {
542 page_offset = iova & (PAGE_SIZE - 1);
543 page = ib_virt_dma_to_page(iova);
548 /* See IBA oA19-28 */
549 err = mr_check_range(mr, iova, sizeof(value));
551 rxe_dbg_mr(mr, "iova out of range");
552 return RESPST_ERR_RKEY_VIOLATION;
554 page_offset = rxe_mr_iova_to_page_offset(mr, iova);
555 index = rxe_mr_iova_to_index(mr, iova);
556 page = xa_load(&mr->page_list, index);
558 return RESPST_ERR_RKEY_VIOLATION;
561 /* See IBA A19.4.2 */
562 if (unlikely(page_offset & 0x7)) {
563 rxe_dbg_mr(mr, "misaligned address");
564 return RESPST_ERR_MISALIGNED_ATOMIC;
567 va = kmap_local_page(page);
569 /* Do atomic write after all prior operations have completed */
570 smp_store_release(&va[page_offset >> 3], value);
577 int rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value)
579 return RESPST_ERR_UNSUPPORTED_OPCODE;
583 int advance_dma_data(struct rxe_dma_info *dma, unsigned int length)
585 struct rxe_sge *sge = &dma->sge[dma->cur_sge];
586 int offset = dma->sge_offset;
587 int resid = dma->resid;
592 if (offset >= sge->length) {
596 if (dma->cur_sge >= dma->num_sge)
602 if (bytes > sge->length - offset)
603 bytes = sge->length - offset;
610 dma->sge_offset = offset;
616 struct rxe_mr *lookup_mr(struct rxe_pd *pd, int access, u32 key,
617 enum rxe_mr_lookup_type type)
620 struct rxe_dev *rxe = to_rdev(pd->ibpd.device);
621 int index = key >> 8;
623 mr = rxe_pool_get_index(&rxe->mr_pool, index);
627 if (unlikely((type == RXE_LOOKUP_LOCAL && mr->lkey != key) ||
628 (type == RXE_LOOKUP_REMOTE && mr->rkey != key) ||
629 mr_pd(mr) != pd || ((access & mr->access) != access) ||
630 mr->state != RXE_MR_STATE_VALID)) {
638 int rxe_invalidate_mr(struct rxe_qp *qp, u32 key)
640 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
645 mr = rxe_pool_get_index(&rxe->mr_pool, key >> 8);
647 rxe_dbg_qp(qp, "No MR for key %#x\n", key);
652 remote = mr->access & RXE_ACCESS_REMOTE;
653 if (remote ? (key != mr->rkey) : (key != mr->lkey)) {
654 rxe_dbg_mr(mr, "wr key (%#x) doesn't match mr key (%#x)\n",
655 key, (remote ? mr->rkey : mr->lkey));
660 if (atomic_read(&mr->num_mw) > 0) {
661 rxe_dbg_mr(mr, "Attempt to invalidate an MR while bound to MWs\n");
666 if (unlikely(mr->ibmr.type != IB_MR_TYPE_MEM_REG)) {
667 rxe_dbg_mr(mr, "Type (%d) is wrong\n", mr->ibmr.type);
672 mr->state = RXE_MR_STATE_FREE;
681 /* user can (re)register fast MR by executing a REG_MR WQE.
682 * user is expected to hold a reference on the ib mr until the
684 * Once a fast MR is created this is the only way to change the
685 * private keys. It is the responsibility of the user to maintain
686 * the ib mr keys in sync with rxe mr keys.
688 int rxe_reg_fast_mr(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
690 struct rxe_mr *mr = to_rmr(wqe->wr.wr.reg.mr);
691 u32 key = wqe->wr.wr.reg.key;
692 u32 access = wqe->wr.wr.reg.access;
694 /* user can only register MR in free state */
695 if (unlikely(mr->state != RXE_MR_STATE_FREE)) {
696 rxe_dbg_mr(mr, "mr->lkey = 0x%x not free\n", mr->lkey);
700 /* user can only register mr with qp in same protection domain */
701 if (unlikely(qp->ibqp.pd != mr->ibmr.pd)) {
702 rxe_dbg_mr(mr, "qp->pd and mr->pd don't match\n");
706 /* user is only allowed to change key portion of l/rkey */
707 if (unlikely((mr->lkey & ~0xff) != (key & ~0xff))) {
708 rxe_dbg_mr(mr, "key = 0x%x has wrong index mr->lkey = 0x%x\n",
716 mr->ibmr.iova = wqe->wr.wr.reg.mr->iova;
717 mr->state = RXE_MR_STATE_VALID;
722 void rxe_mr_cleanup(struct rxe_pool_elem *elem)
724 struct rxe_mr *mr = container_of(elem, typeof(*mr), elem);
727 ib_umem_release(mr->umem);
729 if (mr->ibmr.type != IB_MR_TYPE_DMA)
730 xa_destroy(&mr->page_list);