drivers/misc/mic/scif/scif_dma.c

   1 /*
   2  * Intel MIC Platform Software Stack (MPSS)
   3  *
   4  * Copyright(c) 2015 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License, version 2, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  13  * General Public License for more details.
  14  *
  15  * Intel SCIF driver.
  16  *
  17  */
  18 #include "scif_main.h"
  19 #include "scif_map.h"
  20
  21 /*
  22  * struct scif_dma_comp_cb - SCIF DMA completion callback
  23  *
  24  * @dma_completion_func: DMA completion callback
  25  * @cb_cookie: DMA completion callback cookie
  26  * @temp_buf: Temporary buffer
  27  * @temp_buf_to_free: Temporary buffer to be freed
  28  * @is_cache: Is a kmem_cache allocated buffer
  29  * @dst_offset: Destination registration offset
  30  * @dst_window: Destination registration window
  31  * @len: Length of the temp buffer
  32  * @temp_phys: DMA address of the temp buffer
  33  * @sdev: The SCIF device
  34  * @header_padding: padding for cache line alignment
  35  */
  36 struct scif_dma_comp_cb {
  37         void (*dma_completion_func)(void *cookie);
  38         void *cb_cookie;
  39         u8 *temp_buf;
  40         u8 *temp_buf_to_free;
  41         bool is_cache;
  42         s64 dst_offset;
  43         struct scif_window *dst_window;
  44         size_t len;
  45         dma_addr_t temp_phys;
  46         struct scif_dev *sdev;
  47         int header_padding;
  48 };
  49
  50 /**
  51  * struct scif_copy_work - Work for DMA copy
  52  *
  53  * @src_offset: Starting source offset
  54  * @dst_offset: Starting destination offset
  55  * @src_window: Starting src registered window
  56  * @dst_window: Starting dst registered window
  57  * @loopback: true if this is a loopback DMA transfer
  58  * @len: Length of the transfer
  59  * @comp_cb: DMA copy completion callback
  60  * @remote_dev: The remote SCIF peer device
  61  * @fence_type: polling or interrupt based
  62  * @ordered: is this a tail byte ordered DMA transfer
  63  */
  64 struct scif_copy_work {
  65         s64 src_offset;
  66         s64 dst_offset;
  67         struct scif_window *src_window;
  68         struct scif_window *dst_window;
  69         int loopback;
  70         size_t len;
  71         struct scif_dma_comp_cb   *comp_cb;
  72         struct scif_dev *remote_dev;
  73         int fence_type;
  74         bool ordered;
  75 };
  76
  77 /**
  78  * scif_reserve_dma_chan:
  79  * @ep: Endpoint Descriptor.
  80  *
  81  * This routine reserves a DMA channel for a particular
  82  * endpoint. All DMA transfers for an endpoint are always
  83  * programmed on the same DMA channel.
  84  */
  85 int scif_reserve_dma_chan(struct scif_endpt *ep)
  86 {
  87         int err = 0;
  88         struct scif_dev *scifdev;
  89         struct scif_hw_dev *sdev;
  90         struct dma_chan *chan;
  91
  92         /* Loopback DMAs are not supported on the management node */
  93         if (!scif_info.nodeid && scifdev_self(ep->remote_dev))
  94                 return 0;
  95         if (scif_info.nodeid)
  96                 scifdev = &scif_dev[0];
  97         else
  98                 scifdev = ep->remote_dev;
  99         sdev = scifdev->sdev;
 100         if (!sdev->num_dma_ch)
 101                 return -ENODEV;
 102         chan = sdev->dma_ch[scifdev->dma_ch_idx];
 103         scifdev->dma_ch_idx = (scifdev->dma_ch_idx + 1) % sdev->num_dma_ch;
 104         mutex_lock(&ep->rma_info.rma_lock);
 105         ep->rma_info.dma_chan = chan;
 106         mutex_unlock(&ep->rma_info.rma_lock);
 107         return err;
 108 }
 109
 110 #ifdef CONFIG_MMU_NOTIFIER
 111 /**
 112  * scif_rma_destroy_tcw:
 113  *
 114  * This routine destroys temporary cached windows
 115  */
 116 static
 117 void __scif_rma_destroy_tcw(struct scif_mmu_notif *mmn,
 118                             u64 start, u64 len)
 119 {
 120         struct list_head *item, *tmp;
 121         struct scif_window *window;
 122         u64 start_va, end_va;
 123         u64 end = start + len;
 124
 125         if (end <= start)
 126                 return;
 127
 128         list_for_each_safe(item, tmp, &mmn->tc_reg_list) {
 129                 window = list_entry(item, struct scif_window, list);
 130                 if (!len)
 131                         break;
 132                 start_va = window->va_for_temp;
 133                 end_va = start_va + (window->nr_pages << PAGE_SHIFT);
 134                 if (start < start_va && end <= start_va)
 135                         break;
 136                 if (start >= end_va)
 137                         continue;
 138                 __scif_rma_destroy_tcw_helper(window);
 139         }
 140 }
 141
 142 static void scif_rma_destroy_tcw(struct scif_mmu_notif *mmn, u64 start, u64 len)
 143 {
 144         struct scif_endpt *ep = mmn->ep;
 145
 146         spin_lock(&ep->rma_info.tc_lock);
 147         __scif_rma_destroy_tcw(mmn, start, len);
 148         spin_unlock(&ep->rma_info.tc_lock);
 149 }
 150
 151 static void scif_rma_destroy_tcw_ep(struct scif_endpt *ep)
 152 {
 153         struct list_head *item, *tmp;
 154         struct scif_mmu_notif *mmn;
 155
 156         list_for_each_safe(item, tmp, &ep->rma_info.mmn_list) {
 157                 mmn = list_entry(item, struct scif_mmu_notif, list);
 158                 scif_rma_destroy_tcw(mmn, 0, ULONG_MAX);
 159         }
 160 }
 161
 162 static void __scif_rma_destroy_tcw_ep(struct scif_endpt *ep)
 163 {
 164         struct list_head *item, *tmp;
 165         struct scif_mmu_notif *mmn;
 166
 167         spin_lock(&ep->rma_info.tc_lock);
 168         list_for_each_safe(item, tmp, &ep->rma_info.mmn_list) {
 169                 mmn = list_entry(item, struct scif_mmu_notif, list);
 170                 __scif_rma_destroy_tcw(mmn, 0, ULONG_MAX);
 171         }
 172         spin_unlock(&ep->rma_info.tc_lock);
 173 }
 174
 175 static bool scif_rma_tc_can_cache(struct scif_endpt *ep, size_t cur_bytes)
 176 {
 177         if ((cur_bytes >> PAGE_SHIFT) > scif_info.rma_tc_limit)
 178                 return false;
 179         if ((atomic_read(&ep->rma_info.tcw_total_pages)
 180                         + (cur_bytes >> PAGE_SHIFT)) >
 181                         scif_info.rma_tc_limit) {
 182                 dev_info(scif_info.mdev.this_device,
 183                          "%s %d total=%d, current=%zu reached max\n",
 184                          __func__, __LINE__,
 185                          atomic_read(&ep->rma_info.tcw_total_pages),
 186                          (1 + (cur_bytes >> PAGE_SHIFT)));
 187                 scif_rma_destroy_tcw_invalid();
 188                 __scif_rma_destroy_tcw_ep(ep);
 189         }
 190         return true;
 191 }
 192
 193 static void scif_mmu_notifier_release(struct mmu_notifier *mn,
 194                                       struct mm_struct *mm)
 195 {
 196         struct scif_mmu_notif   *mmn;
 197
 198         mmn = container_of(mn, struct scif_mmu_notif, ep_mmu_notifier);
 199         scif_rma_destroy_tcw(mmn, 0, ULONG_MAX);
 200         schedule_work(&scif_info.misc_work);
 201 }
 202
 203 static void scif_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn,
 204                                                      struct mm_struct *mm,
 205                                                      unsigned long start,
 206                                                      unsigned long end)
 207 {
 208         struct scif_mmu_notif   *mmn;
 209
 210         mmn = container_of(mn, struct scif_mmu_notif, ep_mmu_notifier);
 211         scif_rma_destroy_tcw(mmn, start, end - start);
 212 }
 213
 214 static void scif_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn,
 215                                                    struct mm_struct *mm,
 216                                                    unsigned long start,
 217                                                    unsigned long end)
 218 {
 219         /*
 220          * Nothing to do here, everything needed was done in
 221          * invalidate_range_start.
 222          */
 223 }
 224
 225 static const struct mmu_notifier_ops scif_mmu_notifier_ops = {
 226         .release = scif_mmu_notifier_release,
 227         .clear_flush_young = NULL,
 228         .invalidate_range_start = scif_mmu_notifier_invalidate_range_start,
 229         .invalidate_range_end = scif_mmu_notifier_invalidate_range_end};
 230
 231 static void scif_ep_unregister_mmu_notifier(struct scif_endpt *ep)
 232 {
 233         struct scif_endpt_rma_info *rma = &ep->rma_info;
 234         struct scif_mmu_notif *mmn = NULL;
 235         struct list_head *item, *tmp;
 236
 237         mutex_lock(&ep->rma_info.mmn_lock);
 238         list_for_each_safe(item, tmp, &rma->mmn_list) {
 239                 mmn = list_entry(item, struct scif_mmu_notif, list);
 240                 mmu_notifier_unregister(&mmn->ep_mmu_notifier, mmn->mm);
 241                 list_del(item);
 242                 kfree(mmn);
 243         }
 244         mutex_unlock(&ep->rma_info.mmn_lock);
 245 }
 246
 247 static void scif_init_mmu_notifier(struct scif_mmu_notif *mmn,
 248                                    struct mm_struct *mm, struct scif_endpt *ep)
 249 {
 250         mmn->ep = ep;
 251         mmn->mm = mm;
 252         mmn->ep_mmu_notifier.ops = &scif_mmu_notifier_ops;
 253         INIT_LIST_HEAD(&mmn->list);
 254         INIT_LIST_HEAD(&mmn->tc_reg_list);
 255 }
 256
 257 static struct scif_mmu_notif *
 258 scif_find_mmu_notifier(struct mm_struct *mm, struct scif_endpt_rma_info *rma)
 259 {
 260         struct scif_mmu_notif *mmn;
 261
 262         list_for_each_entry(mmn, &rma->mmn_list, list)
 263                 if (mmn->mm == mm)
 264                         return mmn;
 265         return NULL;
 266 }
 267
 268 static struct scif_mmu_notif *
 269 scif_add_mmu_notifier(struct mm_struct *mm, struct scif_endpt *ep)
 270 {
 271         struct scif_mmu_notif *mmn
 272                  = kzalloc(sizeof(*mmn), GFP_KERNEL);
 273
 274         if (!mmn)
 275                 return ERR_PTR(-ENOMEM);
 276
 277         scif_init_mmu_notifier(mmn, current->mm, ep);
 278         if (mmu_notifier_register(&mmn->ep_mmu_notifier, current->mm)) {
 279                 kfree(mmn);
 280                 return ERR_PTR(-EBUSY);
 281         }
 282         list_add(&mmn->list, &ep->rma_info.mmn_list);
 283         return mmn;
 284 }
 285
 286 /*
 287  * Called from the misc thread to destroy temporary cached windows and
 288  * unregister the MMU notifier for the SCIF endpoint.
 289  */
 290 void scif_mmu_notif_handler(struct work_struct *work)
 291 {
 292         struct list_head *pos, *tmpq;
 293         struct scif_endpt *ep;
 294 restart:
 295         scif_rma_destroy_tcw_invalid();
 296         spin_lock(&scif_info.rmalock);
 297         list_for_each_safe(pos, tmpq, &scif_info.mmu_notif_cleanup) {
 298                 ep = list_entry(pos, struct scif_endpt, mmu_list);
 299                 list_del(&ep->mmu_list);
 300                 spin_unlock(&scif_info.rmalock);
 301                 scif_rma_destroy_tcw_ep(ep);
 302                 scif_ep_unregister_mmu_notifier(ep);
 303                 goto restart;
 304         }
 305         spin_unlock(&scif_info.rmalock);
 306 }
 307
 308 static bool scif_is_set_reg_cache(int flags)
 309 {
 310         return !!(flags & SCIF_RMA_USECACHE);
 311 }
 312 #else
 313 static struct scif_mmu_notif *
 314 scif_find_mmu_notifier(struct mm_struct *mm,
 315                        struct scif_endpt_rma_info *rma)
 316 {
 317         return NULL;
 318 }
 319
 320 static struct scif_mmu_notif *
 321 scif_add_mmu_notifier(struct mm_struct *mm, struct scif_endpt *ep)
 322 {
 323         return NULL;
 324 }
 325
 326 void scif_mmu_notif_handler(struct work_struct *work)
 327 {
 328 }
 329
 330 static bool scif_is_set_reg_cache(int flags)
 331 {
 332         return false;
 333 }
 334
 335 static bool scif_rma_tc_can_cache(struct scif_endpt *ep, size_t cur_bytes)
 336 {
 337         return false;
 338 }
 339 #endif
 340
 341 /**
 342  * scif_register_temp:
 343  * @epd: End Point Descriptor.
 344  * @addr: virtual address to/from which to copy
 345  * @len: length of range to copy
 346  * @out_offset: computed offset returned by reference.
 347  * @out_window: allocated registered window returned by reference.
 348  *
 349  * Create a temporary registered window. The peer will not know about this
 350  * window. This API is used for scif_vreadfrom()/scif_vwriteto() API's.
 351  */
 352 static int
 353 scif_register_temp(scif_epd_t epd, unsigned long addr, size_t len, int prot,
 354                    off_t *out_offset, struct scif_window **out_window)
 355 {
 356         struct scif_endpt *ep = (struct scif_endpt *)epd;
 357         int err;
 358         scif_pinned_pages_t pinned_pages;
 359         size_t aligned_len;
 360
 361         aligned_len = ALIGN(len, PAGE_SIZE);
 362
 363         err = __scif_pin_pages((void *)(addr & PAGE_MASK),
 364                                aligned_len, &prot, 0, &pinned_pages);
 365         if (err)
 366                 return err;
 367
 368         pinned_pages->prot = prot;
 369
 370         /* Compute the offset for this registration */
 371         err = scif_get_window_offset(ep, 0, 0,
 372                                      aligned_len >> PAGE_SHIFT,
 373                                      (s64 *)out_offset);
 374         if (err)
 375                 goto error_unpin;
 376
 377         /* Allocate and prepare self registration window */
 378         *out_window = scif_create_window(ep, aligned_len >> PAGE_SHIFT,
 379                                         *out_offset, true);
 380         if (!*out_window) {
 381                 scif_free_window_offset(ep, NULL, *out_offset);
 382                 err = -ENOMEM;
 383                 goto error_unpin;
 384         }
 385
 386         (*out_window)->pinned_pages = pinned_pages;
 387         (*out_window)->nr_pages = pinned_pages->nr_pages;
 388         (*out_window)->prot = pinned_pages->prot;
 389
 390         (*out_window)->va_for_temp = addr & PAGE_MASK;
 391         err = scif_map_window(ep->remote_dev, *out_window);
 392         if (err) {
 393                 /* Something went wrong! Rollback */
 394                 scif_destroy_window(ep, *out_window);
 395                 *out_window = NULL;
 396         } else {
 397                 *out_offset |= (addr - (*out_window)->va_for_temp);
 398         }
 399         return err;
 400 error_unpin:
 401         if (err)
 402                 dev_err(&ep->remote_dev->sdev->dev,
 403                         "%s %d err %d\n", __func__, __LINE__, err);
 404         scif_unpin_pages(pinned_pages);
 405         return err;
 406 }
 407
 408 #define SCIF_DMA_TO (3 * HZ)
 409
 410 /*
 411  * scif_sync_dma - Program a DMA without an interrupt descriptor
 412  *
 413  * @dev - The address of the pointer to the device instance used
 414  * for DMA registration.
 415  * @chan - DMA channel to be used.
 416  * @sync_wait: Wait for DMA to complete?
 417  *
 418  * Return 0 on success and -errno on error.
 419  */
 420 static int scif_sync_dma(struct scif_hw_dev *sdev, struct dma_chan *chan,
 421                          bool sync_wait)
 422 {
 423         int err = 0;
 424         struct dma_async_tx_descriptor *tx = NULL;
 425         enum dma_ctrl_flags flags = DMA_PREP_FENCE;
 426         dma_cookie_t cookie;
 427         struct dma_device *ddev;
 428
 429         if (!chan) {
 430                 err = -EIO;
 431                 dev_err(&sdev->dev, "%s %d err %d\n",
 432                         __func__, __LINE__, err);
 433                 return err;
 434         }
 435         ddev = chan->device;
 436
 437         tx = ddev->device_prep_dma_memcpy(chan, 0, 0, 0, flags);
 438         if (!tx) {
 439                 err = -ENOMEM;
 440                 dev_err(&sdev->dev, "%s %d err %d\n",
 441                         __func__, __LINE__, err);
 442                 goto release;
 443         }
 444         cookie = tx->tx_submit(tx);
 445
 446         if (dma_submit_error(cookie)) {
 447                 err = -ENOMEM;
 448                 dev_err(&sdev->dev, "%s %d err %d\n",
 449                         __func__, __LINE__, err);
 450                 goto release;
 451         }
 452         if (!sync_wait) {
 453                 dma_async_issue_pending(chan);
 454         } else {
 455                 if (dma_sync_wait(chan, cookie) == DMA_COMPLETE) {
 456                         err = 0;
 457                 } else {
 458                         err = -EIO;
 459                         dev_err(&sdev->dev, "%s %d err %d\n",
 460                                 __func__, __LINE__, err);
 461                 }
 462         }
 463 release:
 464         return err;
 465 }
 466
 467 static void scif_dma_callback(void *arg)
 468 {
 469         struct completion *done = (struct completion *)arg;
 470
 471         complete(done);
 472 }
 473
 474 #define SCIF_DMA_SYNC_WAIT true
 475 #define SCIF_DMA_POLL BIT(0)
 476 #define SCIF_DMA_INTR BIT(1)
 477
 478 /*
 479  * scif_async_dma - Program a DMA with an interrupt descriptor
 480  *
 481  * @dev - The address of the pointer to the device instance used
 482  * for DMA registration.
 483  * @chan - DMA channel to be used.
 484  * Return 0 on success and -errno on error.
 485  */
 486 static int scif_async_dma(struct scif_hw_dev *sdev, struct dma_chan *chan)
 487 {
 488         int err = 0;
 489         struct dma_device *ddev;
 490         struct dma_async_tx_descriptor *tx = NULL;
 491         enum dma_ctrl_flags flags = DMA_PREP_INTERRUPT | DMA_PREP_FENCE;
 492         DECLARE_COMPLETION_ONSTACK(done_wait);
 493         dma_cookie_t cookie;
 494         enum dma_status status;
 495
 496         if (!chan) {
 497                 err = -EIO;
 498                 dev_err(&sdev->dev, "%s %d err %d\n",
 499                         __func__, __LINE__, err);
 500                 return err;
 501         }
 502         ddev = chan->device;
 503
 504         tx = ddev->device_prep_dma_memcpy(chan, 0, 0, 0, flags);
 505         if (!tx) {
 506                 err = -ENOMEM;
 507                 dev_err(&sdev->dev, "%s %d err %d\n",
 508                         __func__, __LINE__, err);
 509                 goto release;
 510         }
 511         reinit_completion(&done_wait);
 512         tx->callback = scif_dma_callback;
 513         tx->callback_param = &done_wait;
 514         cookie = tx->tx_submit(tx);
 515
 516         if (dma_submit_error(cookie)) {
 517                 err = -ENOMEM;
 518                 dev_err(&sdev->dev, "%s %d err %d\n",
 519                         __func__, __LINE__, err);
 520                 goto release;
 521         }
 522         dma_async_issue_pending(chan);
 523
 524         err = wait_for_completion_timeout(&done_wait, SCIF_DMA_TO);
 525         if (!err) {
 526                 err = -EIO;
 527                 dev_err(&sdev->dev, "%s %d err %d\n",
 528                         __func__, __LINE__, err);
 529                 goto release;
 530         }
 531         err = 0;
 532         status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
 533         if (status != DMA_COMPLETE) {
 534                 err = -EIO;
 535                 dev_err(&sdev->dev, "%s %d err %d\n",
 536                         __func__, __LINE__, err);
 537                 goto release;
 538         }
 539 release:
 540         return err;
 541 }
 542
 543 /*
 544  * scif_drain_dma_poll - Drain all outstanding DMA operations for a particular
 545  * DMA channel via polling.
 546  *
 547  * @sdev - The SCIF device
 548  * @chan - DMA channel
 549  * Return 0 on success and -errno on error.
 550  */
 551 static int scif_drain_dma_poll(struct scif_hw_dev *sdev, struct dma_chan *chan)
 552 {
 553         if (!chan)
 554                 return -EINVAL;
 555         return scif_sync_dma(sdev, chan, SCIF_DMA_SYNC_WAIT);
 556 }
 557
 558 /*
 559  * scif_drain_dma_intr - Drain all outstanding DMA operations for a particular
 560  * DMA channel via interrupt based blocking wait.
 561  *
 562  * @sdev - The SCIF device
 563  * @chan - DMA channel
 564  * Return 0 on success and -errno on error.
 565  */
 566 int scif_drain_dma_intr(struct scif_hw_dev *sdev, struct dma_chan *chan)
 567 {
 568         if (!chan)
 569                 return -EINVAL;
 570         return scif_async_dma(sdev, chan);
 571 }
 572
 573 /**
 574  * scif_rma_destroy_windows:
 575  *
 576  * This routine destroys all windows queued for cleanup
 577  */
 578 void scif_rma_destroy_windows(void)
 579 {
 580         struct list_head *item, *tmp;
 581         struct scif_window *window;
 582         struct scif_endpt *ep;
 583         struct dma_chan *chan;
 584
 585         might_sleep();
 586 restart:
 587         spin_lock(&scif_info.rmalock);
 588         list_for_each_safe(item, tmp, &scif_info.rma) {
 589                 window = list_entry(item, struct scif_window,
 590                                     list);
 591                 ep = (struct scif_endpt *)window->ep;
 592                 chan = ep->rma_info.dma_chan;
 593
 594                 list_del_init(&window->list);
 595                 spin_unlock(&scif_info.rmalock);
 596                 if (!chan || !scifdev_alive(ep) ||
 597                     !scif_drain_dma_intr(ep->remote_dev->sdev,
 598                                          ep->rma_info.dma_chan))
 599                         /* Remove window from global list */
 600                         window->unreg_state = OP_COMPLETED;
 601                 else
 602                         dev_warn(&ep->remote_dev->sdev->dev,
 603                                  "DMA engine hung?\n");
 604                 if (window->unreg_state == OP_COMPLETED) {
 605                         if (window->type == SCIF_WINDOW_SELF)
 606                                 scif_destroy_window(ep, window);
 607                         else
 608                                 scif_destroy_remote_window(window);
 609                         atomic_dec(&ep->rma_info.tw_refcount);
 610                 }
 611                 goto restart;
 612         }
 613         spin_unlock(&scif_info.rmalock);
 614 }
 615
 616 /**
 617  * scif_rma_destroy_tcw:
 618  *
 619  * This routine destroys temporary cached registered windows
 620  * which have been queued for cleanup.
 621  */
 622 void scif_rma_destroy_tcw_invalid(void)
 623 {
 624         struct list_head *item, *tmp;
 625         struct scif_window *window;
 626         struct scif_endpt *ep;
 627         struct dma_chan *chan;
 628
 629         might_sleep();
 630 restart:
 631         spin_lock(&scif_info.rmalock);
 632         list_for_each_safe(item, tmp, &scif_info.rma_tc) {
 633                 window = list_entry(item, struct scif_window, list);
 634                 ep = (struct scif_endpt *)window->ep;
 635                 chan = ep->rma_info.dma_chan;
 636                 list_del_init(&window->list);
 637                 spin_unlock(&scif_info.rmalock);
 638                 mutex_lock(&ep->rma_info.rma_lock);
 639                 if (!chan || !scifdev_alive(ep) ||
 640                     !scif_drain_dma_intr(ep->remote_dev->sdev,
 641                                          ep->rma_info.dma_chan)) {
 642                         atomic_sub(window->nr_pages,
 643                                    &ep->rma_info.tcw_total_pages);
 644                         scif_destroy_window(ep, window);
 645                         atomic_dec(&ep->rma_info.tcw_refcount);
 646                 } else {
 647                         dev_warn(&ep->remote_dev->sdev->dev,
 648                                  "DMA engine hung?\n");
 649                 }
 650                 mutex_unlock(&ep->rma_info.rma_lock);
 651                 goto restart;
 652         }
 653         spin_unlock(&scif_info.rmalock);
 654 }
 655
 656 static inline
 657 void *_get_local_va(off_t off, struct scif_window *window, size_t len)
 658 {
 659         int page_nr = (off - window->offset) >> PAGE_SHIFT;
 660         off_t page_off = off & ~PAGE_MASK;
 661         void *va = NULL;
 662
 663         if (window->type == SCIF_WINDOW_SELF) {
 664                 struct page **pages = window->pinned_pages->pages;
 665
 666                 va = page_address(pages[page_nr]) + page_off;
 667         }
 668         return va;
 669 }
 670
 671 static inline
 672 void *ioremap_remote(off_t off, struct scif_window *window,
 673                      size_t len, struct scif_dev *dev,
 674                      struct scif_window_iter *iter)
 675 {
 676         dma_addr_t phys = scif_off_to_dma_addr(window, off, NULL, iter);
 677
 678         /*
 679          * If the DMA address is not card relative then we need the DMA
 680          * addresses to be an offset into the bar. The aperture base was already
 681          * added so subtract it here since scif_ioremap is going to add it again
 682          */
 683         if (!scifdev_self(dev) && window->type == SCIF_WINDOW_PEER &&
 684             dev->sdev->aper && !dev->sdev->card_rel_da)
 685                 phys = phys - dev->sdev->aper->pa;
 686         return scif_ioremap(phys, len, dev);
 687 }
 688
 689 static inline void
 690 iounmap_remote(void *virt, size_t size, struct scif_copy_work *work)
 691 {
 692         scif_iounmap(virt, size, work->remote_dev);
 693 }
 694
 695 /*
 696  * Takes care of ordering issue caused by
 697  * 1. Hardware:  Only in the case of cpu copy from mgmt node to card
 698  * because of WC memory.
 699  * 2. Software: If memcpy reorders copy instructions for optimization.
 700  * This could happen at both mgmt node and card.
 701  */
 702 static inline void
 703 scif_ordered_memcpy_toio(char *dst, const char *src, size_t count)
 704 {
 705         if (!count)
 706                 return;
 707
 708         memcpy_toio((void __iomem __force *)dst, src, --count);
 709         /* Order the last byte with the previous stores */
 710         wmb();
 711         *(dst + count) = *(src + count);
 712 }
 713
 714 static inline void scif_unaligned_cpy_toio(char *dst, const char *src,
 715                                            size_t count, bool ordered)
 716 {
 717         if (ordered)
 718                 scif_ordered_memcpy_toio(dst, src, count);
 719         else
 720                 memcpy_toio((void __iomem __force *)dst, src, count);
 721 }
 722
 723 static inline
 724 void scif_ordered_memcpy_fromio(char *dst, const char *src, size_t count)
 725 {
 726         if (!count)
 727                 return;
 728
 729         memcpy_fromio(dst, (void __iomem __force *)src, --count);
 730         /* Order the last byte with the previous loads */
 731         rmb();
 732         *(dst + count) = *(src + count);
 733 }
 734
 735 static inline void scif_unaligned_cpy_fromio(char *dst, const char *src,
 736                                              size_t count, bool ordered)
 737 {
 738         if (ordered)
 739                 scif_ordered_memcpy_fromio(dst, src, count);
 740         else
 741                 memcpy_fromio(dst, (void __iomem __force *)src, count);
 742 }
 743
 744 #define SCIF_RMA_ERROR_CODE (~(dma_addr_t)0x0)
 745
 746 /*
 747  * scif_off_to_dma_addr:
 748  * Obtain the dma_addr given the window and the offset.
 749  * @window: Registered window.
 750  * @off: Window offset.
 751  * @nr_bytes: Return the number of contiguous bytes till next DMA addr index.
 752  * @index: Return the index of the dma_addr array found.
 753  * @start_off: start offset of index of the dma addr array found.
 754  * The nr_bytes provides the callee an estimate of the maximum possible
 755  * DMA xfer possible while the index/start_off provide faster lookups
 756  * for the next iteration.
 757  */
 758 dma_addr_t scif_off_to_dma_addr(struct scif_window *window, s64 off,
 759                                 size_t *nr_bytes, struct scif_window_iter *iter)
 760 {
 761         int i, page_nr;
 762         s64 start, end;
 763         off_t page_off;
 764
 765         if (window->nr_pages == window->nr_contig_chunks) {
 766                 page_nr = (off - window->offset) >> PAGE_SHIFT;
 767                 page_off = off & ~PAGE_MASK;
 768
 769                 if (nr_bytes)
 770                         *nr_bytes = PAGE_SIZE - page_off;
 771                 return window->dma_addr[page_nr] | page_off;
 772         }
 773         if (iter) {
 774                 i = iter->index;
 775                 start = iter->offset;
 776         } else {
 777                 i =  0;
 778                 start =  window->offset;
 779         }
 780         for (; i < window->nr_contig_chunks; i++) {
 781                 end = start + (window->num_pages[i] << PAGE_SHIFT);
 782                 if (off >= start && off < end) {
 783                         if (iter) {
 784                                 iter->index = i;
 785                                 iter->offset = start;
 786                         }
 787                         if (nr_bytes)
 788                                 *nr_bytes = end - off;
 789                         return (window->dma_addr[i] + (off - start));
 790                 }
 791                 start += (window->num_pages[i] << PAGE_SHIFT);
 792         }
 793         dev_err(scif_info.mdev.this_device,
 794                 "%s %d BUG. Addr not found? window %p off 0x%llx\n",
 795                 __func__, __LINE__, window, off);
 796         return SCIF_RMA_ERROR_CODE;
 797 }
 798
 799 /*
 800  * Copy between rma window and temporary buffer
 801  */
 802 static void scif_rma_local_cpu_copy(s64 offset, struct scif_window *window,
 803                                     u8 *temp, size_t rem_len, bool to_temp)
 804 {
 805         void *window_virt;
 806         size_t loop_len;
 807         int offset_in_page;
 808         s64 end_offset;
 809
 810         offset_in_page = offset & ~PAGE_MASK;
 811         loop_len = PAGE_SIZE - offset_in_page;
 812
 813         if (rem_len < loop_len)
 814                 loop_len = rem_len;
 815
 816         window_virt = _get_local_va(offset, window, loop_len);
 817         if (!window_virt)
 818                 return;
 819         if (to_temp)
 820                 memcpy(temp, window_virt, loop_len);
 821         else
 822                 memcpy(window_virt, temp, loop_len);
 823
 824         offset += loop_len;
 825         temp += loop_len;
 826         rem_len -= loop_len;
 827
 828         end_offset = window->offset +
 829                 (window->nr_pages << PAGE_SHIFT);
 830         while (rem_len) {
 831                 if (offset == end_offset) {
 832                         window = list_next_entry(window, list);
 833                         end_offset = window->offset +
 834                                 (window->nr_pages << PAGE_SHIFT);
 835                 }
 836                 loop_len = min(PAGE_SIZE, rem_len);
 837                 window_virt = _get_local_va(offset, window, loop_len);
 838                 if (!window_virt)
 839                         return;
 840                 if (to_temp)
 841                         memcpy(temp, window_virt, loop_len);
 842                 else
 843                         memcpy(window_virt, temp, loop_len);
 844                 offset  += loop_len;
 845                 temp    += loop_len;
 846                 rem_len -= loop_len;
 847         }
 848 }
 849
 850 /**
 851  * scif_rma_completion_cb:
 852  * @data: RMA cookie
 853  *
 854  * RMA interrupt completion callback.
 855  */
 856 static void scif_rma_completion_cb(void *data)
 857 {
 858         struct scif_dma_comp_cb *comp_cb = data;
 859
 860         /* Free DMA Completion CB. */
 861         if (comp_cb->dst_window)
 862                 scif_rma_local_cpu_copy(comp_cb->dst_offset,
 863                                         comp_cb->dst_window,
 864                                         comp_cb->temp_buf +
 865                                         comp_cb->header_padding,
 866                                         comp_cb->len, false);
 867         scif_unmap_single(comp_cb->temp_phys, comp_cb->sdev,
 868                           SCIF_KMEM_UNALIGNED_BUF_SIZE);
 869         if (comp_cb->is_cache)
 870                 kmem_cache_free(unaligned_cache,
 871                                 comp_cb->temp_buf_to_free);
 872         else
 873                 kfree(comp_cb->temp_buf_to_free);
 874 }
 875
 876 /* Copies between temporary buffer and offsets provided in work */
 877 static int
 878 scif_rma_list_dma_copy_unaligned(struct scif_copy_work *work,
 879                                  u8 *temp, struct dma_chan *chan,
 880                                  bool src_local)
 881 {
 882         struct scif_dma_comp_cb *comp_cb = work->comp_cb;
 883         dma_addr_t window_dma_addr, temp_dma_addr;
 884         dma_addr_t temp_phys = comp_cb->temp_phys;
 885         size_t loop_len, nr_contig_bytes = 0, remaining_len = work->len;
 886         int offset_in_ca, ret = 0;
 887         s64 end_offset, offset;
 888         struct scif_window *window;
 889         void *window_virt_addr;
 890         size_t tail_len;
 891         struct dma_async_tx_descriptor *tx;
 892         struct dma_device *dev = chan->device;
 893         dma_cookie_t cookie;
 894
 895         if (src_local) {
 896                 offset = work->dst_offset;
 897                 window = work->dst_window;
 898         } else {
 899                 offset = work->src_offset;
 900                 window = work->src_window;
 901         }
 902
 903         offset_in_ca = offset & (L1_CACHE_BYTES - 1);
 904         if (offset_in_ca) {
 905                 loop_len = L1_CACHE_BYTES - offset_in_ca;
 906                 loop_len = min(loop_len, remaining_len);
 907                 window_virt_addr = ioremap_remote(offset, window,
 908                                                   loop_len,
 909                                                   work->remote_dev,
 910                                                   NULL);
 911                 if (!window_virt_addr)
 912                         return -ENOMEM;
 913                 if (src_local)
 914                         scif_unaligned_cpy_toio(window_virt_addr, temp,
 915                                                 loop_len,
 916                                                 work->ordered &&
 917                                                 !(remaining_len - loop_len));
 918                 else
 919                         scif_unaligned_cpy_fromio(temp, window_virt_addr,
 920                                                   loop_len, work->ordered &&
 921                                                   !(remaining_len - loop_len));
 922                 iounmap_remote(window_virt_addr, loop_len, work);
 923
 924                 offset += loop_len;
 925                 temp += loop_len;
 926                 temp_phys += loop_len;
 927                 remaining_len -= loop_len;
 928         }
 929
 930         offset_in_ca = offset & ~PAGE_MASK;
 931         end_offset = window->offset +
 932                 (window->nr_pages << PAGE_SHIFT);
 933
 934         tail_len = remaining_len & (L1_CACHE_BYTES - 1);
 935         remaining_len -= tail_len;
 936         while (remaining_len) {
 937                 if (offset == end_offset) {
 938                         window = list_next_entry(window, list);
 939                         end_offset = window->offset +
 940                                 (window->nr_pages << PAGE_SHIFT);
 941                 }
 942                 if (scif_is_mgmt_node())
 943                         temp_dma_addr = temp_phys;
 944                 else
 945                         /* Fix if we ever enable IOMMU on the card */
 946                         temp_dma_addr = (dma_addr_t)virt_to_phys(temp);
 947                 window_dma_addr = scif_off_to_dma_addr(window, offset,
 948                                                        &nr_contig_bytes,
 949                                                        NULL);
 950                 loop_len = min(nr_contig_bytes, remaining_len);
 951                 if (src_local) {
 952                         if (work->ordered && !tail_len &&
 953                             !(remaining_len - loop_len) &&
 954                             loop_len != L1_CACHE_BYTES) {
 955                                 /*
 956                                  * Break up the last chunk of the transfer into
 957                                  * two steps. if there is no tail to guarantee
 958                                  * DMA ordering. SCIF_DMA_POLLING inserts
 959                                  * a status update descriptor in step 1 which
 960                                  * acts as a double sided synchronization fence
 961                                  * for the DMA engine to ensure that the last
 962                                  * cache line in step 2 is updated last.
 963                                  */
 964                                 /* Step 1) DMA: Body Length - L1_CACHE_BYTES. */
 965                                 tx =
 966                                 dev->device_prep_dma_memcpy(chan,
 967                                                             window_dma_addr,
 968                                                             temp_dma_addr,
 969                                                             loop_len -
 970                                                             L1_CACHE_BYTES,
 971                                                             DMA_PREP_FENCE);
 972                                 if (!tx) {
 973                                         ret = -ENOMEM;
 974                                         goto err;
 975                                 }
 976                                 cookie = tx->tx_submit(tx);
 977                                 if (dma_submit_error(cookie)) {
 978                                         ret = -ENOMEM;
 979                                         goto err;
 980                                 }
 981                                 dma_async_issue_pending(chan);
 982                                 offset += (loop_len - L1_CACHE_BYTES);
 983                                 temp_dma_addr += (loop_len - L1_CACHE_BYTES);
 984                                 window_dma_addr += (loop_len - L1_CACHE_BYTES);
 985                                 remaining_len -= (loop_len - L1_CACHE_BYTES);
 986                                 loop_len = remaining_len;
 987
 988                                 /* Step 2) DMA: L1_CACHE_BYTES */
 989                                 tx =
 990                                 dev->device_prep_dma_memcpy(chan,
 991                                                             window_dma_addr,
 992                                                             temp_dma_addr,
 993                                                             loop_len, 0);
 994                                 if (!tx) {
 995                                         ret = -ENOMEM;
 996                                         goto err;
 997                                 }
 998                                 cookie = tx->tx_submit(tx);
 999                                 if (dma_submit_error(cookie)) {
1000                                         ret = -ENOMEM;
1001                                         goto err;
1002                                 }
1003                                 dma_async_issue_pending(chan);
1004                         } else {
1005                                 tx =
1006                                 dev->device_prep_dma_memcpy(chan,
1007                                                             window_dma_addr,
1008                                                             temp_dma_addr,
1009                                                             loop_len, 0);
1010                                 if (!tx) {
1011                                         ret = -ENOMEM;
1012                                         goto err;
1013                                 }
1014                                 cookie = tx->tx_submit(tx);
1015                                 if (dma_submit_error(cookie)) {
1016                                         ret = -ENOMEM;
1017                                         goto err;
1018                                 }
1019                                 dma_async_issue_pending(chan);
1020                         }
1021                 } else {
1022                         tx = dev->device_prep_dma_memcpy(chan, temp_dma_addr,
1023                                         window_dma_addr, loop_len, 0);
1024                         if (!tx) {
1025                                 ret = -ENOMEM;
1026                                 goto err;
1027                         }
1028                         cookie = tx->tx_submit(tx);
1029                         if (dma_submit_error(cookie)) {
1030                                 ret = -ENOMEM;
1031                                 goto err;
1032                         }
1033                         dma_async_issue_pending(chan);
1034                 }
1035                 if (ret < 0)
1036                         goto err;
1037                 offset += loop_len;
1038                 temp += loop_len;
1039                 temp_phys += loop_len;
1040                 remaining_len -= loop_len;
1041                 offset_in_ca = 0;
1042         }
1043         if (tail_len) {
1044                 if (offset == end_offset) {
1045                         window = list_next_entry(window, list);
1046                         end_offset = window->offset +
1047                                 (window->nr_pages << PAGE_SHIFT);
1048                 }
1049                 window_virt_addr = ioremap_remote(offset, window, tail_len,
1050                                                   work->remote_dev,
1051                                                   NULL);
1052                 if (!window_virt_addr)
1053                         return -ENOMEM;
1054                 /*
1055                  * The CPU copy for the tail bytes must be initiated only once
1056                  * previous DMA transfers for this endpoint have completed
1057                  * to guarantee ordering.
1058                  */
1059                 if (work->ordered) {
1060                         struct scif_dev *rdev = work->remote_dev;
1061
1062                         ret = scif_drain_dma_intr(rdev->sdev, chan);
1063                         if (ret)
1064                                 return ret;
1065                 }
1066                 if (src_local)
1067                         scif_unaligned_cpy_toio(window_virt_addr, temp,
1068                                                 tail_len, work->ordered);
1069                 else
1070                         scif_unaligned_cpy_fromio(temp, window_virt_addr,
1071                                                   tail_len, work->ordered);
1072                 iounmap_remote(window_virt_addr, tail_len, work);
1073         }
1074         tx = dev->device_prep_dma_memcpy(chan, 0, 0, 0, DMA_PREP_INTERRUPT);
1075         if (!tx) {
1076                 ret = -ENOMEM;
1077                 return ret;
1078         }
1079         tx->callback = &scif_rma_completion_cb;
1080         tx->callback_param = comp_cb;
1081         cookie = tx->tx_submit(tx);
1082
1083         if (dma_submit_error(cookie)) {
1084                 ret = -ENOMEM;
1085                 return ret;
1086         }
1087         dma_async_issue_pending(chan);
1088         return 0;
1089 err:
1090         dev_err(scif_info.mdev.this_device,
1091                 "%s %d Desc Prog Failed ret %d\n",
1092                 __func__, __LINE__, ret);
1093         return ret;
1094 }
1095
1096 /*
1097  * _scif_rma_list_dma_copy_aligned:
1098  *
1099  * Traverse all the windows and perform DMA copy.
1100  */
1101 static int _scif_rma_list_dma_copy_aligned(struct scif_copy_work *work,
1102                                            struct dma_chan *chan)
1103 {
1104         dma_addr_t src_dma_addr, dst_dma_addr;
1105         size_t loop_len, remaining_len, src_contig_bytes = 0;
1106         size_t dst_contig_bytes = 0;
1107         struct scif_window_iter src_win_iter;
1108         struct scif_window_iter dst_win_iter;
1109         s64 end_src_offset, end_dst_offset;
1110         struct scif_window *src_window = work->src_window;
1111         struct scif_window *dst_window = work->dst_window;
1112         s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1113         int ret = 0;
1114         struct dma_async_tx_descriptor *tx;
1115         struct dma_device *dev = chan->device;
1116         dma_cookie_t cookie;
1117
1118         remaining_len = work->len;
1119
1120         scif_init_window_iter(src_window, &src_win_iter);
1121         scif_init_window_iter(dst_window, &dst_win_iter);
1122         end_src_offset = src_window->offset +
1123                 (src_window->nr_pages << PAGE_SHIFT);
1124         end_dst_offset = dst_window->offset +
1125                 (dst_window->nr_pages << PAGE_SHIFT);
1126         while (remaining_len) {
1127                 if (src_offset == end_src_offset) {
1128                         src_window = list_next_entry(src_window, list);
1129                         end_src_offset = src_window->offset +
1130                                 (src_window->nr_pages << PAGE_SHIFT);
1131                         scif_init_window_iter(src_window, &src_win_iter);
1132                 }
1133                 if (dst_offset == end_dst_offset) {
1134                         dst_window = list_next_entry(dst_window, list);
1135                         end_dst_offset = dst_window->offset +
1136                                 (dst_window->nr_pages << PAGE_SHIFT);
1137                         scif_init_window_iter(dst_window, &dst_win_iter);
1138                 }
1139
1140                 /* compute dma addresses for transfer */
1141                 src_dma_addr = scif_off_to_dma_addr(src_window, src_offset,
1142                                                     &src_contig_bytes,
1143                                                     &src_win_iter);
1144                 dst_dma_addr = scif_off_to_dma_addr(dst_window, dst_offset,
1145                                                     &dst_contig_bytes,
1146                                                     &dst_win_iter);
1147                 loop_len = min(src_contig_bytes, dst_contig_bytes);
1148                 loop_len = min(loop_len, remaining_len);
1149                 if (work->ordered && !(remaining_len - loop_len)) {
1150                         /*
1151                          * Break up the last chunk of the transfer into two
1152                          * steps to ensure that the last byte in step 2 is
1153                          * updated last.
1154                          */
1155                         /* Step 1) DMA: Body Length - 1 */
1156                         tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1157                                                          src_dma_addr,
1158                                                          loop_len - 1,
1159                                                          DMA_PREP_FENCE);
1160                         if (!tx) {
1161                                 ret = -ENOMEM;
1162                                 goto err;
1163                         }
1164                         cookie = tx->tx_submit(tx);
1165                         if (dma_submit_error(cookie)) {
1166                                 ret = -ENOMEM;
1167                                 goto err;
1168                         }
1169                         src_offset += (loop_len - 1);
1170                         dst_offset += (loop_len - 1);
1171                         src_dma_addr += (loop_len - 1);
1172                         dst_dma_addr += (loop_len - 1);
1173                         remaining_len -= (loop_len - 1);
1174                         loop_len = remaining_len;
1175
1176                         /* Step 2) DMA: 1 BYTES */
1177                         tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1178                                         src_dma_addr, loop_len, 0);
1179                         if (!tx) {
1180                                 ret = -ENOMEM;
1181                                 goto err;
1182                         }
1183                         cookie = tx->tx_submit(tx);
1184                         if (dma_submit_error(cookie)) {
1185                                 ret = -ENOMEM;
1186                                 goto err;
1187                         }
1188                         dma_async_issue_pending(chan);
1189                 } else {
1190                         tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1191                                         src_dma_addr, loop_len, 0);
1192                         if (!tx) {
1193                                 ret = -ENOMEM;
1194                                 goto err;
1195                         }
1196                         cookie = tx->tx_submit(tx);
1197                         if (dma_submit_error(cookie)) {
1198                                 ret = -ENOMEM;
1199                                 goto err;
1200                         }
1201                 }
1202                 src_offset += loop_len;
1203                 dst_offset += loop_len;
1204                 remaining_len -= loop_len;
1205         }
1206         return ret;
1207 err:
1208         dev_err(scif_info.mdev.this_device,
1209                 "%s %d Desc Prog Failed ret %d\n",
1210                 __func__, __LINE__, ret);
1211         return ret;
1212 }
1213
1214 /*
1215  * scif_rma_list_dma_copy_aligned:
1216  *
1217  * Traverse all the windows and perform DMA copy.
1218  */
1219 static int scif_rma_list_dma_copy_aligned(struct scif_copy_work *work,
1220                                           struct dma_chan *chan)
1221 {
1222         dma_addr_t src_dma_addr, dst_dma_addr;
1223         size_t loop_len, remaining_len, tail_len, src_contig_bytes = 0;
1224         size_t dst_contig_bytes = 0;
1225         int src_cache_off;
1226         s64 end_src_offset, end_dst_offset;
1227         struct scif_window_iter src_win_iter;
1228         struct scif_window_iter dst_win_iter;
1229         void *src_virt, *dst_virt;
1230         struct scif_window *src_window = work->src_window;
1231         struct scif_window *dst_window = work->dst_window;
1232         s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1233         int ret = 0;
1234         struct dma_async_tx_descriptor *tx;
1235         struct dma_device *dev = chan->device;
1236         dma_cookie_t cookie;
1237
1238         remaining_len = work->len;
1239         scif_init_window_iter(src_window, &src_win_iter);
1240         scif_init_window_iter(dst_window, &dst_win_iter);
1241
1242         src_cache_off = src_offset & (L1_CACHE_BYTES - 1);
1243         if (src_cache_off != 0) {
1244                 /* Head */
1245                 loop_len = L1_CACHE_BYTES - src_cache_off;
1246                 loop_len = min(loop_len, remaining_len);
1247                 src_dma_addr = __scif_off_to_dma_addr(src_window, src_offset);
1248                 dst_dma_addr = __scif_off_to_dma_addr(dst_window, dst_offset);
1249                 if (src_window->type == SCIF_WINDOW_SELF)
1250                         src_virt = _get_local_va(src_offset, src_window,
1251                                                  loop_len);
1252                 else
1253                         src_virt = ioremap_remote(src_offset, src_window,
1254                                                   loop_len,
1255                                                   work->remote_dev, NULL);
1256                 if (!src_virt)
1257                         return -ENOMEM;
1258                 if (dst_window->type == SCIF_WINDOW_SELF)
1259                         dst_virt = _get_local_va(dst_offset, dst_window,
1260                                                  loop_len);
1261                 else
1262                         dst_virt = ioremap_remote(dst_offset, dst_window,
1263                                                   loop_len,
1264                                                   work->remote_dev, NULL);
1265                 if (!dst_virt) {
1266                         if (src_window->type != SCIF_WINDOW_SELF)
1267                                 iounmap_remote(src_virt, loop_len, work);
1268                         return -ENOMEM;
1269                 }
1270                 if (src_window->type == SCIF_WINDOW_SELF)
1271                         scif_unaligned_cpy_toio(dst_virt, src_virt, loop_len,
1272                                                 remaining_len == loop_len ?
1273                                                 work->ordered : false);
1274                 else
1275                         scif_unaligned_cpy_fromio(dst_virt, src_virt, loop_len,
1276                                                   remaining_len == loop_len ?
1277                                                   work->ordered : false);
1278                 if (src_window->type != SCIF_WINDOW_SELF)
1279                         iounmap_remote(src_virt, loop_len, work);
1280                 if (dst_window->type != SCIF_WINDOW_SELF)
1281                         iounmap_remote(dst_virt, loop_len, work);
1282                 src_offset += loop_len;
1283                 dst_offset += loop_len;
1284                 remaining_len -= loop_len;
1285         }
1286
1287         end_src_offset = src_window->offset +
1288                 (src_window->nr_pages << PAGE_SHIFT);
1289         end_dst_offset = dst_window->offset +
1290                 (dst_window->nr_pages << PAGE_SHIFT);
1291         tail_len = remaining_len & (L1_CACHE_BYTES - 1);
1292         remaining_len -= tail_len;
1293         while (remaining_len) {
1294                 if (src_offset == end_src_offset) {
1295                         src_window = list_next_entry(src_window, list);
1296                         end_src_offset = src_window->offset +
1297                                 (src_window->nr_pages << PAGE_SHIFT);
1298                         scif_init_window_iter(src_window, &src_win_iter);
1299                 }
1300                 if (dst_offset == end_dst_offset) {
1301                         dst_window = list_next_entry(dst_window, list);
1302                         end_dst_offset = dst_window->offset +
1303                                 (dst_window->nr_pages << PAGE_SHIFT);
1304                         scif_init_window_iter(dst_window, &dst_win_iter);
1305                 }
1306
1307                 /* compute dma addresses for transfer */
1308                 src_dma_addr = scif_off_to_dma_addr(src_window, src_offset,
1309                                                     &src_contig_bytes,
1310                                                     &src_win_iter);
1311                 dst_dma_addr = scif_off_to_dma_addr(dst_window, dst_offset,
1312                                                     &dst_contig_bytes,
1313                                                     &dst_win_iter);
1314                 loop_len = min(src_contig_bytes, dst_contig_bytes);
1315                 loop_len = min(loop_len, remaining_len);
1316                 if (work->ordered && !tail_len &&
1317                     !(remaining_len - loop_len)) {
1318                         /*
1319                          * Break up the last chunk of the transfer into two
1320                          * steps. if there is no tail to gurantee DMA ordering.
1321                          * Passing SCIF_DMA_POLLING inserts a status update
1322                          * descriptor in step 1 which acts as a double sided
1323                          * synchronization fence for the DMA engine to ensure
1324                          * that the last cache line in step 2 is updated last.
1325                          */
1326                         /* Step 1) DMA: Body Length - L1_CACHE_BYTES. */
1327                         tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1328                                                          src_dma_addr,
1329                                                          loop_len -
1330                                                          L1_CACHE_BYTES,
1331                                                          DMA_PREP_FENCE);
1332                         if (!tx) {
1333                                 ret = -ENOMEM;
1334                                 goto err;
1335                         }
1336                         cookie = tx->tx_submit(tx);
1337                         if (dma_submit_error(cookie)) {
1338                                 ret = -ENOMEM;
1339                                 goto err;
1340                         }
1341                         dma_async_issue_pending(chan);
1342                         src_offset += (loop_len - L1_CACHE_BYTES);
1343                         dst_offset += (loop_len - L1_CACHE_BYTES);
1344                         src_dma_addr += (loop_len - L1_CACHE_BYTES);
1345                         dst_dma_addr += (loop_len - L1_CACHE_BYTES);
1346                         remaining_len -= (loop_len - L1_CACHE_BYTES);
1347                         loop_len = remaining_len;
1348
1349                         /* Step 2) DMA: L1_CACHE_BYTES */
1350                         tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1351                                                          src_dma_addr,
1352                                                          loop_len, 0);
1353                         if (!tx) {
1354                                 ret = -ENOMEM;
1355                                 goto err;
1356                         }
1357                         cookie = tx->tx_submit(tx);
1358                         if (dma_submit_error(cookie)) {
1359                                 ret = -ENOMEM;
1360                                 goto err;
1361                         }
1362                         dma_async_issue_pending(chan);
1363                 } else {
1364                         tx = dev->device_prep_dma_memcpy(chan, dst_dma_addr,
1365                                                          src_dma_addr,
1366                                                          loop_len, 0);
1367                         if (!tx) {
1368                                 ret = -ENOMEM;
1369                                 goto err;
1370                         }
1371                         cookie = tx->tx_submit(tx);
1372                         if (dma_submit_error(cookie)) {
1373                                 ret = -ENOMEM;
1374                                 goto err;
1375                         }
1376                         dma_async_issue_pending(chan);
1377                 }
1378                 src_offset += loop_len;
1379                 dst_offset += loop_len;
1380                 remaining_len -= loop_len;
1381         }
1382         remaining_len = tail_len;
1383         if (remaining_len) {
1384                 loop_len = remaining_len;
1385                 if (src_offset == end_src_offset)
1386                         src_window = list_next_entry(src_window, list);
1387                 if (dst_offset == end_dst_offset)
1388                         dst_window = list_next_entry(dst_window, list);
1389
1390                 src_dma_addr = __scif_off_to_dma_addr(src_window, src_offset);
1391                 dst_dma_addr = __scif_off_to_dma_addr(dst_window, dst_offset);
1392                 /*
1393                  * The CPU copy for the tail bytes must be initiated only once
1394                  * previous DMA transfers for this endpoint have completed to
1395                  * guarantee ordering.
1396                  */
1397                 if (work->ordered) {
1398                         struct scif_dev *rdev = work->remote_dev;
1399
1400                         ret = scif_drain_dma_poll(rdev->sdev, chan);
1401                         if (ret)
1402                                 return ret;
1403                 }
1404                 if (src_window->type == SCIF_WINDOW_SELF)
1405                         src_virt = _get_local_va(src_offset, src_window,
1406                                                  loop_len);
1407                 else
1408                         src_virt = ioremap_remote(src_offset, src_window,
1409                                                   loop_len,
1410                                                   work->remote_dev, NULL);
1411                 if (!src_virt)
1412                         return -ENOMEM;
1413
1414                 if (dst_window->type == SCIF_WINDOW_SELF)
1415                         dst_virt = _get_local_va(dst_offset, dst_window,
1416                                                  loop_len);
1417                 else
1418                         dst_virt = ioremap_remote(dst_offset, dst_window,
1419                                                   loop_len,
1420                                                   work->remote_dev, NULL);
1421                 if (!dst_virt) {
1422                         if (src_window->type != SCIF_WINDOW_SELF)
1423                                 iounmap_remote(src_virt, loop_len, work);
1424                         return -ENOMEM;
1425                 }
1426
1427                 if (src_window->type == SCIF_WINDOW_SELF)
1428                         scif_unaligned_cpy_toio(dst_virt, src_virt, loop_len,
1429                                                 work->ordered);
1430                 else
1431                         scif_unaligned_cpy_fromio(dst_virt, src_virt,
1432                                                   loop_len, work->ordered);
1433                 if (src_window->type != SCIF_WINDOW_SELF)
1434                         iounmap_remote(src_virt, loop_len, work);
1435
1436                 if (dst_window->type != SCIF_WINDOW_SELF)
1437                         iounmap_remote(dst_virt, loop_len, work);
1438                 remaining_len -= loop_len;
1439         }
1440         return ret;
1441 err:
1442         dev_err(scif_info.mdev.this_device,
1443                 "%s %d Desc Prog Failed ret %d\n",
1444                 __func__, __LINE__, ret);
1445         return ret;
1446 }
1447
1448 /*
1449  * scif_rma_list_cpu_copy:
1450  *
1451  * Traverse all the windows and perform CPU copy.
1452  */
1453 static int scif_rma_list_cpu_copy(struct scif_copy_work *work)
1454 {
1455         void *src_virt, *dst_virt;
1456         size_t loop_len, remaining_len;
1457         int src_page_off, dst_page_off;
1458         s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1459         struct scif_window *src_window = work->src_window;
1460         struct scif_window *dst_window = work->dst_window;
1461         s64 end_src_offset, end_dst_offset;
1462         int ret = 0;
1463         struct scif_window_iter src_win_iter;
1464         struct scif_window_iter dst_win_iter;
1465
1466         remaining_len = work->len;
1467
1468         scif_init_window_iter(src_window, &src_win_iter);
1469         scif_init_window_iter(dst_window, &dst_win_iter);
1470         while (remaining_len) {
1471                 src_page_off = src_offset & ~PAGE_MASK;
1472                 dst_page_off = dst_offset & ~PAGE_MASK;
1473                 loop_len = min(PAGE_SIZE -
1474                                max(src_page_off, dst_page_off),
1475                                remaining_len);
1476
1477                 if (src_window->type == SCIF_WINDOW_SELF)
1478                         src_virt = _get_local_va(src_offset, src_window,
1479                                                  loop_len);
1480                 else
1481                         src_virt = ioremap_remote(src_offset, src_window,
1482                                                   loop_len,
1483                                                   work->remote_dev,
1484                                                   &src_win_iter);
1485                 if (!src_virt) {
1486                         ret = -ENOMEM;
1487                         goto error;
1488                 }
1489
1490                 if (dst_window->type == SCIF_WINDOW_SELF)
1491                         dst_virt = _get_local_va(dst_offset, dst_window,
1492                                                  loop_len);
1493                 else
1494                         dst_virt = ioremap_remote(dst_offset, dst_window,
1495                                                   loop_len,
1496                                                   work->remote_dev,
1497                                                   &dst_win_iter);
1498                 if (!dst_virt) {
1499                         if (src_window->type == SCIF_WINDOW_PEER)
1500                                 iounmap_remote(src_virt, loop_len, work);
1501                         ret = -ENOMEM;
1502                         goto error;
1503                 }
1504
1505                 if (work->loopback) {
1506                         memcpy(dst_virt, src_virt, loop_len);
1507                 } else {
1508                         if (src_window->type == SCIF_WINDOW_SELF)
1509                                 memcpy_toio((void __iomem __force *)dst_virt,
1510                                             src_virt, loop_len);
1511                         else
1512                                 memcpy_fromio(dst_virt,
1513                                               (void __iomem __force *)src_virt,
1514                                               loop_len);
1515                 }
1516                 if (src_window->type == SCIF_WINDOW_PEER)
1517                         iounmap_remote(src_virt, loop_len, work);
1518
1519                 if (dst_window->type == SCIF_WINDOW_PEER)
1520                         iounmap_remote(dst_virt, loop_len, work);
1521
1522                 src_offset += loop_len;
1523                 dst_offset += loop_len;
1524                 remaining_len -= loop_len;
1525                 if (remaining_len) {
1526                         end_src_offset = src_window->offset +
1527                                 (src_window->nr_pages << PAGE_SHIFT);
1528                         end_dst_offset = dst_window->offset +
1529                                 (dst_window->nr_pages << PAGE_SHIFT);
1530                         if (src_offset == end_src_offset) {
1531                                 src_window = list_next_entry(src_window, list);
1532                                 scif_init_window_iter(src_window,
1533                                                       &src_win_iter);
1534                         }
1535                         if (dst_offset == end_dst_offset) {
1536                                 dst_window = list_next_entry(dst_window, list);
1537                                 scif_init_window_iter(dst_window,
1538                                                       &dst_win_iter);
1539                         }
1540                 }
1541         }
1542 error:
1543         return ret;
1544 }
1545
1546 static int scif_rma_list_dma_copy_wrapper(struct scif_endpt *epd,
1547                                           struct scif_copy_work *work,
1548                                           struct dma_chan *chan, off_t loffset)
1549 {
1550         int src_cache_off, dst_cache_off;
1551         s64 src_offset = work->src_offset, dst_offset = work->dst_offset;
1552         u8 *temp = NULL;
1553         bool src_local = true, dst_local = false;
1554         struct scif_dma_comp_cb *comp_cb;
1555         dma_addr_t src_dma_addr, dst_dma_addr;
1556         int err;
1557
1558         if (is_dma_copy_aligned(chan->device, 1, 1, 1))
1559                 return _scif_rma_list_dma_copy_aligned(work, chan);
1560
1561         src_cache_off = src_offset & (L1_CACHE_BYTES - 1);
1562         dst_cache_off = dst_offset & (L1_CACHE_BYTES - 1);
1563
1564         if (dst_cache_off == src_cache_off)
1565                 return scif_rma_list_dma_copy_aligned(work, chan);
1566
1567         if (work->loopback)
1568                 return scif_rma_list_cpu_copy(work);
1569         src_dma_addr = __scif_off_to_dma_addr(work->src_window, src_offset);
1570         dst_dma_addr = __scif_off_to_dma_addr(work->dst_window, dst_offset);
1571         src_local = work->src_window->type == SCIF_WINDOW_SELF;
1572         dst_local = work->dst_window->type == SCIF_WINDOW_SELF;
1573
1574         dst_local = dst_local;
1575         /* Allocate dma_completion cb */
1576         comp_cb = kzalloc(sizeof(*comp_cb), GFP_KERNEL);
1577         if (!comp_cb)
1578                 goto error;
1579
1580         work->comp_cb = comp_cb;
1581         comp_cb->cb_cookie = comp_cb;
1582         comp_cb->dma_completion_func = &scif_rma_completion_cb;
1583
1584         if (work->len + (L1_CACHE_BYTES << 1) < SCIF_KMEM_UNALIGNED_BUF_SIZE) {
1585                 comp_cb->is_cache = false;
1586                 /* Allocate padding bytes to align to a cache line */
1587                 temp = kmalloc(work->len + (L1_CACHE_BYTES << 1),
1588                                GFP_KERNEL);
1589                 if (!temp)
1590                         goto free_comp_cb;
1591                 comp_cb->temp_buf_to_free = temp;
1592                 /* kmalloc(..) does not guarantee cache line alignment */
1593                 if (!IS_ALIGNED((u64)temp, L1_CACHE_BYTES))
1594                         temp = PTR_ALIGN(temp, L1_CACHE_BYTES);
1595         } else {
1596                 comp_cb->is_cache = true;
1597                 temp = kmem_cache_alloc(unaligned_cache, GFP_KERNEL);
1598                 if (!temp)
1599                         goto free_comp_cb;
1600                 comp_cb->temp_buf_to_free = temp;
1601         }
1602
1603         if (src_local) {
1604                 temp += dst_cache_off;
1605                 scif_rma_local_cpu_copy(work->src_offset, work->src_window,
1606                                         temp, work->len, true);
1607         } else {
1608                 comp_cb->dst_window = work->dst_window;
1609                 comp_cb->dst_offset = work->dst_offset;
1610                 work->src_offset = work->src_offset - src_cache_off;
1611                 comp_cb->len = work->len;
1612                 work->len = ALIGN(work->len + src_cache_off, L1_CACHE_BYTES);
1613                 comp_cb->header_padding = src_cache_off;
1614         }
1615         comp_cb->temp_buf = temp;
1616
1617         err = scif_map_single(&comp_cb->temp_phys, temp,
1618                               work->remote_dev, SCIF_KMEM_UNALIGNED_BUF_SIZE);
1619         if (err)
1620                 goto free_temp_buf;
1621         comp_cb->sdev = work->remote_dev;
1622         if (scif_rma_list_dma_copy_unaligned(work, temp, chan, src_local) < 0)
1623                 goto free_temp_buf;
1624         if (!src_local)
1625                 work->fence_type = SCIF_DMA_INTR;
1626         return 0;
1627 free_temp_buf:
1628         if (comp_cb->is_cache)
1629                 kmem_cache_free(unaligned_cache, comp_cb->temp_buf_to_free);
1630         else
1631                 kfree(comp_cb->temp_buf_to_free);
1632 free_comp_cb:
1633         kfree(comp_cb);
1634 error:
1635         return -ENOMEM;
1636 }
1637
1638 /**
1639  * scif_rma_copy:
1640  * @epd: end point descriptor.
1641  * @loffset: offset in local registered address space to/from which to copy
1642  * @addr: user virtual address to/from which to copy
1643  * @len: length of range to copy
1644  * @roffset: offset in remote registered address space to/from which to copy
1645  * @flags: flags
1646  * @dir: LOCAL->REMOTE or vice versa.
1647  * @last_chunk: true if this is the last chunk of a larger transfer
1648  *
1649  * Validate parameters, check if src/dst registered ranges requested for copy
1650  * are valid and initiate either CPU or DMA copy.
1651  */
1652 static int scif_rma_copy(scif_epd_t epd, off_t loffset, unsigned long addr,
1653                          size_t len, off_t roffset, int flags,
1654                          enum scif_rma_dir dir, bool last_chunk)
1655 {
1656         struct scif_endpt *ep = (struct scif_endpt *)epd;
1657         struct scif_rma_req remote_req;
1658         struct scif_rma_req req;
1659         struct scif_window *local_window = NULL;
1660         struct scif_window *remote_window = NULL;
1661         struct scif_copy_work copy_work;
1662         bool loopback;
1663         int err = 0;
1664         struct dma_chan *chan;
1665         struct scif_mmu_notif *mmn = NULL;
1666         bool cache = false;
1667         struct device *spdev;
1668
1669         err = scif_verify_epd(ep);
1670         if (err)
1671                 return err;
1672
1673         if (flags && !(flags & (SCIF_RMA_USECPU | SCIF_RMA_USECACHE |
1674                                 SCIF_RMA_SYNC | SCIF_RMA_ORDERED)))
1675                 return -EINVAL;
1676
1677         loopback = scifdev_self(ep->remote_dev) ? true : false;
1678         copy_work.fence_type = ((flags & SCIF_RMA_SYNC) && last_chunk) ?
1679                                 SCIF_DMA_POLL : 0;
1680         copy_work.ordered = !!((flags & SCIF_RMA_ORDERED) && last_chunk);
1681
1682         /* Use CPU for Mgmt node <-> Mgmt node copies */
1683         if (loopback && scif_is_mgmt_node()) {
1684                 flags |= SCIF_RMA_USECPU;
1685                 copy_work.fence_type = 0x0;
1686         }
1687
1688         cache = scif_is_set_reg_cache(flags);
1689
1690         remote_req.out_window = &remote_window;
1691         remote_req.offset = roffset;
1692         remote_req.nr_bytes = len;
1693         /*
1694          * If transfer is from local to remote then the remote window
1695          * must be writeable and vice versa.
1696          */
1697         remote_req.prot = dir == SCIF_LOCAL_TO_REMOTE ? VM_WRITE : VM_READ;
1698         remote_req.type = SCIF_WINDOW_PARTIAL;
1699         remote_req.head = &ep->rma_info.remote_reg_list;
1700
1701         spdev = scif_get_peer_dev(ep->remote_dev);
1702         if (IS_ERR(spdev)) {
1703                 err = PTR_ERR(spdev);
1704                 return err;
1705         }
1706
1707         if (addr && cache) {
1708                 mutex_lock(&ep->rma_info.mmn_lock);
1709                 mmn = scif_find_mmu_notifier(current->mm, &ep->rma_info);
1710                 if (!mmn)
1711                         mmn = scif_add_mmu_notifier(current->mm, ep);
1712                 mutex_unlock(&ep->rma_info.mmn_lock);
1713                 if (IS_ERR(mmn)) {
1714                         scif_put_peer_dev(spdev);
1715                         return PTR_ERR(mmn);
1716                 }
1717                 cache = cache && !scif_rma_tc_can_cache(ep, len);
1718         }
1719         mutex_lock(&ep->rma_info.rma_lock);
1720         if (addr) {
1721                 req.out_window = &local_window;
1722                 req.nr_bytes = ALIGN(len + (addr & ~PAGE_MASK),
1723                                      PAGE_SIZE);
1724                 req.va_for_temp = addr & PAGE_MASK;
1725                 req.prot = (dir == SCIF_LOCAL_TO_REMOTE ?
1726                             VM_READ : VM_WRITE | VM_READ);
1727                 /* Does a valid local window exist? */
1728                 if (mmn) {
1729                         spin_lock(&ep->rma_info.tc_lock);
1730                         req.head = &mmn->tc_reg_list;
1731                         err = scif_query_tcw(ep, &req);
1732                         spin_unlock(&ep->rma_info.tc_lock);
1733                 }
1734                 if (!mmn || err) {
1735                         err = scif_register_temp(epd, req.va_for_temp,
1736                                                  req.nr_bytes, req.prot,
1737                                                  &loffset, &local_window);
1738                         if (err) {
1739                                 mutex_unlock(&ep->rma_info.rma_lock);
1740                                 goto error;
1741                         }
1742                         if (!cache)
1743                                 goto skip_cache;
1744                         atomic_inc(&ep->rma_info.tcw_refcount);
1745                         atomic_add_return(local_window->nr_pages,
1746                                           &ep->rma_info.tcw_total_pages);
1747                         if (mmn) {
1748                                 spin_lock(&ep->rma_info.tc_lock);
1749                                 scif_insert_tcw(local_window,
1750                                                 &mmn->tc_reg_list);
1751                                 spin_unlock(&ep->rma_info.tc_lock);
1752                         }
1753                 }
1754 skip_cache:
1755                 loffset = local_window->offset +
1756                                 (addr - local_window->va_for_temp);
1757         } else {
1758                 req.out_window = &local_window;
1759                 req.offset = loffset;
1760                 /*
1761                  * If transfer is from local to remote then the self window
1762                  * must be readable and vice versa.
1763                  */
1764                 req.prot = dir == SCIF_LOCAL_TO_REMOTE ? VM_READ : VM_WRITE;
1765                 req.nr_bytes = len;
1766                 req.type = SCIF_WINDOW_PARTIAL;
1767                 req.head = &ep->rma_info.reg_list;
1768                 /* Does a valid local window exist? */
1769                 err = scif_query_window(&req);
1770                 if (err) {
1771                         mutex_unlock(&ep->rma_info.rma_lock);
1772                         goto error;
1773                 }
1774         }
1775
1776         /* Does a valid remote window exist? */
1777         err = scif_query_window(&remote_req);
1778         if (err) {
1779                 mutex_unlock(&ep->rma_info.rma_lock);
1780                 goto error;
1781         }
1782
1783         /*
1784          * Prepare copy_work for submitting work to the DMA kernel thread
1785          * or CPU copy routine.
1786          */
1787         copy_work.len = len;
1788         copy_work.loopback = loopback;
1789         copy_work.remote_dev = ep->remote_dev;
1790         if (dir == SCIF_LOCAL_TO_REMOTE) {
1791                 copy_work.src_offset = loffset;
1792                 copy_work.src_window = local_window;
1793                 copy_work.dst_offset = roffset;
1794                 copy_work.dst_window = remote_window;
1795         } else {
1796                 copy_work.src_offset = roffset;
1797                 copy_work.src_window = remote_window;
1798                 copy_work.dst_offset = loffset;
1799                 copy_work.dst_window = local_window;
1800         }
1801
1802         if (flags & SCIF_RMA_USECPU) {
1803                 scif_rma_list_cpu_copy(&copy_work);
1804         } else {
1805                 chan = ep->rma_info.dma_chan;
1806                 err = scif_rma_list_dma_copy_wrapper(epd, &copy_work,
1807                                                      chan, loffset);
1808         }
1809         if (addr && !cache)
1810                 atomic_inc(&ep->rma_info.tw_refcount);
1811
1812         mutex_unlock(&ep->rma_info.rma_lock);
1813
1814         if (last_chunk) {
1815                 struct scif_dev *rdev = ep->remote_dev;
1816
1817                 if (copy_work.fence_type == SCIF_DMA_POLL)
1818                         err = scif_drain_dma_poll(rdev->sdev,
1819                                                   ep->rma_info.dma_chan);
1820                 else if (copy_work.fence_type == SCIF_DMA_INTR)
1821                         err = scif_drain_dma_intr(rdev->sdev,
1822                                                   ep->rma_info.dma_chan);
1823         }
1824
1825         if (addr && !cache)
1826                 scif_queue_for_cleanup(local_window, &scif_info.rma);
1827         scif_put_peer_dev(spdev);
1828         return err;
1829 error:
1830         if (err) {
1831                 if (addr && local_window && !cache)
1832                         scif_destroy_window(ep, local_window);
1833                 dev_err(scif_info.mdev.this_device,
1834                         "%s %d err %d len 0x%lx\n",
1835                         __func__, __LINE__, err, len);
1836         }
1837         scif_put_peer_dev(spdev);
1838         return err;
1839 }
1840
1841 int scif_readfrom(scif_epd_t epd, off_t loffset, size_t len,
1842                   off_t roffset, int flags)
1843 {
1844         int err;
1845
1846         dev_dbg(scif_info.mdev.this_device,
1847                 "SCIFAPI readfrom: ep %p loffset 0x%lx len 0x%lx offset 0x%lx flags 0x%x\n",
1848                 epd, loffset, len, roffset, flags);
1849         if (scif_unaligned(loffset, roffset)) {
1850                 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1851                         err = scif_rma_copy(epd, loffset, 0x0,
1852                                             SCIF_MAX_UNALIGNED_BUF_SIZE,
1853                                             roffset, flags,
1854                                             SCIF_REMOTE_TO_LOCAL, false);
1855                         if (err)
1856                                 goto readfrom_err;
1857                         loffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1858                         roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1859                         len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1860                 }
1861         }
1862         err = scif_rma_copy(epd, loffset, 0x0, len,
1863                             roffset, flags, SCIF_REMOTE_TO_LOCAL, true);
1864 readfrom_err:
1865         return err;
1866 }
1867 EXPORT_SYMBOL_GPL(scif_readfrom);
1868
1869 int scif_writeto(scif_epd_t epd, off_t loffset, size_t len,
1870                  off_t roffset, int flags)
1871 {
1872         int err;
1873
1874         dev_dbg(scif_info.mdev.this_device,
1875                 "SCIFAPI writeto: ep %p loffset 0x%lx len 0x%lx roffset 0x%lx flags 0x%x\n",
1876                 epd, loffset, len, roffset, flags);
1877         if (scif_unaligned(loffset, roffset)) {
1878                 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1879                         err = scif_rma_copy(epd, loffset, 0x0,
1880                                             SCIF_MAX_UNALIGNED_BUF_SIZE,
1881                                             roffset, flags,
1882                                             SCIF_LOCAL_TO_REMOTE, false);
1883                         if (err)
1884                                 goto writeto_err;
1885                         loffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1886                         roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1887                         len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1888                 }
1889         }
1890         err = scif_rma_copy(epd, loffset, 0x0, len,
1891                             roffset, flags, SCIF_LOCAL_TO_REMOTE, true);
1892 writeto_err:
1893         return err;
1894 }
1895 EXPORT_SYMBOL_GPL(scif_writeto);
1896
1897 int scif_vreadfrom(scif_epd_t epd, void *addr, size_t len,
1898                    off_t roffset, int flags)
1899 {
1900         int err;
1901
1902         dev_dbg(scif_info.mdev.this_device,
1903                 "SCIFAPI vreadfrom: ep %p addr %p len 0x%lx roffset 0x%lx flags 0x%x\n",
1904                 epd, addr, len, roffset, flags);
1905         if (scif_unaligned((off_t __force)addr, roffset)) {
1906                 if (len > SCIF_MAX_UNALIGNED_BUF_SIZE)
1907                         flags &= ~SCIF_RMA_USECACHE;
1908
1909                 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1910                         err = scif_rma_copy(epd, 0, (u64)addr,
1911                                             SCIF_MAX_UNALIGNED_BUF_SIZE,
1912                                             roffset, flags,
1913                                             SCIF_REMOTE_TO_LOCAL, false);
1914                         if (err)
1915                                 goto vreadfrom_err;
1916                         addr += SCIF_MAX_UNALIGNED_BUF_SIZE;
1917                         roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1918                         len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1919                 }
1920         }
1921         err = scif_rma_copy(epd, 0, (u64)addr, len,
1922                             roffset, flags, SCIF_REMOTE_TO_LOCAL, true);
1923 vreadfrom_err:
1924         return err;
1925 }
1926 EXPORT_SYMBOL_GPL(scif_vreadfrom);
1927
1928 int scif_vwriteto(scif_epd_t epd, void *addr, size_t len,
1929                   off_t roffset, int flags)
1930 {
1931         int err;
1932
1933         dev_dbg(scif_info.mdev.this_device,
1934                 "SCIFAPI vwriteto: ep %p addr %p len 0x%lx roffset 0x%lx flags 0x%x\n",
1935                 epd, addr, len, roffset, flags);
1936         if (scif_unaligned((off_t __force)addr, roffset)) {
1937                 if (len > SCIF_MAX_UNALIGNED_BUF_SIZE)
1938                         flags &= ~SCIF_RMA_USECACHE;
1939
1940                 while (len > SCIF_MAX_UNALIGNED_BUF_SIZE) {
1941                         err = scif_rma_copy(epd, 0, (u64)addr,
1942                                             SCIF_MAX_UNALIGNED_BUF_SIZE,
1943                                             roffset, flags,
1944                                             SCIF_LOCAL_TO_REMOTE, false);
1945                         if (err)
1946                                 goto vwriteto_err;
1947                         addr += SCIF_MAX_UNALIGNED_BUF_SIZE;
1948                         roffset += SCIF_MAX_UNALIGNED_BUF_SIZE;
1949                         len -= SCIF_MAX_UNALIGNED_BUF_SIZE;
1950                 }
1951         }
1952         err = scif_rma_copy(epd, 0, (u64)addr, len,
1953                             roffset, flags, SCIF_LOCAL_TO_REMOTE, true);
1954 vwriteto_err:
1955         return err;
1956 }
1957 EXPORT_SYMBOL_GPL(scif_vwriteto);