drivers/ntb/test/ntb_perf.c

   1 /*
   2  * This file is provided under a dual BSD/GPLv2 license.  When using or
   3  *   redistributing this file, you may do so under either license.
   4  *
   5  *   GPL LICENSE SUMMARY
   6  *
   7  *   Copyright(c) 2015 Intel Corporation. All rights reserved.
   8  *   Copyright(c) 2017 T-Platforms. All Rights Reserved.
   9  *
  10  *   This program is free software; you can redistribute it and/or modify
  11  *   it under the terms of version 2 of the GNU General Public License as
  12  *   published by the Free Software Foundation.
  13  *
  14  *   BSD LICENSE
  15  *
  16  *   Copyright(c) 2015 Intel Corporation. All rights reserved.
  17  *   Copyright(c) 2017 T-Platforms. All Rights Reserved.
  18  *
  19  *   Redistribution and use in source and binary forms, with or without
  20  *   modification, are permitted provided that the following conditions
  21  *   are met:
  22  *
  23  *     * Redistributions of source code must retain the above copyright
  24  *       notice, this list of conditions and the following disclaimer.
  25  *     * Redistributions in binary form must reproduce the above copy
  26  *       notice, this list of conditions and the following disclaimer in
  27  *       the documentation and/or other materials provided with the
  28  *       distribution.
  29  *     * Neither the name of Intel Corporation nor the names of its
  30  *       contributors may be used to endorse or promote products derived
  31  *       from this software without specific prior written permission.
  32  *
  33  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  34  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  35  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  36  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  37  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  38  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  39  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  40  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  41  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  42  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  43  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  44  *
  45  * PCIe NTB Perf Linux driver
  46  */
  47
  48 /*
  49  * How to use this tool, by example.
  50  *
  51  * Assuming $DBG_DIR is something like:
  52  * '/sys/kernel/debug/ntb_perf/0000:00:03.0'
  53  * Suppose aside from local device there is at least one remote device
  54  * connected to NTB with index 0.
  55  *-----------------------------------------------------------------------------
  56  * Eg: install driver with specified chunk/total orders and dma-enabled flag
  57  *
  58  * root@self# insmod ntb_perf.ko chunk_order=19 total_order=28 use_dma
  59  *-----------------------------------------------------------------------------
  60  * Eg: check NTB ports (index) and MW mapping information
  61  *
  62  * root@self# cat $DBG_DIR/info
  63  *-----------------------------------------------------------------------------
  64  * Eg: start performance test with peer (index 0) and get the test metrics
  65  *
  66  * root@self# echo 0 > $DBG_DIR/run
  67  * root@self# cat $DBG_DIR/run
  68  */
  69
  70 #include <linux/init.h>
  71 #include <linux/kernel.h>
  72 #include <linux/module.h>
  73 #include <linux/sched.h>
  74 #include <linux/wait.h>
  75 #include <linux/dma-mapping.h>
  76 #include <linux/dmaengine.h>
  77 #include <linux/pci.h>
  78 #include <linux/ktime.h>
  79 #include <linux/slab.h>
  80 #include <linux/delay.h>
  81 #include <linux/sizes.h>
  82 #include <linux/workqueue.h>
  83 #include <linux/debugfs.h>
  84 #include <linux/random.h>
  85 #include <linux/ntb.h>
  86
  87 #define DRIVER_NAME             "ntb_perf"
  88 #define DRIVER_VERSION          "2.0"
  89
  90 MODULE_LICENSE("Dual BSD/GPL");
  91 MODULE_VERSION(DRIVER_VERSION);
  92 MODULE_AUTHOR("Dave Jiang <dave.jiang@intel.com>");
  93 MODULE_DESCRIPTION("PCIe NTB Performance Measurement Tool");
  94
  95 #define MAX_THREADS_CNT         32
  96 #define DEF_THREADS_CNT         1
  97 #define MAX_CHUNK_SIZE          SZ_1M
  98 #define MAX_CHUNK_ORDER         20 /* no larger than 1M */
  99
 100 #define DMA_TRIES               100
 101 #define DMA_MDELAY              10
 102
 103 #define MSG_TRIES               500
 104 #define MSG_UDELAY_LOW          1000
 105 #define MSG_UDELAY_HIGH         2000
 106
 107 #define PERF_BUF_LEN 1024
 108
 109 static unsigned long max_mw_size;
 110 module_param(max_mw_size, ulong, 0644);
 111 MODULE_PARM_DESC(max_mw_size, "Upper limit of memory window size");
 112
 113 static unsigned char chunk_order = 19; /* 512K */
 114 module_param(chunk_order, byte, 0644);
 115 MODULE_PARM_DESC(chunk_order, "Data chunk order [2^n] to transfer");
 116
 117 static unsigned char total_order = 30; /* 1G */
 118 module_param(total_order, byte, 0644);
 119 MODULE_PARM_DESC(total_order, "Total data order [2^n] to transfer");
 120
 121 static bool use_dma; /* default to 0 */
 122 module_param(use_dma, bool, 0644);
 123 MODULE_PARM_DESC(use_dma, "Use DMA engine to measure performance");
 124
 125 /*==============================================================================
 126  *                         Perf driver data definition
 127  *==============================================================================
 128  */
 129
 130 enum perf_cmd {
 131         PERF_CMD_INVAL = -1,/* invalid spad command */
 132         PERF_CMD_SSIZE = 0, /* send out buffer size */
 133         PERF_CMD_RSIZE = 1, /* recv in  buffer size */
 134         PERF_CMD_SXLAT = 2, /* send in  buffer xlat */
 135         PERF_CMD_RXLAT = 3, /* recv out buffer xlat */
 136         PERF_CMD_CLEAR = 4, /* clear allocated memory */
 137         PERF_STS_DONE  = 5, /* init is done */
 138         PERF_STS_LNKUP = 6, /* link up state flag */
 139 };
 140
 141 struct perf_ctx;
 142
 143 struct perf_peer {
 144         struct perf_ctx *perf;
 145         int pidx;
 146         int gidx;
 147
 148         /* Outbound MW params */
 149         u64 outbuf_xlat;
 150         resource_size_t outbuf_size;
 151         void __iomem *outbuf;
 152
 153         /* Inbound MW params */
 154         dma_addr_t inbuf_xlat;
 155         resource_size_t inbuf_size;
 156         void            *inbuf;
 157
 158         /* NTB connection setup service */
 159         struct work_struct      service;
 160         unsigned long           sts;
 161
 162         struct completion init_comp;
 163 };
 164 #define to_peer_service(__work) \
 165         container_of(__work, struct perf_peer, service)
 166
 167 struct perf_thread {
 168         struct perf_ctx *perf;
 169         int tidx;
 170
 171         /* DMA-based test sync parameters */
 172         atomic_t dma_sync;
 173         wait_queue_head_t dma_wait;
 174         struct dma_chan *dma_chan;
 175
 176         /* Data source and measured statistics */
 177         void *src;
 178         u64 copied;
 179         ktime_t duration;
 180         int status;
 181         struct work_struct work;
 182 };
 183 #define to_thread_work(__work) \
 184         container_of(__work, struct perf_thread, work)
 185
 186 struct perf_ctx {
 187         struct ntb_dev *ntb;
 188
 189         /* Global device index and peers descriptors */
 190         int gidx;
 191         int pcnt;
 192         struct perf_peer *peers;
 193
 194         /* Performance measuring work-threads interface */
 195         unsigned long busy_flag;
 196         wait_queue_head_t twait;
 197         atomic_t tsync;
 198         u8 tcnt;
 199         struct perf_peer *test_peer;
 200         struct perf_thread threads[MAX_THREADS_CNT];
 201
 202         /* Scratchpad/Message IO operations */
 203         int (*cmd_send)(struct perf_peer *peer, enum perf_cmd cmd, u64 data);
 204         int (*cmd_recv)(struct perf_ctx *perf, int *pidx, enum perf_cmd *cmd,
 205                         u64 *data);
 206
 207         struct dentry *dbgfs_dir;
 208 };
 209
 210 /*
 211  * Scratchpads-base commands interface
 212  */
 213 #define PERF_SPAD_CNT(_pcnt) \
 214         (3*((_pcnt) + 1))
 215 #define PERF_SPAD_CMD(_gidx) \
 216         (3*(_gidx))
 217 #define PERF_SPAD_LDATA(_gidx) \
 218         (3*(_gidx) + 1)
 219 #define PERF_SPAD_HDATA(_gidx) \
 220         (3*(_gidx) + 2)
 221 #define PERF_SPAD_NOTIFY(_gidx) \
 222         (BIT_ULL(_gidx))
 223
 224 /*
 225  * Messages-base commands interface
 226  */
 227 #define PERF_MSG_CNT            3
 228 #define PERF_MSG_CMD            0
 229 #define PERF_MSG_LDATA          1
 230 #define PERF_MSG_HDATA          2
 231
 232 /*==============================================================================
 233  *                           Static data declarations
 234  *==============================================================================
 235  */
 236
 237 static struct dentry *perf_dbgfs_topdir;
 238
 239 static struct workqueue_struct *perf_wq __read_mostly;
 240
 241 /*==============================================================================
 242  *                  NTB cross-link commands execution service
 243  *==============================================================================
 244  */
 245
 246 static void perf_terminate_test(struct perf_ctx *perf);
 247
 248 static inline bool perf_link_is_up(struct perf_peer *peer)
 249 {
 250         u64 link;
 251
 252         link = ntb_link_is_up(peer->perf->ntb, NULL, NULL);
 253         return !!(link & BIT_ULL_MASK(peer->pidx));
 254 }
 255
 256 static int perf_spad_cmd_send(struct perf_peer *peer, enum perf_cmd cmd,
 257                               u64 data)
 258 {
 259         struct perf_ctx *perf = peer->perf;
 260         int try;
 261         u32 sts;
 262
 263         dev_dbg(&perf->ntb->dev, "CMD send: %d 0x%llx\n", cmd, data);
 264
 265         /*
 266          * Perform predefined number of attempts before give up.
 267          * We are sending the data to the port specific scratchpad, so
 268          * to prevent a multi-port access race-condition. Additionally
 269          * there is no need in local locking since only thread-safe
 270          * service work is using this method.
 271          */
 272         for (try = 0; try < MSG_TRIES; try++) {
 273                 if (!perf_link_is_up(peer))
 274                         return -ENOLINK;
 275
 276                 sts = ntb_peer_spad_read(perf->ntb, peer->pidx,
 277                                          PERF_SPAD_CMD(perf->gidx));
 278                 if (sts != PERF_CMD_INVAL) {
 279                         usleep_range(MSG_UDELAY_LOW, MSG_UDELAY_HIGH);
 280                         continue;
 281                 }
 282
 283                 ntb_peer_spad_write(perf->ntb, peer->pidx,
 284                                     PERF_SPAD_LDATA(perf->gidx),
 285                                     lower_32_bits(data));
 286                 ntb_peer_spad_write(perf->ntb, peer->pidx,
 287                                     PERF_SPAD_HDATA(perf->gidx),
 288                                     upper_32_bits(data));
 289                 mmiowb();
 290                 ntb_peer_spad_write(perf->ntb, peer->pidx,
 291                                     PERF_SPAD_CMD(perf->gidx),
 292                                     cmd);
 293                 mmiowb();
 294                 ntb_peer_db_set(perf->ntb, PERF_SPAD_NOTIFY(peer->gidx));
 295
 296                 dev_dbg(&perf->ntb->dev, "DB ring peer %#llx\n",
 297                         PERF_SPAD_NOTIFY(peer->gidx));
 298
 299                 break;
 300         }
 301
 302         return try < MSG_TRIES ? 0 : -EAGAIN;
 303 }
 304
 305 static int perf_spad_cmd_recv(struct perf_ctx *perf, int *pidx,
 306                               enum perf_cmd *cmd, u64 *data)
 307 {
 308         struct perf_peer *peer;
 309         u32 val;
 310
 311         ntb_db_clear(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx));
 312
 313         /*
 314          * We start scanning all over, since cleared DB may have been set
 315          * by any peer. Yes, it makes peer with smaller index being
 316          * serviced with greater priority, but it's convenient for spad
 317          * and message code unification and simplicity.
 318          */
 319         for (*pidx = 0; *pidx < perf->pcnt; (*pidx)++) {
 320                 peer = &perf->peers[*pidx];
 321
 322                 if (!perf_link_is_up(peer))
 323                         continue;
 324
 325                 val = ntb_spad_read(perf->ntb, PERF_SPAD_CMD(peer->gidx));
 326                 if (val == PERF_CMD_INVAL)
 327                         continue;
 328
 329                 *cmd = val;
 330
 331                 val = ntb_spad_read(perf->ntb, PERF_SPAD_LDATA(peer->gidx));
 332                 *data = val;
 333
 334                 val = ntb_spad_read(perf->ntb, PERF_SPAD_HDATA(peer->gidx));
 335                 *data |= (u64)val << 32;
 336
 337                 /* Next command can be retrieved from now */
 338                 ntb_spad_write(perf->ntb, PERF_SPAD_CMD(peer->gidx),
 339                                PERF_CMD_INVAL);
 340
 341                 dev_dbg(&perf->ntb->dev, "CMD recv: %d 0x%llx\n", *cmd, *data);
 342
 343                 return 0;
 344         }
 345
 346         return -ENODATA;
 347 }
 348
 349 static int perf_msg_cmd_send(struct perf_peer *peer, enum perf_cmd cmd,
 350                              u64 data)
 351 {
 352         struct perf_ctx *perf = peer->perf;
 353         int try, ret;
 354         u64 outbits;
 355
 356         dev_dbg(&perf->ntb->dev, "CMD send: %d 0x%llx\n", cmd, data);
 357
 358         /*
 359          * Perform predefined number of attempts before give up. Message
 360          * registers are free of race-condition problem when accessed
 361          * from different ports, so we don't need splitting registers
 362          * by global device index. We also won't have local locking,
 363          * since the method is used from service work only.
 364          */
 365         outbits = ntb_msg_outbits(perf->ntb);
 366         for (try = 0; try < MSG_TRIES; try++) {
 367                 if (!perf_link_is_up(peer))
 368                         return -ENOLINK;
 369
 370                 ret = ntb_msg_clear_sts(perf->ntb, outbits);
 371                 if (ret)
 372                         return ret;
 373
 374                 ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_LDATA,
 375                                    lower_32_bits(data));
 376
 377                 if (ntb_msg_read_sts(perf->ntb) & outbits) {
 378                         usleep_range(MSG_UDELAY_LOW, MSG_UDELAY_HIGH);
 379                         continue;
 380                 }
 381
 382                 ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_HDATA,
 383                                    upper_32_bits(data));
 384                 mmiowb();
 385
 386                 /* This call shall trigger peer message event */
 387                 ntb_peer_msg_write(perf->ntb, peer->pidx, PERF_MSG_CMD, cmd);
 388
 389                 break;
 390         }
 391
 392         return try < MSG_TRIES ? 0 : -EAGAIN;
 393 }
 394
 395 static int perf_msg_cmd_recv(struct perf_ctx *perf, int *pidx,
 396                              enum perf_cmd *cmd, u64 *data)
 397 {
 398         u64 inbits;
 399         u32 val;
 400
 401         inbits = ntb_msg_inbits(perf->ntb);
 402
 403         if (hweight64(ntb_msg_read_sts(perf->ntb) & inbits) < 3)
 404                 return -ENODATA;
 405
 406         val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_CMD);
 407         *cmd = val;
 408
 409         val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_LDATA);
 410         *data = val;
 411
 412         val = ntb_msg_read(perf->ntb, pidx, PERF_MSG_HDATA);
 413         *data |= (u64)val << 32;
 414
 415         /* Next command can be retrieved from now */
 416         ntb_msg_clear_sts(perf->ntb, inbits);
 417
 418         dev_dbg(&perf->ntb->dev, "CMD recv: %d 0x%llx\n", *cmd, *data);
 419
 420         return 0;
 421 }
 422
 423 static int perf_cmd_send(struct perf_peer *peer, enum perf_cmd cmd, u64 data)
 424 {
 425         struct perf_ctx *perf = peer->perf;
 426
 427         if (cmd == PERF_CMD_SSIZE || cmd == PERF_CMD_SXLAT)
 428                 return perf->cmd_send(peer, cmd, data);
 429
 430         dev_err(&perf->ntb->dev, "Send invalid command\n");
 431         return -EINVAL;
 432 }
 433
 434 static int perf_cmd_exec(struct perf_peer *peer, enum perf_cmd cmd)
 435 {
 436         switch (cmd) {
 437         case PERF_CMD_SSIZE:
 438         case PERF_CMD_RSIZE:
 439         case PERF_CMD_SXLAT:
 440         case PERF_CMD_RXLAT:
 441         case PERF_CMD_CLEAR:
 442                 break;
 443         default:
 444                 dev_err(&peer->perf->ntb->dev, "Exec invalid command\n");
 445                 return -EINVAL;
 446         }
 447
 448         /* No need of memory barrier, since bit ops have invernal lock */
 449         set_bit(cmd, &peer->sts);
 450
 451         dev_dbg(&peer->perf->ntb->dev, "CMD exec: %d\n", cmd);
 452
 453         (void)queue_work(system_highpri_wq, &peer->service);
 454
 455         return 0;
 456 }
 457
 458 static int perf_cmd_recv(struct perf_ctx *perf)
 459 {
 460         struct perf_peer *peer;
 461         int ret, pidx, cmd;
 462         u64 data;
 463
 464         while (!(ret = perf->cmd_recv(perf, &pidx, &cmd, &data))) {
 465                 peer = &perf->peers[pidx];
 466
 467                 switch (cmd) {
 468                 case PERF_CMD_SSIZE:
 469                         peer->inbuf_size = data;
 470                         return perf_cmd_exec(peer, PERF_CMD_RSIZE);
 471                 case PERF_CMD_SXLAT:
 472                         peer->outbuf_xlat = data;
 473                         return perf_cmd_exec(peer, PERF_CMD_RXLAT);
 474                 default:
 475                         dev_err(&perf->ntb->dev, "Recv invalid command\n");
 476                         return -EINVAL;
 477                 }
 478         }
 479
 480         /* Return 0 if no data left to process, otherwise an error */
 481         return ret == -ENODATA ? 0 : ret;
 482 }
 483
 484 static void perf_link_event(void *ctx)
 485 {
 486         struct perf_ctx *perf = ctx;
 487         struct perf_peer *peer;
 488         bool lnk_up;
 489         int pidx;
 490
 491         for (pidx = 0; pidx < perf->pcnt; pidx++) {
 492                 peer = &perf->peers[pidx];
 493
 494                 lnk_up = perf_link_is_up(peer);
 495
 496                 if (lnk_up &&
 497                     !test_and_set_bit(PERF_STS_LNKUP, &peer->sts)) {
 498                         perf_cmd_exec(peer, PERF_CMD_SSIZE);
 499                 } else if (!lnk_up &&
 500                            test_and_clear_bit(PERF_STS_LNKUP, &peer->sts)) {
 501                         perf_cmd_exec(peer, PERF_CMD_CLEAR);
 502                 }
 503         }
 504 }
 505
 506 static void perf_db_event(void *ctx, int vec)
 507 {
 508         struct perf_ctx *perf = ctx;
 509
 510         dev_dbg(&perf->ntb->dev, "DB vec %d mask %#llx bits %#llx\n", vec,
 511                 ntb_db_vector_mask(perf->ntb, vec), ntb_db_read(perf->ntb));
 512
 513         /* Just receive all available commands */
 514         (void)perf_cmd_recv(perf);
 515 }
 516
 517 static void perf_msg_event(void *ctx)
 518 {
 519         struct perf_ctx *perf = ctx;
 520
 521         dev_dbg(&perf->ntb->dev, "Msg status bits %#llx\n",
 522                 ntb_msg_read_sts(perf->ntb));
 523
 524         /* Messages are only sent one-by-one */
 525         (void)perf_cmd_recv(perf);
 526 }
 527
 528 static const struct ntb_ctx_ops perf_ops = {
 529         .link_event = perf_link_event,
 530         .db_event = perf_db_event,
 531         .msg_event = perf_msg_event
 532 };
 533
 534 static void perf_free_outbuf(struct perf_peer *peer)
 535 {
 536         (void)ntb_peer_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);
 537 }
 538
 539 static int perf_setup_outbuf(struct perf_peer *peer)
 540 {
 541         struct perf_ctx *perf = peer->perf;
 542         int ret;
 543
 544         /* Outbuf size can be unaligned due to custom max_mw_size */
 545         ret = ntb_peer_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,
 546                                     peer->outbuf_xlat, peer->outbuf_size);
 547         if (ret) {
 548                 dev_err(&perf->ntb->dev, "Failed to set outbuf translation\n");
 549                 return ret;
 550         }
 551
 552         /* Initialization is finally done */
 553         set_bit(PERF_STS_DONE, &peer->sts);
 554         complete_all(&peer->init_comp);
 555
 556         return 0;
 557 }
 558
 559 static void perf_free_inbuf(struct perf_peer *peer)
 560 {
 561         if (!peer->inbuf)
 562                 return;
 563
 564         (void)ntb_mw_clear_trans(peer->perf->ntb, peer->pidx, peer->gidx);
 565         dma_free_coherent(&peer->perf->ntb->pdev->dev, peer->inbuf_size,
 566                           peer->inbuf, peer->inbuf_xlat);
 567         peer->inbuf = NULL;
 568 }
 569
 570 static int perf_setup_inbuf(struct perf_peer *peer)
 571 {
 572         resource_size_t xlat_align, size_align, size_max;
 573         struct perf_ctx *perf = peer->perf;
 574         int ret;
 575
 576         /* Get inbound MW parameters */
 577         ret = ntb_mw_get_align(perf->ntb, peer->pidx, perf->gidx,
 578                                &xlat_align, &size_align, &size_max);
 579         if (ret) {
 580                 dev_err(&perf->ntb->dev, "Couldn't get inbuf restrictions\n");
 581                 return ret;
 582         }
 583
 584         if (peer->inbuf_size > size_max) {
 585                 dev_err(&perf->ntb->dev, "Too big inbuf size %pa > %pa\n",
 586                         &peer->inbuf_size, &size_max);
 587                 return -EINVAL;
 588         }
 589
 590         peer->inbuf_size = round_up(peer->inbuf_size, size_align);
 591
 592         perf_free_inbuf(peer);
 593
 594         peer->inbuf = dma_alloc_coherent(&perf->ntb->pdev->dev,
 595                                          peer->inbuf_size, &peer->inbuf_xlat,
 596                                          GFP_KERNEL);
 597         if (!peer->inbuf) {
 598                 dev_err(&perf->ntb->dev, "Failed to alloc inbuf of %pa\n",
 599                         &peer->inbuf_size);
 600                 return -ENOMEM;
 601         }
 602         if (!IS_ALIGNED(peer->inbuf_xlat, xlat_align)) {
 603                 ret = -EINVAL;
 604                 dev_err(&perf->ntb->dev, "Unaligned inbuf allocated\n");
 605                 goto err_free_inbuf;
 606         }
 607
 608         ret = ntb_mw_set_trans(perf->ntb, peer->pidx, peer->gidx,
 609                                peer->inbuf_xlat, peer->inbuf_size);
 610         if (ret) {
 611                 dev_err(&perf->ntb->dev, "Failed to set inbuf translation\n");
 612                 goto err_free_inbuf;
 613         }
 614
 615         /*
 616          * We submit inbuf xlat transmission cmd for execution here to follow
 617          * the code architecture, even though this method is called from service
 618          * work itself so the command will be executed right after it returns.
 619          */
 620         (void)perf_cmd_exec(peer, PERF_CMD_SXLAT);
 621
 622         return 0;
 623
 624 err_free_inbuf:
 625         perf_free_inbuf(peer);
 626
 627         return ret;
 628 }
 629
 630 static void perf_service_work(struct work_struct *work)
 631 {
 632         struct perf_peer *peer = to_peer_service(work);
 633
 634         if (test_and_clear_bit(PERF_CMD_SSIZE, &peer->sts))
 635                 perf_cmd_send(peer, PERF_CMD_SSIZE, peer->outbuf_size);
 636
 637         if (test_and_clear_bit(PERF_CMD_RSIZE, &peer->sts))
 638                 perf_setup_inbuf(peer);
 639
 640         if (test_and_clear_bit(PERF_CMD_SXLAT, &peer->sts))
 641                 perf_cmd_send(peer, PERF_CMD_SXLAT, peer->inbuf_xlat);
 642
 643         if (test_and_clear_bit(PERF_CMD_RXLAT, &peer->sts))
 644                 perf_setup_outbuf(peer);
 645
 646         if (test_and_clear_bit(PERF_CMD_CLEAR, &peer->sts)) {
 647                 init_completion(&peer->init_comp);
 648                 clear_bit(PERF_STS_DONE, &peer->sts);
 649                 if (test_bit(0, &peer->perf->busy_flag) &&
 650                     peer == peer->perf->test_peer) {
 651                         dev_warn(&peer->perf->ntb->dev,
 652                                 "Freeing while test on-fly\n");
 653                         perf_terminate_test(peer->perf);
 654                 }
 655                 perf_free_outbuf(peer);
 656                 perf_free_inbuf(peer);
 657         }
 658 }
 659
 660 static int perf_init_service(struct perf_ctx *perf)
 661 {
 662         u64 mask;
 663
 664         if (ntb_peer_mw_count(perf->ntb) < perf->pcnt) {
 665                 dev_err(&perf->ntb->dev, "Not enough memory windows\n");
 666                 return -EINVAL;
 667         }
 668
 669         if (ntb_msg_count(perf->ntb) >= PERF_MSG_CNT) {
 670                 perf->cmd_send = perf_msg_cmd_send;
 671                 perf->cmd_recv = perf_msg_cmd_recv;
 672
 673                 dev_dbg(&perf->ntb->dev, "Message service initialized\n");
 674
 675                 return 0;
 676         }
 677
 678         dev_dbg(&perf->ntb->dev, "Message service unsupported\n");
 679
 680         mask = GENMASK_ULL(perf->pcnt, 0);
 681         if (ntb_spad_count(perf->ntb) >= PERF_SPAD_CNT(perf->pcnt) &&
 682             (ntb_db_valid_mask(perf->ntb) & mask) == mask) {
 683                 perf->cmd_send = perf_spad_cmd_send;
 684                 perf->cmd_recv = perf_spad_cmd_recv;
 685
 686                 dev_dbg(&perf->ntb->dev, "Scratchpad service initialized\n");
 687
 688                 return 0;
 689         }
 690
 691         dev_dbg(&perf->ntb->dev, "Scratchpad service unsupported\n");
 692
 693         dev_err(&perf->ntb->dev, "Command services unsupported\n");
 694
 695         return -EINVAL;
 696 }
 697
 698 static int perf_enable_service(struct perf_ctx *perf)
 699 {
 700         u64 mask, incmd_bit;
 701         int ret, sidx, scnt;
 702
 703         mask = ntb_db_valid_mask(perf->ntb);
 704         (void)ntb_db_set_mask(perf->ntb, mask);
 705
 706         ret = ntb_set_ctx(perf->ntb, perf, &perf_ops);
 707         if (ret)
 708                 return ret;
 709
 710         if (perf->cmd_send == perf_msg_cmd_send) {
 711                 u64 inbits, outbits;
 712
 713                 inbits = ntb_msg_inbits(perf->ntb);
 714                 outbits = ntb_msg_outbits(perf->ntb);
 715                 (void)ntb_msg_set_mask(perf->ntb, inbits | outbits);
 716
 717                 incmd_bit = BIT_ULL(__ffs64(inbits));
 718                 ret = ntb_msg_clear_mask(perf->ntb, incmd_bit);
 719
 720                 dev_dbg(&perf->ntb->dev, "MSG sts unmasked %#llx\n", incmd_bit);
 721         } else {
 722                 scnt = ntb_spad_count(perf->ntb);
 723                 for (sidx = 0; sidx < scnt; sidx++)
 724                         ntb_spad_write(perf->ntb, sidx, PERF_CMD_INVAL);
 725                 incmd_bit = PERF_SPAD_NOTIFY(perf->gidx);
 726                 ret = ntb_db_clear_mask(perf->ntb, incmd_bit);
 727
 728                 dev_dbg(&perf->ntb->dev, "DB bits unmasked %#llx\n", incmd_bit);
 729         }
 730         if (ret) {
 731                 ntb_clear_ctx(perf->ntb);
 732                 return ret;
 733         }
 734
 735         ntb_link_enable(perf->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
 736         /* Might be not necessary */
 737         ntb_link_event(perf->ntb);
 738
 739         return 0;
 740 }
 741
 742 static void perf_disable_service(struct perf_ctx *perf)
 743 {
 744         int pidx;
 745
 746         ntb_link_disable(perf->ntb);
 747
 748         if (perf->cmd_send == perf_msg_cmd_send) {
 749                 u64 inbits;
 750
 751                 inbits = ntb_msg_inbits(perf->ntb);
 752                 (void)ntb_msg_set_mask(perf->ntb, inbits);
 753         } else {
 754                 (void)ntb_db_set_mask(perf->ntb, PERF_SPAD_NOTIFY(perf->gidx));
 755         }
 756
 757         ntb_clear_ctx(perf->ntb);
 758
 759         for (pidx = 0; pidx < perf->pcnt; pidx++)
 760                 perf_cmd_exec(&perf->peers[pidx], PERF_CMD_CLEAR);
 761
 762         for (pidx = 0; pidx < perf->pcnt; pidx++)
 763                 flush_work(&perf->peers[pidx].service);
 764 }
 765
 766 /*==============================================================================
 767  *                      Performance measuring work-thread
 768  *==============================================================================
 769  */
 770
 771 static void perf_dma_copy_callback(void *data)
 772 {
 773         struct perf_thread *pthr = data;
 774
 775         atomic_dec(&pthr->dma_sync);
 776         wake_up(&pthr->dma_wait);
 777 }
 778
 779 static int perf_copy_chunk(struct perf_thread *pthr,
 780                            void __iomem *dst, void *src, size_t len)
 781 {
 782         struct dma_async_tx_descriptor *tx;
 783         struct dmaengine_unmap_data *unmap;
 784         struct device *dma_dev;
 785         int try = 0, ret = 0;
 786
 787         if (!use_dma) {
 788                 memcpy_toio(dst, src, len);
 789                 goto ret_check_tsync;
 790         }
 791
 792         dma_dev = pthr->dma_chan->device->dev;
 793
 794         if (!is_dma_copy_aligned(pthr->dma_chan->device, offset_in_page(src),
 795                                  offset_in_page(dst), len))
 796                 return -EIO;
 797
 798         unmap = dmaengine_get_unmap_data(dma_dev, 2, GFP_NOWAIT);
 799         if (!unmap)
 800                 return -ENOMEM;
 801
 802         unmap->len = len;
 803         unmap->addr[0] = dma_map_page(dma_dev, virt_to_page(src),
 804                 offset_in_page(src), len, DMA_TO_DEVICE);
 805         if (dma_mapping_error(dma_dev, unmap->addr[0])) {
 806                 ret = -EIO;
 807                 goto err_free_resource;
 808         }
 809         unmap->to_cnt = 1;
 810
 811         unmap->addr[1] = dma_map_page(dma_dev, virt_to_page(dst),
 812                 offset_in_page(dst), len, DMA_FROM_DEVICE);
 813         if (dma_mapping_error(dma_dev, unmap->addr[1])) {
 814                 ret = -EIO;
 815                 goto err_free_resource;
 816         }
 817         unmap->from_cnt = 1;
 818
 819         do {
 820                 tx = dmaengine_prep_dma_memcpy(pthr->dma_chan, unmap->addr[1],
 821                         unmap->addr[0], len, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 822                 if (!tx)
 823                         msleep(DMA_MDELAY);
 824         } while (!tx && (try++ < DMA_TRIES));
 825
 826         if (!tx) {
 827                 ret = -EIO;
 828                 goto err_free_resource;
 829         }
 830
 831         tx->callback = perf_dma_copy_callback;
 832         tx->callback_param = pthr;
 833         dma_set_unmap(tx, unmap);
 834
 835         ret = dma_submit_error(dmaengine_submit(tx));
 836         if (ret) {
 837                 dmaengine_unmap_put(unmap);
 838                 goto err_free_resource;
 839         }
 840
 841         dmaengine_unmap_put(unmap);
 842
 843         atomic_inc(&pthr->dma_sync);
 844         dma_async_issue_pending(pthr->dma_chan);
 845
 846 ret_check_tsync:
 847         return likely(atomic_read(&pthr->perf->tsync) > 0) ? 0 : -EINTR;
 848
 849 err_free_resource:
 850         dmaengine_unmap_put(unmap);
 851
 852         return ret;
 853 }
 854
 855 static bool perf_dma_filter(struct dma_chan *chan, void *data)
 856 {
 857         struct perf_ctx *perf = data;
 858         int node;
 859
 860         node = dev_to_node(&perf->ntb->dev);
 861
 862         return node == NUMA_NO_NODE || node == dev_to_node(chan->device->dev);
 863 }
 864
 865 static int perf_init_test(struct perf_thread *pthr)
 866 {
 867         struct perf_ctx *perf = pthr->perf;
 868         dma_cap_mask_t dma_mask;
 869
 870         pthr->src = kmalloc_node(perf->test_peer->outbuf_size, GFP_KERNEL,
 871                                  dev_to_node(&perf->ntb->dev));
 872         if (!pthr->src)
 873                 return -ENOMEM;
 874
 875         get_random_bytes(pthr->src, perf->test_peer->outbuf_size);
 876
 877         if (!use_dma)
 878                 return 0;
 879
 880         dma_cap_zero(dma_mask);
 881         dma_cap_set(DMA_MEMCPY, dma_mask);
 882         pthr->dma_chan = dma_request_channel(dma_mask, perf_dma_filter, perf);
 883         if (!pthr->dma_chan) {
 884                 dev_err(&perf->ntb->dev, "%d: Failed to get DMA channel\n",
 885                         pthr->tidx);
 886                 atomic_dec(&perf->tsync);
 887                 wake_up(&perf->twait);
 888                 kfree(pthr->src);
 889                 return -ENODEV;
 890         }
 891
 892         atomic_set(&pthr->dma_sync, 0);
 893
 894         return 0;
 895 }
 896
 897 static int perf_run_test(struct perf_thread *pthr)
 898 {
 899         struct perf_peer *peer = pthr->perf->test_peer;
 900         struct perf_ctx *perf = pthr->perf;
 901         void __iomem *flt_dst, *bnd_dst;
 902         u64 total_size, chunk_size;
 903         void *flt_src;
 904         int ret = 0;
 905
 906         total_size = 1ULL << total_order;
 907         chunk_size = 1ULL << chunk_order;
 908         chunk_size = min_t(u64, peer->outbuf_size, chunk_size);
 909
 910         flt_src = pthr->src;
 911         bnd_dst = peer->outbuf + peer->outbuf_size;
 912         flt_dst = peer->outbuf;
 913
 914         pthr->duration = ktime_get();
 915
 916         /* Copied field is cleared on test launch stage */
 917         while (pthr->copied < total_size) {
 918                 ret = perf_copy_chunk(pthr, flt_dst, flt_src, chunk_size);
 919                 if (ret) {
 920                         dev_err(&perf->ntb->dev, "%d: Got error %d on test\n",
 921                                 pthr->tidx, ret);
 922                         return ret;
 923                 }
 924
 925                 pthr->copied += chunk_size;
 926
 927                 flt_dst += chunk_size;
 928                 flt_src += chunk_size;
 929                 if (flt_dst >= bnd_dst || flt_dst < peer->outbuf) {
 930                         flt_dst = peer->outbuf;
 931                         flt_src = pthr->src;
 932                 }
 933
 934                 /* Give up CPU to give a chance for other threads to use it */
 935                 schedule();
 936         }
 937
 938         return 0;
 939 }
 940
 941 static int perf_sync_test(struct perf_thread *pthr)
 942 {
 943         struct perf_ctx *perf = pthr->perf;
 944
 945         if (!use_dma)
 946                 goto no_dma_ret;
 947
 948         wait_event(pthr->dma_wait,
 949                    (atomic_read(&pthr->dma_sync) == 0 ||
 950                     atomic_read(&perf->tsync) < 0));
 951
 952         if (atomic_read(&perf->tsync) < 0)
 953                 return -EINTR;
 954
 955 no_dma_ret:
 956         pthr->duration = ktime_sub(ktime_get(), pthr->duration);
 957
 958         dev_dbg(&perf->ntb->dev, "%d: copied %llu bytes\n",
 959                 pthr->tidx, pthr->copied);
 960
 961         dev_dbg(&perf->ntb->dev, "%d: lasted %llu usecs\n",
 962                 pthr->tidx, ktime_to_us(pthr->duration));
 963
 964         dev_dbg(&perf->ntb->dev, "%d: %llu MBytes/s\n", pthr->tidx,
 965                 div64_u64(pthr->copied, ktime_to_us(pthr->duration)));
 966
 967         return 0;
 968 }
 969
 970 static void perf_clear_test(struct perf_thread *pthr)
 971 {
 972         struct perf_ctx *perf = pthr->perf;
 973
 974         if (!use_dma)
 975                 goto no_dma_notify;
 976
 977         /*
 978          * If test finished without errors, termination isn't needed.
 979          * We call it anyway just to be sure of the transfers completion.
 980          */
 981         (void)dmaengine_terminate_sync(pthr->dma_chan);
 982
 983         dma_release_channel(pthr->dma_chan);
 984
 985 no_dma_notify:
 986         atomic_dec(&perf->tsync);
 987         wake_up(&perf->twait);
 988         kfree(pthr->src);
 989 }
 990
 991 static void perf_thread_work(struct work_struct *work)
 992 {
 993         struct perf_thread *pthr = to_thread_work(work);
 994         int ret;
 995
 996         /*
 997          * Perform stages in compliance with use_dma flag value.
 998          * Test status is changed only if error happened, otherwise
 999          * status -ENODATA is kept while test is on-fly. Results
1000          * synchronization is performed only if test fininshed
1001          * without an error or interruption.
1002          */
1003         ret = perf_init_test(pthr);
1004         if (ret) {
1005                 pthr->status = ret;
1006                 return;
1007         }
1008
1009         ret = perf_run_test(pthr);
1010         if (ret) {
1011                 pthr->status = ret;
1012                 goto err_clear_test;
1013         }
1014
1015         pthr->status = perf_sync_test(pthr);
1016
1017 err_clear_test:
1018         perf_clear_test(pthr);
1019 }
1020
1021 static int perf_set_tcnt(struct perf_ctx *perf, u8 tcnt)
1022 {
1023         if (tcnt == 0 || tcnt > MAX_THREADS_CNT)
1024                 return -EINVAL;
1025
1026         if (test_and_set_bit_lock(0, &perf->busy_flag))
1027                 return -EBUSY;
1028
1029         perf->tcnt = tcnt;
1030
1031         clear_bit_unlock(0, &perf->busy_flag);
1032
1033         return 0;
1034 }
1035
1036 static void perf_terminate_test(struct perf_ctx *perf)
1037 {
1038         int tidx;
1039
1040         atomic_set(&perf->tsync, -1);
1041         wake_up(&perf->twait);
1042
1043         for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
1044                 wake_up(&perf->threads[tidx].dma_wait);
1045                 cancel_work_sync(&perf->threads[tidx].work);
1046         }
1047 }
1048
1049 static int perf_submit_test(struct perf_peer *peer)
1050 {
1051         struct perf_ctx *perf = peer->perf;
1052         struct perf_thread *pthr;
1053         int tidx, ret;
1054
1055         ret = wait_for_completion_interruptible(&peer->init_comp);
1056         if (ret < 0)
1057                 return ret;
1058
1059         if (test_and_set_bit_lock(0, &perf->busy_flag))
1060                 return -EBUSY;
1061
1062         perf->test_peer = peer;
1063         atomic_set(&perf->tsync, perf->tcnt);
1064
1065         for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
1066                 pthr = &perf->threads[tidx];
1067
1068                 pthr->status = -ENODATA;
1069                 pthr->copied = 0;
1070                 pthr->duration = ktime_set(0, 0);
1071                 if (tidx < perf->tcnt)
1072                         (void)queue_work(perf_wq, &pthr->work);
1073         }
1074
1075         ret = wait_event_interruptible(perf->twait,
1076                                        atomic_read(&perf->tsync) <= 0);
1077         if (ret == -ERESTARTSYS) {
1078                 perf_terminate_test(perf);
1079                 ret = -EINTR;
1080         }
1081
1082         clear_bit_unlock(0, &perf->busy_flag);
1083
1084         return ret;
1085 }
1086
1087 static int perf_read_stats(struct perf_ctx *perf, char *buf,
1088                            size_t size, ssize_t *pos)
1089 {
1090         struct perf_thread *pthr;
1091         int tidx;
1092
1093         if (test_and_set_bit_lock(0, &perf->busy_flag))
1094                 return -EBUSY;
1095
1096         (*pos) += scnprintf(buf + *pos, size - *pos,
1097                 "    Peer %d test statistics:\n", perf->test_peer->pidx);
1098
1099         for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
1100                 pthr = &perf->threads[tidx];
1101
1102                 if (pthr->status == -ENODATA)
1103                         continue;
1104
1105                 if (pthr->status) {
1106                         (*pos) += scnprintf(buf + *pos, size - *pos,
1107                                 "%d: error status %d\n", tidx, pthr->status);
1108                         continue;
1109                 }
1110
1111                 (*pos) += scnprintf(buf + *pos, size - *pos,
1112                         "%d: copied %llu bytes in %llu usecs, %llu MBytes/s\n",
1113                         tidx, pthr->copied, ktime_to_us(pthr->duration),
1114                         div64_u64(pthr->copied, ktime_to_us(pthr->duration)));
1115         }
1116
1117         clear_bit_unlock(0, &perf->busy_flag);
1118
1119         return 0;
1120 }
1121
1122 static void perf_init_threads(struct perf_ctx *perf)
1123 {
1124         struct perf_thread *pthr;
1125         int tidx;
1126
1127         perf->tcnt = DEF_THREADS_CNT;
1128         perf->test_peer = &perf->peers[0];
1129         init_waitqueue_head(&perf->twait);
1130
1131         for (tidx = 0; tidx < MAX_THREADS_CNT; tidx++) {
1132                 pthr = &perf->threads[tidx];
1133
1134                 pthr->perf = perf;
1135                 pthr->tidx = tidx;
1136                 pthr->status = -ENODATA;
1137                 init_waitqueue_head(&pthr->dma_wait);
1138                 INIT_WORK(&pthr->work, perf_thread_work);
1139         }
1140 }
1141
1142 static void perf_clear_threads(struct perf_ctx *perf)
1143 {
1144         perf_terminate_test(perf);
1145 }
1146
1147 /*==============================================================================
1148  *                               DebugFS nodes
1149  *==============================================================================
1150  */
1151
1152 static ssize_t perf_dbgfs_read_info(struct file *filep, char __user *ubuf,
1153                                     size_t size, loff_t *offp)
1154 {
1155         struct perf_ctx *perf = filep->private_data;
1156         struct perf_peer *peer;
1157         size_t buf_size;
1158         ssize_t pos = 0;
1159         int ret, pidx;
1160         char *buf;
1161
1162         buf_size = min_t(size_t, size, 0x1000U);
1163
1164         buf = kmalloc(buf_size, GFP_KERNEL);
1165         if (!buf)
1166                 return -ENOMEM;
1167
1168         pos += scnprintf(buf + pos, buf_size - pos,
1169                 "    Performance measuring tool info:\n\n");
1170
1171         pos += scnprintf(buf + pos, buf_size - pos,
1172                 "Local port %d, Global index %d\n", ntb_port_number(perf->ntb),
1173                 perf->gidx);
1174         pos += scnprintf(buf + pos, buf_size - pos, "Test status: ");
1175         if (test_bit(0, &perf->busy_flag)) {
1176                 pos += scnprintf(buf + pos, buf_size - pos,
1177                         "on-fly with port %d (%d)\n",
1178                         ntb_peer_port_number(perf->ntb, perf->test_peer->pidx),
1179                         perf->test_peer->pidx);
1180         } else {
1181                 pos += scnprintf(buf + pos, buf_size - pos, "idle\n");
1182         }
1183
1184         for (pidx = 0; pidx < perf->pcnt; pidx++) {
1185                 peer = &perf->peers[pidx];
1186
1187                 pos += scnprintf(buf + pos, buf_size - pos,
1188                         "Port %d (%d), Global index %d:\n",
1189                         ntb_peer_port_number(perf->ntb, peer->pidx), peer->pidx,
1190                         peer->gidx);
1191
1192                 pos += scnprintf(buf + pos, buf_size - pos,
1193                         "\tLink status: %s\n",
1194                         test_bit(PERF_STS_LNKUP, &peer->sts) ? "up" : "down");
1195
1196                 pos += scnprintf(buf + pos, buf_size - pos,
1197                         "\tOut buffer addr 0x%pK\n", peer->outbuf);
1198
1199                 pos += scnprintf(buf + pos, buf_size - pos,
1200                         "\tOut buffer size %pa\n", &peer->outbuf_size);
1201
1202                 pos += scnprintf(buf + pos, buf_size - pos,
1203                         "\tOut buffer xlat 0x%016llx[p]\n", peer->outbuf_xlat);
1204
1205                 if (!peer->inbuf) {
1206                         pos += scnprintf(buf + pos, buf_size - pos,
1207                                 "\tIn buffer addr: unallocated\n");
1208                         continue;
1209                 }
1210
1211                 pos += scnprintf(buf + pos, buf_size - pos,
1212                         "\tIn buffer addr 0x%pK\n", peer->inbuf);
1213
1214                 pos += scnprintf(buf + pos, buf_size - pos,
1215                         "\tIn buffer size %pa\n", &peer->inbuf_size);
1216
1217                 pos += scnprintf(buf + pos, buf_size - pos,
1218                         "\tIn buffer xlat %pad[p]\n", &peer->inbuf_xlat);
1219         }
1220
1221         ret = simple_read_from_buffer(ubuf, size, offp, buf, pos);
1222         kfree(buf);
1223
1224         return ret;
1225 }
1226
1227 static const struct file_operations perf_dbgfs_info = {
1228         .open = simple_open,
1229         .read = perf_dbgfs_read_info
1230 };
1231
1232 static ssize_t perf_dbgfs_read_run(struct file *filep, char __user *ubuf,
1233                                    size_t size, loff_t *offp)
1234 {
1235         struct perf_ctx *perf = filep->private_data;
1236         ssize_t ret, pos = 0;
1237         char *buf;
1238
1239         buf = kmalloc(PERF_BUF_LEN, GFP_KERNEL);
1240         if (!buf)
1241                 return -ENOMEM;
1242
1243         ret = perf_read_stats(perf, buf, PERF_BUF_LEN, &pos);
1244         if (ret)
1245                 goto err_free;
1246
1247         ret = simple_read_from_buffer(ubuf, size, offp, buf, pos);
1248 err_free:
1249         kfree(buf);
1250
1251         return ret;
1252 }
1253
1254 static ssize_t perf_dbgfs_write_run(struct file *filep, const char __user *ubuf,
1255                                     size_t size, loff_t *offp)
1256 {
1257         struct perf_ctx *perf = filep->private_data;
1258         struct perf_peer *peer;
1259         int pidx, ret;
1260
1261         ret = kstrtoint_from_user(ubuf, size, 0, &pidx);
1262         if (ret)
1263                 return ret;
1264
1265         if (pidx < 0 || pidx >= perf->pcnt)
1266                 return -EINVAL;
1267
1268         peer = &perf->peers[pidx];
1269
1270         ret = perf_submit_test(peer);
1271         if (ret)
1272                 return ret;
1273
1274         return size;
1275 }
1276
1277 static const struct file_operations perf_dbgfs_run = {
1278         .open = simple_open,
1279         .read = perf_dbgfs_read_run,
1280         .write = perf_dbgfs_write_run
1281 };
1282
1283 static ssize_t perf_dbgfs_read_tcnt(struct file *filep, char __user *ubuf,
1284                                     size_t size, loff_t *offp)
1285 {
1286         struct perf_ctx *perf = filep->private_data;
1287         char buf[8];
1288         ssize_t pos;
1289
1290         pos = scnprintf(buf, sizeof(buf), "%hhu\n", perf->tcnt);
1291
1292         return simple_read_from_buffer(ubuf, size, offp, buf, pos);
1293 }
1294
1295 static ssize_t perf_dbgfs_write_tcnt(struct file *filep,
1296                                      const char __user *ubuf,
1297                                      size_t size, loff_t *offp)
1298 {
1299         struct perf_ctx *perf = filep->private_data;
1300         int ret;
1301         u8 val;
1302
1303         ret = kstrtou8_from_user(ubuf, size, 0, &val);
1304         if (ret)
1305                 return ret;
1306
1307         ret = perf_set_tcnt(perf, val);
1308         if (ret)
1309                 return ret;
1310
1311         return size;
1312 }
1313
1314 static const struct file_operations perf_dbgfs_tcnt = {
1315         .open = simple_open,
1316         .read = perf_dbgfs_read_tcnt,
1317         .write = perf_dbgfs_write_tcnt
1318 };
1319
1320 static void perf_setup_dbgfs(struct perf_ctx *perf)
1321 {
1322         struct pci_dev *pdev = perf->ntb->pdev;
1323
1324         perf->dbgfs_dir = debugfs_create_dir(pci_name(pdev), perf_dbgfs_topdir);
1325         if (!perf->dbgfs_dir) {
1326                 dev_warn(&perf->ntb->dev, "DebugFS unsupported\n");
1327                 return;
1328         }
1329
1330         debugfs_create_file("info", 0600, perf->dbgfs_dir, perf,
1331                             &perf_dbgfs_info);
1332
1333         debugfs_create_file("run", 0600, perf->dbgfs_dir, perf,
1334                             &perf_dbgfs_run);
1335
1336         debugfs_create_file("threads_count", 0600, perf->dbgfs_dir, perf,
1337                             &perf_dbgfs_tcnt);
1338
1339         /* They are made read-only for test exec safety and integrity */
1340         debugfs_create_u8("chunk_order", 0500, perf->dbgfs_dir, &chunk_order);
1341
1342         debugfs_create_u8("total_order", 0500, perf->dbgfs_dir, &total_order);
1343
1344         debugfs_create_bool("use_dma", 0500, perf->dbgfs_dir, &use_dma);
1345 }
1346
1347 static void perf_clear_dbgfs(struct perf_ctx *perf)
1348 {
1349         debugfs_remove_recursive(perf->dbgfs_dir);
1350 }
1351
1352 /*==============================================================================
1353  *                        Basic driver initialization
1354  *==============================================================================
1355  */
1356
1357 static struct perf_ctx *perf_create_data(struct ntb_dev *ntb)
1358 {
1359         struct perf_ctx *perf;
1360
1361         perf = devm_kzalloc(&ntb->dev, sizeof(*perf), GFP_KERNEL);
1362         if (!perf)
1363                 return ERR_PTR(-ENOMEM);
1364
1365         perf->pcnt = ntb_peer_port_count(ntb);
1366         perf->peers = devm_kcalloc(&ntb->dev, perf->pcnt, sizeof(*perf->peers),
1367                                   GFP_KERNEL);
1368         if (!perf->peers)
1369                 return ERR_PTR(-ENOMEM);
1370
1371         perf->ntb = ntb;
1372
1373         return perf;
1374 }
1375
1376 static int perf_setup_peer_mw(struct perf_peer *peer)
1377 {
1378         struct perf_ctx *perf = peer->perf;
1379         phys_addr_t phys_addr;
1380         int ret;
1381
1382         /* Get outbound MW parameters and map it */
1383         ret = ntb_peer_mw_get_addr(perf->ntb, perf->gidx, &phys_addr,
1384                                    &peer->outbuf_size);
1385         if (ret)
1386                 return ret;
1387
1388         peer->outbuf = devm_ioremap_wc(&perf->ntb->dev, phys_addr,
1389                                         peer->outbuf_size);
1390         if (!peer->outbuf)
1391                 return -ENOMEM;
1392
1393         if (max_mw_size && peer->outbuf_size > max_mw_size) {
1394                 peer->outbuf_size = max_mw_size;
1395                 dev_warn(&peer->perf->ntb->dev,
1396                         "Peer %d outbuf reduced to %pa\n", peer->pidx,
1397                         &peer->outbuf_size);
1398         }
1399
1400         return 0;
1401 }
1402
1403 static int perf_init_peers(struct perf_ctx *perf)
1404 {
1405         struct perf_peer *peer;
1406         int pidx, lport, ret;
1407
1408         lport = ntb_port_number(perf->ntb);
1409         perf->gidx = -1;
1410         for (pidx = 0; pidx < perf->pcnt; pidx++) {
1411                 peer = &perf->peers[pidx];
1412
1413                 peer->perf = perf;
1414                 peer->pidx = pidx;
1415                 if (lport < ntb_peer_port_number(perf->ntb, pidx)) {
1416                         if (perf->gidx == -1)
1417                                 perf->gidx = pidx;
1418                         peer->gidx = pidx + 1;
1419                 } else {
1420                         peer->gidx = pidx;
1421                 }
1422                 INIT_WORK(&peer->service, perf_service_work);
1423                 init_completion(&peer->init_comp);
1424         }
1425         if (perf->gidx == -1)
1426                 perf->gidx = pidx;
1427
1428         /*
1429          * Hardware with only two ports may not have unique port
1430          * numbers. In this case, the gidxs should all be zero.
1431          */
1432         if (perf->pcnt == 1 &&  ntb_port_number(perf->ntb) == 0 &&
1433             ntb_peer_port_number(perf->ntb, 0) == 0) {
1434                 perf->gidx = 0;
1435                 perf->peers[0].gidx = 0;
1436         }
1437
1438         for (pidx = 0; pidx < perf->pcnt; pidx++) {
1439                 ret = perf_setup_peer_mw(&perf->peers[pidx]);
1440                 if (ret)
1441                         return ret;
1442         }
1443
1444         dev_dbg(&perf->ntb->dev, "Global port index %d\n", perf->gidx);
1445
1446         return 0;
1447 }
1448
1449 static int perf_probe(struct ntb_client *client, struct ntb_dev *ntb)
1450 {
1451         struct perf_ctx *perf;
1452         int ret;
1453
1454         perf = perf_create_data(ntb);
1455         if (IS_ERR(perf))
1456                 return PTR_ERR(perf);
1457
1458         ret = perf_init_peers(perf);
1459         if (ret)
1460                 return ret;
1461
1462         perf_init_threads(perf);
1463
1464         ret = perf_init_service(perf);
1465         if (ret)
1466                 return ret;
1467
1468         ret = perf_enable_service(perf);
1469         if (ret)
1470                 return ret;
1471
1472         perf_setup_dbgfs(perf);
1473
1474         return 0;
1475 }
1476
1477 static void perf_remove(struct ntb_client *client, struct ntb_dev *ntb)
1478 {
1479         struct perf_ctx *perf = ntb->ctx;
1480
1481         perf_clear_dbgfs(perf);
1482
1483         perf_disable_service(perf);
1484
1485         perf_clear_threads(perf);
1486 }
1487
1488 static struct ntb_client perf_client = {
1489         .ops = {
1490                 .probe = perf_probe,
1491                 .remove = perf_remove
1492         }
1493 };
1494
1495 static int __init perf_init(void)
1496 {
1497         int ret;
1498
1499         if (chunk_order > MAX_CHUNK_ORDER) {
1500                 chunk_order = MAX_CHUNK_ORDER;
1501                 pr_info("Chunk order reduced to %hhu\n", chunk_order);
1502         }
1503
1504         if (total_order < chunk_order) {
1505                 total_order = chunk_order;
1506                 pr_info("Total data order reduced to %hhu\n", total_order);
1507         }
1508
1509         perf_wq = alloc_workqueue("perf_wq", WQ_UNBOUND | WQ_SYSFS, 0);
1510         if (!perf_wq)
1511                 return -ENOMEM;
1512
1513         if (debugfs_initialized())
1514                 perf_dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1515
1516         ret = ntb_register_client(&perf_client);
1517         if (ret) {
1518                 debugfs_remove_recursive(perf_dbgfs_topdir);
1519                 destroy_workqueue(perf_wq);
1520         }
1521
1522         return ret;
1523 }
1524 module_init(perf_init);
1525
1526 static void __exit perf_exit(void)
1527 {
1528         ntb_unregister_client(&perf_client);
1529         debugfs_remove_recursive(perf_dbgfs_topdir);
1530         destroy_workqueue(perf_wq);
1531 }
1532 module_exit(perf_exit);