drivers/ntb/test/ntb_tool.c

   1 /*
   2  * This file is provided under a dual BSD/GPLv2 license.  When using or
   3  *   redistributing this file, you may do so under either license.
   4  *
   5  *   GPL LICENSE SUMMARY
   6  *
   7  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
   8  *   Copyright (C) 2017 T-Platforms All Rights Reserved.
   9  *
  10  *   This program is free software; you can redistribute it and/or modify
  11  *   it under the terms of version 2 of the GNU General Public License as
  12  *   published by the Free Software Foundation.
  13  *
  14  *   This program is distributed in the hope that it will be useful, but
  15  *   WITHOUT ANY WARRANTY; without even the implied warranty of
  16  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17  *   General Public License for more details.
  18  *
  19  *   BSD LICENSE
  20  *
  21  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
  22  *   Copyright (C) 2017 T-Platforms All Rights Reserved.
  23  *
  24  *   Redistribution and use in source and binary forms, with or without
  25  *   modification, are permitted provided that the following conditions
  26  *   are met:
  27  *
  28  *     * Redistributions of source code must retain the above copyright
  29  *       notice, this list of conditions and the following disclaimer.
  30  *     * Redistributions in binary form must reproduce the above copy
  31  *       notice, this list of conditions and the following disclaimer in
  32  *       the documentation and/or other materials provided with the
  33  *       distribution.
  34  *     * Neither the name of Intel Corporation nor the names of its
  35  *       contributors may be used to endorse or promote products derived
  36  *       from this software without specific prior written permission.
  37  *
  38  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  39  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  40  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  41  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  42  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  43  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  44  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  45  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  46  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  47  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  48  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  49  *
  50  * PCIe NTB Debugging Tool Linux driver
  51  */
  52
  53 /*
  54  * How to use this tool, by example.
  55  *
  56  * Assuming $DBG_DIR is something like:
  57  * '/sys/kernel/debug/ntb_tool/0000:00:03.0'
  58  * Suppose aside from local device there is at least one remote device
  59  * connected to NTB with index 0.
  60  *-----------------------------------------------------------------------------
  61  * Eg: check local/peer device information.
  62  *
  63  * # Get local device port number
  64  * root@self# cat $DBG_DIR/port
  65  *
  66  * # Check local device functionality
  67  * root@self# ls $DBG_DIR
  68  * db            msg1          msg_sts     peer4/        port
  69  * db_event      msg2          peer0/      peer5/        spad0
  70  * db_mask       msg3          peer1/      peer_db       spad1
  71  * link          msg_event     peer2/      peer_db_mask  spad2
  72  * msg0          msg_mask      peer3/      peer_spad     spad3
  73  * # As one can see it supports:
  74  * # 1) four inbound message registers
  75  * # 2) four inbound scratchpads
  76  * # 3) up to six peer devices
  77  *
  78  * # Check peer device port number
  79  * root@self# cat $DBG_DIR/peer0/port
  80  *
  81  * # Check peer device(s) functionality to be used
  82  * root@self# ls $DBG_DIR/peer0
  83  * link             mw_trans0       mw_trans6        port
  84  * link_event       mw_trans1       mw_trans7        spad0
  85  * msg0             mw_trans2       peer_mw_trans0   spad1
  86  * msg1             mw_trans3       peer_mw_trans1   spad2
  87  * msg2             mw_trans4       peer_mw_trans2   spad3
  88  * msg3             mw_trans5       peer_mw_trans3
  89  * # As one can see we got:
  90  * # 1) four outbound message registers
  91  * # 2) four outbound scratchpads
  92  * # 3) eight inbound memory windows
  93  * # 4) four outbound memory windows
  94  *-----------------------------------------------------------------------------
  95  * Eg: NTB link tests
  96  *
  97  * # Set local link up/down
  98  * root@self# echo Y > $DBG_DIR/link
  99  * root@self# echo N > $DBG_DIR/link
 100  *
 101  * # Check if link with peer device is up/down:
 102  * root@self# cat $DBG_DIR/peer0/link
 103  *
 104  * # Block until the link is up/down
 105  * root@self# echo Y > $DBG_DIR/peer0/link_event
 106  * root@self# echo N > $DBG_DIR/peer0/link_event
 107  *-----------------------------------------------------------------------------
 108  * Eg: Doorbell registers tests (some functionality might be absent)
 109  *
 110  * # Set/clear/get local doorbell
 111  * root@self# echo 's 1' > $DBG_DIR/db
 112  * root@self# echo 'c 1' > $DBG_DIR/db
 113  * root@self# cat  $DBG_DIR/db
 114  *
 115  * # Set/clear/get local doorbell mask
 116  * root@self# echo 's 1' > $DBG_DIR/db_mask
 117  * root@self# echo 'c 1' > $DBG_DIR/db_mask
 118  * root@self# cat $DBG_DIR/db_mask
 119  *
 120  * # Ring/clear/get peer doorbell
 121  * root@peer# echo 's 1' > $DBG_DIR/peer_db
 122  * root@peer# echo 'c 1' > $DBG_DIR/peer_db
 123  * root@peer# cat $DBG_DIR/peer_db
 124  *
 125  * # Set/clear/get peer doorbell mask
 126  * root@self# echo 's 1' > $DBG_DIR/peer_db_mask
 127  * root@self# echo 'c 1' > $DBG_DIR/peer_db_mask
 128  * root@self# cat $DBG_DIR/peer_db_mask
 129  *
 130  * # Block until local doorbell is set with specified value
 131  * root@self# echo 1 > $DBG_DIR/db_event
 132  *-----------------------------------------------------------------------------
 133  * Eg: Message registers tests (functionality might be absent)
 134  *
 135  * # Set/clear/get in/out message registers status
 136  * root@self# echo 's 1' > $DBG_DIR/msg_sts
 137  * root@self# echo 'c 1' > $DBG_DIR/msg_sts
 138  * root@self# cat $DBG_DIR/msg_sts
 139  *
 140  * # Set/clear in/out message registers mask
 141  * root@self# echo 's 1' > $DBG_DIR/msg_mask
 142  * root@self# echo 'c 1' > $DBG_DIR/msg_mask
 143  *
 144  * # Get inbound message register #0 value and source of port index
 145  * root@self# cat  $DBG_DIR/msg0
 146  *
 147  * # Send some data to peer over outbound message register #0
 148  * root@self# echo 0x01020304 > $DBG_DIR/peer0/msg0
 149  *-----------------------------------------------------------------------------
 150  * Eg: Scratchpad registers tests (functionality might be absent)
 151  *
 152  * # Write/read to/from local scratchpad register #0
 153  * root@peer# echo 0x01020304 > $DBG_DIR/spad0
 154  * root@peer# cat $DBG_DIR/spad0
 155  *
 156  * # Write/read to/from peer scratchpad register #0
 157  * root@peer# echo 0x01020304 > $DBG_DIR/peer0/spad0
 158  * root@peer# cat $DBG_DIR/peer0/spad0
 159  *-----------------------------------------------------------------------------
 160  * Eg: Memory windows tests
 161  *
 162  * # Create inbound memory window buffer of specified size/get its base address
 163  * root@peer# echo 16384 > $DBG_DIR/peer0/mw_trans0
 164  * root@peer# cat $DBG_DIR/peer0/mw_trans0
 165  *
 166  * # Write/read data to/from inbound memory window
 167  * root@peer# echo Hello > $DBG_DIR/peer0/mw0
 168  * root@peer# head -c 7 $DBG_DIR/peer0/mw0
 169  *
 170  * # Map outbound memory window/check it settings (on peer device)
 171  * root@peer# echo 0xADD0BA5E:16384 > $DBG_DIR/peer0/peer_mw_trans0
 172  * root@peer# cat $DBG_DIR/peer0/peer_mw_trans0
 173  *
 174  * # Write/read data to/from outbound memory window (on peer device)
 175  * root@peer# echo olleH > $DBG_DIR/peer0/peer_mw0
 176  * root@peer# head -c 7 $DBG_DIR/peer0/peer_mw0
 177  */
 178
 179 #include <linux/init.h>
 180 #include <linux/kernel.h>
 181 #include <linux/module.h>
 182
 183 #include <linux/debugfs.h>
 184 #include <linux/dma-mapping.h>
 185 #include <linux/pci.h>
 186 #include <linux/slab.h>
 187 #include <linux/uaccess.h>
 188
 189 #include <linux/ntb.h>
 190
 191 #define DRIVER_NAME             "ntb_tool"
 192 #define DRIVER_VERSION          "2.0"
 193
 194 MODULE_LICENSE("Dual BSD/GPL");
 195 MODULE_VERSION(DRIVER_VERSION);
 196 MODULE_AUTHOR("Allen Hubbe <Allen.Hubbe@emc.com>");
 197 MODULE_DESCRIPTION("PCIe NTB Debugging Tool");
 198
 199 /*
 200  * Inbound and outbound memory windows descriptor. Union members selection
 201  * depends on the MW type the structure describes. mm_base/dma_base are the
 202  * virtual and DMA address of an inbound MW. io_base/tr_base are the MMIO
 203  * mapped virtual and xlat addresses of an outbound MW respectively.
 204  */
 205 struct tool_mw {
 206         int widx;
 207         int pidx;
 208         struct tool_ctx *tc;
 209         union {
 210                 u8 *mm_base;
 211                 u8 __iomem *io_base;
 212         };
 213         union {
 214                 dma_addr_t dma_base;
 215                 u64 tr_base;
 216         };
 217         resource_size_t size;
 218         struct dentry *dbgfs_file;
 219 };
 220
 221 /*
 222  * Wrapper structure is used to distinguish the outbound MW peers reference
 223  * within the corresponding DebugFS directory IO operation.
 224  */
 225 struct tool_mw_wrap {
 226         int pidx;
 227         struct tool_mw *mw;
 228 };
 229
 230 struct tool_msg {
 231         int midx;
 232         int pidx;
 233         struct tool_ctx *tc;
 234 };
 235
 236 struct tool_spad {
 237         int sidx;
 238         int pidx;
 239         struct tool_ctx *tc;
 240 };
 241
 242 struct tool_peer {
 243         int pidx;
 244         struct tool_ctx *tc;
 245         int inmw_cnt;
 246         struct tool_mw *inmws;
 247         int outmw_cnt;
 248         struct tool_mw_wrap *outmws;
 249         int outmsg_cnt;
 250         struct tool_msg *outmsgs;
 251         int outspad_cnt;
 252         struct tool_spad *outspads;
 253         struct dentry *dbgfs_dir;
 254 };
 255
 256 struct tool_ctx {
 257         struct ntb_dev *ntb;
 258         wait_queue_head_t link_wq;
 259         wait_queue_head_t db_wq;
 260         wait_queue_head_t msg_wq;
 261         int outmw_cnt;
 262         struct tool_mw *outmws;
 263         int peer_cnt;
 264         struct tool_peer *peers;
 265         int inmsg_cnt;
 266         struct tool_msg *inmsgs;
 267         int inspad_cnt;
 268         struct tool_spad *inspads;
 269         struct dentry *dbgfs_dir;
 270 };
 271
 272 #define TOOL_FOPS_RDWR(__name, __read, __write) \
 273         const struct file_operations __name = { \
 274                 .owner = THIS_MODULE,           \
 275                 .open = simple_open,            \
 276                 .read = __read,                 \
 277                 .write = __write,               \
 278         }
 279
 280 #define TOOL_BUF_LEN 32
 281
 282 static struct dentry *tool_dbgfs_topdir;
 283
 284 /*==============================================================================
 285  *                               NTB events handlers
 286  *==============================================================================
 287  */
 288
 289 static void tool_link_event(void *ctx)
 290 {
 291         struct tool_ctx *tc = ctx;
 292         enum ntb_speed speed;
 293         enum ntb_width width;
 294         int up;
 295
 296         up = ntb_link_is_up(tc->ntb, &speed, &width);
 297
 298         dev_dbg(&tc->ntb->dev, "link is %s speed %d width %d\n",
 299                 up ? "up" : "down", speed, width);
 300
 301         wake_up(&tc->link_wq);
 302 }
 303
 304 static void tool_db_event(void *ctx, int vec)
 305 {
 306         struct tool_ctx *tc = ctx;
 307         u64 db_bits, db_mask;
 308
 309         db_mask = ntb_db_vector_mask(tc->ntb, vec);
 310         db_bits = ntb_db_read(tc->ntb);
 311
 312         dev_dbg(&tc->ntb->dev, "doorbell vec %d mask %#llx bits %#llx\n",
 313                 vec, db_mask, db_bits);
 314
 315         wake_up(&tc->db_wq);
 316 }
 317
 318 static void tool_msg_event(void *ctx)
 319 {
 320         struct tool_ctx *tc = ctx;
 321         u64 msg_sts;
 322
 323         msg_sts = ntb_msg_read_sts(tc->ntb);
 324
 325         dev_dbg(&tc->ntb->dev, "message bits %#llx\n", msg_sts);
 326
 327         wake_up(&tc->msg_wq);
 328 }
 329
 330 static const struct ntb_ctx_ops tool_ops = {
 331         .link_event = tool_link_event,
 332         .db_event = tool_db_event,
 333         .msg_event = tool_msg_event
 334 };
 335
 336 /*==============================================================================
 337  *                        Common read/write methods
 338  *==============================================================================
 339  */
 340
 341 static ssize_t tool_fn_read(struct tool_ctx *tc, char __user *ubuf,
 342                             size_t size, loff_t *offp,
 343                             u64 (*fn_read)(struct ntb_dev *))
 344 {
 345         size_t buf_size;
 346         char buf[TOOL_BUF_LEN];
 347         ssize_t pos;
 348
 349         if (!fn_read)
 350                 return -EINVAL;
 351
 352         buf_size = min(size, sizeof(buf));
 353
 354         pos = scnprintf(buf, buf_size, "%#llx\n", fn_read(tc->ntb));
 355
 356         return simple_read_from_buffer(ubuf, size, offp, buf, pos);
 357 }
 358
 359 static ssize_t tool_fn_write(struct tool_ctx *tc,
 360                              const char __user *ubuf,
 361                              size_t size, loff_t *offp,
 362                              int (*fn_set)(struct ntb_dev *, u64),
 363                              int (*fn_clear)(struct ntb_dev *, u64))
 364 {
 365         char *buf, cmd;
 366         ssize_t ret;
 367         u64 bits;
 368         int n;
 369
 370         buf = kmalloc(size + 1, GFP_KERNEL);
 371         if (!buf)
 372                 return -ENOMEM;
 373
 374         ret = simple_write_to_buffer(buf, size, offp, ubuf, size);
 375         if (ret < 0) {
 376                 kfree(buf);
 377                 return ret;
 378         }
 379
 380         buf[size] = 0;
 381
 382         n = sscanf(buf, "%c %lli", &cmd, &bits);
 383
 384         kfree(buf);
 385
 386         if (n != 2) {
 387                 ret = -EINVAL;
 388         } else if (cmd == 's') {
 389                 if (!fn_set)
 390                         ret = -EINVAL;
 391                 else
 392                         ret = fn_set(tc->ntb, bits);
 393         } else if (cmd == 'c') {
 394                 if (!fn_clear)
 395                         ret = -EINVAL;
 396                 else
 397                         ret = fn_clear(tc->ntb, bits);
 398         } else {
 399                 ret = -EINVAL;
 400         }
 401
 402         return ret ? : size;
 403 }
 404
 405 /*==============================================================================
 406  *                            Port read/write methods
 407  *==============================================================================
 408  */
 409
 410 static ssize_t tool_port_read(struct file *filep, char __user *ubuf,
 411                               size_t size, loff_t *offp)
 412 {
 413         struct tool_ctx *tc = filep->private_data;
 414         char buf[TOOL_BUF_LEN];
 415         int pos;
 416
 417         pos = scnprintf(buf, sizeof(buf), "%d\n", ntb_port_number(tc->ntb));
 418
 419         return simple_read_from_buffer(ubuf, size, offp, buf, pos);
 420 }
 421
 422 static TOOL_FOPS_RDWR(tool_port_fops,
 423                       tool_port_read,
 424                       NULL);
 425
 426 static ssize_t tool_peer_port_read(struct file *filep, char __user *ubuf,
 427                                    size_t size, loff_t *offp)
 428 {
 429         struct tool_peer *peer = filep->private_data;
 430         struct tool_ctx *tc = peer->tc;
 431         char buf[TOOL_BUF_LEN];
 432         int pos;
 433
 434         pos = scnprintf(buf, sizeof(buf), "%d\n",
 435                 ntb_peer_port_number(tc->ntb, peer->pidx));
 436
 437         return simple_read_from_buffer(ubuf, size, offp, buf, pos);
 438 }
 439
 440 static TOOL_FOPS_RDWR(tool_peer_port_fops,
 441                       tool_peer_port_read,
 442                       NULL);
 443
 444 static int tool_init_peers(struct tool_ctx *tc)
 445 {
 446         int pidx;
 447
 448         tc->peer_cnt = ntb_peer_port_count(tc->ntb);
 449         tc->peers = devm_kcalloc(&tc->ntb->dev, tc->peer_cnt,
 450                                  sizeof(*tc->peers), GFP_KERNEL);
 451         if (tc->peers == NULL)
 452                 return -ENOMEM;
 453
 454         for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
 455                 tc->peers[pidx].pidx = pidx;
 456                 tc->peers[pidx].tc = tc;
 457         }
 458
 459         return 0;
 460 }
 461
 462 /*==============================================================================
 463  *                       Link state read/write methods
 464  *==============================================================================
 465  */
 466
 467 static ssize_t tool_link_write(struct file *filep, const char __user *ubuf,
 468                                size_t size, loff_t *offp)
 469 {
 470         struct tool_ctx *tc = filep->private_data;
 471         bool val;
 472         int ret;
 473
 474         ret = kstrtobool_from_user(ubuf, size, &val);
 475         if (ret)
 476                 return ret;
 477
 478         if (val)
 479                 ret = ntb_link_enable(tc->ntb, NTB_SPEED_AUTO, NTB_WIDTH_AUTO);
 480         else
 481                 ret = ntb_link_disable(tc->ntb);
 482
 483         if (ret)
 484                 return ret;
 485
 486         return size;
 487 }
 488
 489 static TOOL_FOPS_RDWR(tool_link_fops,
 490                       NULL,
 491                       tool_link_write);
 492
 493 static ssize_t tool_peer_link_read(struct file *filep, char __user *ubuf,
 494                                    size_t size, loff_t *offp)
 495 {
 496         struct tool_peer *peer = filep->private_data;
 497         struct tool_ctx *tc = peer->tc;
 498         char buf[3];
 499
 500         if (ntb_link_is_up(tc->ntb, NULL, NULL) & BIT(peer->pidx))
 501                 buf[0] = 'Y';
 502         else
 503                 buf[0] = 'N';
 504         buf[1] = '\n';
 505         buf[2] = '\0';
 506
 507         return simple_read_from_buffer(ubuf, size, offp, buf, 2);
 508 }
 509
 510 static TOOL_FOPS_RDWR(tool_peer_link_fops,
 511                       tool_peer_link_read,
 512                       NULL);
 513
 514 static ssize_t tool_peer_link_event_write(struct file *filep,
 515                                           const char __user *ubuf,
 516                                           size_t size, loff_t *offp)
 517 {
 518         struct tool_peer *peer = filep->private_data;
 519         struct tool_ctx *tc = peer->tc;
 520         u64 link_msk;
 521         bool val;
 522         int ret;
 523
 524         ret = kstrtobool_from_user(ubuf, size, &val);
 525         if (ret)
 526                 return ret;
 527
 528         link_msk = BIT_ULL_MASK(peer->pidx);
 529
 530         if (wait_event_interruptible(tc->link_wq,
 531                 !!(ntb_link_is_up(tc->ntb, NULL, NULL) & link_msk) == val))
 532                 return -ERESTART;
 533
 534         return size;
 535 }
 536
 537 static TOOL_FOPS_RDWR(tool_peer_link_event_fops,
 538                       NULL,
 539                       tool_peer_link_event_write);
 540
 541 /*==============================================================================
 542  *                  Memory windows read/write/setting methods
 543  *==============================================================================
 544  */
 545
 546 static ssize_t tool_mw_read(struct file *filep, char __user *ubuf,
 547                             size_t size, loff_t *offp)
 548 {
 549         struct tool_mw *inmw = filep->private_data;
 550
 551         if (inmw->mm_base == NULL)
 552                 return -ENXIO;
 553
 554         return simple_read_from_buffer(ubuf, size, offp,
 555                                        inmw->mm_base, inmw->size);
 556 }
 557
 558 static ssize_t tool_mw_write(struct file *filep, const char __user *ubuf,
 559                              size_t size, loff_t *offp)
 560 {
 561         struct tool_mw *inmw = filep->private_data;
 562
 563         if (inmw->mm_base == NULL)
 564                 return -ENXIO;
 565
 566         return simple_write_to_buffer(inmw->mm_base, inmw->size, offp,
 567                                       ubuf, size);
 568 }
 569
 570 static TOOL_FOPS_RDWR(tool_mw_fops,
 571                       tool_mw_read,
 572                       tool_mw_write);
 573
 574 static int tool_setup_mw(struct tool_ctx *tc, int pidx, int widx,
 575                          size_t req_size)
 576 {
 577         resource_size_t size, addr_align, size_align;
 578         struct tool_mw *inmw = &tc->peers[pidx].inmws[widx];
 579         char buf[TOOL_BUF_LEN];
 580         int ret;
 581
 582         if (inmw->mm_base != NULL)
 583                 return 0;
 584
 585         ret = ntb_mw_get_align(tc->ntb, pidx, widx, &addr_align,
 586                                 &size_align, &size);
 587         if (ret)
 588                 return ret;
 589
 590         inmw->size = min_t(resource_size_t, req_size, size);
 591         inmw->size = round_up(inmw->size, addr_align);
 592         inmw->size = round_up(inmw->size, size_align);
 593         inmw->mm_base = dma_alloc_coherent(&tc->ntb->pdev->dev, inmw->size,
 594                                            &inmw->dma_base, GFP_KERNEL);
 595         if (!inmw->mm_base)
 596                 return -ENOMEM;
 597
 598         if (!IS_ALIGNED(inmw->dma_base, addr_align)) {
 599                 ret = -ENOMEM;
 600                 goto err_free_dma;
 601         }
 602
 603         ret = ntb_mw_set_trans(tc->ntb, pidx, widx, inmw->dma_base, inmw->size);
 604         if (ret)
 605                 goto err_free_dma;
 606
 607         snprintf(buf, sizeof(buf), "mw%d", widx);
 608         inmw->dbgfs_file = debugfs_create_file(buf, 0600,
 609                                                tc->peers[pidx].dbgfs_dir, inmw,
 610                                                &tool_mw_fops);
 611
 612         return 0;
 613
 614 err_free_dma:
 615         dma_free_coherent(&tc->ntb->pdev->dev, inmw->size, inmw->mm_base,
 616                           inmw->dma_base);
 617         inmw->mm_base = NULL;
 618         inmw->dma_base = 0;
 619         inmw->size = 0;
 620
 621         return ret;
 622 }
 623
 624 static void tool_free_mw(struct tool_ctx *tc, int pidx, int widx)
 625 {
 626         struct tool_mw *inmw = &tc->peers[pidx].inmws[widx];
 627
 628         debugfs_remove(inmw->dbgfs_file);
 629
 630         if (inmw->mm_base != NULL) {
 631                 ntb_mw_clear_trans(tc->ntb, pidx, widx);
 632                 dma_free_coherent(&tc->ntb->pdev->dev, inmw->size,
 633                                   inmw->mm_base, inmw->dma_base);
 634         }
 635
 636         inmw->mm_base = NULL;
 637         inmw->dma_base = 0;
 638         inmw->size = 0;
 639         inmw->dbgfs_file = NULL;
 640 }
 641
 642 static ssize_t tool_mw_trans_read(struct file *filep, char __user *ubuf,
 643                                   size_t size, loff_t *offp)
 644 {
 645         struct tool_mw *inmw = filep->private_data;
 646         resource_size_t addr_align;
 647         resource_size_t size_align;
 648         resource_size_t size_max;
 649         ssize_t ret, off = 0;
 650         size_t buf_size;
 651         char *buf;
 652
 653         buf_size = min_t(size_t, size, 512);
 654
 655         buf = kmalloc(buf_size, GFP_KERNEL);
 656         if (!buf)
 657                 return -ENOMEM;
 658
 659         ret = ntb_mw_get_align(inmw->tc->ntb, inmw->pidx, inmw->widx,
 660                                &addr_align, &size_align, &size_max);
 661         if (ret)
 662                 goto err;
 663
 664         off += scnprintf(buf + off, buf_size - off,
 665                          "Inbound MW     \t%d\n",
 666                          inmw->widx);
 667
 668         off += scnprintf(buf + off, buf_size - off,
 669                          "Port           \t%d (%d)\n",
 670                          ntb_peer_port_number(inmw->tc->ntb, inmw->pidx),
 671                          inmw->pidx);
 672
 673         off += scnprintf(buf + off, buf_size - off,
 674                          "Window Address \t0x%pK\n", inmw->mm_base);
 675
 676         off += scnprintf(buf + off, buf_size - off,
 677                          "DMA Address    \t%pad\n",
 678                          &inmw->dma_base);
 679
 680         off += scnprintf(buf + off, buf_size - off,
 681                          "Window Size    \t%pa[p]\n",
 682                          &inmw->size);
 683
 684         off += scnprintf(buf + off, buf_size - off,
 685                          "Alignment      \t%pa[p]\n",
 686                          &addr_align);
 687
 688         off += scnprintf(buf + off, buf_size - off,
 689                          "Size Alignment \t%pa[p]\n",
 690                          &size_align);
 691
 692         off += scnprintf(buf + off, buf_size - off,
 693                          "Size Max       \t%pa[p]\n",
 694                          &size_max);
 695
 696         ret = simple_read_from_buffer(ubuf, size, offp, buf, off);
 697
 698 err:
 699         kfree(buf);
 700
 701         return ret;
 702 }
 703
 704 static ssize_t tool_mw_trans_write(struct file *filep, const char __user *ubuf,
 705                                    size_t size, loff_t *offp)
 706 {
 707         struct tool_mw *inmw = filep->private_data;
 708         unsigned int val;
 709         int ret;
 710
 711         ret = kstrtouint_from_user(ubuf, size, 0, &val);
 712         if (ret)
 713                 return ret;
 714
 715         tool_free_mw(inmw->tc, inmw->pidx, inmw->widx);
 716         if (val) {
 717                 ret = tool_setup_mw(inmw->tc, inmw->pidx, inmw->widx, val);
 718                 if (ret)
 719                         return ret;
 720         }
 721
 722         return size;
 723 }
 724
 725 static TOOL_FOPS_RDWR(tool_mw_trans_fops,
 726                       tool_mw_trans_read,
 727                       tool_mw_trans_write);
 728
 729 static ssize_t tool_peer_mw_read(struct file *filep, char __user *ubuf,
 730                                  size_t size, loff_t *offp)
 731 {
 732         struct tool_mw *outmw = filep->private_data;
 733         loff_t pos = *offp;
 734         ssize_t ret;
 735         void *buf;
 736
 737         if (outmw->io_base == NULL)
 738                 return -EIO;
 739
 740         if (pos >= outmw->size || !size)
 741                 return 0;
 742
 743         if (size > outmw->size - pos)
 744                 size = outmw->size - pos;
 745
 746         buf = kmalloc(size, GFP_KERNEL);
 747         if (!buf)
 748                 return -ENOMEM;
 749
 750         memcpy_fromio(buf, outmw->io_base + pos, size);
 751         ret = copy_to_user(ubuf, buf, size);
 752         if (ret == size) {
 753                 ret = -EFAULT;
 754                 goto err_free;
 755         }
 756
 757         size -= ret;
 758         *offp = pos + size;
 759         ret = size;
 760
 761 err_free:
 762         kfree(buf);
 763
 764         return ret;
 765 }
 766
 767 static ssize_t tool_peer_mw_write(struct file *filep, const char __user *ubuf,
 768                                   size_t size, loff_t *offp)
 769 {
 770         struct tool_mw *outmw = filep->private_data;
 771         ssize_t ret;
 772         loff_t pos = *offp;
 773         void *buf;
 774
 775         if (outmw->io_base == NULL)
 776                 return -EIO;
 777
 778         if (pos >= outmw->size || !size)
 779                 return 0;
 780         if (size > outmw->size - pos)
 781                 size = outmw->size - pos;
 782
 783         buf = kmalloc(size, GFP_KERNEL);
 784         if (!buf)
 785                 return -ENOMEM;
 786
 787         ret = copy_from_user(buf, ubuf, size);
 788         if (ret == size) {
 789                 ret = -EFAULT;
 790                 goto err_free;
 791         }
 792
 793         size -= ret;
 794         *offp = pos + size;
 795         ret = size;
 796
 797         memcpy_toio(outmw->io_base + pos, buf, size);
 798
 799 err_free:
 800         kfree(buf);
 801
 802         return ret;
 803 }
 804
 805 static TOOL_FOPS_RDWR(tool_peer_mw_fops,
 806                       tool_peer_mw_read,
 807                       tool_peer_mw_write);
 808
 809 static int tool_setup_peer_mw(struct tool_ctx *tc, int pidx, int widx,
 810                               u64 req_addr, size_t req_size)
 811 {
 812         struct tool_mw *outmw = &tc->outmws[widx];
 813         resource_size_t map_size;
 814         phys_addr_t map_base;
 815         char buf[TOOL_BUF_LEN];
 816         int ret;
 817
 818         if (outmw->io_base != NULL)
 819                 return 0;
 820
 821         ret = ntb_peer_mw_get_addr(tc->ntb, widx, &map_base, &map_size);
 822         if (ret)
 823                 return ret;
 824
 825         ret = ntb_peer_mw_set_trans(tc->ntb, pidx, widx, req_addr, req_size);
 826         if (ret)
 827                 return ret;
 828
 829         outmw->io_base = ioremap_wc(map_base, map_size);
 830         if (outmw->io_base == NULL) {
 831                 ret = -EFAULT;
 832                 goto err_clear_trans;
 833         }
 834
 835         outmw->tr_base = req_addr;
 836         outmw->size = req_size;
 837         outmw->pidx = pidx;
 838
 839         snprintf(buf, sizeof(buf), "peer_mw%d", widx);
 840         outmw->dbgfs_file = debugfs_create_file(buf, 0600,
 841                                                tc->peers[pidx].dbgfs_dir, outmw,
 842                                                &tool_peer_mw_fops);
 843
 844         return 0;
 845
 846 err_clear_trans:
 847         ntb_peer_mw_clear_trans(tc->ntb, pidx, widx);
 848
 849         return ret;
 850 }
 851
 852 static void tool_free_peer_mw(struct tool_ctx *tc, int widx)
 853 {
 854         struct tool_mw *outmw = &tc->outmws[widx];
 855
 856         debugfs_remove(outmw->dbgfs_file);
 857
 858         if (outmw->io_base != NULL) {
 859                 iounmap(tc->outmws[widx].io_base);
 860                 ntb_peer_mw_clear_trans(tc->ntb, outmw->pidx, widx);
 861         }
 862
 863         outmw->io_base = NULL;
 864         outmw->tr_base = 0;
 865         outmw->size = 0;
 866         outmw->pidx = -1;
 867         outmw->dbgfs_file = NULL;
 868 }
 869
 870 static ssize_t tool_peer_mw_trans_read(struct file *filep, char __user *ubuf,
 871                                         size_t size, loff_t *offp)
 872 {
 873         struct tool_mw_wrap *outmw_wrap = filep->private_data;
 874         struct tool_mw *outmw = outmw_wrap->mw;
 875         resource_size_t map_size;
 876         phys_addr_t map_base;
 877         ssize_t off = 0;
 878         size_t buf_size;
 879         char *buf;
 880         int ret;
 881
 882         ret = ntb_peer_mw_get_addr(outmw->tc->ntb, outmw->widx,
 883                                   &map_base, &map_size);
 884         if (ret)
 885                 return ret;
 886
 887         buf_size = min_t(size_t, size, 512);
 888
 889         buf = kmalloc(buf_size, GFP_KERNEL);
 890         if (!buf)
 891                 return -ENOMEM;
 892
 893         off += scnprintf(buf + off, buf_size - off,
 894                          "Outbound MW:        \t%d\n", outmw->widx);
 895
 896         if (outmw->io_base != NULL) {
 897                 off += scnprintf(buf + off, buf_size - off,
 898                         "Port attached       \t%d (%d)\n",
 899                         ntb_peer_port_number(outmw->tc->ntb, outmw->pidx),
 900                         outmw->pidx);
 901         } else {
 902                 off += scnprintf(buf + off, buf_size - off,
 903                                  "Port attached       \t-1 (-1)\n");
 904         }
 905
 906         off += scnprintf(buf + off, buf_size - off,
 907                          "Virtual address     \t0x%pK\n", outmw->io_base);
 908
 909         off += scnprintf(buf + off, buf_size - off,
 910                          "Phys Address        \t%pa[p]\n", &map_base);
 911
 912         off += scnprintf(buf + off, buf_size - off,
 913                          "Mapping Size        \t%pa[p]\n", &map_size);
 914
 915         off += scnprintf(buf + off, buf_size - off,
 916                          "Translation Address \t0x%016llx\n", outmw->tr_base);
 917
 918         off += scnprintf(buf + off, buf_size - off,
 919                          "Window Size         \t%pa[p]\n", &outmw->size);
 920
 921         ret = simple_read_from_buffer(ubuf, size, offp, buf, off);
 922         kfree(buf);
 923
 924         return ret;
 925 }
 926
 927 static ssize_t tool_peer_mw_trans_write(struct file *filep,
 928                                         const char __user *ubuf,
 929                                         size_t size, loff_t *offp)
 930 {
 931         struct tool_mw_wrap *outmw_wrap = filep->private_data;
 932         struct tool_mw *outmw = outmw_wrap->mw;
 933         size_t buf_size, wsize;
 934         char buf[TOOL_BUF_LEN];
 935         int ret, n;
 936         u64 addr;
 937
 938         buf_size = min(size, (sizeof(buf) - 1));
 939         if (copy_from_user(buf, ubuf, buf_size))
 940                 return -EFAULT;
 941
 942         buf[buf_size] = '\0';
 943
 944         n = sscanf(buf, "%lli:%zi", &addr, &wsize);
 945         if (n != 2)
 946                 return -EINVAL;
 947
 948         tool_free_peer_mw(outmw->tc, outmw->widx);
 949         if (wsize) {
 950                 ret = tool_setup_peer_mw(outmw->tc, outmw_wrap->pidx,
 951                                          outmw->widx, addr, wsize);
 952                 if (ret)
 953                         return ret;
 954         }
 955
 956         return size;
 957 }
 958
 959 static TOOL_FOPS_RDWR(tool_peer_mw_trans_fops,
 960                       tool_peer_mw_trans_read,
 961                       tool_peer_mw_trans_write);
 962
 963 static int tool_init_mws(struct tool_ctx *tc)
 964 {
 965         int widx, pidx;
 966
 967         /* Initialize outbound memory windows */
 968         tc->outmw_cnt = ntb_peer_mw_count(tc->ntb);
 969         tc->outmws = devm_kcalloc(&tc->ntb->dev, tc->outmw_cnt,
 970                                   sizeof(*tc->outmws), GFP_KERNEL);
 971         if (tc->outmws == NULL)
 972                 return -ENOMEM;
 973
 974         for (widx = 0; widx < tc->outmw_cnt; widx++) {
 975                 tc->outmws[widx].widx = widx;
 976                 tc->outmws[widx].pidx = -1;
 977                 tc->outmws[widx].tc = tc;
 978         }
 979
 980         /* Initialize inbound memory windows and outbound MWs wrapper */
 981         for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
 982                 tc->peers[pidx].inmw_cnt = ntb_mw_count(tc->ntb, pidx);
 983                 tc->peers[pidx].inmws =
 984                         devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].inmw_cnt,
 985                                     sizeof(*tc->peers[pidx].inmws), GFP_KERNEL);
 986                 if (tc->peers[pidx].inmws == NULL)
 987                         return -ENOMEM;
 988
 989                 for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++) {
 990                         tc->peers[pidx].inmws[widx].widx = widx;
 991                         tc->peers[pidx].inmws[widx].pidx = pidx;
 992                         tc->peers[pidx].inmws[widx].tc = tc;
 993                 }
 994
 995                 tc->peers[pidx].outmw_cnt = ntb_peer_mw_count(tc->ntb);
 996                 tc->peers[pidx].outmws =
 997                         devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].outmw_cnt,
 998                                    sizeof(*tc->peers[pidx].outmws), GFP_KERNEL);
 999
1000                 for (widx = 0; widx < tc->peers[pidx].outmw_cnt; widx++) {
1001                         tc->peers[pidx].outmws[widx].pidx = pidx;
1002                         tc->peers[pidx].outmws[widx].mw = &tc->outmws[widx];
1003                 }
1004         }
1005
1006         return 0;
1007 }
1008
1009 static void tool_clear_mws(struct tool_ctx *tc)
1010 {
1011         int widx, pidx;
1012
1013         /* Free outbound memory windows */
1014         for (widx = 0; widx < tc->outmw_cnt; widx++)
1015                 tool_free_peer_mw(tc, widx);
1016
1017         /* Free outbound memory windows */
1018         for (pidx = 0; pidx < tc->peer_cnt; pidx++)
1019                 for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++)
1020                         tool_free_mw(tc, pidx, widx);
1021 }
1022
1023 /*==============================================================================
1024  *                       Doorbell read/write methods
1025  *==============================================================================
1026  */
1027
1028 static ssize_t tool_db_read(struct file *filep, char __user *ubuf,
1029                             size_t size, loff_t *offp)
1030 {
1031         struct tool_ctx *tc = filep->private_data;
1032
1033         return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->db_read);
1034 }
1035
1036 static ssize_t tool_db_write(struct file *filep, const char __user *ubuf,
1037                              size_t size, loff_t *offp)
1038 {
1039         struct tool_ctx *tc = filep->private_data;
1040
1041         return tool_fn_write(tc, ubuf, size, offp, tc->ntb->ops->db_set,
1042                              tc->ntb->ops->db_clear);
1043 }
1044
1045 static TOOL_FOPS_RDWR(tool_db_fops,
1046                       tool_db_read,
1047                       tool_db_write);
1048
1049 static ssize_t tool_db_valid_mask_read(struct file *filep, char __user *ubuf,
1050                                        size_t size, loff_t *offp)
1051 {
1052         struct tool_ctx *tc = filep->private_data;
1053
1054         return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->db_valid_mask);
1055 }
1056
1057 static TOOL_FOPS_RDWR(tool_db_valid_mask_fops,
1058                       tool_db_valid_mask_read,
1059                       NULL);
1060
1061 static ssize_t tool_db_mask_read(struct file *filep, char __user *ubuf,
1062                                  size_t size, loff_t *offp)
1063 {
1064         struct tool_ctx *tc = filep->private_data;
1065
1066         return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->db_read_mask);
1067 }
1068
1069 static ssize_t tool_db_mask_write(struct file *filep, const char __user *ubuf,
1070                                size_t size, loff_t *offp)
1071 {
1072         struct tool_ctx *tc = filep->private_data;
1073
1074         return tool_fn_write(tc, ubuf, size, offp, tc->ntb->ops->db_set_mask,
1075                              tc->ntb->ops->db_clear_mask);
1076 }
1077
1078 static TOOL_FOPS_RDWR(tool_db_mask_fops,
1079                       tool_db_mask_read,
1080                       tool_db_mask_write);
1081
1082 static ssize_t tool_peer_db_read(struct file *filep, char __user *ubuf,
1083                                  size_t size, loff_t *offp)
1084 {
1085         struct tool_ctx *tc = filep->private_data;
1086
1087         return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->peer_db_read);
1088 }
1089
1090 static ssize_t tool_peer_db_write(struct file *filep, const char __user *ubuf,
1091                                   size_t size, loff_t *offp)
1092 {
1093         struct tool_ctx *tc = filep->private_data;
1094
1095         return tool_fn_write(tc, ubuf, size, offp, tc->ntb->ops->peer_db_set,
1096                              tc->ntb->ops->peer_db_clear);
1097 }
1098
1099 static TOOL_FOPS_RDWR(tool_peer_db_fops,
1100                       tool_peer_db_read,
1101                       tool_peer_db_write);
1102
1103 static ssize_t tool_peer_db_mask_read(struct file *filep, char __user *ubuf,
1104                                    size_t size, loff_t *offp)
1105 {
1106         struct tool_ctx *tc = filep->private_data;
1107
1108         return tool_fn_read(tc, ubuf, size, offp,
1109                             tc->ntb->ops->peer_db_read_mask);
1110 }
1111
1112 static ssize_t tool_peer_db_mask_write(struct file *filep,
1113                                        const char __user *ubuf,
1114                                        size_t size, loff_t *offp)
1115 {
1116         struct tool_ctx *tc = filep->private_data;
1117
1118         return tool_fn_write(tc, ubuf, size, offp,
1119                              tc->ntb->ops->peer_db_set_mask,
1120                              tc->ntb->ops->peer_db_clear_mask);
1121 }
1122
1123 static TOOL_FOPS_RDWR(tool_peer_db_mask_fops,
1124                       tool_peer_db_mask_read,
1125                       tool_peer_db_mask_write);
1126
1127 static ssize_t tool_db_event_write(struct file *filep,
1128                                    const char __user *ubuf,
1129                                    size_t size, loff_t *offp)
1130 {
1131         struct tool_ctx *tc = filep->private_data;
1132         u64 val;
1133         int ret;
1134
1135         ret = kstrtou64_from_user(ubuf, size, 0, &val);
1136         if (ret)
1137                 return ret;
1138
1139         if (wait_event_interruptible(tc->db_wq, ntb_db_read(tc->ntb) == val))
1140                 return -ERESTART;
1141
1142         return size;
1143 }
1144
1145 static TOOL_FOPS_RDWR(tool_db_event_fops,
1146                       NULL,
1147                       tool_db_event_write);
1148
1149 /*==============================================================================
1150  *                       Scratchpads read/write methods
1151  *==============================================================================
1152  */
1153
1154 static ssize_t tool_spad_read(struct file *filep, char __user *ubuf,
1155                               size_t size, loff_t *offp)
1156 {
1157         struct tool_spad *spad = filep->private_data;
1158         char buf[TOOL_BUF_LEN];
1159         ssize_t pos;
1160
1161         if (!spad->tc->ntb->ops->spad_read)
1162                 return -EINVAL;
1163
1164         pos = scnprintf(buf, sizeof(buf), "%#x\n",
1165                 ntb_spad_read(spad->tc->ntb, spad->sidx));
1166
1167         return simple_read_from_buffer(ubuf, size, offp, buf, pos);
1168 }
1169
1170 static ssize_t tool_spad_write(struct file *filep, const char __user *ubuf,
1171                                size_t size, loff_t *offp)
1172 {
1173         struct tool_spad *spad = filep->private_data;
1174         u32 val;
1175         int ret;
1176
1177         if (!spad->tc->ntb->ops->spad_write) {
1178                 dev_dbg(&spad->tc->ntb->dev, "no spad write fn\n");
1179                 return -EINVAL;
1180         }
1181
1182         ret = kstrtou32_from_user(ubuf, size, 0, &val);
1183         if (ret)
1184                 return ret;
1185
1186         ret = ntb_spad_write(spad->tc->ntb, spad->sidx, val);
1187
1188         return ret ?: size;
1189 }
1190
1191 static TOOL_FOPS_RDWR(tool_spad_fops,
1192                       tool_spad_read,
1193                       tool_spad_write);
1194
1195 static ssize_t tool_peer_spad_read(struct file *filep, char __user *ubuf,
1196                                    size_t size, loff_t *offp)
1197 {
1198         struct tool_spad *spad = filep->private_data;
1199         char buf[TOOL_BUF_LEN];
1200         ssize_t pos;
1201
1202         if (!spad->tc->ntb->ops->peer_spad_read)
1203                 return -EINVAL;
1204
1205         pos = scnprintf(buf, sizeof(buf), "%#x\n",
1206                 ntb_peer_spad_read(spad->tc->ntb, spad->pidx, spad->sidx));
1207
1208         return simple_read_from_buffer(ubuf, size, offp, buf, pos);
1209 }
1210
1211 static ssize_t tool_peer_spad_write(struct file *filep, const char __user *ubuf,
1212                                     size_t size, loff_t *offp)
1213 {
1214         struct tool_spad *spad = filep->private_data;
1215         u32 val;
1216         int ret;
1217
1218         if (!spad->tc->ntb->ops->peer_spad_write) {
1219                 dev_dbg(&spad->tc->ntb->dev, "no spad write fn\n");
1220                 return -EINVAL;
1221         }
1222
1223         ret = kstrtou32_from_user(ubuf, size, 0, &val);
1224         if (ret)
1225                 return ret;
1226
1227         ret = ntb_peer_spad_write(spad->tc->ntb, spad->pidx, spad->sidx, val);
1228
1229         return ret ?: size;
1230 }
1231
1232 static TOOL_FOPS_RDWR(tool_peer_spad_fops,
1233                       tool_peer_spad_read,
1234                       tool_peer_spad_write);
1235
1236 static int tool_init_spads(struct tool_ctx *tc)
1237 {
1238         int sidx, pidx;
1239
1240         /* Initialize inbound scratchpad structures */
1241         tc->inspad_cnt = ntb_spad_count(tc->ntb);
1242         tc->inspads = devm_kcalloc(&tc->ntb->dev, tc->inspad_cnt,
1243                                    sizeof(*tc->inspads), GFP_KERNEL);
1244         if (tc->inspads == NULL)
1245                 return -ENOMEM;
1246
1247         for (sidx = 0; sidx < tc->inspad_cnt; sidx++) {
1248                 tc->inspads[sidx].sidx = sidx;
1249                 tc->inspads[sidx].pidx = -1;
1250                 tc->inspads[sidx].tc = tc;
1251         }
1252
1253         /* Initialize outbound scratchpad structures */
1254         for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
1255                 tc->peers[pidx].outspad_cnt = ntb_spad_count(tc->ntb);
1256                 tc->peers[pidx].outspads =
1257                         devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].outspad_cnt,
1258                                 sizeof(*tc->peers[pidx].outspads), GFP_KERNEL);
1259                 if (tc->peers[pidx].outspads == NULL)
1260                         return -ENOMEM;
1261
1262                 for (sidx = 0; sidx < tc->peers[pidx].outspad_cnt; sidx++) {
1263                         tc->peers[pidx].outspads[sidx].sidx = sidx;
1264                         tc->peers[pidx].outspads[sidx].pidx = pidx;
1265                         tc->peers[pidx].outspads[sidx].tc = tc;
1266                 }
1267         }
1268
1269         return 0;
1270 }
1271
1272 /*==============================================================================
1273  *                       Messages read/write methods
1274  *==============================================================================
1275  */
1276
1277 static ssize_t tool_inmsg_read(struct file *filep, char __user *ubuf,
1278                                size_t size, loff_t *offp)
1279 {
1280         struct tool_msg *msg = filep->private_data;
1281         char buf[TOOL_BUF_LEN];
1282         ssize_t pos;
1283         u32 data;
1284         int pidx;
1285
1286         data = ntb_msg_read(msg->tc->ntb, &pidx, msg->midx);
1287
1288         pos = scnprintf(buf, sizeof(buf), "0x%08x<-%d\n", data, pidx);
1289
1290         return simple_read_from_buffer(ubuf, size, offp, buf, pos);
1291 }
1292
1293 static TOOL_FOPS_RDWR(tool_inmsg_fops,
1294                       tool_inmsg_read,
1295                       NULL);
1296
1297 static ssize_t tool_outmsg_write(struct file *filep,
1298                                  const char __user *ubuf,
1299                                  size_t size, loff_t *offp)
1300 {
1301         struct tool_msg *msg = filep->private_data;
1302         u32 val;
1303         int ret;
1304
1305         ret = kstrtou32_from_user(ubuf, size, 0, &val);
1306         if (ret)
1307                 return ret;
1308
1309         ret = ntb_peer_msg_write(msg->tc->ntb, msg->pidx, msg->midx, val);
1310
1311         return ret ? : size;
1312 }
1313
1314 static TOOL_FOPS_RDWR(tool_outmsg_fops,
1315                       NULL,
1316                       tool_outmsg_write);
1317
1318 static ssize_t tool_msg_sts_read(struct file *filep, char __user *ubuf,
1319                                  size_t size, loff_t *offp)
1320 {
1321         struct tool_ctx *tc = filep->private_data;
1322
1323         return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->msg_read_sts);
1324 }
1325
1326 static ssize_t tool_msg_sts_write(struct file *filep, const char __user *ubuf,
1327                                   size_t size, loff_t *offp)
1328 {
1329         struct tool_ctx *tc = filep->private_data;
1330
1331         return tool_fn_write(tc, ubuf, size, offp, NULL,
1332                              tc->ntb->ops->msg_clear_sts);
1333 }
1334
1335 static TOOL_FOPS_RDWR(tool_msg_sts_fops,
1336                       tool_msg_sts_read,
1337                       tool_msg_sts_write);
1338
1339 static ssize_t tool_msg_inbits_read(struct file *filep, char __user *ubuf,
1340                                     size_t size, loff_t *offp)
1341 {
1342         struct tool_ctx *tc = filep->private_data;
1343
1344         return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->msg_inbits);
1345 }
1346
1347 static TOOL_FOPS_RDWR(tool_msg_inbits_fops,
1348                       tool_msg_inbits_read,
1349                       NULL);
1350
1351 static ssize_t tool_msg_outbits_read(struct file *filep, char __user *ubuf,
1352                                      size_t size, loff_t *offp)
1353 {
1354         struct tool_ctx *tc = filep->private_data;
1355
1356         return tool_fn_read(tc, ubuf, size, offp, tc->ntb->ops->msg_outbits);
1357 }
1358
1359 static TOOL_FOPS_RDWR(tool_msg_outbits_fops,
1360                       tool_msg_outbits_read,
1361                       NULL);
1362
1363 static ssize_t tool_msg_mask_write(struct file *filep, const char __user *ubuf,
1364                                    size_t size, loff_t *offp)
1365 {
1366         struct tool_ctx *tc = filep->private_data;
1367
1368         return tool_fn_write(tc, ubuf, size, offp,
1369                              tc->ntb->ops->msg_set_mask,
1370                              tc->ntb->ops->msg_clear_mask);
1371 }
1372
1373 static TOOL_FOPS_RDWR(tool_msg_mask_fops,
1374                       NULL,
1375                       tool_msg_mask_write);
1376
1377 static ssize_t tool_msg_event_write(struct file *filep,
1378                                     const char __user *ubuf,
1379                                     size_t size, loff_t *offp)
1380 {
1381         struct tool_ctx *tc = filep->private_data;
1382         u64 val;
1383         int ret;
1384
1385         ret = kstrtou64_from_user(ubuf, size, 0, &val);
1386         if (ret)
1387                 return ret;
1388
1389         if (wait_event_interruptible(tc->msg_wq,
1390                 ntb_msg_read_sts(tc->ntb) == val))
1391                 return -ERESTART;
1392
1393         return size;
1394 }
1395
1396 static TOOL_FOPS_RDWR(tool_msg_event_fops,
1397                       NULL,
1398                       tool_msg_event_write);
1399
1400 static int tool_init_msgs(struct tool_ctx *tc)
1401 {
1402         int midx, pidx;
1403
1404         /* Initialize inbound message structures */
1405         tc->inmsg_cnt = ntb_msg_count(tc->ntb);
1406         tc->inmsgs = devm_kcalloc(&tc->ntb->dev, tc->inmsg_cnt,
1407                                    sizeof(*tc->inmsgs), GFP_KERNEL);
1408         if (tc->inmsgs == NULL)
1409                 return -ENOMEM;
1410
1411         for (midx = 0; midx < tc->inmsg_cnt; midx++) {
1412                 tc->inmsgs[midx].midx = midx;
1413                 tc->inmsgs[midx].pidx = -1;
1414                 tc->inmsgs[midx].tc = tc;
1415         }
1416
1417         /* Initialize outbound message structures */
1418         for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
1419                 tc->peers[pidx].outmsg_cnt = ntb_msg_count(tc->ntb);
1420                 tc->peers[pidx].outmsgs =
1421                         devm_kcalloc(&tc->ntb->dev, tc->peers[pidx].outmsg_cnt,
1422                                 sizeof(*tc->peers[pidx].outmsgs), GFP_KERNEL);
1423                 if (tc->peers[pidx].outmsgs == NULL)
1424                         return -ENOMEM;
1425
1426                 for (midx = 0; midx < tc->peers[pidx].outmsg_cnt; midx++) {
1427                         tc->peers[pidx].outmsgs[midx].midx = midx;
1428                         tc->peers[pidx].outmsgs[midx].pidx = pidx;
1429                         tc->peers[pidx].outmsgs[midx].tc = tc;
1430                 }
1431         }
1432
1433         return 0;
1434 }
1435
1436 /*==============================================================================
1437  *                          Initialization methods
1438  *==============================================================================
1439  */
1440
1441 static struct tool_ctx *tool_create_data(struct ntb_dev *ntb)
1442 {
1443         struct tool_ctx *tc;
1444
1445         tc = devm_kzalloc(&ntb->dev, sizeof(*tc), GFP_KERNEL);
1446         if (tc == NULL)
1447                 return ERR_PTR(-ENOMEM);
1448
1449         tc->ntb = ntb;
1450         init_waitqueue_head(&tc->link_wq);
1451         init_waitqueue_head(&tc->db_wq);
1452         init_waitqueue_head(&tc->msg_wq);
1453
1454         if (ntb_db_is_unsafe(ntb))
1455                 dev_dbg(&ntb->dev, "doorbell is unsafe\n");
1456
1457         if (ntb_spad_is_unsafe(ntb))
1458                 dev_dbg(&ntb->dev, "scratchpad is unsafe\n");
1459
1460         return tc;
1461 }
1462
1463 static void tool_clear_data(struct tool_ctx *tc)
1464 {
1465         wake_up(&tc->link_wq);
1466         wake_up(&tc->db_wq);
1467         wake_up(&tc->msg_wq);
1468 }
1469
1470 static int tool_init_ntb(struct tool_ctx *tc)
1471 {
1472         return ntb_set_ctx(tc->ntb, tc, &tool_ops);
1473 }
1474
1475 static void tool_clear_ntb(struct tool_ctx *tc)
1476 {
1477         ntb_clear_ctx(tc->ntb);
1478         ntb_link_disable(tc->ntb);
1479 }
1480
1481 static void tool_setup_dbgfs(struct tool_ctx *tc)
1482 {
1483         int pidx, widx, sidx, midx;
1484         char buf[TOOL_BUF_LEN];
1485
1486         /* This modules is useless without dbgfs... */
1487         if (!tool_dbgfs_topdir) {
1488                 tc->dbgfs_dir = NULL;
1489                 return;
1490         }
1491
1492         tc->dbgfs_dir = debugfs_create_dir(dev_name(&tc->ntb->dev),
1493                                            tool_dbgfs_topdir);
1494         if (!tc->dbgfs_dir)
1495                 return;
1496
1497         debugfs_create_file("port", 0600, tc->dbgfs_dir,
1498                             tc, &tool_port_fops);
1499
1500         debugfs_create_file("link", 0600, tc->dbgfs_dir,
1501                             tc, &tool_link_fops);
1502
1503         debugfs_create_file("db", 0600, tc->dbgfs_dir,
1504                             tc, &tool_db_fops);
1505
1506         debugfs_create_file("db_valid_mask", 0600, tc->dbgfs_dir,
1507                             tc, &tool_db_valid_mask_fops);
1508
1509         debugfs_create_file("db_mask", 0600, tc->dbgfs_dir,
1510                             tc, &tool_db_mask_fops);
1511
1512         debugfs_create_file("db_event", 0600, tc->dbgfs_dir,
1513                             tc, &tool_db_event_fops);
1514
1515         debugfs_create_file("peer_db", 0600, tc->dbgfs_dir,
1516                             tc, &tool_peer_db_fops);
1517
1518         debugfs_create_file("peer_db_mask", 0600, tc->dbgfs_dir,
1519                             tc, &tool_peer_db_mask_fops);
1520
1521         if (tc->inspad_cnt != 0) {
1522                 for (sidx = 0; sidx < tc->inspad_cnt; sidx++) {
1523                         snprintf(buf, sizeof(buf), "spad%d", sidx);
1524
1525                         debugfs_create_file(buf, 0600, tc->dbgfs_dir,
1526                                            &tc->inspads[sidx], &tool_spad_fops);
1527                 }
1528         }
1529
1530         if (tc->inmsg_cnt != 0) {
1531                 for (midx = 0; midx < tc->inmsg_cnt; midx++) {
1532                         snprintf(buf, sizeof(buf), "msg%d", midx);
1533                         debugfs_create_file(buf, 0600, tc->dbgfs_dir,
1534                                            &tc->inmsgs[midx], &tool_inmsg_fops);
1535                 }
1536
1537                 debugfs_create_file("msg_sts", 0600, tc->dbgfs_dir,
1538                                     tc, &tool_msg_sts_fops);
1539
1540                 debugfs_create_file("msg_inbits", 0600, tc->dbgfs_dir,
1541                                     tc, &tool_msg_inbits_fops);
1542
1543                 debugfs_create_file("msg_outbits", 0600, tc->dbgfs_dir,
1544                                     tc, &tool_msg_outbits_fops);
1545
1546                 debugfs_create_file("msg_mask", 0600, tc->dbgfs_dir,
1547                                     tc, &tool_msg_mask_fops);
1548
1549                 debugfs_create_file("msg_event", 0600, tc->dbgfs_dir,
1550                                     tc, &tool_msg_event_fops);
1551         }
1552
1553         for (pidx = 0; pidx < tc->peer_cnt; pidx++) {
1554                 snprintf(buf, sizeof(buf), "peer%d", pidx);
1555                 tc->peers[pidx].dbgfs_dir =
1556                         debugfs_create_dir(buf, tc->dbgfs_dir);
1557
1558                 debugfs_create_file("port", 0600,
1559                                     tc->peers[pidx].dbgfs_dir,
1560                                     &tc->peers[pidx], &tool_peer_port_fops);
1561
1562                 debugfs_create_file("link", 0200,
1563                                     tc->peers[pidx].dbgfs_dir,
1564                                     &tc->peers[pidx], &tool_peer_link_fops);
1565
1566                 debugfs_create_file("link_event", 0200,
1567                                   tc->peers[pidx].dbgfs_dir,
1568                                   &tc->peers[pidx], &tool_peer_link_event_fops);
1569
1570                 for (widx = 0; widx < tc->peers[pidx].inmw_cnt; widx++) {
1571                         snprintf(buf, sizeof(buf), "mw_trans%d", widx);
1572                         debugfs_create_file(buf, 0600,
1573                                             tc->peers[pidx].dbgfs_dir,
1574                                             &tc->peers[pidx].inmws[widx],
1575                                             &tool_mw_trans_fops);
1576                 }
1577
1578                 for (widx = 0; widx < tc->peers[pidx].outmw_cnt; widx++) {
1579                         snprintf(buf, sizeof(buf), "peer_mw_trans%d", widx);
1580                         debugfs_create_file(buf, 0600,
1581                                             tc->peers[pidx].dbgfs_dir,
1582                                             &tc->peers[pidx].outmws[widx],
1583                                             &tool_peer_mw_trans_fops);
1584                 }
1585
1586                 for (sidx = 0; sidx < tc->peers[pidx].outspad_cnt; sidx++) {
1587                         snprintf(buf, sizeof(buf), "spad%d", sidx);
1588
1589                         debugfs_create_file(buf, 0600,
1590                                             tc->peers[pidx].dbgfs_dir,
1591                                             &tc->peers[pidx].outspads[sidx],
1592                                             &tool_peer_spad_fops);
1593                 }
1594
1595                 for (midx = 0; midx < tc->peers[pidx].outmsg_cnt; midx++) {
1596                         snprintf(buf, sizeof(buf), "msg%d", midx);
1597                         debugfs_create_file(buf, 0600,
1598                                             tc->peers[pidx].dbgfs_dir,
1599                                             &tc->peers[pidx].outmsgs[midx],
1600                                             &tool_outmsg_fops);
1601                 }
1602         }
1603 }
1604
1605 static void tool_clear_dbgfs(struct tool_ctx *tc)
1606 {
1607         debugfs_remove_recursive(tc->dbgfs_dir);
1608 }
1609
1610 static int tool_probe(struct ntb_client *self, struct ntb_dev *ntb)
1611 {
1612         struct tool_ctx *tc;
1613         int ret;
1614
1615         tc = tool_create_data(ntb);
1616         if (IS_ERR(tc))
1617                 return PTR_ERR(tc);
1618
1619         ret = tool_init_peers(tc);
1620         if (ret != 0)
1621                 goto err_clear_data;
1622
1623         ret = tool_init_mws(tc);
1624         if (ret != 0)
1625                 goto err_clear_data;
1626
1627         ret = tool_init_spads(tc);
1628         if (ret != 0)
1629                 goto err_clear_mws;
1630
1631         ret = tool_init_msgs(tc);
1632         if (ret != 0)
1633                 goto err_clear_mws;
1634
1635         ret = tool_init_ntb(tc);
1636         if (ret != 0)
1637                 goto err_clear_mws;
1638
1639         tool_setup_dbgfs(tc);
1640
1641         return 0;
1642
1643 err_clear_mws:
1644         tool_clear_mws(tc);
1645
1646 err_clear_data:
1647         tool_clear_data(tc);
1648
1649         return ret;
1650 }
1651
1652 static void tool_remove(struct ntb_client *self, struct ntb_dev *ntb)
1653 {
1654         struct tool_ctx *tc = ntb->ctx;
1655
1656         tool_clear_dbgfs(tc);
1657
1658         tool_clear_ntb(tc);
1659
1660         tool_clear_mws(tc);
1661
1662         tool_clear_data(tc);
1663 }
1664
1665 static struct ntb_client tool_client = {
1666         .ops = {
1667                 .probe = tool_probe,
1668                 .remove = tool_remove,
1669         }
1670 };
1671
1672 static int __init tool_init(void)
1673 {
1674         int ret;
1675
1676         if (debugfs_initialized())
1677                 tool_dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, NULL);
1678
1679         ret = ntb_register_client(&tool_client);
1680         if (ret)
1681                 debugfs_remove_recursive(tool_dbgfs_topdir);
1682
1683         return ret;
1684 }
1685 module_init(tool_init);
1686
1687 static void __exit tool_exit(void)
1688 {
1689         ntb_unregister_client(&tool_client);
1690         debugfs_remove_recursive(tool_dbgfs_topdir);
1691 }
1692 module_exit(tool_exit);