drivers/pci/endpoint/functions/pci-epf-vntb.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Endpoint Function Driver to implement Non-Transparent Bridge functionality
   4  * Between PCI RC and EP
   5  *
   6  * Copyright (C) 2020 Texas Instruments
   7  * Copyright (C) 2022 NXP
   8  *
   9  * Based on pci-epf-ntb.c
  10  * Author: Frank Li <Frank.Li@nxp.com>
  11  * Author: Kishon Vijay Abraham I <kishon@ti.com>
  12  */
  13
  14 /*
  15  * +------------+         +---------------------------------------+
  16  * |            |         |                                       |
  17  * +------------+         |                        +--------------+
  18  * | NTB        |         |                        | NTB          |
  19  * | NetDev     |         |                        | NetDev       |
  20  * +------------+         |                        +--------------+
  21  * | NTB        |         |                        | NTB          |
  22  * | Transfer   |         |                        | Transfer     |
  23  * +------------+         |                        +--------------+
  24  * |            |         |                        |              |
  25  * |  PCI NTB   |         |                        |              |
  26  * |    EPF     |         |                        |              |
  27  * |   Driver   |         |                        | PCI Virtual  |
  28  * |            |         +---------------+        | NTB Driver   |
  29  * |            |         | PCI EP NTB    |<------>|              |
  30  * |            |         |  FN Driver    |        |              |
  31  * +------------+         +---------------+        +--------------+
  32  * |            |         |               |        |              |
  33  * |  PCI Bus   | <-----> |  PCI EP Bus   |        |  Virtual PCI |
  34  * |            |  PCI    |               |        |     Bus      |
  35  * +------------+         +---------------+--------+--------------+
  36  * PCIe Root Port                        PCI EP
  37  */
  38
  39 #include <linux/delay.h>
  40 #include <linux/io.h>
  41 #include <linux/module.h>
  42 #include <linux/slab.h>
  43
  44 #include <linux/pci-epc.h>
  45 #include <linux/pci-epf.h>
  46 #include <linux/ntb.h>
  47
  48 static struct workqueue_struct *kpcintb_workqueue;
  49
  50 #define COMMAND_CONFIGURE_DOORBELL      1
  51 #define COMMAND_TEARDOWN_DOORBELL       2
  52 #define COMMAND_CONFIGURE_MW            3
  53 #define COMMAND_TEARDOWN_MW             4
  54 #define COMMAND_LINK_UP                 5
  55 #define COMMAND_LINK_DOWN               6
  56
  57 #define COMMAND_STATUS_OK               1
  58 #define COMMAND_STATUS_ERROR            2
  59
  60 #define LINK_STATUS_UP                  BIT(0)
  61
  62 #define SPAD_COUNT                      64
  63 #define DB_COUNT                        4
  64 #define NTB_MW_OFFSET                   2
  65 #define DB_COUNT_MASK                   GENMASK(15, 0)
  66 #define MSIX_ENABLE                     BIT(16)
  67 #define MAX_DB_COUNT                    32
  68 #define MAX_MW                          4
  69
  70 enum epf_ntb_bar {
  71         BAR_CONFIG,
  72         BAR_DB,
  73         BAR_MW0,
  74         BAR_MW1,
  75         BAR_MW2,
  76 };
  77
  78 /*
  79  * +--------------------------------------------------+ Base
  80  * |                                                  |
  81  * |                                                  |
  82  * |                                                  |
  83  * |          Common Control Register                 |
  84  * |                                                  |
  85  * |                                                  |
  86  * |                                                  |
  87  * +-----------------------+--------------------------+ Base+span_offset
  88  * |                       |                          |
  89  * |    Peer Span Space    |    Span Space            |
  90  * |                       |                          |
  91  * |                       |                          |
  92  * +-----------------------+--------------------------+ Base+span_offset
  93  * |                       |                          |     +span_count * 4
  94  * |                       |                          |
  95  * |     Span Space        |   Peer Span Space        |
  96  * |                       |                          |
  97  * +-----------------------+--------------------------+
  98  *       Virtual PCI             PCIe Endpoint
  99  *       NTB Driver               NTB Driver
 100  */
 101 struct epf_ntb_ctrl {
 102         u32     command;
 103         u32     argument;
 104         u16     command_status;
 105         u16     link_status;
 106         u32     topology;
 107         u64     addr;
 108         u64     size;
 109         u32     num_mws;
 110         u32     reserved;
 111         u32     spad_offset;
 112         u32     spad_count;
 113         u32     db_entry_size;
 114         u32     db_data[MAX_DB_COUNT];
 115         u32     db_offset[MAX_DB_COUNT];
 116 } __packed;
 117
 118 struct epf_ntb {
 119         struct ntb_dev ntb;
 120         struct pci_epf *epf;
 121         struct config_group group;
 122
 123         u32 num_mws;
 124         u32 db_count;
 125         u32 spad_count;
 126         u64 mws_size[MAX_MW];
 127         u64 db;
 128         u32 vbus_number;
 129         u16 vntb_pid;
 130         u16 vntb_vid;
 131
 132         bool linkup;
 133         u32 spad_size;
 134
 135         enum pci_barno epf_ntb_bar[6];
 136
 137         struct epf_ntb_ctrl *reg;
 138
 139         phys_addr_t epf_db_phy;
 140         void __iomem *epf_db;
 141
 142         phys_addr_t vpci_mw_phy[MAX_MW];
 143         void __iomem *vpci_mw_addr[MAX_MW];
 144
 145         struct delayed_work cmd_handler;
 146 };
 147
 148 #define to_epf_ntb(epf_group) container_of((epf_group), struct epf_ntb, group)
 149 #define ntb_ndev(__ntb) container_of(__ntb, struct epf_ntb, ntb)
 150
 151 static struct pci_epf_header epf_ntb_header = {
 152         .vendorid       = PCI_ANY_ID,
 153         .deviceid       = PCI_ANY_ID,
 154         .baseclass_code = PCI_BASE_CLASS_MEMORY,
 155         .interrupt_pin  = PCI_INTERRUPT_INTA,
 156 };
 157
 158 /**
 159  * epf_ntb_link_up() - Raise link_up interrupt to Virtual Host (VHOST)
 160  * @ntb: NTB device that facilitates communication between HOST and VHOST
 161  * @link_up: true or false indicating Link is UP or Down
 162  *
 163  * Once NTB function in HOST invoke ntb_link_enable(),
 164  * this NTB function driver will trigger a link event to VHOST.
 165  *
 166  * Returns: Zero for success, or an error code in case of failure
 167  */
 168 static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up)
 169 {
 170         if (link_up)
 171                 ntb->reg->link_status |= LINK_STATUS_UP;
 172         else
 173                 ntb->reg->link_status &= ~LINK_STATUS_UP;
 174
 175         ntb_link_event(&ntb->ntb);
 176         return 0;
 177 }
 178
 179 /**
 180  * epf_ntb_configure_mw() - Configure the Outbound Address Space for VHOST
 181  *   to access the memory window of HOST
 182  * @ntb: NTB device that facilitates communication between HOST and VHOST
 183  * @mw: Index of the memory window (either 0, 1, 2 or 3)
 184  *
 185  *                          EP Outbound Window
 186  * +--------+              +-----------+
 187  * |        |              |           |
 188  * |        |              |           |
 189  * |        |              |           |
 190  * |        |              |           |
 191  * |        |              +-----------+
 192  * | Virtual|              | Memory Win|
 193  * | NTB    | -----------> |           |
 194  * | Driver |              |           |
 195  * |        |              +-----------+
 196  * |        |              |           |
 197  * |        |              |           |
 198  * +--------+              +-----------+
 199  *  VHOST                   PCI EP
 200  *
 201  * Returns: Zero for success, or an error code in case of failure
 202  */
 203 static int epf_ntb_configure_mw(struct epf_ntb *ntb, u32 mw)
 204 {
 205         phys_addr_t phys_addr;
 206         u8 func_no, vfunc_no;
 207         u64 addr, size;
 208         int ret = 0;
 209
 210         phys_addr = ntb->vpci_mw_phy[mw];
 211         addr = ntb->reg->addr;
 212         size = ntb->reg->size;
 213
 214         func_no = ntb->epf->func_no;
 215         vfunc_no = ntb->epf->vfunc_no;
 216
 217         ret = pci_epc_map_addr(ntb->epf->epc, func_no, vfunc_no, phys_addr, addr, size);
 218         if (ret)
 219                 dev_err(&ntb->epf->epc->dev,
 220                         "Failed to map memory window %d address\n", mw);
 221         return ret;
 222 }
 223
 224 /**
 225  * epf_ntb_teardown_mw() - Teardown the configured OB ATU
 226  * @ntb: NTB device that facilitates communication between HOST and VHOST
 227  * @mw: Index of the memory window (either 0, 1, 2 or 3)
 228  *
 229  * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using
 230  * pci_epc_unmap_addr()
 231  */
 232 static void epf_ntb_teardown_mw(struct epf_ntb *ntb, u32 mw)
 233 {
 234         pci_epc_unmap_addr(ntb->epf->epc,
 235                            ntb->epf->func_no,
 236                            ntb->epf->vfunc_no,
 237                            ntb->vpci_mw_phy[mw]);
 238 }
 239
 240 /**
 241  * epf_ntb_cmd_handler() - Handle commands provided by the NTB HOST
 242  * @work: work_struct for the epf_ntb_epc
 243  *
 244  * Workqueue function that gets invoked for the two epf_ntb_epc
 245  * periodically (once every 5ms) to see if it has received any commands
 246  * from NTB HOST. The HOST can send commands to configure doorbell or
 247  * configure memory window or to update link status.
 248  */
 249 static void epf_ntb_cmd_handler(struct work_struct *work)
 250 {
 251         struct epf_ntb_ctrl *ctrl;
 252         u32 command, argument;
 253         struct epf_ntb *ntb;
 254         struct device *dev;
 255         int ret;
 256         int i;
 257
 258         ntb = container_of(work, struct epf_ntb, cmd_handler.work);
 259
 260         for (i = 1; i < ntb->db_count; i++) {
 261                 if (readl(ntb->epf_db + i * 4)) {
 262                         if (readl(ntb->epf_db + i * 4))
 263                                 ntb->db |= 1 << (i - 1);
 264
 265                         ntb_db_event(&ntb->ntb, i);
 266                         writel(0, ntb->epf_db + i * 4);
 267                 }
 268         }
 269
 270         ctrl = ntb->reg;
 271         command = ctrl->command;
 272         if (!command)
 273                 goto reset_handler;
 274         argument = ctrl->argument;
 275
 276         ctrl->command = 0;
 277         ctrl->argument = 0;
 278
 279         ctrl = ntb->reg;
 280         dev = &ntb->epf->dev;
 281
 282         switch (command) {
 283         case COMMAND_CONFIGURE_DOORBELL:
 284                 ctrl->command_status = COMMAND_STATUS_OK;
 285                 break;
 286         case COMMAND_TEARDOWN_DOORBELL:
 287                 ctrl->command_status = COMMAND_STATUS_OK;
 288                 break;
 289         case COMMAND_CONFIGURE_MW:
 290                 ret = epf_ntb_configure_mw(ntb, argument);
 291                 if (ret < 0)
 292                         ctrl->command_status = COMMAND_STATUS_ERROR;
 293                 else
 294                         ctrl->command_status = COMMAND_STATUS_OK;
 295                 break;
 296         case COMMAND_TEARDOWN_MW:
 297                 epf_ntb_teardown_mw(ntb, argument);
 298                 ctrl->command_status = COMMAND_STATUS_OK;
 299                 break;
 300         case COMMAND_LINK_UP:
 301                 ntb->linkup = true;
 302                 ret = epf_ntb_link_up(ntb, true);
 303                 if (ret < 0)
 304                         ctrl->command_status = COMMAND_STATUS_ERROR;
 305                 else
 306                         ctrl->command_status = COMMAND_STATUS_OK;
 307                 goto reset_handler;
 308         case COMMAND_LINK_DOWN:
 309                 ntb->linkup = false;
 310                 ret = epf_ntb_link_up(ntb, false);
 311                 if (ret < 0)
 312                         ctrl->command_status = COMMAND_STATUS_ERROR;
 313                 else
 314                         ctrl->command_status = COMMAND_STATUS_OK;
 315                 break;
 316         default:
 317                 dev_err(dev, "UNKNOWN command: %d\n", command);
 318                 break;
 319         }
 320
 321 reset_handler:
 322         queue_delayed_work(kpcintb_workqueue, &ntb->cmd_handler,
 323                            msecs_to_jiffies(5));
 324 }
 325
 326 /**
 327  * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR
 328  * @ntb: EPC associated with one of the HOST which holds peer's outbound
 329  *       address.
 330  *
 331  * Clear BAR0 of EP CONTROLLER 1 which contains the HOST1's config and
 332  * self scratchpad region (removes inbound ATU configuration). While BAR0 is
 333  * the default self scratchpad BAR, an NTB could have other BARs for self
 334  * scratchpad (because of reserved BARs). This function can get the exact BAR
 335  * used for self scratchpad from epf_ntb_bar[BAR_CONFIG].
 336  *
 337  * Please note the self scratchpad region and config region is combined to
 338  * a single region and mapped using the same BAR. Also note VHOST's peer
 339  * scratchpad is HOST's self scratchpad.
 340  *
 341  * Returns: void
 342  */
 343 static void epf_ntb_config_sspad_bar_clear(struct epf_ntb *ntb)
 344 {
 345         struct pci_epf_bar *epf_bar;
 346         enum pci_barno barno;
 347
 348         barno = ntb->epf_ntb_bar[BAR_CONFIG];
 349         epf_bar = &ntb->epf->bar[barno];
 350
 351         pci_epc_clear_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar);
 352 }
 353
 354 /**
 355  * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR
 356  * @ntb: NTB device that facilitates communication between HOST and VHOST
 357  *
 358  * Map BAR0 of EP CONTROLLER which contains the VHOST's config and
 359  * self scratchpad region.
 360  *
 361  * Please note the self scratchpad region and config region is combined to
 362  * a single region and mapped using the same BAR.
 363  *
 364  * Returns: Zero for success, or an error code in case of failure
 365  */
 366 static int epf_ntb_config_sspad_bar_set(struct epf_ntb *ntb)
 367 {
 368         struct pci_epf_bar *epf_bar;
 369         enum pci_barno barno;
 370         u8 func_no, vfunc_no;
 371         struct device *dev;
 372         int ret;
 373
 374         dev = &ntb->epf->dev;
 375         func_no = ntb->epf->func_no;
 376         vfunc_no = ntb->epf->vfunc_no;
 377         barno = ntb->epf_ntb_bar[BAR_CONFIG];
 378         epf_bar = &ntb->epf->bar[barno];
 379
 380         ret = pci_epc_set_bar(ntb->epf->epc, func_no, vfunc_no, epf_bar);
 381         if (ret) {
 382                 dev_err(dev, "inft: Config/Status/SPAD BAR set failed\n");
 383                 return ret;
 384         }
 385         return 0;
 386 }
 387
 388 /**
 389  * epf_ntb_config_spad_bar_free() - Free the physical memory associated with
 390  *   config + scratchpad region
 391  * @ntb: NTB device that facilitates communication between HOST and VHOST
 392  */
 393 static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb)
 394 {
 395         enum pci_barno barno;
 396
 397         barno = ntb->epf_ntb_bar[BAR_CONFIG];
 398         pci_epf_free_space(ntb->epf, ntb->reg, barno, 0);
 399 }
 400
 401 /**
 402  * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad
 403  *   region
 404  * @ntb: NTB device that facilitates communication between HOST and VHOST
 405  *
 406  * Allocate the Local Memory mentioned in the above diagram. The size of
 407  * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION
 408  * is obtained from "spad-count" configfs entry.
 409  *
 410  * Returns: Zero for success, or an error code in case of failure
 411  */
 412 static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb)
 413 {
 414         size_t align;
 415         enum pci_barno barno;
 416         struct epf_ntb_ctrl *ctrl;
 417         u32 spad_size, ctrl_size;
 418         u64 size;
 419         struct pci_epf *epf = ntb->epf;
 420         struct device *dev = &epf->dev;
 421         u32 spad_count;
 422         void *base;
 423         int i;
 424         const struct pci_epc_features *epc_features = pci_epc_get_features(epf->epc,
 425                                                                 epf->func_no,
 426                                                                 epf->vfunc_no);
 427         barno = ntb->epf_ntb_bar[BAR_CONFIG];
 428         size = epc_features->bar_fixed_size[barno];
 429         align = epc_features->align;
 430
 431         if ((!IS_ALIGNED(size, align)))
 432                 return -EINVAL;
 433
 434         spad_count = ntb->spad_count;
 435
 436         ctrl_size = sizeof(struct epf_ntb_ctrl);
 437         spad_size = 2 * spad_count * 4;
 438
 439         if (!align) {
 440                 ctrl_size = roundup_pow_of_two(ctrl_size);
 441                 spad_size = roundup_pow_of_two(spad_size);
 442         } else {
 443                 ctrl_size = ALIGN(ctrl_size, align);
 444                 spad_size = ALIGN(spad_size, align);
 445         }
 446
 447         if (!size)
 448                 size = ctrl_size + spad_size;
 449         else if (size < ctrl_size + spad_size)
 450                 return -EINVAL;
 451
 452         base = pci_epf_alloc_space(epf, size, barno, align, 0);
 453         if (!base) {
 454                 dev_err(dev, "Config/Status/SPAD alloc region fail\n");
 455                 return -ENOMEM;
 456         }
 457
 458         ntb->reg = base;
 459
 460         ctrl = ntb->reg;
 461         ctrl->spad_offset = ctrl_size;
 462
 463         ctrl->spad_count = spad_count;
 464         ctrl->num_mws = ntb->num_mws;
 465         ntb->spad_size = spad_size;
 466
 467         ctrl->db_entry_size = 4;
 468
 469         for (i = 0; i < ntb->db_count; i++) {
 470                 ntb->reg->db_data[i] = 1 + i;
 471                 ntb->reg->db_offset[i] = 0;
 472         }
 473
 474         return 0;
 475 }
 476
 477 /**
 478  * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capability
 479  * @ntb: NTB device that facilitates communication between HOST and VHOST
 480  *
 481  * Configure MSI/MSI-X capability for each interface with number of
 482  * interrupts equal to "db_count" configfs entry.
 483  *
 484  * Returns: Zero for success, or an error code in case of failure
 485  */
 486 static int epf_ntb_configure_interrupt(struct epf_ntb *ntb)
 487 {
 488         const struct pci_epc_features *epc_features;
 489         struct device *dev;
 490         u32 db_count;
 491         int ret;
 492
 493         dev = &ntb->epf->dev;
 494
 495         epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no);
 496
 497         if (!(epc_features->msix_capable || epc_features->msi_capable)) {
 498                 dev_err(dev, "MSI or MSI-X is required for doorbell\n");
 499                 return -EINVAL;
 500         }
 501
 502         db_count = ntb->db_count;
 503         if (db_count > MAX_DB_COUNT) {
 504                 dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT);
 505                 return -EINVAL;
 506         }
 507
 508         ntb->db_count = db_count;
 509
 510         if (epc_features->msi_capable) {
 511                 ret = pci_epc_set_msi(ntb->epf->epc,
 512                                       ntb->epf->func_no,
 513                                       ntb->epf->vfunc_no,
 514                                       16);
 515                 if (ret) {
 516                         dev_err(dev, "MSI configuration failed\n");
 517                         return ret;
 518                 }
 519         }
 520
 521         return 0;
 522 }
 523
 524 /**
 525  * epf_ntb_db_bar_init() - Configure Doorbell window BARs
 526  * @ntb: NTB device that facilitates communication between HOST and VHOST
 527  *
 528  * Returns: Zero for success, or an error code in case of failure
 529  */
 530 static int epf_ntb_db_bar_init(struct epf_ntb *ntb)
 531 {
 532         const struct pci_epc_features *epc_features;
 533         u32 align;
 534         struct device *dev = &ntb->epf->dev;
 535         int ret;
 536         struct pci_epf_bar *epf_bar;
 537         void __iomem *mw_addr;
 538         enum pci_barno barno;
 539         size_t size = 4 * ntb->db_count;
 540
 541         epc_features = pci_epc_get_features(ntb->epf->epc,
 542                                             ntb->epf->func_no,
 543                                             ntb->epf->vfunc_no);
 544         align = epc_features->align;
 545
 546         if (size < 128)
 547                 size = 128;
 548
 549         if (align)
 550                 size = ALIGN(size, align);
 551         else
 552                 size = roundup_pow_of_two(size);
 553
 554         barno = ntb->epf_ntb_bar[BAR_DB];
 555
 556         mw_addr = pci_epf_alloc_space(ntb->epf, size, barno, align, 0);
 557         if (!mw_addr) {
 558                 dev_err(dev, "Failed to allocate OB address\n");
 559                 return -ENOMEM;
 560         }
 561
 562         ntb->epf_db = mw_addr;
 563
 564         epf_bar = &ntb->epf->bar[barno];
 565
 566         ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar);
 567         if (ret) {
 568                 dev_err(dev, "Doorbell BAR set failed\n");
 569                         goto err_alloc_peer_mem;
 570         }
 571         return ret;
 572
 573 err_alloc_peer_mem:
 574         pci_epf_free_space(ntb->epf, mw_addr, barno, 0);
 575         return -1;
 576 }
 577
 578 static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws);
 579
 580 /**
 581  * epf_ntb_db_bar_clear() - Clear doorbell BAR and free memory
 582  *   allocated in peer's outbound address space
 583  * @ntb: NTB device that facilitates communication between HOST and VHOST
 584  */
 585 static void epf_ntb_db_bar_clear(struct epf_ntb *ntb)
 586 {
 587         enum pci_barno barno;
 588
 589         barno = ntb->epf_ntb_bar[BAR_DB];
 590         pci_epf_free_space(ntb->epf, ntb->epf_db, barno, 0);
 591         pci_epc_clear_bar(ntb->epf->epc,
 592                           ntb->epf->func_no,
 593                           ntb->epf->vfunc_no,
 594                           &ntb->epf->bar[barno]);
 595 }
 596
 597 /**
 598  * epf_ntb_mw_bar_init() - Configure Memory window BARs
 599  * @ntb: NTB device that facilitates communication between HOST and VHOST
 600  *
 601  * Returns: Zero for success, or an error code in case of failure
 602  */
 603 static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)
 604 {
 605         int ret = 0;
 606         int i;
 607         u64 size;
 608         enum pci_barno barno;
 609         struct device *dev = &ntb->epf->dev;
 610
 611         for (i = 0; i < ntb->num_mws; i++) {
 612                 size = ntb->mws_size[i];
 613                 barno = ntb->epf_ntb_bar[BAR_MW0 + i];
 614
 615                 ntb->epf->bar[barno].barno = barno;
 616                 ntb->epf->bar[barno].size = size;
 617                 ntb->epf->bar[barno].addr = NULL;
 618                 ntb->epf->bar[barno].phys_addr = 0;
 619                 ntb->epf->bar[barno].flags |= upper_32_bits(size) ?
 620                                 PCI_BASE_ADDRESS_MEM_TYPE_64 :
 621                                 PCI_BASE_ADDRESS_MEM_TYPE_32;
 622
 623                 ret = pci_epc_set_bar(ntb->epf->epc,
 624                                       ntb->epf->func_no,
 625                                       ntb->epf->vfunc_no,
 626                                       &ntb->epf->bar[barno]);
 627                 if (ret) {
 628                         dev_err(dev, "MW set failed\n");
 629                         goto err_alloc_mem;
 630                 }
 631
 632                 /* Allocate EPC outbound memory windows to vpci vntb device */
 633                 ntb->vpci_mw_addr[i] = pci_epc_mem_alloc_addr(ntb->epf->epc,
 634                                                               &ntb->vpci_mw_phy[i],
 635                                                               size);
 636                 if (!ntb->vpci_mw_addr[i]) {
 637                         ret = -ENOMEM;
 638                         dev_err(dev, "Failed to allocate source address\n");
 639                         goto err_set_bar;
 640                 }
 641         }
 642
 643         return ret;
 644
 645 err_set_bar:
 646         pci_epc_clear_bar(ntb->epf->epc,
 647                           ntb->epf->func_no,
 648                           ntb->epf->vfunc_no,
 649                           &ntb->epf->bar[barno]);
 650 err_alloc_mem:
 651         epf_ntb_mw_bar_clear(ntb, i);
 652         return ret;
 653 }
 654
 655 /**
 656  * epf_ntb_mw_bar_clear() - Clear Memory window BARs
 657  * @ntb: NTB device that facilitates communication between HOST and VHOST
 658  * @num_mws: the number of Memory window BARs that to be cleared
 659  */
 660 static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws)
 661 {
 662         enum pci_barno barno;
 663         int i;
 664
 665         for (i = 0; i < num_mws; i++) {
 666                 barno = ntb->epf_ntb_bar[BAR_MW0 + i];
 667                 pci_epc_clear_bar(ntb->epf->epc,
 668                                   ntb->epf->func_no,
 669                                   ntb->epf->vfunc_no,
 670                                   &ntb->epf->bar[barno]);
 671
 672                 pci_epc_mem_free_addr(ntb->epf->epc,
 673                                       ntb->vpci_mw_phy[i],
 674                                       ntb->vpci_mw_addr[i],
 675                                       ntb->mws_size[i]);
 676         }
 677 }
 678
 679 /**
 680  * epf_ntb_epc_destroy() - Cleanup NTB EPC interface
 681  * @ntb: NTB device that facilitates communication between HOST and VHOST
 682  *
 683  * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces
 684  */
 685 static void epf_ntb_epc_destroy(struct epf_ntb *ntb)
 686 {
 687         pci_epc_remove_epf(ntb->epf->epc, ntb->epf, 0);
 688         pci_epc_put(ntb->epf->epc);
 689 }
 690
 691 /**
 692  * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB
 693  * constructs (scratchpad region, doorbell, memorywindow)
 694  * @ntb: NTB device that facilitates communication between HOST and VHOST
 695  *
 696  * Returns: Zero for success, or an error code in case of failure
 697  */
 698 static int epf_ntb_init_epc_bar(struct epf_ntb *ntb)
 699 {
 700         const struct pci_epc_features *epc_features;
 701         enum pci_barno barno;
 702         enum epf_ntb_bar bar;
 703         struct device *dev;
 704         u32 num_mws;
 705         int i;
 706
 707         barno = BAR_0;
 708         num_mws = ntb->num_mws;
 709         dev = &ntb->epf->dev;
 710         epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no);
 711
 712         /* These are required BARs which are mandatory for NTB functionality */
 713         for (bar = BAR_CONFIG; bar <= BAR_MW0; bar++, barno++) {
 714                 barno = pci_epc_get_next_free_bar(epc_features, barno);
 715                 if (barno < 0) {
 716                         dev_err(dev, "Fail to get NTB function BAR\n");
 717                         return barno;
 718                 }
 719                 ntb->epf_ntb_bar[bar] = barno;
 720         }
 721
 722         /* These are optional BARs which don't impact NTB functionality */
 723         for (bar = BAR_MW1, i = 1; i < num_mws; bar++, barno++, i++) {
 724                 barno = pci_epc_get_next_free_bar(epc_features, barno);
 725                 if (barno < 0) {
 726                         ntb->num_mws = i;
 727                         dev_dbg(dev, "BAR not available for > MW%d\n", i + 1);
 728                 }
 729                 ntb->epf_ntb_bar[bar] = barno;
 730         }
 731
 732         return 0;
 733 }
 734
 735 /**
 736  * epf_ntb_epc_init() - Initialize NTB interface
 737  * @ntb: NTB device that facilitates communication between HOST and VHOST
 738  *
 739  * Wrapper to initialize a particular EPC interface and start the workqueue
 740  * to check for commands from HOST. This function will write to the
 741  * EP controller HW for configuring it.
 742  *
 743  * Returns: Zero for success, or an error code in case of failure
 744  */
 745 static int epf_ntb_epc_init(struct epf_ntb *ntb)
 746 {
 747         u8 func_no, vfunc_no;
 748         struct pci_epc *epc;
 749         struct pci_epf *epf;
 750         struct device *dev;
 751         int ret;
 752
 753         epf = ntb->epf;
 754         dev = &epf->dev;
 755         epc = epf->epc;
 756         func_no = ntb->epf->func_no;
 757         vfunc_no = ntb->epf->vfunc_no;
 758
 759         ret = epf_ntb_config_sspad_bar_set(ntb);
 760         if (ret) {
 761                 dev_err(dev, "Config/self SPAD BAR init failed");
 762                 return ret;
 763         }
 764
 765         ret = epf_ntb_configure_interrupt(ntb);
 766         if (ret) {
 767                 dev_err(dev, "Interrupt configuration failed\n");
 768                 goto err_config_interrupt;
 769         }
 770
 771         ret = epf_ntb_db_bar_init(ntb);
 772         if (ret) {
 773                 dev_err(dev, "DB BAR init failed\n");
 774                 goto err_db_bar_init;
 775         }
 776
 777         ret = epf_ntb_mw_bar_init(ntb);
 778         if (ret) {
 779                 dev_err(dev, "MW BAR init failed\n");
 780                 goto err_mw_bar_init;
 781         }
 782
 783         if (vfunc_no <= 1) {
 784                 ret = pci_epc_write_header(epc, func_no, vfunc_no, epf->header);
 785                 if (ret) {
 786                         dev_err(dev, "Configuration header write failed\n");
 787                         goto err_write_header;
 788                 }
 789         }
 790
 791         INIT_DELAYED_WORK(&ntb->cmd_handler, epf_ntb_cmd_handler);
 792         queue_work(kpcintb_workqueue, &ntb->cmd_handler.work);
 793
 794         return 0;
 795
 796 err_write_header:
 797         epf_ntb_mw_bar_clear(ntb, ntb->num_mws);
 798 err_mw_bar_init:
 799         epf_ntb_db_bar_clear(ntb);
 800 err_db_bar_init:
 801 err_config_interrupt:
 802         epf_ntb_config_sspad_bar_clear(ntb);
 803
 804         return ret;
 805 }
 806
 807
 808 /**
 809  * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces
 810  * @ntb: NTB device that facilitates communication between HOST and VHOST
 811  *
 812  * Wrapper to cleanup all NTB interfaces.
 813  */
 814 static void epf_ntb_epc_cleanup(struct epf_ntb *ntb)
 815 {
 816         epf_ntb_db_bar_clear(ntb);
 817         epf_ntb_mw_bar_clear(ntb, ntb->num_mws);
 818 }
 819
 820 #define EPF_NTB_R(_name)                                                \
 821 static ssize_t epf_ntb_##_name##_show(struct config_item *item,         \
 822                                       char *page)                       \
 823 {                                                                       \
 824         struct config_group *group = to_config_group(item);             \
 825         struct epf_ntb *ntb = to_epf_ntb(group);                        \
 826                                                                         \
 827         return sprintf(page, "%d\n", ntb->_name);                       \
 828 }
 829
 830 #define EPF_NTB_W(_name)                                                \
 831 static ssize_t epf_ntb_##_name##_store(struct config_item *item,        \
 832                                        const char *page, size_t len)    \
 833 {                                                                       \
 834         struct config_group *group = to_config_group(item);             \
 835         struct epf_ntb *ntb = to_epf_ntb(group);                        \
 836         u32 val;                                                        \
 837         int ret;                                                        \
 838                                                                         \
 839         ret = kstrtou32(page, 0, &val);                                 \
 840         if (ret)                                                        \
 841                 return ret;                                             \
 842                                                                         \
 843         ntb->_name = val;                                               \
 844                                                                         \
 845         return len;                                                     \
 846 }
 847
 848 #define EPF_NTB_MW_R(_name)                                             \
 849 static ssize_t epf_ntb_##_name##_show(struct config_item *item,         \
 850                                       char *page)                       \
 851 {                                                                       \
 852         struct config_group *group = to_config_group(item);             \
 853         struct epf_ntb *ntb = to_epf_ntb(group);                        \
 854         struct device *dev = &ntb->epf->dev;                            \
 855         int win_no;                                                     \
 856                                                                         \
 857         if (sscanf(#_name, "mw%d", &win_no) != 1)                       \
 858                 return -EINVAL;                                         \
 859                                                                         \
 860         if (win_no <= 0 || win_no > ntb->num_mws) {                     \
 861                 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
 862                 return -EINVAL;                                         \
 863         }                                                               \
 864                                                                         \
 865         return sprintf(page, "%lld\n", ntb->mws_size[win_no - 1]);      \
 866 }
 867
 868 #define EPF_NTB_MW_W(_name)                                             \
 869 static ssize_t epf_ntb_##_name##_store(struct config_item *item,        \
 870                                        const char *page, size_t len)    \
 871 {                                                                       \
 872         struct config_group *group = to_config_group(item);             \
 873         struct epf_ntb *ntb = to_epf_ntb(group);                        \
 874         struct device *dev = &ntb->epf->dev;                            \
 875         int win_no;                                                     \
 876         u64 val;                                                        \
 877         int ret;                                                        \
 878                                                                         \
 879         ret = kstrtou64(page, 0, &val);                                 \
 880         if (ret)                                                        \
 881                 return ret;                                             \
 882                                                                         \
 883         if (sscanf(#_name, "mw%d", &win_no) != 1)                       \
 884                 return -EINVAL;                                         \
 885                                                                         \
 886         if (win_no <= 0 || win_no > ntb->num_mws) {                     \
 887                 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
 888                 return -EINVAL;                                         \
 889         }                                                               \
 890                                                                         \
 891         ntb->mws_size[win_no - 1] = val;                                \
 892                                                                         \
 893         return len;                                                     \
 894 }
 895
 896 static ssize_t epf_ntb_num_mws_store(struct config_item *item,
 897                                      const char *page, size_t len)
 898 {
 899         struct config_group *group = to_config_group(item);
 900         struct epf_ntb *ntb = to_epf_ntb(group);
 901         u32 val;
 902         int ret;
 903
 904         ret = kstrtou32(page, 0, &val);
 905         if (ret)
 906                 return ret;
 907
 908         if (val > MAX_MW)
 909                 return -EINVAL;
 910
 911         ntb->num_mws = val;
 912
 913         return len;
 914 }
 915
 916 EPF_NTB_R(spad_count)
 917 EPF_NTB_W(spad_count)
 918 EPF_NTB_R(db_count)
 919 EPF_NTB_W(db_count)
 920 EPF_NTB_R(num_mws)
 921 EPF_NTB_R(vbus_number)
 922 EPF_NTB_W(vbus_number)
 923 EPF_NTB_R(vntb_pid)
 924 EPF_NTB_W(vntb_pid)
 925 EPF_NTB_R(vntb_vid)
 926 EPF_NTB_W(vntb_vid)
 927 EPF_NTB_MW_R(mw1)
 928 EPF_NTB_MW_W(mw1)
 929 EPF_NTB_MW_R(mw2)
 930 EPF_NTB_MW_W(mw2)
 931 EPF_NTB_MW_R(mw3)
 932 EPF_NTB_MW_W(mw3)
 933 EPF_NTB_MW_R(mw4)
 934 EPF_NTB_MW_W(mw4)
 935
 936 CONFIGFS_ATTR(epf_ntb_, spad_count);
 937 CONFIGFS_ATTR(epf_ntb_, db_count);
 938 CONFIGFS_ATTR(epf_ntb_, num_mws);
 939 CONFIGFS_ATTR(epf_ntb_, mw1);
 940 CONFIGFS_ATTR(epf_ntb_, mw2);
 941 CONFIGFS_ATTR(epf_ntb_, mw3);
 942 CONFIGFS_ATTR(epf_ntb_, mw4);
 943 CONFIGFS_ATTR(epf_ntb_, vbus_number);
 944 CONFIGFS_ATTR(epf_ntb_, vntb_pid);
 945 CONFIGFS_ATTR(epf_ntb_, vntb_vid);
 946
 947 static struct configfs_attribute *epf_ntb_attrs[] = {
 948         &epf_ntb_attr_spad_count,
 949         &epf_ntb_attr_db_count,
 950         &epf_ntb_attr_num_mws,
 951         &epf_ntb_attr_mw1,
 952         &epf_ntb_attr_mw2,
 953         &epf_ntb_attr_mw3,
 954         &epf_ntb_attr_mw4,
 955         &epf_ntb_attr_vbus_number,
 956         &epf_ntb_attr_vntb_pid,
 957         &epf_ntb_attr_vntb_vid,
 958         NULL,
 959 };
 960
 961 static const struct config_item_type ntb_group_type = {
 962         .ct_attrs       = epf_ntb_attrs,
 963         .ct_owner       = THIS_MODULE,
 964 };
 965
 966 /**
 967  * epf_ntb_add_cfs() - Add configfs directory specific to NTB
 968  * @epf: NTB endpoint function device
 969  * @group: A pointer to the config_group structure referencing a group of
 970  *         config_items of a specific type that belong to a specific sub-system.
 971  *
 972  * Add configfs directory specific to NTB. This directory will hold
 973  * NTB specific properties like db_count, spad_count, num_mws etc.,
 974  *
 975  * Returns: Pointer to config_group
 976  */
 977 static struct config_group *epf_ntb_add_cfs(struct pci_epf *epf,
 978                                             struct config_group *group)
 979 {
 980         struct epf_ntb *ntb = epf_get_drvdata(epf);
 981         struct config_group *ntb_group = &ntb->group;
 982         struct device *dev = &epf->dev;
 983
 984         config_group_init_type_name(ntb_group, dev_name(dev), &ntb_group_type);
 985
 986         return ntb_group;
 987 }
 988
 989 /*==== virtual PCI bus driver, which only load virtual NTB PCI driver ====*/
 990
 991 static u32 pci_space[] = {
 992         0xffffffff,     /*DeviceID, Vendor ID*/
 993         0,              /*Status, Command*/
 994         0xffffffff,     /*Class code, subclass, prog if, revision id*/
 995         0x40,           /*bist, header type, latency Timer, cache line size*/
 996         0,              /*BAR 0*/
 997         0,              /*BAR 1*/
 998         0,              /*BAR 2*/
 999         0,              /*BAR 3*/
1000         0,              /*BAR 4*/
1001         0,              /*BAR 5*/
1002         0,              /*Cardbus cis point*/
1003         0,              /*Subsystem ID Subystem vendor id*/
1004         0,              /*ROM Base Address*/
1005         0,              /*Reserved, Cap. Point*/
1006         0,              /*Reserved,*/
1007         0,              /*Max Lat, Min Gnt, interrupt pin, interrupt line*/
1008 };
1009
1010 static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val)
1011 {
1012         if (devfn == 0) {
1013                 memcpy(val, ((u8 *)pci_space) + where, size);
1014                 return PCIBIOS_SUCCESSFUL;
1015         }
1016         return PCIBIOS_DEVICE_NOT_FOUND;
1017 }
1018
1019 static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val)
1020 {
1021         return 0;
1022 }
1023
1024 static struct pci_ops vpci_ops = {
1025         .read = pci_read,
1026         .write = pci_write,
1027 };
1028
1029 static int vpci_scan_bus(void *sysdata)
1030 {
1031         struct pci_bus *vpci_bus;
1032         struct epf_ntb *ndev = sysdata;
1033
1034         vpci_bus = pci_scan_bus(ndev->vbus_number, &vpci_ops, sysdata);
1035         if (vpci_bus)
1036                 pr_err("create pci bus\n");
1037
1038         pci_bus_add_devices(vpci_bus);
1039
1040         return 0;
1041 }
1042
1043 /*==================== Virtual PCIe NTB driver ==========================*/
1044
1045 static int vntb_epf_mw_count(struct ntb_dev *ntb, int pidx)
1046 {
1047         struct epf_ntb *ndev = ntb_ndev(ntb);
1048
1049         return ndev->num_mws;
1050 }
1051
1052 static int vntb_epf_spad_count(struct ntb_dev *ntb)
1053 {
1054         return ntb_ndev(ntb)->spad_count;
1055 }
1056
1057 static int vntb_epf_peer_mw_count(struct ntb_dev *ntb)
1058 {
1059         return ntb_ndev(ntb)->num_mws;
1060 }
1061
1062 static u64 vntb_epf_db_valid_mask(struct ntb_dev *ntb)
1063 {
1064         return BIT_ULL(ntb_ndev(ntb)->db_count) - 1;
1065 }
1066
1067 static int vntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
1068 {
1069         return 0;
1070 }
1071
1072 static int vntb_epf_mw_set_trans(struct ntb_dev *ndev, int pidx, int idx,
1073                 dma_addr_t addr, resource_size_t size)
1074 {
1075         struct epf_ntb *ntb = ntb_ndev(ndev);
1076         struct pci_epf_bar *epf_bar;
1077         enum pci_barno barno;
1078         int ret;
1079         struct device *dev;
1080
1081         dev = &ntb->ntb.dev;
1082         barno = ntb->epf_ntb_bar[BAR_MW0 + idx];
1083         epf_bar = &ntb->epf->bar[barno];
1084         epf_bar->phys_addr = addr;
1085         epf_bar->barno = barno;
1086         epf_bar->size = size;
1087
1088         ret = pci_epc_set_bar(ntb->epf->epc, 0, 0, epf_bar);
1089         if (ret) {
1090                 dev_err(dev, "failure set mw trans\n");
1091                 return ret;
1092         }
1093         return 0;
1094 }
1095
1096 static int vntb_epf_mw_clear_trans(struct ntb_dev *ntb, int pidx, int idx)
1097 {
1098         return 0;
1099 }
1100
1101 static int vntb_epf_peer_mw_get_addr(struct ntb_dev *ndev, int idx,
1102                                 phys_addr_t *base, resource_size_t *size)
1103 {
1104
1105         struct epf_ntb *ntb = ntb_ndev(ndev);
1106
1107         if (base)
1108                 *base = ntb->vpci_mw_phy[idx];
1109
1110         if (size)
1111                 *size = ntb->mws_size[idx];
1112
1113         return 0;
1114 }
1115
1116 static int vntb_epf_link_enable(struct ntb_dev *ntb,
1117                         enum ntb_speed max_speed,
1118                         enum ntb_width max_width)
1119 {
1120         return 0;
1121 }
1122
1123 static u32 vntb_epf_spad_read(struct ntb_dev *ndev, int idx)
1124 {
1125         struct epf_ntb *ntb = ntb_ndev(ndev);
1126         int off = ntb->reg->spad_offset, ct = ntb->reg->spad_count * 4;
1127         u32 val;
1128         void __iomem *base = ntb->reg;
1129
1130         val = readl(base + off + ct + idx * 4);
1131         return val;
1132 }
1133
1134 static int vntb_epf_spad_write(struct ntb_dev *ndev, int idx, u32 val)
1135 {
1136         struct epf_ntb *ntb = ntb_ndev(ndev);
1137         struct epf_ntb_ctrl *ctrl = ntb->reg;
1138         int off = ctrl->spad_offset, ct = ctrl->spad_count * 4;
1139         void __iomem *base = ntb->reg;
1140
1141         writel(val, base + off + ct + idx * 4);
1142         return 0;
1143 }
1144
1145 static u32 vntb_epf_peer_spad_read(struct ntb_dev *ndev, int pidx, int idx)
1146 {
1147         struct epf_ntb *ntb = ntb_ndev(ndev);
1148         struct epf_ntb_ctrl *ctrl = ntb->reg;
1149         int off = ctrl->spad_offset;
1150         void __iomem *base = ntb->reg;
1151         u32 val;
1152
1153         val = readl(base + off + idx * 4);
1154         return val;
1155 }
1156
1157 static int vntb_epf_peer_spad_write(struct ntb_dev *ndev, int pidx, int idx, u32 val)
1158 {
1159         struct epf_ntb *ntb = ntb_ndev(ndev);
1160         struct epf_ntb_ctrl *ctrl = ntb->reg;
1161         int off = ctrl->spad_offset;
1162         void __iomem *base = ntb->reg;
1163
1164         writel(val, base + off + idx * 4);
1165         return 0;
1166 }
1167
1168 static int vntb_epf_peer_db_set(struct ntb_dev *ndev, u64 db_bits)
1169 {
1170         u32 interrupt_num = ffs(db_bits) + 1;
1171         struct epf_ntb *ntb = ntb_ndev(ndev);
1172         u8 func_no, vfunc_no;
1173         int ret;
1174
1175         func_no = ntb->epf->func_no;
1176         vfunc_no = ntb->epf->vfunc_no;
1177
1178         ret = pci_epc_raise_irq(ntb->epf->epc,
1179                                 func_no,
1180                                 vfunc_no,
1181                                 PCI_EPC_IRQ_MSI,
1182                                 interrupt_num + 1);
1183         if (ret)
1184                 dev_err(&ntb->ntb.dev, "Failed to raise IRQ\n");
1185
1186         return ret;
1187 }
1188
1189 static u64 vntb_epf_db_read(struct ntb_dev *ndev)
1190 {
1191         struct epf_ntb *ntb = ntb_ndev(ndev);
1192
1193         return ntb->db;
1194 }
1195
1196 static int vntb_epf_mw_get_align(struct ntb_dev *ndev, int pidx, int idx,
1197                         resource_size_t *addr_align,
1198                         resource_size_t *size_align,
1199                         resource_size_t *size_max)
1200 {
1201         struct epf_ntb *ntb = ntb_ndev(ndev);
1202
1203         if (addr_align)
1204                 *addr_align = SZ_4K;
1205
1206         if (size_align)
1207                 *size_align = 1;
1208
1209         if (size_max)
1210                 *size_max = ntb->mws_size[idx];
1211
1212         return 0;
1213 }
1214
1215 static u64 vntb_epf_link_is_up(struct ntb_dev *ndev,
1216                         enum ntb_speed *speed,
1217                         enum ntb_width *width)
1218 {
1219         struct epf_ntb *ntb = ntb_ndev(ndev);
1220
1221         return ntb->reg->link_status;
1222 }
1223
1224 static int vntb_epf_db_clear_mask(struct ntb_dev *ndev, u64 db_bits)
1225 {
1226         return 0;
1227 }
1228
1229 static int vntb_epf_db_clear(struct ntb_dev *ndev, u64 db_bits)
1230 {
1231         struct epf_ntb *ntb = ntb_ndev(ndev);
1232
1233         ntb->db &= ~db_bits;
1234         return 0;
1235 }
1236
1237 static int vntb_epf_link_disable(struct ntb_dev *ntb)
1238 {
1239         return 0;
1240 }
1241
1242 static const struct ntb_dev_ops vntb_epf_ops = {
1243         .mw_count               = vntb_epf_mw_count,
1244         .spad_count             = vntb_epf_spad_count,
1245         .peer_mw_count          = vntb_epf_peer_mw_count,
1246         .db_valid_mask          = vntb_epf_db_valid_mask,
1247         .db_set_mask            = vntb_epf_db_set_mask,
1248         .mw_set_trans           = vntb_epf_mw_set_trans,
1249         .mw_clear_trans         = vntb_epf_mw_clear_trans,
1250         .peer_mw_get_addr       = vntb_epf_peer_mw_get_addr,
1251         .link_enable            = vntb_epf_link_enable,
1252         .spad_read              = vntb_epf_spad_read,
1253         .spad_write             = vntb_epf_spad_write,
1254         .peer_spad_read         = vntb_epf_peer_spad_read,
1255         .peer_spad_write        = vntb_epf_peer_spad_write,
1256         .peer_db_set            = vntb_epf_peer_db_set,
1257         .db_read                = vntb_epf_db_read,
1258         .mw_get_align           = vntb_epf_mw_get_align,
1259         .link_is_up             = vntb_epf_link_is_up,
1260         .db_clear_mask          = vntb_epf_db_clear_mask,
1261         .db_clear               = vntb_epf_db_clear,
1262         .link_disable           = vntb_epf_link_disable,
1263 };
1264
1265 static int pci_vntb_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1266 {
1267         int ret;
1268         struct epf_ntb *ndev = (struct epf_ntb *)pdev->sysdata;
1269         struct device *dev = &pdev->dev;
1270
1271         ndev->ntb.pdev = pdev;
1272         ndev->ntb.topo = NTB_TOPO_NONE;
1273         ndev->ntb.ops =  &vntb_epf_ops;
1274
1275         ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
1276         if (ret) {
1277                 dev_err(dev, "Cannot set DMA mask\n");
1278                 return -EINVAL;
1279         }
1280
1281         ret = ntb_register_device(&ndev->ntb);
1282         if (ret) {
1283                 dev_err(dev, "Failed to register NTB device\n");
1284                 return ret;
1285         }
1286
1287         dev_dbg(dev, "PCI Virtual NTB driver loaded\n");
1288         return 0;
1289 }
1290
1291 static struct pci_device_id pci_vntb_table[] = {
1292         {
1293                 PCI_DEVICE(0xffff, 0xffff),
1294         },
1295         {},
1296 };
1297
1298 static struct pci_driver vntb_pci_driver = {
1299         .name           = "pci-vntb",
1300         .id_table       = pci_vntb_table,
1301         .probe          = pci_vntb_probe,
1302 };
1303
1304 /* ============ PCIe EPF Driver Bind ====================*/
1305
1306 /**
1307  * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality
1308  * @epf: NTB endpoint function device
1309  *
1310  * Initialize both the endpoint controllers associated with NTB function device.
1311  * Invoked when a primary interface or secondary interface is bound to EPC
1312  * device. This function will succeed only when EPC is bound to both the
1313  * interfaces.
1314  *
1315  * Returns: Zero for success, or an error code in case of failure
1316  */
1317 static int epf_ntb_bind(struct pci_epf *epf)
1318 {
1319         struct epf_ntb *ntb = epf_get_drvdata(epf);
1320         struct device *dev = &epf->dev;
1321         int ret;
1322
1323         if (!epf->epc) {
1324                 dev_dbg(dev, "PRIMARY EPC interface not yet bound\n");
1325                 return 0;
1326         }
1327
1328         ret = epf_ntb_init_epc_bar(ntb);
1329         if (ret) {
1330                 dev_err(dev, "Failed to create NTB EPC\n");
1331                 goto err_bar_init;
1332         }
1333
1334         ret = epf_ntb_config_spad_bar_alloc(ntb);
1335         if (ret) {
1336                 dev_err(dev, "Failed to allocate BAR memory\n");
1337                 goto err_bar_alloc;
1338         }
1339
1340         ret = epf_ntb_epc_init(ntb);
1341         if (ret) {
1342                 dev_err(dev, "Failed to initialize EPC\n");
1343                 goto err_bar_alloc;
1344         }
1345
1346         epf_set_drvdata(epf, ntb);
1347
1348         pci_space[0] = (ntb->vntb_pid << 16) | ntb->vntb_vid;
1349         pci_vntb_table[0].vendor = ntb->vntb_vid;
1350         pci_vntb_table[0].device = ntb->vntb_pid;
1351
1352         ret = pci_register_driver(&vntb_pci_driver);
1353         if (ret) {
1354                 dev_err(dev, "failure register vntb pci driver\n");
1355                 goto err_bar_alloc;
1356         }
1357
1358         vpci_scan_bus(ntb);
1359
1360         return 0;
1361
1362 err_bar_alloc:
1363         epf_ntb_config_spad_bar_free(ntb);
1364
1365 err_bar_init:
1366         epf_ntb_epc_destroy(ntb);
1367
1368         return ret;
1369 }
1370
1371 /**
1372  * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind()
1373  * @epf: NTB endpoint function device
1374  *
1375  * Cleanup the initialization from epf_ntb_bind()
1376  */
1377 static void epf_ntb_unbind(struct pci_epf *epf)
1378 {
1379         struct epf_ntb *ntb = epf_get_drvdata(epf);
1380
1381         epf_ntb_epc_cleanup(ntb);
1382         epf_ntb_config_spad_bar_free(ntb);
1383         epf_ntb_epc_destroy(ntb);
1384
1385         pci_unregister_driver(&vntb_pci_driver);
1386 }
1387
1388 // EPF driver probe
1389 static struct pci_epf_ops epf_ntb_ops = {
1390         .bind   = epf_ntb_bind,
1391         .unbind = epf_ntb_unbind,
1392         .add_cfs = epf_ntb_add_cfs,
1393 };
1394
1395 /**
1396  * epf_ntb_probe() - Probe NTB function driver
1397  * @epf: NTB endpoint function device
1398  *
1399  * Probe NTB function driver when endpoint function bus detects a NTB
1400  * endpoint function.
1401  *
1402  * Returns: Zero for success, or an error code in case of failure
1403  */
1404 static int epf_ntb_probe(struct pci_epf *epf)
1405 {
1406         struct epf_ntb *ntb;
1407         struct device *dev;
1408
1409         dev = &epf->dev;
1410
1411         ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL);
1412         if (!ntb)
1413                 return -ENOMEM;
1414
1415         epf->header = &epf_ntb_header;
1416         ntb->epf = epf;
1417         ntb->vbus_number = 0xff;
1418         epf_set_drvdata(epf, ntb);
1419
1420         dev_info(dev, "pci-ep epf driver loaded\n");
1421         return 0;
1422 }
1423
1424 static const struct pci_epf_device_id epf_ntb_ids[] = {
1425         {
1426                 .name = "pci_epf_vntb",
1427         },
1428         {},
1429 };
1430
1431 static struct pci_epf_driver epf_ntb_driver = {
1432         .driver.name    = "pci_epf_vntb",
1433         .probe          = epf_ntb_probe,
1434         .id_table       = epf_ntb_ids,
1435         .ops            = &epf_ntb_ops,
1436         .owner          = THIS_MODULE,
1437 };
1438
1439 static int __init epf_ntb_init(void)
1440 {
1441         int ret;
1442
1443         kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM |
1444                                             WQ_HIGHPRI, 0);
1445         ret = pci_epf_register_driver(&epf_ntb_driver);
1446         if (ret) {
1447                 destroy_workqueue(kpcintb_workqueue);
1448                 pr_err("Failed to register pci epf ntb driver --> %d\n", ret);
1449                 return ret;
1450         }
1451
1452         return 0;
1453 }
1454 module_init(epf_ntb_init);
1455
1456 static void __exit epf_ntb_exit(void)
1457 {
1458         pci_epf_unregister_driver(&epf_ntb_driver);
1459         destroy_workqueue(kpcintb_workqueue);
1460 }
1461 module_exit(epf_ntb_exit);
1462
1463 MODULE_DESCRIPTION("PCI EPF NTB DRIVER");
1464 MODULE_AUTHOR("Frank Li <Frank.li@nxp.com>");
1465 MODULE_LICENSE("GPL v2");