drivers/ntb/hw/intel/ntb_hw_intel.c

   1 /*
   2  * This file is provided under a dual BSD/GPLv2 license.  When using or
   3  *   redistributing this file, you may do so under either license.
   4  *
   5  *   GPL LICENSE SUMMARY
   6  *
   7  *   Copyright(c) 2012 Intel Corporation. All rights reserved.
   8  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
   9  *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  10  *
  11  *   This program is free software; you can redistribute it and/or modify
  12  *   it under the terms of version 2 of the GNU General Public License as
  13  *   published by the Free Software Foundation.
  14  *
  15  *   BSD LICENSE
  16  *
  17  *   Copyright(c) 2012 Intel Corporation. All rights reserved.
  18  *   Copyright (C) 2015 EMC Corporation. All Rights Reserved.
  19  *   Copyright (C) 2016 T-Platforms. All Rights Reserved.
  20  *
  21  *   Redistribution and use in source and binary forms, with or without
  22  *   modification, are permitted provided that the following conditions
  23  *   are met:
  24  *
  25  *     * Redistributions of source code must retain the above copyright
  26  *       notice, this list of conditions and the following disclaimer.
  27  *     * Redistributions in binary form must reproduce the above copy
  28  *       notice, this list of conditions and the following disclaimer in
  29  *       the documentation and/or other materials provided with the
  30  *       distribution.
  31  *     * Neither the name of Intel Corporation nor the names of its
  32  *       contributors may be used to endorse or promote products derived
  33  *       from this software without specific prior written permission.
  34  *
  35  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  36  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  37  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  38  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  39  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  40  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  41  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  42  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  43  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  44  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  45  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  46  *
  47  * Intel PCIe NTB Linux driver
  48  *
  49  * Contact Information:
  50  * Jon Mason <jon.mason@intel.com>
  51  */
  52
  53 #include <linux/debugfs.h>
  54 #include <linux/delay.h>
  55 #include <linux/init.h>
  56 #include <linux/interrupt.h>
  57 #include <linux/module.h>
  58 #include <linux/pci.h>
  59 #include <linux/random.h>
  60 #include <linux/slab.h>
  61 #include <linux/ntb.h>
  62
  63 #include "ntb_hw_intel.h"
  64
  65 #define NTB_NAME        "ntb_hw_intel"
  66 #define NTB_DESC        "Intel(R) PCI-E Non-Transparent Bridge Driver"
  67 #define NTB_VER         "2.0"
  68
  69 MODULE_DESCRIPTION(NTB_DESC);
  70 MODULE_VERSION(NTB_VER);
  71 MODULE_LICENSE("Dual BSD/GPL");
  72 MODULE_AUTHOR("Intel Corporation");
  73
  74 #define bar0_off(base, bar) ((base) + ((bar) << 2))
  75 #define bar2_off(base, bar) bar0_off(base, (bar) - 2)
  76
  77 static const struct intel_ntb_reg atom_reg;
  78 static const struct intel_ntb_alt_reg atom_pri_reg;
  79 static const struct intel_ntb_alt_reg atom_sec_reg;
  80 static const struct intel_ntb_alt_reg atom_b2b_reg;
  81 static const struct intel_ntb_xlat_reg atom_pri_xlat;
  82 static const struct intel_ntb_xlat_reg atom_sec_xlat;
  83 static const struct intel_ntb_reg xeon_reg;
  84 static const struct intel_ntb_alt_reg xeon_pri_reg;
  85 static const struct intel_ntb_alt_reg xeon_sec_reg;
  86 static const struct intel_ntb_alt_reg xeon_b2b_reg;
  87 static const struct intel_ntb_xlat_reg xeon_pri_xlat;
  88 static const struct intel_ntb_xlat_reg xeon_sec_xlat;
  89 static struct intel_b2b_addr xeon_b2b_usd_addr;
  90 static struct intel_b2b_addr xeon_b2b_dsd_addr;
  91 static const struct intel_ntb_reg skx_reg;
  92 static const struct intel_ntb_alt_reg skx_pri_reg;
  93 static const struct intel_ntb_alt_reg skx_b2b_reg;
  94 static const struct intel_ntb_xlat_reg skx_sec_xlat;
  95 static const struct ntb_dev_ops intel_ntb_ops;
  96 static const struct ntb_dev_ops intel_ntb3_ops;
  97
  98 static const struct file_operations intel_ntb_debugfs_info;
  99 static struct dentry *debugfs_dir;
 100
 101 static int b2b_mw_idx = -1;
 102 module_param(b2b_mw_idx, int, 0644);
 103 MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb.  A "
 104                  "value of zero or positive starts from first mw idx, and a "
 105                  "negative value starts from last mw idx.  Both sides MUST "
 106                  "set the same value here!");
 107
 108 static unsigned int b2b_mw_share;
 109 module_param(b2b_mw_share, uint, 0644);
 110 MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the "
 111                  "ntb so that the peer ntb only occupies the first half of "
 112                  "the mw, so the second half can still be used as a mw.  Both "
 113                  "sides MUST set the same value here!");
 114
 115 module_param_named(xeon_b2b_usd_bar2_addr64,
 116                    xeon_b2b_usd_addr.bar2_addr64, ullong, 0644);
 117 MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
 118                  "XEON B2B USD BAR 2 64-bit address");
 119
 120 module_param_named(xeon_b2b_usd_bar4_addr64,
 121                    xeon_b2b_usd_addr.bar4_addr64, ullong, 0644);
 122 MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr64,
 123                  "XEON B2B USD BAR 4 64-bit address");
 124
 125 module_param_named(xeon_b2b_usd_bar4_addr32,
 126                    xeon_b2b_usd_addr.bar4_addr32, ullong, 0644);
 127 MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr32,
 128                  "XEON B2B USD split-BAR 4 32-bit address");
 129
 130 module_param_named(xeon_b2b_usd_bar5_addr32,
 131                    xeon_b2b_usd_addr.bar5_addr32, ullong, 0644);
 132 MODULE_PARM_DESC(xeon_b2b_usd_bar5_addr32,
 133                  "XEON B2B USD split-BAR 5 32-bit address");
 134
 135 module_param_named(xeon_b2b_dsd_bar2_addr64,
 136                    xeon_b2b_dsd_addr.bar2_addr64, ullong, 0644);
 137 MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
 138                  "XEON B2B DSD BAR 2 64-bit address");
 139
 140 module_param_named(xeon_b2b_dsd_bar4_addr64,
 141                    xeon_b2b_dsd_addr.bar4_addr64, ullong, 0644);
 142 MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr64,
 143                  "XEON B2B DSD BAR 4 64-bit address");
 144
 145 module_param_named(xeon_b2b_dsd_bar4_addr32,
 146                    xeon_b2b_dsd_addr.bar4_addr32, ullong, 0644);
 147 MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr32,
 148                  "XEON B2B DSD split-BAR 4 32-bit address");
 149
 150 module_param_named(xeon_b2b_dsd_bar5_addr32,
 151                    xeon_b2b_dsd_addr.bar5_addr32, ullong, 0644);
 152 MODULE_PARM_DESC(xeon_b2b_dsd_bar5_addr32,
 153                  "XEON B2B DSD split-BAR 5 32-bit address");
 154
 155 static inline enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd);
 156 static int xeon_init_isr(struct intel_ntb_dev *ndev);
 157
 158 #ifndef ioread64
 159 #ifdef readq
 160 #define ioread64 readq
 161 #else
 162 #define ioread64 _ioread64
 163 static inline u64 _ioread64(void __iomem *mmio)
 164 {
 165         u64 low, high;
 166
 167         low = ioread32(mmio);
 168         high = ioread32(mmio + sizeof(u32));
 169         return low | (high << 32);
 170 }
 171 #endif
 172 #endif
 173
 174 #ifndef iowrite64
 175 #ifdef writeq
 176 #define iowrite64 writeq
 177 #else
 178 #define iowrite64 _iowrite64
 179 static inline void _iowrite64(u64 val, void __iomem *mmio)
 180 {
 181         iowrite32(val, mmio);
 182         iowrite32(val >> 32, mmio + sizeof(u32));
 183 }
 184 #endif
 185 #endif
 186
 187 static inline int pdev_is_atom(struct pci_dev *pdev)
 188 {
 189         switch (pdev->device) {
 190         case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD:
 191                 return 1;
 192         }
 193         return 0;
 194 }
 195
 196 static inline int pdev_is_xeon(struct pci_dev *pdev)
 197 {
 198         switch (pdev->device) {
 199         case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
 200         case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
 201         case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
 202         case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
 203         case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
 204         case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
 205         case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
 206         case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
 207         case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
 208         case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
 209         case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
 210         case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
 211         case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
 212         case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
 213         case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
 214                 return 1;
 215         }
 216         return 0;
 217 }
 218
 219 static inline int pdev_is_skx_xeon(struct pci_dev *pdev)
 220 {
 221         if (pdev->device == PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)
 222                 return 1;
 223
 224         return 0;
 225 }
 226
 227 static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev)
 228 {
 229         ndev->unsafe_flags = 0;
 230         ndev->unsafe_flags_ignore = 0;
 231
 232         /* Only B2B has a workaround to avoid SDOORBELL */
 233         if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP)
 234                 if (!ntb_topo_is_b2b(ndev->ntb.topo))
 235                         ndev->unsafe_flags |= NTB_UNSAFE_DB;
 236
 237         /* No low level workaround to avoid SB01BASE */
 238         if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) {
 239                 ndev->unsafe_flags |= NTB_UNSAFE_DB;
 240                 ndev->unsafe_flags |= NTB_UNSAFE_SPAD;
 241         }
 242 }
 243
 244 static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev,
 245                                  unsigned long flag)
 246 {
 247         return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore);
 248 }
 249
 250 static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev,
 251                                      unsigned long flag)
 252 {
 253         flag &= ndev->unsafe_flags;
 254         ndev->unsafe_flags_ignore |= flag;
 255
 256         return !!flag;
 257 }
 258
 259 static int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx)
 260 {
 261         if (idx < 0 || idx >= ndev->mw_count)
 262                 return -EINVAL;
 263         return ndev->reg->mw_bar[idx];
 264 }
 265
 266 static inline int ndev_db_addr(struct intel_ntb_dev *ndev,
 267                                phys_addr_t *db_addr, resource_size_t *db_size,
 268                                phys_addr_t reg_addr, unsigned long reg)
 269 {
 270         if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 271                 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 272
 273         if (db_addr) {
 274                 *db_addr = reg_addr + reg;
 275                 dev_dbg(&ndev->ntb.pdev->dev, "Peer db addr %llx\n", *db_addr);
 276         }
 277
 278         if (db_size) {
 279                 *db_size = ndev->reg->db_size;
 280                 dev_dbg(&ndev->ntb.pdev->dev, "Peer db size %llx\n", *db_size);
 281         }
 282
 283         return 0;
 284 }
 285
 286 static inline u64 ndev_db_read(struct intel_ntb_dev *ndev,
 287                                void __iomem *mmio)
 288 {
 289         if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 290                 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 291
 292         return ndev->reg->db_ioread(mmio);
 293 }
 294
 295 static inline int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits,
 296                                 void __iomem *mmio)
 297 {
 298         if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 299                 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 300
 301         if (db_bits & ~ndev->db_valid_mask)
 302                 return -EINVAL;
 303
 304         ndev->reg->db_iowrite(db_bits, mmio);
 305
 306         return 0;
 307 }
 308
 309 static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits,
 310                                    void __iomem *mmio)
 311 {
 312         unsigned long irqflags;
 313
 314         if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 315                 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 316
 317         if (db_bits & ~ndev->db_valid_mask)
 318                 return -EINVAL;
 319
 320         spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
 321         {
 322                 ndev->db_mask |= db_bits;
 323                 ndev->reg->db_iowrite(ndev->db_mask, mmio);
 324         }
 325         spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
 326
 327         return 0;
 328 }
 329
 330 static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits,
 331                                      void __iomem *mmio)
 332 {
 333         unsigned long irqflags;
 334
 335         if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
 336                 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
 337
 338         if (db_bits & ~ndev->db_valid_mask)
 339                 return -EINVAL;
 340
 341         spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
 342         {
 343                 ndev->db_mask &= ~db_bits;
 344                 ndev->reg->db_iowrite(ndev->db_mask, mmio);
 345         }
 346         spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
 347
 348         return 0;
 349 }
 350
 351 static inline u64 ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector)
 352 {
 353         u64 shift, mask;
 354
 355         shift = ndev->db_vec_shift;
 356         mask = BIT_ULL(shift) - 1;
 357
 358         return mask << (shift * db_vector);
 359 }
 360
 361 static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx,
 362                                  phys_addr_t *spad_addr, phys_addr_t reg_addr,
 363                                  unsigned long reg)
 364 {
 365         if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
 366                 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
 367
 368         if (idx < 0 || idx >= ndev->spad_count)
 369                 return -EINVAL;
 370
 371         if (spad_addr) {
 372                 *spad_addr = reg_addr + reg + (idx << 2);
 373                 dev_dbg(&ndev->ntb.pdev->dev, "Peer spad addr %llx\n",
 374                         *spad_addr);
 375         }
 376
 377         return 0;
 378 }
 379
 380 static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx,
 381                                  void __iomem *mmio)
 382 {
 383         if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
 384                 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
 385
 386         if (idx < 0 || idx >= ndev->spad_count)
 387                 return 0;
 388
 389         return ioread32(mmio + (idx << 2));
 390 }
 391
 392 static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val,
 393                                   void __iomem *mmio)
 394 {
 395         if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
 396                 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
 397
 398         if (idx < 0 || idx >= ndev->spad_count)
 399                 return -EINVAL;
 400
 401         iowrite32(val, mmio + (idx << 2));
 402
 403         return 0;
 404 }
 405
 406 static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec)
 407 {
 408         u64 vec_mask;
 409
 410         vec_mask = ndev_vec_mask(ndev, vec);
 411
 412         if ((ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) && (vec == 31))
 413                 vec_mask |= ndev->db_link_mask;
 414
 415         dev_dbg(&ndev->ntb.pdev->dev, "vec %d vec_mask %llx\n", vec, vec_mask);
 416
 417         ndev->last_ts = jiffies;
 418
 419         if (vec_mask & ndev->db_link_mask) {
 420                 if (ndev->reg->poll_link(ndev))
 421                         ntb_link_event(&ndev->ntb);
 422         }
 423
 424         if (vec_mask & ndev->db_valid_mask)
 425                 ntb_db_event(&ndev->ntb, vec);
 426
 427         return IRQ_HANDLED;
 428 }
 429
 430 static irqreturn_t ndev_vec_isr(int irq, void *dev)
 431 {
 432         struct intel_ntb_vec *nvec = dev;
 433
 434         dev_dbg(&nvec->ndev->ntb.pdev->dev, "irq: %d  nvec->num: %d\n",
 435                 irq, nvec->num);
 436
 437         return ndev_interrupt(nvec->ndev, nvec->num);
 438 }
 439
 440 static irqreturn_t ndev_irq_isr(int irq, void *dev)
 441 {
 442         struct intel_ntb_dev *ndev = dev;
 443
 444         return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq);
 445 }
 446
 447 static int ndev_init_isr(struct intel_ntb_dev *ndev,
 448                          int msix_min, int msix_max,
 449                          int msix_shift, int total_shift)
 450 {
 451         struct pci_dev *pdev;
 452         int rc, i, msix_count, node;
 453
 454         pdev = ndev->ntb.pdev;
 455
 456         node = dev_to_node(&pdev->dev);
 457
 458         /* Mask all doorbell interrupts */
 459         ndev->db_mask = ndev->db_valid_mask;
 460         ndev->reg->db_iowrite(ndev->db_mask,
 461                               ndev->self_mmio +
 462                               ndev->self_reg->db_mask);
 463
 464         /* Try to set up msix irq */
 465
 466         ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec),
 467                                  GFP_KERNEL, node);
 468         if (!ndev->vec)
 469                 goto err_msix_vec_alloc;
 470
 471         ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix),
 472                                   GFP_KERNEL, node);
 473         if (!ndev->msix)
 474                 goto err_msix_alloc;
 475
 476         for (i = 0; i < msix_max; ++i)
 477                 ndev->msix[i].entry = i;
 478
 479         msix_count = pci_enable_msix_range(pdev, ndev->msix,
 480                                            msix_min, msix_max);
 481         if (msix_count < 0)
 482                 goto err_msix_enable;
 483
 484         for (i = 0; i < msix_count; ++i) {
 485                 ndev->vec[i].ndev = ndev;
 486                 ndev->vec[i].num = i;
 487                 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
 488                                  "ndev_vec_isr", &ndev->vec[i]);
 489                 if (rc)
 490                         goto err_msix_request;
 491         }
 492
 493         dev_dbg(&pdev->dev, "Using %d msix interrupts\n", msix_count);
 494         ndev->db_vec_count = msix_count;
 495         ndev->db_vec_shift = msix_shift;
 496         return 0;
 497
 498 err_msix_request:
 499         while (i-- > 0)
 500                 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
 501         pci_disable_msix(pdev);
 502 err_msix_enable:
 503         kfree(ndev->msix);
 504 err_msix_alloc:
 505         kfree(ndev->vec);
 506 err_msix_vec_alloc:
 507         ndev->msix = NULL;
 508         ndev->vec = NULL;
 509
 510         /* Try to set up msi irq */
 511
 512         rc = pci_enable_msi(pdev);
 513         if (rc)
 514                 goto err_msi_enable;
 515
 516         rc = request_irq(pdev->irq, ndev_irq_isr, 0,
 517                          "ndev_irq_isr", ndev);
 518         if (rc)
 519                 goto err_msi_request;
 520
 521         dev_dbg(&pdev->dev, "Using msi interrupts\n");
 522         ndev->db_vec_count = 1;
 523         ndev->db_vec_shift = total_shift;
 524         return 0;
 525
 526 err_msi_request:
 527         pci_disable_msi(pdev);
 528 err_msi_enable:
 529
 530         /* Try to set up intx irq */
 531
 532         pci_intx(pdev, 1);
 533
 534         rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
 535                          "ndev_irq_isr", ndev);
 536         if (rc)
 537                 goto err_intx_request;
 538
 539         dev_dbg(&pdev->dev, "Using intx interrupts\n");
 540         ndev->db_vec_count = 1;
 541         ndev->db_vec_shift = total_shift;
 542         return 0;
 543
 544 err_intx_request:
 545         return rc;
 546 }
 547
 548 static void ndev_deinit_isr(struct intel_ntb_dev *ndev)
 549 {
 550         struct pci_dev *pdev;
 551         int i;
 552
 553         pdev = ndev->ntb.pdev;
 554
 555         /* Mask all doorbell interrupts */
 556         ndev->db_mask = ndev->db_valid_mask;
 557         ndev->reg->db_iowrite(ndev->db_mask,
 558                               ndev->self_mmio +
 559                               ndev->self_reg->db_mask);
 560
 561         if (ndev->msix) {
 562                 i = ndev->db_vec_count;
 563                 while (i--)
 564                         free_irq(ndev->msix[i].vector, &ndev->vec[i]);
 565                 pci_disable_msix(pdev);
 566                 kfree(ndev->msix);
 567                 kfree(ndev->vec);
 568         } else {
 569                 free_irq(pdev->irq, ndev);
 570                 if (pci_dev_msi_enabled(pdev))
 571                         pci_disable_msi(pdev);
 572         }
 573 }
 574
 575 static ssize_t ndev_ntb3_debugfs_read(struct file *filp, char __user *ubuf,
 576                                       size_t count, loff_t *offp)
 577 {
 578         struct intel_ntb_dev *ndev;
 579         void __iomem *mmio;
 580         char *buf;
 581         size_t buf_size;
 582         ssize_t ret, off;
 583         union { u64 v64; u32 v32; u16 v16; } u;
 584
 585         ndev = filp->private_data;
 586         mmio = ndev->self_mmio;
 587
 588         buf_size = min(count, 0x800ul);
 589
 590         buf = kmalloc(buf_size, GFP_KERNEL);
 591         if (!buf)
 592                 return -ENOMEM;
 593
 594         off = 0;
 595
 596         off += scnprintf(buf + off, buf_size - off,
 597                          "NTB Device Information:\n");
 598
 599         off += scnprintf(buf + off, buf_size - off,
 600                          "Connection Topology -\t%s\n",
 601                          ntb_topo_string(ndev->ntb.topo));
 602
 603         off += scnprintf(buf + off, buf_size - off,
 604                          "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
 605         off += scnprintf(buf + off, buf_size - off,
 606                          "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
 607
 608         if (!ndev->reg->link_is_up(ndev))
 609                 off += scnprintf(buf + off, buf_size - off,
 610                                  "Link Status -\t\tDown\n");
 611         else {
 612                 off += scnprintf(buf + off, buf_size - off,
 613                                  "Link Status -\t\tUp\n");
 614                 off += scnprintf(buf + off, buf_size - off,
 615                                  "Link Speed -\t\tPCI-E Gen %u\n",
 616                                  NTB_LNK_STA_SPEED(ndev->lnk_sta));
 617                 off += scnprintf(buf + off, buf_size - off,
 618                                  "Link Width -\t\tx%u\n",
 619                                  NTB_LNK_STA_WIDTH(ndev->lnk_sta));
 620         }
 621
 622         off += scnprintf(buf + off, buf_size - off,
 623                          "Memory Window Count -\t%u\n", ndev->mw_count);
 624         off += scnprintf(buf + off, buf_size - off,
 625                          "Scratchpad Count -\t%u\n", ndev->spad_count);
 626         off += scnprintf(buf + off, buf_size - off,
 627                          "Doorbell Count -\t%u\n", ndev->db_count);
 628         off += scnprintf(buf + off, buf_size - off,
 629                          "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
 630         off += scnprintf(buf + off, buf_size - off,
 631                          "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
 632
 633         off += scnprintf(buf + off, buf_size - off,
 634                          "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
 635         off += scnprintf(buf + off, buf_size - off,
 636                          "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
 637         off += scnprintf(buf + off, buf_size - off,
 638                          "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
 639
 640         u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
 641         off += scnprintf(buf + off, buf_size - off,
 642                          "Doorbell Mask -\t\t%#llx\n", u.v64);
 643
 644         u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
 645         off += scnprintf(buf + off, buf_size - off,
 646                          "Doorbell Bell -\t\t%#llx\n", u.v64);
 647
 648         off += scnprintf(buf + off, buf_size - off,
 649                          "\nNTB Incoming XLAT:\n");
 650
 651         u.v64 = ioread64(mmio + SKX_IMBAR1XBASE_OFFSET);
 652         off += scnprintf(buf + off, buf_size - off,
 653                          "IMBAR1XBASE -\t\t%#018llx\n", u.v64);
 654
 655         u.v64 = ioread64(mmio + SKX_IMBAR2XBASE_OFFSET);
 656         off += scnprintf(buf + off, buf_size - off,
 657                          "IMBAR2XBASE -\t\t%#018llx\n", u.v64);
 658
 659         u.v64 = ioread64(mmio + SKX_IMBAR1XLMT_OFFSET);
 660         off += scnprintf(buf + off, buf_size - off,
 661                          "IMBAR1XLMT -\t\t\t%#018llx\n", u.v64);
 662
 663         u.v64 = ioread64(mmio + SKX_IMBAR2XLMT_OFFSET);
 664         off += scnprintf(buf + off, buf_size - off,
 665                          "IMBAR2XLMT -\t\t\t%#018llx\n", u.v64);
 666
 667         if (ntb_topo_is_b2b(ndev->ntb.topo)) {
 668                 off += scnprintf(buf + off, buf_size - off,
 669                                  "\nNTB Outgoing B2B XLAT:\n");
 670
 671                 u.v64 = ioread64(mmio + SKX_EMBAR1XBASE_OFFSET);
 672                 off += scnprintf(buf + off, buf_size - off,
 673                                  "EMBAR1XBASE -\t\t%#018llx\n", u.v64);
 674
 675                 u.v64 = ioread64(mmio + SKX_EMBAR2XBASE_OFFSET);
 676                 off += scnprintf(buf + off, buf_size - off,
 677                                  "EMBAR2XBASE -\t\t%#018llx\n", u.v64);
 678
 679                 u.v64 = ioread64(mmio + SKX_EMBAR1XLMT_OFFSET);
 680                 off += scnprintf(buf + off, buf_size - off,
 681                                  "EMBAR1XLMT -\t\t%#018llx\n", u.v64);
 682
 683                 u.v64 = ioread64(mmio + SKX_EMBAR2XLMT_OFFSET);
 684                 off += scnprintf(buf + off, buf_size - off,
 685                                  "EMBAR2XLMT -\t\t%#018llx\n", u.v64);
 686
 687                 off += scnprintf(buf + off, buf_size - off,
 688                                  "\nNTB Secondary BAR:\n");
 689
 690                 u.v64 = ioread64(mmio + SKX_EMBAR0_OFFSET);
 691                 off += scnprintf(buf + off, buf_size - off,
 692                                  "EMBAR0 -\t\t%#018llx\n", u.v64);
 693
 694                 u.v64 = ioread64(mmio + SKX_EMBAR1_OFFSET);
 695                 off += scnprintf(buf + off, buf_size - off,
 696                                  "EMBAR1 -\t\t%#018llx\n", u.v64);
 697
 698                 u.v64 = ioread64(mmio + SKX_EMBAR2_OFFSET);
 699                 off += scnprintf(buf + off, buf_size - off,
 700                                  "EMBAR2 -\t\t%#018llx\n", u.v64);
 701         }
 702
 703         off += scnprintf(buf + off, buf_size - off,
 704                          "\nNTB Statistics:\n");
 705
 706         u.v16 = ioread16(mmio + SKX_USMEMMISS_OFFSET);
 707         off += scnprintf(buf + off, buf_size - off,
 708                          "Upstream Memory Miss -\t%u\n", u.v16);
 709
 710         off += scnprintf(buf + off, buf_size - off,
 711                          "\nNTB Hardware Errors:\n");
 712
 713         if (!pci_read_config_word(ndev->ntb.pdev,
 714                                   SKX_DEVSTS_OFFSET, &u.v16))
 715                 off += scnprintf(buf + off, buf_size - off,
 716                                  "DEVSTS -\t\t%#06x\n", u.v16);
 717
 718         if (!pci_read_config_word(ndev->ntb.pdev,
 719                                   SKX_LINK_STATUS_OFFSET, &u.v16))
 720                 off += scnprintf(buf + off, buf_size - off,
 721                                  "LNKSTS -\t\t%#06x\n", u.v16);
 722
 723         if (!pci_read_config_dword(ndev->ntb.pdev,
 724                                    SKX_UNCERRSTS_OFFSET, &u.v32))
 725                 off += scnprintf(buf + off, buf_size - off,
 726                                  "UNCERRSTS -\t\t%#06x\n", u.v32);
 727
 728         if (!pci_read_config_dword(ndev->ntb.pdev,
 729                                    SKX_CORERRSTS_OFFSET, &u.v32))
 730                 off += scnprintf(buf + off, buf_size - off,
 731                                  "CORERRSTS -\t\t%#06x\n", u.v32);
 732
 733         ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
 734         kfree(buf);
 735         return ret;
 736 }
 737
 738 static ssize_t ndev_ntb_debugfs_read(struct file *filp, char __user *ubuf,
 739                                      size_t count, loff_t *offp)
 740 {
 741         struct intel_ntb_dev *ndev;
 742         struct pci_dev *pdev;
 743         void __iomem *mmio;
 744         char *buf;
 745         size_t buf_size;
 746         ssize_t ret, off;
 747         union { u64 v64; u32 v32; u16 v16; u8 v8; } u;
 748
 749         ndev = filp->private_data;
 750         pdev = ndev->ntb.pdev;
 751         mmio = ndev->self_mmio;
 752
 753         buf_size = min(count, 0x800ul);
 754
 755         buf = kmalloc(buf_size, GFP_KERNEL);
 756         if (!buf)
 757                 return -ENOMEM;
 758
 759         off = 0;
 760
 761         off += scnprintf(buf + off, buf_size - off,
 762                          "NTB Device Information:\n");
 763
 764         off += scnprintf(buf + off, buf_size - off,
 765                          "Connection Topology -\t%s\n",
 766                          ntb_topo_string(ndev->ntb.topo));
 767
 768         if (ndev->b2b_idx != UINT_MAX) {
 769                 off += scnprintf(buf + off, buf_size - off,
 770                                  "B2B MW Idx -\t\t%u\n", ndev->b2b_idx);
 771                 off += scnprintf(buf + off, buf_size - off,
 772                                  "B2B Offset -\t\t%#lx\n", ndev->b2b_off);
 773         }
 774
 775         off += scnprintf(buf + off, buf_size - off,
 776                          "BAR4 Split -\t\t%s\n",
 777                          ndev->bar4_split ? "yes" : "no");
 778
 779         off += scnprintf(buf + off, buf_size - off,
 780                          "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
 781         off += scnprintf(buf + off, buf_size - off,
 782                          "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
 783
 784         if (!ndev->reg->link_is_up(ndev)) {
 785                 off += scnprintf(buf + off, buf_size - off,
 786                                  "Link Status -\t\tDown\n");
 787         } else {
 788                 off += scnprintf(buf + off, buf_size - off,
 789                                  "Link Status -\t\tUp\n");
 790                 off += scnprintf(buf + off, buf_size - off,
 791                                  "Link Speed -\t\tPCI-E Gen %u\n",
 792                                  NTB_LNK_STA_SPEED(ndev->lnk_sta));
 793                 off += scnprintf(buf + off, buf_size - off,
 794                                  "Link Width -\t\tx%u\n",
 795                                  NTB_LNK_STA_WIDTH(ndev->lnk_sta));
 796         }
 797
 798         off += scnprintf(buf + off, buf_size - off,
 799                          "Memory Window Count -\t%u\n", ndev->mw_count);
 800         off += scnprintf(buf + off, buf_size - off,
 801                          "Scratchpad Count -\t%u\n", ndev->spad_count);
 802         off += scnprintf(buf + off, buf_size - off,
 803                          "Doorbell Count -\t%u\n", ndev->db_count);
 804         off += scnprintf(buf + off, buf_size - off,
 805                          "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
 806         off += scnprintf(buf + off, buf_size - off,
 807                          "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
 808
 809         off += scnprintf(buf + off, buf_size - off,
 810                          "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
 811         off += scnprintf(buf + off, buf_size - off,
 812                          "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
 813         off += scnprintf(buf + off, buf_size - off,
 814                          "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
 815
 816         u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
 817         off += scnprintf(buf + off, buf_size - off,
 818                          "Doorbell Mask -\t\t%#llx\n", u.v64);
 819
 820         u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
 821         off += scnprintf(buf + off, buf_size - off,
 822                          "Doorbell Bell -\t\t%#llx\n", u.v64);
 823
 824         off += scnprintf(buf + off, buf_size - off,
 825                          "\nNTB Window Size:\n");
 826
 827         pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &u.v8);
 828         off += scnprintf(buf + off, buf_size - off,
 829                          "PBAR23SZ %hhu\n", u.v8);
 830         if (!ndev->bar4_split) {
 831                 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &u.v8);
 832                 off += scnprintf(buf + off, buf_size - off,
 833                                  "PBAR45SZ %hhu\n", u.v8);
 834         } else {
 835                 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &u.v8);
 836                 off += scnprintf(buf + off, buf_size - off,
 837                                  "PBAR4SZ %hhu\n", u.v8);
 838                 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &u.v8);
 839                 off += scnprintf(buf + off, buf_size - off,
 840                                  "PBAR5SZ %hhu\n", u.v8);
 841         }
 842
 843         pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &u.v8);
 844         off += scnprintf(buf + off, buf_size - off,
 845                          "SBAR23SZ %hhu\n", u.v8);
 846         if (!ndev->bar4_split) {
 847                 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &u.v8);
 848                 off += scnprintf(buf + off, buf_size - off,
 849                                  "SBAR45SZ %hhu\n", u.v8);
 850         } else {
 851                 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &u.v8);
 852                 off += scnprintf(buf + off, buf_size - off,
 853                                  "SBAR4SZ %hhu\n", u.v8);
 854                 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &u.v8);
 855                 off += scnprintf(buf + off, buf_size - off,
 856                                  "SBAR5SZ %hhu\n", u.v8);
 857         }
 858
 859         off += scnprintf(buf + off, buf_size - off,
 860                          "\nNTB Incoming XLAT:\n");
 861
 862         u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2));
 863         off += scnprintf(buf + off, buf_size - off,
 864                          "XLAT23 -\t\t%#018llx\n", u.v64);
 865
 866         if (ndev->bar4_split) {
 867                 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
 868                 off += scnprintf(buf + off, buf_size - off,
 869                                  "XLAT4 -\t\t\t%#06x\n", u.v32);
 870
 871                 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 5));
 872                 off += scnprintf(buf + off, buf_size - off,
 873                                  "XLAT5 -\t\t\t%#06x\n", u.v32);
 874         } else {
 875                 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
 876                 off += scnprintf(buf + off, buf_size - off,
 877                                  "XLAT45 -\t\t%#018llx\n", u.v64);
 878         }
 879
 880         u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2));
 881         off += scnprintf(buf + off, buf_size - off,
 882                          "LMT23 -\t\t\t%#018llx\n", u.v64);
 883
 884         if (ndev->bar4_split) {
 885                 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
 886                 off += scnprintf(buf + off, buf_size - off,
 887                                  "LMT4 -\t\t\t%#06x\n", u.v32);
 888                 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 5));
 889                 off += scnprintf(buf + off, buf_size - off,
 890                                  "LMT5 -\t\t\t%#06x\n", u.v32);
 891         } else {
 892                 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
 893                 off += scnprintf(buf + off, buf_size - off,
 894                                  "LMT45 -\t\t\t%#018llx\n", u.v64);
 895         }
 896
 897         if (pdev_is_xeon(pdev)) {
 898                 if (ntb_topo_is_b2b(ndev->ntb.topo)) {
 899                         off += scnprintf(buf + off, buf_size - off,
 900                                          "\nNTB Outgoing B2B XLAT:\n");
 901
 902                         u.v64 = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
 903                         off += scnprintf(buf + off, buf_size - off,
 904                                          "B2B XLAT23 -\t\t%#018llx\n", u.v64);
 905
 906                         if (ndev->bar4_split) {
 907                                 u.v32 = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
 908                                 off += scnprintf(buf + off, buf_size - off,
 909                                                  "B2B XLAT4 -\t\t%#06x\n",
 910                                                  u.v32);
 911                                 u.v32 = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
 912                                 off += scnprintf(buf + off, buf_size - off,
 913                                                  "B2B XLAT5 -\t\t%#06x\n",
 914                                                  u.v32);
 915                         } else {
 916                                 u.v64 = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
 917                                 off += scnprintf(buf + off, buf_size - off,
 918                                                  "B2B XLAT45 -\t\t%#018llx\n",
 919                                                  u.v64);
 920                         }
 921
 922                         u.v64 = ioread64(mmio + XEON_PBAR23LMT_OFFSET);
 923                         off += scnprintf(buf + off, buf_size - off,
 924                                          "B2B LMT23 -\t\t%#018llx\n", u.v64);
 925
 926                         if (ndev->bar4_split) {
 927                                 u.v32 = ioread32(mmio + XEON_PBAR4LMT_OFFSET);
 928                                 off += scnprintf(buf + off, buf_size - off,
 929                                                  "B2B LMT4 -\t\t%#06x\n",
 930                                                  u.v32);
 931                                 u.v32 = ioread32(mmio + XEON_PBAR5LMT_OFFSET);
 932                                 off += scnprintf(buf + off, buf_size - off,
 933                                                  "B2B LMT5 -\t\t%#06x\n",
 934                                                  u.v32);
 935                         } else {
 936                                 u.v64 = ioread64(mmio + XEON_PBAR45LMT_OFFSET);
 937                                 off += scnprintf(buf + off, buf_size - off,
 938                                                  "B2B LMT45 -\t\t%#018llx\n",
 939                                                  u.v64);
 940                         }
 941
 942                         off += scnprintf(buf + off, buf_size - off,
 943                                          "\nNTB Secondary BAR:\n");
 944
 945                         u.v64 = ioread64(mmio + XEON_SBAR0BASE_OFFSET);
 946                         off += scnprintf(buf + off, buf_size - off,
 947                                          "SBAR01 -\t\t%#018llx\n", u.v64);
 948
 949                         u.v64 = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
 950                         off += scnprintf(buf + off, buf_size - off,
 951                                          "SBAR23 -\t\t%#018llx\n", u.v64);
 952
 953                         if (ndev->bar4_split) {
 954                                 u.v32 = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
 955                                 off += scnprintf(buf + off, buf_size - off,
 956                                                  "SBAR4 -\t\t\t%#06x\n", u.v32);
 957                                 u.v32 = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
 958                                 off += scnprintf(buf + off, buf_size - off,
 959                                                  "SBAR5 -\t\t\t%#06x\n", u.v32);
 960                         } else {
 961                                 u.v64 = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
 962                                 off += scnprintf(buf + off, buf_size - off,
 963                                                  "SBAR45 -\t\t%#018llx\n",
 964                                                  u.v64);
 965                         }
 966                 }
 967
 968                 off += scnprintf(buf + off, buf_size - off,
 969                                  "\nXEON NTB Statistics:\n");
 970
 971                 u.v16 = ioread16(mmio + XEON_USMEMMISS_OFFSET);
 972                 off += scnprintf(buf + off, buf_size - off,
 973                                  "Upstream Memory Miss -\t%u\n", u.v16);
 974
 975                 off += scnprintf(buf + off, buf_size - off,
 976                                  "\nXEON NTB Hardware Errors:\n");
 977
 978                 if (!pci_read_config_word(pdev,
 979                                           XEON_DEVSTS_OFFSET, &u.v16))
 980                         off += scnprintf(buf + off, buf_size - off,
 981                                          "DEVSTS -\t\t%#06x\n", u.v16);
 982
 983                 if (!pci_read_config_word(pdev,
 984                                           XEON_LINK_STATUS_OFFSET, &u.v16))
 985                         off += scnprintf(buf + off, buf_size - off,
 986                                          "LNKSTS -\t\t%#06x\n", u.v16);
 987
 988                 if (!pci_read_config_dword(pdev,
 989                                            XEON_UNCERRSTS_OFFSET, &u.v32))
 990                         off += scnprintf(buf + off, buf_size - off,
 991                                          "UNCERRSTS -\t\t%#06x\n", u.v32);
 992
 993                 if (!pci_read_config_dword(pdev,
 994                                            XEON_CORERRSTS_OFFSET, &u.v32))
 995                         off += scnprintf(buf + off, buf_size - off,
 996                                          "CORERRSTS -\t\t%#06x\n", u.v32);
 997         }
 998
 999         ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
1000         kfree(buf);
1001         return ret;
1002 }
1003
1004 static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
1005                                  size_t count, loff_t *offp)
1006 {
1007         struct intel_ntb_dev *ndev = filp->private_data;
1008
1009         if (pdev_is_xeon(ndev->ntb.pdev) ||
1010             pdev_is_atom(ndev->ntb.pdev))
1011                 return ndev_ntb_debugfs_read(filp, ubuf, count, offp);
1012         else if (pdev_is_skx_xeon(ndev->ntb.pdev))
1013                 return ndev_ntb3_debugfs_read(filp, ubuf, count, offp);
1014
1015         return -ENXIO;
1016 }
1017
1018 static void ndev_init_debugfs(struct intel_ntb_dev *ndev)
1019 {
1020         if (!debugfs_dir) {
1021                 ndev->debugfs_dir = NULL;
1022                 ndev->debugfs_info = NULL;
1023         } else {
1024                 ndev->debugfs_dir =
1025                         debugfs_create_dir(pci_name(ndev->ntb.pdev),
1026                                            debugfs_dir);
1027                 if (!ndev->debugfs_dir)
1028                         ndev->debugfs_info = NULL;
1029                 else
1030                         ndev->debugfs_info =
1031                                 debugfs_create_file("info", S_IRUSR,
1032                                                     ndev->debugfs_dir, ndev,
1033                                                     &intel_ntb_debugfs_info);
1034         }
1035 }
1036
1037 static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev)
1038 {
1039         debugfs_remove_recursive(ndev->debugfs_dir);
1040 }
1041
1042 static int intel_ntb_mw_count(struct ntb_dev *ntb, int pidx)
1043 {
1044         if (pidx != NTB_DEF_PEER_IDX)
1045                 return -EINVAL;
1046
1047         return ntb_ndev(ntb)->mw_count;
1048 }
1049
1050 static int intel_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx,
1051                                   resource_size_t *addr_align,
1052                                   resource_size_t *size_align,
1053                                   resource_size_t *size_max)
1054 {
1055         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1056         resource_size_t bar_size, mw_size;
1057         int bar;
1058
1059         if (pidx != NTB_DEF_PEER_IDX)
1060                 return -EINVAL;
1061
1062         if (idx >= ndev->b2b_idx && !ndev->b2b_off)
1063                 idx += 1;
1064
1065         bar = ndev_mw_to_bar(ndev, idx);
1066         if (bar < 0)
1067                 return bar;
1068
1069         bar_size = pci_resource_len(ndev->ntb.pdev, bar);
1070
1071         if (idx == ndev->b2b_idx)
1072                 mw_size = bar_size - ndev->b2b_off;
1073         else
1074                 mw_size = bar_size;
1075
1076         if (addr_align)
1077                 *addr_align = pci_resource_len(ndev->ntb.pdev, bar);
1078
1079         if (size_align)
1080                 *size_align = 1;
1081
1082         if (size_max)
1083                 *size_max = mw_size;
1084
1085         return 0;
1086 }
1087
1088 static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
1089                                   dma_addr_t addr, resource_size_t size)
1090 {
1091         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1092         unsigned long base_reg, xlat_reg, limit_reg;
1093         resource_size_t bar_size, mw_size;
1094         void __iomem *mmio;
1095         u64 base, limit, reg_val;
1096         int bar;
1097
1098         if (pidx != NTB_DEF_PEER_IDX)
1099                 return -EINVAL;
1100
1101         if (idx >= ndev->b2b_idx && !ndev->b2b_off)
1102                 idx += 1;
1103
1104         bar = ndev_mw_to_bar(ndev, idx);
1105         if (bar < 0)
1106                 return bar;
1107
1108         bar_size = pci_resource_len(ndev->ntb.pdev, bar);
1109
1110         if (idx == ndev->b2b_idx)
1111                 mw_size = bar_size - ndev->b2b_off;
1112         else
1113                 mw_size = bar_size;
1114
1115         /* hardware requires that addr is aligned to bar size */
1116         if (addr & (bar_size - 1))
1117                 return -EINVAL;
1118
1119         /* make sure the range fits in the usable mw size */
1120         if (size > mw_size)
1121                 return -EINVAL;
1122
1123         mmio = ndev->self_mmio;
1124         base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar);
1125         xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar);
1126         limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);
1127
1128         if (bar < 4 || !ndev->bar4_split) {
1129                 base = ioread64(mmio + base_reg) & NTB_BAR_MASK_64;
1130
1131                 /* Set the limit if supported, if size is not mw_size */
1132                 if (limit_reg && size != mw_size)
1133                         limit = base + size;
1134                 else
1135                         limit = 0;
1136
1137                 /* set and verify setting the translation address */
1138                 iowrite64(addr, mmio + xlat_reg);
1139                 reg_val = ioread64(mmio + xlat_reg);
1140                 if (reg_val != addr) {
1141                         iowrite64(0, mmio + xlat_reg);
1142                         return -EIO;
1143                 }
1144
1145                 /* set and verify setting the limit */
1146                 iowrite64(limit, mmio + limit_reg);
1147                 reg_val = ioread64(mmio + limit_reg);
1148                 if (reg_val != limit) {
1149                         iowrite64(base, mmio + limit_reg);
1150                         iowrite64(0, mmio + xlat_reg);
1151                         return -EIO;
1152                 }
1153         } else {
1154                 /* split bar addr range must all be 32 bit */
1155                 if (addr & (~0ull << 32))
1156                         return -EINVAL;
1157                 if ((addr + size) & (~0ull << 32))
1158                         return -EINVAL;
1159
1160                 base = ioread32(mmio + base_reg) & NTB_BAR_MASK_32;
1161
1162                 /* Set the limit if supported, if size is not mw_size */
1163                 if (limit_reg && size != mw_size)
1164                         limit = base + size;
1165                 else
1166                         limit = 0;
1167
1168                 /* set and verify setting the translation address */
1169                 iowrite32(addr, mmio + xlat_reg);
1170                 reg_val = ioread32(mmio + xlat_reg);
1171                 if (reg_val != addr) {
1172                         iowrite32(0, mmio + xlat_reg);
1173                         return -EIO;
1174                 }
1175
1176                 /* set and verify setting the limit */
1177                 iowrite32(limit, mmio + limit_reg);
1178                 reg_val = ioread32(mmio + limit_reg);
1179                 if (reg_val != limit) {
1180                         iowrite32(base, mmio + limit_reg);
1181                         iowrite32(0, mmio + xlat_reg);
1182                         return -EIO;
1183                 }
1184         }
1185
1186         return 0;
1187 }
1188
1189 static u64 intel_ntb_link_is_up(struct ntb_dev *ntb,
1190                                 enum ntb_speed *speed,
1191                                 enum ntb_width *width)
1192 {
1193         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1194
1195         if (ndev->reg->link_is_up(ndev)) {
1196                 if (speed)
1197                         *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
1198                 if (width)
1199                         *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
1200                 return 1;
1201         } else {
1202                 /* TODO MAYBE: is it possible to observe the link speed and
1203                  * width while link is training? */
1204                 if (speed)
1205                         *speed = NTB_SPEED_NONE;
1206                 if (width)
1207                         *width = NTB_WIDTH_NONE;
1208                 return 0;
1209         }
1210 }
1211
1212 static int intel_ntb_link_enable(struct ntb_dev *ntb,
1213                                  enum ntb_speed max_speed,
1214                                  enum ntb_width max_width)
1215 {
1216         struct intel_ntb_dev *ndev;
1217         u32 ntb_ctl;
1218
1219         ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1220
1221         if (ndev->ntb.topo == NTB_TOPO_SEC)
1222                 return -EINVAL;
1223
1224         dev_dbg(&ntb->pdev->dev,
1225                 "Enabling link with max_speed %d max_width %d\n",
1226                 max_speed, max_width);
1227         if (max_speed != NTB_SPEED_AUTO)
1228                 dev_dbg(&ntb->pdev->dev, "ignoring max_speed %d\n", max_speed);
1229         if (max_width != NTB_WIDTH_AUTO)
1230                 dev_dbg(&ntb->pdev->dev, "ignoring max_width %d\n", max_width);
1231
1232         ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1233         ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
1234         ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
1235         ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
1236         if (ndev->bar4_split)
1237                 ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP;
1238         iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
1239
1240         return 0;
1241 }
1242
1243 static int intel_ntb_link_disable(struct ntb_dev *ntb)
1244 {
1245         struct intel_ntb_dev *ndev;
1246         u32 ntb_cntl;
1247
1248         ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1249
1250         if (ndev->ntb.topo == NTB_TOPO_SEC)
1251                 return -EINVAL;
1252
1253         dev_dbg(&ntb->pdev->dev, "Disabling link\n");
1254
1255         /* Bring NTB link down */
1256         ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1257         ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
1258         ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
1259         if (ndev->bar4_split)
1260                 ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP);
1261         ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
1262         iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl);
1263
1264         return 0;
1265 }
1266
1267 static int intel_ntb_peer_mw_count(struct ntb_dev *ntb)
1268 {
1269         /* Numbers of inbound and outbound memory windows match */
1270         return ntb_ndev(ntb)->mw_count;
1271 }
1272
1273 static int intel_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
1274                                      phys_addr_t *base, resource_size_t *size)
1275 {
1276         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1277         int bar;
1278
1279         if (idx >= ndev->b2b_idx && !ndev->b2b_off)
1280                 idx += 1;
1281
1282         bar = ndev_mw_to_bar(ndev, idx);
1283         if (bar < 0)
1284                 return bar;
1285
1286         if (base)
1287                 *base = pci_resource_start(ndev->ntb.pdev, bar) +
1288                         (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
1289
1290         if (size)
1291                 *size = pci_resource_len(ndev->ntb.pdev, bar) -
1292                         (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
1293
1294         return 0;
1295 }
1296
1297 static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb)
1298 {
1299         return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB);
1300 }
1301
1302 static u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb)
1303 {
1304         return ntb_ndev(ntb)->db_valid_mask;
1305 }
1306
1307 static int intel_ntb_db_vector_count(struct ntb_dev *ntb)
1308 {
1309         struct intel_ntb_dev *ndev;
1310
1311         ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1312
1313         return ndev->db_vec_count;
1314 }
1315
1316 static u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
1317 {
1318         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1319
1320         if (db_vector < 0 || db_vector > ndev->db_vec_count)
1321                 return 0;
1322
1323         return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector);
1324 }
1325
1326 static u64 intel_ntb_db_read(struct ntb_dev *ntb)
1327 {
1328         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1329
1330         return ndev_db_read(ndev,
1331                             ndev->self_mmio +
1332                             ndev->self_reg->db_bell);
1333 }
1334
1335 static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
1336 {
1337         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1338
1339         return ndev_db_write(ndev, db_bits,
1340                              ndev->self_mmio +
1341                              ndev->self_reg->db_bell);
1342 }
1343
1344 static int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
1345 {
1346         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1347
1348         return ndev_db_set_mask(ndev, db_bits,
1349                                 ndev->self_mmio +
1350                                 ndev->self_reg->db_mask);
1351 }
1352
1353 static int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
1354 {
1355         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1356
1357         return ndev_db_clear_mask(ndev, db_bits,
1358                                   ndev->self_mmio +
1359                                   ndev->self_reg->db_mask);
1360 }
1361
1362 static int intel_ntb_peer_db_addr(struct ntb_dev *ntb,
1363                                   phys_addr_t *db_addr,
1364                                   resource_size_t *db_size)
1365 {
1366         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1367
1368         return ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr,
1369                             ndev->peer_reg->db_bell);
1370 }
1371
1372 static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
1373 {
1374         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1375
1376         return ndev_db_write(ndev, db_bits,
1377                              ndev->peer_mmio +
1378                              ndev->peer_reg->db_bell);
1379 }
1380
1381 static int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb)
1382 {
1383         return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD);
1384 }
1385
1386 static int intel_ntb_spad_count(struct ntb_dev *ntb)
1387 {
1388         struct intel_ntb_dev *ndev;
1389
1390         ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1391
1392         return ndev->spad_count;
1393 }
1394
1395 static u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx)
1396 {
1397         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1398
1399         return ndev_spad_read(ndev, idx,
1400                               ndev->self_mmio +
1401                               ndev->self_reg->spad);
1402 }
1403
1404 static int intel_ntb_spad_write(struct ntb_dev *ntb,
1405                                 int idx, u32 val)
1406 {
1407         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1408
1409         return ndev_spad_write(ndev, idx, val,
1410                                ndev->self_mmio +
1411                                ndev->self_reg->spad);
1412 }
1413
1414 static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx,
1415                                     phys_addr_t *spad_addr)
1416 {
1417         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1418
1419         return ndev_spad_addr(ndev, sidx, spad_addr, ndev->peer_addr,
1420                               ndev->peer_reg->spad);
1421 }
1422
1423 static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx)
1424 {
1425         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1426
1427         return ndev_spad_read(ndev, sidx,
1428                               ndev->peer_mmio +
1429                               ndev->peer_reg->spad);
1430 }
1431
1432 static int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
1433                                      int sidx, u32 val)
1434 {
1435         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1436
1437         return ndev_spad_write(ndev, sidx, val,
1438                                ndev->peer_mmio +
1439                                ndev->peer_reg->spad);
1440 }
1441
1442 /* ATOM */
1443
1444 static u64 atom_db_ioread(void __iomem *mmio)
1445 {
1446         return ioread64(mmio);
1447 }
1448
1449 static void atom_db_iowrite(u64 bits, void __iomem *mmio)
1450 {
1451         iowrite64(bits, mmio);
1452 }
1453
1454 static int atom_poll_link(struct intel_ntb_dev *ndev)
1455 {
1456         u32 ntb_ctl;
1457
1458         ntb_ctl = ioread32(ndev->self_mmio + ATOM_NTBCNTL_OFFSET);
1459
1460         if (ntb_ctl == ndev->ntb_ctl)
1461                 return 0;
1462
1463         ndev->ntb_ctl = ntb_ctl;
1464
1465         ndev->lnk_sta = ioread32(ndev->self_mmio + ATOM_LINK_STATUS_OFFSET);
1466
1467         return 1;
1468 }
1469
1470 static int atom_link_is_up(struct intel_ntb_dev *ndev)
1471 {
1472         return ATOM_NTB_CTL_ACTIVE(ndev->ntb_ctl);
1473 }
1474
1475 static int atom_link_is_err(struct intel_ntb_dev *ndev)
1476 {
1477         if (ioread32(ndev->self_mmio + ATOM_LTSSMSTATEJMP_OFFSET)
1478             & ATOM_LTSSMSTATEJMP_FORCEDETECT)
1479                 return 1;
1480
1481         if (ioread32(ndev->self_mmio + ATOM_IBSTERRRCRVSTS0_OFFSET)
1482             & ATOM_IBIST_ERR_OFLOW)
1483                 return 1;
1484
1485         return 0;
1486 }
1487
1488 static inline enum ntb_topo atom_ppd_topo(struct intel_ntb_dev *ndev, u32 ppd)
1489 {
1490         struct device *dev = &ndev->ntb.pdev->dev;
1491
1492         switch (ppd & ATOM_PPD_TOPO_MASK) {
1493         case ATOM_PPD_TOPO_B2B_USD:
1494                 dev_dbg(dev, "PPD %d B2B USD\n", ppd);
1495                 return NTB_TOPO_B2B_USD;
1496
1497         case ATOM_PPD_TOPO_B2B_DSD:
1498                 dev_dbg(dev, "PPD %d B2B DSD\n", ppd);
1499                 return NTB_TOPO_B2B_DSD;
1500
1501         case ATOM_PPD_TOPO_PRI_USD:
1502         case ATOM_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
1503         case ATOM_PPD_TOPO_SEC_USD:
1504         case ATOM_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
1505                 dev_dbg(dev, "PPD %d non B2B disabled\n", ppd);
1506                 return NTB_TOPO_NONE;
1507         }
1508
1509         dev_dbg(dev, "PPD %d invalid\n", ppd);
1510         return NTB_TOPO_NONE;
1511 }
1512
1513 static void atom_link_hb(struct work_struct *work)
1514 {
1515         struct intel_ntb_dev *ndev = hb_ndev(work);
1516         struct device *dev = &ndev->ntb.pdev->dev;
1517         unsigned long poll_ts;
1518         void __iomem *mmio;
1519         u32 status32;
1520
1521         poll_ts = ndev->last_ts + ATOM_LINK_HB_TIMEOUT;
1522
1523         /* Delay polling the link status if an interrupt was received,
1524          * unless the cached link status says the link is down.
1525          */
1526         if (time_after(poll_ts, jiffies) && atom_link_is_up(ndev)) {
1527                 schedule_delayed_work(&ndev->hb_timer, poll_ts - jiffies);
1528                 return;
1529         }
1530
1531         if (atom_poll_link(ndev))
1532                 ntb_link_event(&ndev->ntb);
1533
1534         if (atom_link_is_up(ndev) || !atom_link_is_err(ndev)) {
1535                 schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT);
1536                 return;
1537         }
1538
1539         /* Link is down with error: recover the link! */
1540
1541         mmio = ndev->self_mmio;
1542
1543         /* Driver resets the NTB ModPhy lanes - magic! */
1544         iowrite8(0xe0, mmio + ATOM_MODPHY_PCSREG6);
1545         iowrite8(0x40, mmio + ATOM_MODPHY_PCSREG4);
1546         iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG4);
1547         iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG6);
1548
1549         /* Driver waits 100ms to allow the NTB ModPhy to settle */
1550         msleep(100);
1551
1552         /* Clear AER Errors, write to clear */
1553         status32 = ioread32(mmio + ATOM_ERRCORSTS_OFFSET);
1554         dev_dbg(dev, "ERRCORSTS = %x\n", status32);
1555         status32 &= PCI_ERR_COR_REP_ROLL;
1556         iowrite32(status32, mmio + ATOM_ERRCORSTS_OFFSET);
1557
1558         /* Clear unexpected electrical idle event in LTSSM, write to clear */
1559         status32 = ioread32(mmio + ATOM_LTSSMERRSTS0_OFFSET);
1560         dev_dbg(dev, "LTSSMERRSTS0 = %x\n", status32);
1561         status32 |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI;
1562         iowrite32(status32, mmio + ATOM_LTSSMERRSTS0_OFFSET);
1563
1564         /* Clear DeSkew Buffer error, write to clear */
1565         status32 = ioread32(mmio + ATOM_DESKEWSTS_OFFSET);
1566         dev_dbg(dev, "DESKEWSTS = %x\n", status32);
1567         status32 |= ATOM_DESKEWSTS_DBERR;
1568         iowrite32(status32, mmio + ATOM_DESKEWSTS_OFFSET);
1569
1570         status32 = ioread32(mmio + ATOM_IBSTERRRCRVSTS0_OFFSET);
1571         dev_dbg(dev, "IBSTERRRCRVSTS0 = %x\n", status32);
1572         status32 &= ATOM_IBIST_ERR_OFLOW;
1573         iowrite32(status32, mmio + ATOM_IBSTERRRCRVSTS0_OFFSET);
1574
1575         /* Releases the NTB state machine to allow the link to retrain */
1576         status32 = ioread32(mmio + ATOM_LTSSMSTATEJMP_OFFSET);
1577         dev_dbg(dev, "LTSSMSTATEJMP = %x\n", status32);
1578         status32 &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT;
1579         iowrite32(status32, mmio + ATOM_LTSSMSTATEJMP_OFFSET);
1580
1581         /* There is a potential race between the 2 NTB devices recovering at the
1582          * same time.  If the times are the same, the link will not recover and
1583          * the driver will be stuck in this loop forever.  Add a random interval
1584          * to the recovery time to prevent this race.
1585          */
1586         schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_RECOVERY_TIME
1587                               + prandom_u32() % ATOM_LINK_RECOVERY_TIME);
1588 }
1589
1590 static int atom_init_isr(struct intel_ntb_dev *ndev)
1591 {
1592         int rc;
1593
1594         rc = ndev_init_isr(ndev, 1, ATOM_DB_MSIX_VECTOR_COUNT,
1595                            ATOM_DB_MSIX_VECTOR_SHIFT, ATOM_DB_TOTAL_SHIFT);
1596         if (rc)
1597                 return rc;
1598
1599         /* ATOM doesn't have link status interrupt, poll on that platform */
1600         ndev->last_ts = jiffies;
1601         INIT_DELAYED_WORK(&ndev->hb_timer, atom_link_hb);
1602         schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT);
1603
1604         return 0;
1605 }
1606
1607 static void atom_deinit_isr(struct intel_ntb_dev *ndev)
1608 {
1609         cancel_delayed_work_sync(&ndev->hb_timer);
1610         ndev_deinit_isr(ndev);
1611 }
1612
1613 static int atom_init_ntb(struct intel_ntb_dev *ndev)
1614 {
1615         ndev->mw_count = ATOM_MW_COUNT;
1616         ndev->spad_count = ATOM_SPAD_COUNT;
1617         ndev->db_count = ATOM_DB_COUNT;
1618
1619         switch (ndev->ntb.topo) {
1620         case NTB_TOPO_B2B_USD:
1621         case NTB_TOPO_B2B_DSD:
1622                 ndev->self_reg = &atom_pri_reg;
1623                 ndev->peer_reg = &atom_b2b_reg;
1624                 ndev->xlat_reg = &atom_sec_xlat;
1625
1626                 /* Enable Bus Master and Memory Space on the secondary side */
1627                 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1628                           ndev->self_mmio + ATOM_SPCICMD_OFFSET);
1629
1630                 break;
1631
1632         default:
1633                 return -EINVAL;
1634         }
1635
1636         ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1637
1638         return 0;
1639 }
1640
1641 static int atom_init_dev(struct intel_ntb_dev *ndev)
1642 {
1643         u32 ppd;
1644         int rc;
1645
1646         rc = pci_read_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET, &ppd);
1647         if (rc)
1648                 return -EIO;
1649
1650         ndev->ntb.topo = atom_ppd_topo(ndev, ppd);
1651         if (ndev->ntb.topo == NTB_TOPO_NONE)
1652                 return -EINVAL;
1653
1654         rc = atom_init_ntb(ndev);
1655         if (rc)
1656                 return rc;
1657
1658         rc = atom_init_isr(ndev);
1659         if (rc)
1660                 return rc;
1661
1662         if (ndev->ntb.topo != NTB_TOPO_SEC) {
1663                 /* Initiate PCI-E link training */
1664                 rc = pci_write_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET,
1665                                             ppd | ATOM_PPD_INIT_LINK);
1666                 if (rc)
1667                         return rc;
1668         }
1669
1670         return 0;
1671 }
1672
1673 static void atom_deinit_dev(struct intel_ntb_dev *ndev)
1674 {
1675         atom_deinit_isr(ndev);
1676 }
1677
1678 /* Skylake Xeon NTB */
1679
1680 static int skx_poll_link(struct intel_ntb_dev *ndev)
1681 {
1682         u16 reg_val;
1683         int rc;
1684
1685         ndev->reg->db_iowrite(ndev->db_link_mask,
1686                               ndev->self_mmio +
1687                               ndev->self_reg->db_clear);
1688
1689         rc = pci_read_config_word(ndev->ntb.pdev,
1690                                   SKX_LINK_STATUS_OFFSET, &reg_val);
1691         if (rc)
1692                 return 0;
1693
1694         if (reg_val == ndev->lnk_sta)
1695                 return 0;
1696
1697         ndev->lnk_sta = reg_val;
1698
1699         return 1;
1700 }
1701
1702 static u64 skx_db_ioread(void __iomem *mmio)
1703 {
1704         return ioread64(mmio);
1705 }
1706
1707 static void skx_db_iowrite(u64 bits, void __iomem *mmio)
1708 {
1709         iowrite64(bits, mmio);
1710 }
1711
1712 static int skx_init_isr(struct intel_ntb_dev *ndev)
1713 {
1714         int i;
1715
1716         /*
1717          * The MSIX vectors and the interrupt status bits are not lined up
1718          * on Skylake. By default the link status bit is bit 32, however it
1719          * is by default MSIX vector0. We need to fixup to line them up.
1720          * The vectors at reset is 1-32,0. We need to reprogram to 0-32.
1721          */
1722
1723         for (i = 0; i < SKX_DB_MSIX_VECTOR_COUNT; i++)
1724                 iowrite8(i, ndev->self_mmio + SKX_INTVEC_OFFSET + i);
1725
1726         /* move link status down one as workaround */
1727         if (ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) {
1728                 iowrite8(SKX_DB_MSIX_VECTOR_COUNT - 2,
1729                          ndev->self_mmio + SKX_INTVEC_OFFSET +
1730                          (SKX_DB_MSIX_VECTOR_COUNT - 1));
1731         }
1732
1733         return ndev_init_isr(ndev, SKX_DB_MSIX_VECTOR_COUNT,
1734                              SKX_DB_MSIX_VECTOR_COUNT,
1735                              SKX_DB_MSIX_VECTOR_SHIFT,
1736                              SKX_DB_TOTAL_SHIFT);
1737 }
1738
1739 static int skx_setup_b2b_mw(struct intel_ntb_dev *ndev,
1740                             const struct intel_b2b_addr *addr,
1741                             const struct intel_b2b_addr *peer_addr)
1742 {
1743         struct pci_dev *pdev;
1744         void __iomem *mmio;
1745         resource_size_t bar_size;
1746         phys_addr_t bar_addr;
1747         int b2b_bar;
1748         u8 bar_sz;
1749
1750         pdev = ndev->ntb.pdev;
1751         mmio = ndev->self_mmio;
1752
1753         if (ndev->b2b_idx == UINT_MAX) {
1754                 dev_dbg(&pdev->dev, "not using b2b mw\n");
1755                 b2b_bar = 0;
1756                 ndev->b2b_off = 0;
1757         } else {
1758                 b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
1759                 if (b2b_bar < 0)
1760                         return -EIO;
1761
1762                 dev_dbg(&pdev->dev, "using b2b mw bar %d\n", b2b_bar);
1763
1764                 bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
1765
1766                 dev_dbg(&pdev->dev, "b2b bar size %#llx\n", bar_size);
1767
1768                 if (b2b_mw_share && ((bar_size >> 1) >= XEON_B2B_MIN_SIZE)) {
1769                         dev_dbg(&pdev->dev, "b2b using first half of bar\n");
1770                         ndev->b2b_off = bar_size >> 1;
1771                 } else if (bar_size >= XEON_B2B_MIN_SIZE) {
1772                         dev_dbg(&pdev->dev, "b2b using whole bar\n");
1773                         ndev->b2b_off = 0;
1774                         --ndev->mw_count;
1775                 } else {
1776                         dev_dbg(&pdev->dev, "b2b bar size is too small\n");
1777                         return -EIO;
1778                 }
1779         }
1780
1781         /*
1782          * Reset the secondary bar sizes to match the primary bar sizes,
1783          * except disable or halve the size of the b2b secondary bar.
1784          */
1785         pci_read_config_byte(pdev, SKX_IMBAR1SZ_OFFSET, &bar_sz);
1786         dev_dbg(&pdev->dev, "IMBAR1SZ %#x\n", bar_sz);
1787         if (b2b_bar == 1) {
1788                 if (ndev->b2b_off)
1789                         bar_sz -= 1;
1790                 else
1791                         bar_sz = 0;
1792         }
1793
1794         pci_write_config_byte(pdev, SKX_EMBAR1SZ_OFFSET, bar_sz);
1795         pci_read_config_byte(pdev, SKX_EMBAR1SZ_OFFSET, &bar_sz);
1796         dev_dbg(&pdev->dev, "EMBAR1SZ %#x\n", bar_sz);
1797
1798         pci_read_config_byte(pdev, SKX_IMBAR2SZ_OFFSET, &bar_sz);
1799         dev_dbg(&pdev->dev, "IMBAR2SZ %#x\n", bar_sz);
1800         if (b2b_bar == 2) {
1801                 if (ndev->b2b_off)
1802                         bar_sz -= 1;
1803                 else
1804                         bar_sz = 0;
1805         }
1806
1807         pci_write_config_byte(pdev, SKX_EMBAR2SZ_OFFSET, bar_sz);
1808         pci_read_config_byte(pdev, SKX_EMBAR2SZ_OFFSET, &bar_sz);
1809         dev_dbg(&pdev->dev, "EMBAR2SZ %#x\n", bar_sz);
1810
1811         /* SBAR01 hit by first part of the b2b bar */
1812         if (b2b_bar == 0)
1813                 bar_addr = addr->bar0_addr;
1814         else if (b2b_bar == 1)
1815                 bar_addr = addr->bar2_addr64;
1816         else if (b2b_bar == 2)
1817                 bar_addr = addr->bar4_addr64;
1818         else
1819                 return -EIO;
1820
1821         /* setup incoming bar limits == base addrs (zero length windows) */
1822         bar_addr = addr->bar2_addr64 + (b2b_bar == 1 ? ndev->b2b_off : 0);
1823         iowrite64(bar_addr, mmio + SKX_IMBAR1XLMT_OFFSET);
1824         bar_addr = ioread64(mmio + SKX_IMBAR1XLMT_OFFSET);
1825         dev_dbg(&pdev->dev, "IMBAR1XLMT %#018llx\n", bar_addr);
1826
1827         bar_addr = addr->bar4_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1828         iowrite64(bar_addr, mmio + SKX_IMBAR2XLMT_OFFSET);
1829         bar_addr = ioread64(mmio + SKX_IMBAR2XLMT_OFFSET);
1830         dev_dbg(&pdev->dev, "IMBAR2XLMT %#018llx\n", bar_addr);
1831
1832         /* zero incoming translation addrs */
1833         iowrite64(0, mmio + SKX_IMBAR1XBASE_OFFSET);
1834         iowrite64(0, mmio + SKX_IMBAR2XBASE_OFFSET);
1835
1836         ndev->peer_mmio = ndev->self_mmio;
1837
1838         return 0;
1839 }
1840
1841 static int skx_init_ntb(struct intel_ntb_dev *ndev)
1842 {
1843         int rc;
1844
1845
1846         ndev->mw_count = XEON_MW_COUNT;
1847         ndev->spad_count = SKX_SPAD_COUNT;
1848         ndev->db_count = SKX_DB_COUNT;
1849         ndev->db_link_mask = SKX_DB_LINK_BIT;
1850
1851         /* DB fixup for using 31 right now */
1852         if (ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD)
1853                 ndev->db_link_mask |= BIT_ULL(31);
1854
1855         switch (ndev->ntb.topo) {
1856         case NTB_TOPO_B2B_USD:
1857         case NTB_TOPO_B2B_DSD:
1858                 ndev->self_reg = &skx_pri_reg;
1859                 ndev->peer_reg = &skx_b2b_reg;
1860                 ndev->xlat_reg = &skx_sec_xlat;
1861
1862                 if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
1863                         rc = skx_setup_b2b_mw(ndev,
1864                                               &xeon_b2b_dsd_addr,
1865                                               &xeon_b2b_usd_addr);
1866                 } else {
1867                         rc = skx_setup_b2b_mw(ndev,
1868                                               &xeon_b2b_usd_addr,
1869                                               &xeon_b2b_dsd_addr);
1870                 }
1871
1872                 if (rc)
1873                         return rc;
1874
1875                 /* Enable Bus Master and Memory Space on the secondary side */
1876                 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1877                           ndev->self_mmio + SKX_SPCICMD_OFFSET);
1878
1879                 break;
1880
1881         default:
1882                 return -EINVAL;
1883         }
1884
1885         ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1886
1887         ndev->reg->db_iowrite(ndev->db_valid_mask,
1888                               ndev->self_mmio +
1889                               ndev->self_reg->db_mask);
1890
1891         return 0;
1892 }
1893
1894 static int skx_init_dev(struct intel_ntb_dev *ndev)
1895 {
1896         struct pci_dev *pdev;
1897         u8 ppd;
1898         int rc;
1899
1900         pdev = ndev->ntb.pdev;
1901
1902         ndev->reg = &skx_reg;
1903
1904         rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
1905         if (rc)
1906                 return -EIO;
1907
1908         ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
1909         dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd,
1910                 ntb_topo_string(ndev->ntb.topo));
1911         if (ndev->ntb.topo == NTB_TOPO_NONE)
1912                 return -EINVAL;
1913
1914         if (pdev_is_skx_xeon(pdev))
1915                 ndev->hwerr_flags |= NTB_HWERR_MSIX_VECTOR32_BAD;
1916
1917         rc = skx_init_ntb(ndev);
1918         if (rc)
1919                 return rc;
1920
1921         return skx_init_isr(ndev);
1922 }
1923
1924 static int intel_ntb3_link_enable(struct ntb_dev *ntb,
1925                                   enum ntb_speed max_speed,
1926                                   enum ntb_width max_width)
1927 {
1928         struct intel_ntb_dev *ndev;
1929         u32 ntb_ctl;
1930
1931         ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1932
1933         dev_dbg(&ntb->pdev->dev,
1934                 "Enabling link with max_speed %d max_width %d\n",
1935                 max_speed, max_width);
1936
1937         if (max_speed != NTB_SPEED_AUTO)
1938                 dev_dbg(&ntb->pdev->dev, "ignoring max_speed %d\n", max_speed);
1939         if (max_width != NTB_WIDTH_AUTO)
1940                 dev_dbg(&ntb->pdev->dev, "ignoring max_width %d\n", max_width);
1941
1942         ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1943         ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
1944         ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
1945         ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
1946         iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
1947
1948         return 0;
1949 }
1950 static int intel_ntb3_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx,
1951                                    dma_addr_t addr, resource_size_t size)
1952 {
1953         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1954         unsigned long xlat_reg, limit_reg;
1955         resource_size_t bar_size, mw_size;
1956         void __iomem *mmio;
1957         u64 base, limit, reg_val;
1958         int bar;
1959
1960         if (pidx != NTB_DEF_PEER_IDX)
1961                 return -EINVAL;
1962
1963         if (idx >= ndev->b2b_idx && !ndev->b2b_off)
1964                 idx += 1;
1965
1966         bar = ndev_mw_to_bar(ndev, idx);
1967         if (bar < 0)
1968                 return bar;
1969
1970         bar_size = pci_resource_len(ndev->ntb.pdev, bar);
1971
1972         if (idx == ndev->b2b_idx)
1973                 mw_size = bar_size - ndev->b2b_off;
1974         else
1975                 mw_size = bar_size;
1976
1977         /* hardware requires that addr is aligned to bar size */
1978         if (addr & (bar_size - 1))
1979                 return -EINVAL;
1980
1981         /* make sure the range fits in the usable mw size */
1982         if (size > mw_size)
1983                 return -EINVAL;
1984
1985         mmio = ndev->self_mmio;
1986         xlat_reg = ndev->xlat_reg->bar2_xlat + (idx * 0x10);
1987         limit_reg = ndev->xlat_reg->bar2_limit + (idx * 0x10);
1988         base = pci_resource_start(ndev->ntb.pdev, bar);
1989
1990         /* Set the limit if supported, if size is not mw_size */
1991         if (limit_reg && size != mw_size)
1992                 limit = base + size;
1993         else
1994                 limit = base + mw_size;
1995
1996         /* set and verify setting the translation address */
1997         iowrite64(addr, mmio + xlat_reg);
1998         reg_val = ioread64(mmio + xlat_reg);
1999         if (reg_val != addr) {
2000                 iowrite64(0, mmio + xlat_reg);
2001                 return -EIO;
2002         }
2003
2004         dev_dbg(&ntb->pdev->dev, "BAR %d IMBARXBASE: %#Lx\n", bar, reg_val);
2005
2006         /* set and verify setting the limit */
2007         iowrite64(limit, mmio + limit_reg);
2008         reg_val = ioread64(mmio + limit_reg);
2009         if (reg_val != limit) {
2010                 iowrite64(base, mmio + limit_reg);
2011                 iowrite64(0, mmio + xlat_reg);
2012                 return -EIO;
2013         }
2014
2015         dev_dbg(&ntb->pdev->dev, "BAR %d IMBARXLMT: %#Lx\n", bar, reg_val);
2016
2017         /* setup the EP */
2018         limit_reg = ndev->xlat_reg->bar2_limit + (idx * 0x10) + 0x4000;
2019         base = ioread64(mmio + SKX_EMBAR1_OFFSET + (8 * idx));
2020         base &= ~0xf;
2021
2022         if (limit_reg && size != mw_size)
2023                 limit = base + size;
2024         else
2025                 limit = base + mw_size;
2026
2027         /* set and verify setting the limit */
2028         iowrite64(limit, mmio + limit_reg);
2029         reg_val = ioread64(mmio + limit_reg);
2030         if (reg_val != limit) {
2031                 iowrite64(base, mmio + limit_reg);
2032                 iowrite64(0, mmio + xlat_reg);
2033                 return -EIO;
2034         }
2035
2036         dev_dbg(&ntb->pdev->dev, "BAR %d EMBARXLMT: %#Lx\n", bar, reg_val);
2037
2038         return 0;
2039 }
2040
2041 static int intel_ntb3_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
2042 {
2043         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
2044         int bit;
2045
2046         if (db_bits & ~ndev->db_valid_mask)
2047                 return -EINVAL;
2048
2049         while (db_bits) {
2050                 bit = __ffs(db_bits);
2051                 iowrite32(1, ndev->peer_mmio +
2052                                 ndev->peer_reg->db_bell + (bit * 4));
2053                 db_bits &= db_bits - 1;
2054         }
2055
2056         return 0;
2057 }
2058
2059 static u64 intel_ntb3_db_read(struct ntb_dev *ntb)
2060 {
2061         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
2062
2063         return ndev_db_read(ndev,
2064                             ndev->self_mmio +
2065                             ndev->self_reg->db_clear);
2066 }
2067
2068 static int intel_ntb3_db_clear(struct ntb_dev *ntb, u64 db_bits)
2069 {
2070         struct intel_ntb_dev *ndev = ntb_ndev(ntb);
2071
2072         return ndev_db_write(ndev, db_bits,
2073                              ndev->self_mmio +
2074                              ndev->self_reg->db_clear);
2075 }
2076
2077 /* XEON */
2078
2079 static u64 xeon_db_ioread(void __iomem *mmio)
2080 {
2081         return (u64)ioread16(mmio);
2082 }
2083
2084 static void xeon_db_iowrite(u64 bits, void __iomem *mmio)
2085 {
2086         iowrite16((u16)bits, mmio);
2087 }
2088
2089 static int xeon_poll_link(struct intel_ntb_dev *ndev)
2090 {
2091         u16 reg_val;
2092         int rc;
2093
2094         ndev->reg->db_iowrite(ndev->db_link_mask,
2095                               ndev->self_mmio +
2096                               ndev->self_reg->db_bell);
2097
2098         rc = pci_read_config_word(ndev->ntb.pdev,
2099                                   XEON_LINK_STATUS_OFFSET, &reg_val);
2100         if (rc)
2101                 return 0;
2102
2103         if (reg_val == ndev->lnk_sta)
2104                 return 0;
2105
2106         ndev->lnk_sta = reg_val;
2107
2108         return 1;
2109 }
2110
2111 static int xeon_link_is_up(struct intel_ntb_dev *ndev)
2112 {
2113         if (ndev->ntb.topo == NTB_TOPO_SEC)
2114                 return 1;
2115
2116         return NTB_LNK_STA_ACTIVE(ndev->lnk_sta);
2117 }
2118
2119 static inline enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd)
2120 {
2121         switch (ppd & XEON_PPD_TOPO_MASK) {
2122         case XEON_PPD_TOPO_B2B_USD:
2123                 return NTB_TOPO_B2B_USD;
2124
2125         case XEON_PPD_TOPO_B2B_DSD:
2126                 return NTB_TOPO_B2B_DSD;
2127
2128         case XEON_PPD_TOPO_PRI_USD:
2129         case XEON_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
2130                 return NTB_TOPO_PRI;
2131
2132         case XEON_PPD_TOPO_SEC_USD:
2133         case XEON_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
2134                 return NTB_TOPO_SEC;
2135         }
2136
2137         return NTB_TOPO_NONE;
2138 }
2139
2140 static inline int xeon_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd)
2141 {
2142         if (ppd & XEON_PPD_SPLIT_BAR_MASK) {
2143                 dev_dbg(&ndev->ntb.pdev->dev, "PPD %d split bar\n", ppd);
2144                 return 1;
2145         }
2146         return 0;
2147 }
2148
2149 static int xeon_init_isr(struct intel_ntb_dev *ndev)
2150 {
2151         return ndev_init_isr(ndev, XEON_DB_MSIX_VECTOR_COUNT,
2152                              XEON_DB_MSIX_VECTOR_COUNT,
2153                              XEON_DB_MSIX_VECTOR_SHIFT,
2154                              XEON_DB_TOTAL_SHIFT);
2155 }
2156
2157 static void xeon_deinit_isr(struct intel_ntb_dev *ndev)
2158 {
2159         ndev_deinit_isr(ndev);
2160 }
2161
2162 static int xeon_setup_b2b_mw(struct intel_ntb_dev *ndev,
2163                              const struct intel_b2b_addr *addr,
2164                              const struct intel_b2b_addr *peer_addr)
2165 {
2166         struct pci_dev *pdev;
2167         void __iomem *mmio;
2168         resource_size_t bar_size;
2169         phys_addr_t bar_addr;
2170         int b2b_bar;
2171         u8 bar_sz;
2172
2173         pdev = ndev->ntb.pdev;
2174         mmio = ndev->self_mmio;
2175
2176         if (ndev->b2b_idx == UINT_MAX) {
2177                 dev_dbg(&pdev->dev, "not using b2b mw\n");
2178                 b2b_bar = 0;
2179                 ndev->b2b_off = 0;
2180         } else {
2181                 b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
2182                 if (b2b_bar < 0)
2183                         return -EIO;
2184
2185                 dev_dbg(&pdev->dev, "using b2b mw bar %d\n", b2b_bar);
2186
2187                 bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
2188
2189                 dev_dbg(&pdev->dev, "b2b bar size %#llx\n", bar_size);
2190
2191                 if (b2b_mw_share && XEON_B2B_MIN_SIZE <= bar_size >> 1) {
2192                         dev_dbg(&pdev->dev, "b2b using first half of bar\n");
2193                         ndev->b2b_off = bar_size >> 1;
2194                 } else if (XEON_B2B_MIN_SIZE <= bar_size) {
2195                         dev_dbg(&pdev->dev, "b2b using whole bar\n");
2196                         ndev->b2b_off = 0;
2197                         --ndev->mw_count;
2198                 } else {
2199                         dev_dbg(&pdev->dev, "b2b bar size is too small\n");
2200                         return -EIO;
2201                 }
2202         }
2203
2204         /* Reset the secondary bar sizes to match the primary bar sizes,
2205          * except disable or halve the size of the b2b secondary bar.
2206          *
2207          * Note: code for each specific bar size register, because the register
2208          * offsets are not in a consistent order (bar5sz comes after ppd, odd).
2209          */
2210         pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &bar_sz);
2211         dev_dbg(&pdev->dev, "PBAR23SZ %#x\n", bar_sz);
2212         if (b2b_bar == 2) {
2213                 if (ndev->b2b_off)
2214                         bar_sz -= 1;
2215                 else
2216                         bar_sz = 0;
2217         }
2218         pci_write_config_byte(pdev, XEON_SBAR23SZ_OFFSET, bar_sz);
2219         pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &bar_sz);
2220         dev_dbg(&pdev->dev, "SBAR23SZ %#x\n", bar_sz);
2221
2222         if (!ndev->bar4_split) {
2223                 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &bar_sz);
2224                 dev_dbg(&pdev->dev, "PBAR45SZ %#x\n", bar_sz);
2225                 if (b2b_bar == 4) {
2226                         if (ndev->b2b_off)
2227                                 bar_sz -= 1;
2228                         else
2229                                 bar_sz = 0;
2230                 }
2231                 pci_write_config_byte(pdev, XEON_SBAR45SZ_OFFSET, bar_sz);
2232                 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &bar_sz);
2233                 dev_dbg(&pdev->dev, "SBAR45SZ %#x\n", bar_sz);
2234         } else {
2235                 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &bar_sz);
2236                 dev_dbg(&pdev->dev, "PBAR4SZ %#x\n", bar_sz);
2237                 if (b2b_bar == 4) {
2238                         if (ndev->b2b_off)
2239                                 bar_sz -= 1;
2240                         else
2241                                 bar_sz = 0;
2242                 }
2243                 pci_write_config_byte(pdev, XEON_SBAR4SZ_OFFSET, bar_sz);
2244                 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &bar_sz);
2245                 dev_dbg(&pdev->dev, "SBAR4SZ %#x\n", bar_sz);
2246
2247                 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &bar_sz);
2248                 dev_dbg(&pdev->dev, "PBAR5SZ %#x\n", bar_sz);
2249                 if (b2b_bar == 5) {
2250                         if (ndev->b2b_off)
2251                                 bar_sz -= 1;
2252                         else
2253                                 bar_sz = 0;
2254                 }
2255                 pci_write_config_byte(pdev, XEON_SBAR5SZ_OFFSET, bar_sz);
2256                 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &bar_sz);
2257                 dev_dbg(&pdev->dev, "SBAR5SZ %#x\n", bar_sz);
2258         }
2259
2260         /* SBAR01 hit by first part of the b2b bar */
2261         if (b2b_bar == 0)
2262                 bar_addr = addr->bar0_addr;
2263         else if (b2b_bar == 2)
2264                 bar_addr = addr->bar2_addr64;
2265         else if (b2b_bar == 4 && !ndev->bar4_split)
2266                 bar_addr = addr->bar4_addr64;
2267         else if (b2b_bar == 4)
2268                 bar_addr = addr->bar4_addr32;
2269         else if (b2b_bar == 5)
2270                 bar_addr = addr->bar5_addr32;
2271         else
2272                 return -EIO;
2273
2274         dev_dbg(&pdev->dev, "SBAR01 %#018llx\n", bar_addr);
2275         iowrite64(bar_addr, mmio + XEON_SBAR0BASE_OFFSET);
2276
2277         /* Other SBAR are normally hit by the PBAR xlat, except for b2b bar.
2278          * The b2b bar is either disabled above, or configured half-size, and
2279          * it starts at the PBAR xlat + offset.
2280          */
2281
2282         bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
2283         iowrite64(bar_addr, mmio + XEON_SBAR23BASE_OFFSET);
2284         bar_addr = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
2285         dev_dbg(&pdev->dev, "SBAR23 %#018llx\n", bar_addr);
2286
2287         if (!ndev->bar4_split) {
2288                 bar_addr = addr->bar4_addr64 +
2289                         (b2b_bar == 4 ? ndev->b2b_off : 0);
2290                 iowrite64(bar_addr, mmio + XEON_SBAR45BASE_OFFSET);
2291                 bar_addr = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
2292                 dev_dbg(&pdev->dev, "SBAR45 %#018llx\n", bar_addr);
2293         } else {
2294                 bar_addr = addr->bar4_addr32 +
2295                         (b2b_bar == 4 ? ndev->b2b_off : 0);
2296                 iowrite32(bar_addr, mmio + XEON_SBAR4BASE_OFFSET);
2297                 bar_addr = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
2298                 dev_dbg(&pdev->dev, "SBAR4 %#010llx\n", bar_addr);
2299
2300                 bar_addr = addr->bar5_addr32 +
2301                         (b2b_bar == 5 ? ndev->b2b_off : 0);
2302                 iowrite32(bar_addr, mmio + XEON_SBAR5BASE_OFFSET);
2303                 bar_addr = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
2304                 dev_dbg(&pdev->dev, "SBAR5 %#010llx\n", bar_addr);
2305         }
2306
2307         /* setup incoming bar limits == base addrs (zero length windows) */
2308
2309         bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
2310         iowrite64(bar_addr, mmio + XEON_SBAR23LMT_OFFSET);
2311         bar_addr = ioread64(mmio + XEON_SBAR23LMT_OFFSET);
2312         dev_dbg(&pdev->dev, "SBAR23LMT %#018llx\n", bar_addr);
2313
2314         if (!ndev->bar4_split) {
2315                 bar_addr = addr->bar4_addr64 +
2316                         (b2b_bar == 4 ? ndev->b2b_off : 0);
2317                 iowrite64(bar_addr, mmio + XEON_SBAR45LMT_OFFSET);
2318                 bar_addr = ioread64(mmio + XEON_SBAR45LMT_OFFSET);
2319                 dev_dbg(&pdev->dev, "SBAR45LMT %#018llx\n", bar_addr);
2320         } else {
2321                 bar_addr = addr->bar4_addr32 +
2322                         (b2b_bar == 4 ? ndev->b2b_off : 0);
2323                 iowrite32(bar_addr, mmio + XEON_SBAR4LMT_OFFSET);
2324                 bar_addr = ioread32(mmio + XEON_SBAR4LMT_OFFSET);
2325                 dev_dbg(&pdev->dev, "SBAR4LMT %#010llx\n", bar_addr);
2326
2327                 bar_addr = addr->bar5_addr32 +
2328                         (b2b_bar == 5 ? ndev->b2b_off : 0);
2329                 iowrite32(bar_addr, mmio + XEON_SBAR5LMT_OFFSET);
2330                 bar_addr = ioread32(mmio + XEON_SBAR5LMT_OFFSET);
2331                 dev_dbg(&pdev->dev, "SBAR5LMT %#05llx\n", bar_addr);
2332         }
2333
2334         /* zero incoming translation addrs */
2335         iowrite64(0, mmio + XEON_SBAR23XLAT_OFFSET);
2336
2337         if (!ndev->bar4_split) {
2338                 iowrite64(0, mmio + XEON_SBAR45XLAT_OFFSET);
2339         } else {
2340                 iowrite32(0, mmio + XEON_SBAR4XLAT_OFFSET);
2341                 iowrite32(0, mmio + XEON_SBAR5XLAT_OFFSET);
2342         }
2343
2344         /* zero outgoing translation limits (whole bar size windows) */
2345         iowrite64(0, mmio + XEON_PBAR23LMT_OFFSET);
2346         if (!ndev->bar4_split) {
2347                 iowrite64(0, mmio + XEON_PBAR45LMT_OFFSET);
2348         } else {
2349                 iowrite32(0, mmio + XEON_PBAR4LMT_OFFSET);
2350                 iowrite32(0, mmio + XEON_PBAR5LMT_OFFSET);
2351         }
2352
2353         /* set outgoing translation offsets */
2354         bar_addr = peer_addr->bar2_addr64;
2355         iowrite64(bar_addr, mmio + XEON_PBAR23XLAT_OFFSET);
2356         bar_addr = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
2357         dev_dbg(&pdev->dev, "PBAR23XLAT %#018llx\n", bar_addr);
2358
2359         if (!ndev->bar4_split) {
2360                 bar_addr = peer_addr->bar4_addr64;
2361                 iowrite64(bar_addr, mmio + XEON_PBAR45XLAT_OFFSET);
2362                 bar_addr = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
2363                 dev_dbg(&pdev->dev, "PBAR45XLAT %#018llx\n", bar_addr);
2364         } else {
2365                 bar_addr = peer_addr->bar4_addr32;
2366                 iowrite32(bar_addr, mmio + XEON_PBAR4XLAT_OFFSET);
2367                 bar_addr = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
2368                 dev_dbg(&pdev->dev, "PBAR4XLAT %#010llx\n", bar_addr);
2369
2370                 bar_addr = peer_addr->bar5_addr32;
2371                 iowrite32(bar_addr, mmio + XEON_PBAR5XLAT_OFFSET);
2372                 bar_addr = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
2373                 dev_dbg(&pdev->dev, "PBAR5XLAT %#010llx\n", bar_addr);
2374         }
2375
2376         /* set the translation offset for b2b registers */
2377         if (b2b_bar == 0)
2378                 bar_addr = peer_addr->bar0_addr;
2379         else if (b2b_bar == 2)
2380                 bar_addr = peer_addr->bar2_addr64;
2381         else if (b2b_bar == 4 && !ndev->bar4_split)
2382                 bar_addr = peer_addr->bar4_addr64;
2383         else if (b2b_bar == 4)
2384                 bar_addr = peer_addr->bar4_addr32;
2385         else if (b2b_bar == 5)
2386                 bar_addr = peer_addr->bar5_addr32;
2387         else
2388                 return -EIO;
2389
2390         /* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */
2391         dev_dbg(&pdev->dev, "B2BXLAT %#018llx\n", bar_addr);
2392         iowrite32(bar_addr, mmio + XEON_B2B_XLAT_OFFSETL);
2393         iowrite32(bar_addr >> 32, mmio + XEON_B2B_XLAT_OFFSETU);
2394
2395         if (b2b_bar) {
2396                 /* map peer ntb mmio config space registers */
2397                 ndev->peer_mmio = pci_iomap(pdev, b2b_bar,
2398                                             XEON_B2B_MIN_SIZE);
2399                 if (!ndev->peer_mmio)
2400                         return -EIO;
2401
2402                 ndev->peer_addr = pci_resource_start(pdev, b2b_bar);
2403         }
2404
2405         return 0;
2406 }
2407
2408 static int xeon_init_ntb(struct intel_ntb_dev *ndev)
2409 {
2410         struct device *dev = &ndev->ntb.pdev->dev;
2411         int rc;
2412         u32 ntb_ctl;
2413
2414         if (ndev->bar4_split)
2415                 ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT;
2416         else
2417                 ndev->mw_count = XEON_MW_COUNT;
2418
2419         ndev->spad_count = XEON_SPAD_COUNT;
2420         ndev->db_count = XEON_DB_COUNT;
2421         ndev->db_link_mask = XEON_DB_LINK_BIT;
2422
2423         switch (ndev->ntb.topo) {
2424         case NTB_TOPO_PRI:
2425                 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
2426                         dev_err(dev, "NTB Primary config disabled\n");
2427                         return -EINVAL;
2428                 }
2429
2430                 /* enable link to allow secondary side device to appear */
2431                 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
2432                 ntb_ctl &= ~NTB_CTL_DISABLE;
2433                 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
2434
2435                 /* use half the spads for the peer */
2436                 ndev->spad_count >>= 1;
2437                 ndev->self_reg = &xeon_pri_reg;
2438                 ndev->peer_reg = &xeon_sec_reg;
2439                 ndev->xlat_reg = &xeon_sec_xlat;
2440                 break;
2441
2442         case NTB_TOPO_SEC:
2443                 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
2444                         dev_err(dev, "NTB Secondary config disabled\n");
2445                         return -EINVAL;
2446                 }
2447                 /* use half the spads for the peer */
2448                 ndev->spad_count >>= 1;
2449                 ndev->self_reg = &xeon_sec_reg;
2450                 ndev->peer_reg = &xeon_pri_reg;
2451                 ndev->xlat_reg = &xeon_pri_xlat;
2452                 break;
2453
2454         case NTB_TOPO_B2B_USD:
2455         case NTB_TOPO_B2B_DSD:
2456                 ndev->self_reg = &xeon_pri_reg;
2457                 ndev->peer_reg = &xeon_b2b_reg;
2458                 ndev->xlat_reg = &xeon_sec_xlat;
2459
2460                 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
2461                         ndev->peer_reg = &xeon_pri_reg;
2462
2463                         if (b2b_mw_idx < 0)
2464                                 ndev->b2b_idx = b2b_mw_idx + ndev->mw_count;
2465                         else
2466                                 ndev->b2b_idx = b2b_mw_idx;
2467
2468                         if (ndev->b2b_idx >= ndev->mw_count) {
2469                                 dev_dbg(dev,
2470                                         "b2b_mw_idx %d invalid for mw_count %u\n",
2471                                         b2b_mw_idx, ndev->mw_count);
2472                                 return -EINVAL;
2473                         }
2474
2475                         dev_dbg(dev, "setting up b2b mw idx %d means %d\n",
2476                                 b2b_mw_idx, ndev->b2b_idx);
2477
2478                 } else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) {
2479                         dev_warn(dev, "Reduce doorbell count by 1\n");
2480                         ndev->db_count -= 1;
2481                 }
2482
2483                 if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
2484                         rc = xeon_setup_b2b_mw(ndev,
2485                                                &xeon_b2b_dsd_addr,
2486                                                &xeon_b2b_usd_addr);
2487                 } else {
2488                         rc = xeon_setup_b2b_mw(ndev,
2489                                                &xeon_b2b_usd_addr,
2490                                                &xeon_b2b_dsd_addr);
2491                 }
2492                 if (rc)
2493                         return rc;
2494
2495                 /* Enable Bus Master and Memory Space on the secondary side */
2496                 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
2497                           ndev->self_mmio + XEON_SPCICMD_OFFSET);
2498
2499                 break;
2500
2501         default:
2502                 return -EINVAL;
2503         }
2504
2505         ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
2506
2507         ndev->reg->db_iowrite(ndev->db_valid_mask,
2508                               ndev->self_mmio +
2509                               ndev->self_reg->db_mask);
2510
2511         return 0;
2512 }
2513
2514 static int xeon_init_dev(struct intel_ntb_dev *ndev)
2515 {
2516         struct pci_dev *pdev;
2517         u8 ppd;
2518         int rc, mem;
2519
2520         pdev = ndev->ntb.pdev;
2521
2522         switch (pdev->device) {
2523         /* There is a Xeon hardware errata related to writes to SDOORBELL or
2524          * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space,
2525          * which may hang the system.  To workaround this use the second memory
2526          * window to access the interrupt and scratch pad registers on the
2527          * remote system.
2528          */
2529         case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
2530         case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
2531         case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
2532         case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
2533         case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
2534         case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
2535         case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
2536         case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
2537         case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
2538         case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
2539         case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
2540         case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
2541         case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
2542         case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
2543         case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
2544                 ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP;
2545                 break;
2546         }
2547
2548         switch (pdev->device) {
2549         /* There is a hardware errata related to accessing any register in
2550          * SB01BASE in the presence of bidirectional traffic crossing the NTB.
2551          */
2552         case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
2553         case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
2554         case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
2555         case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
2556         case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
2557         case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
2558         case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
2559         case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
2560         case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
2561                 ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP;
2562                 break;
2563         }
2564
2565         switch (pdev->device) {
2566         /* HW Errata on bit 14 of b2bdoorbell register.  Writes will not be
2567          * mirrored to the remote system.  Shrink the number of bits by one,
2568          * since bit 14 is the last bit.
2569          */
2570         case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
2571         case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
2572         case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
2573         case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
2574         case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
2575         case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
2576         case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
2577         case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
2578         case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
2579         case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
2580         case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
2581         case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
2582         case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
2583         case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
2584         case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
2585                 ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14;
2586                 break;
2587         }
2588
2589         ndev->reg = &xeon_reg;
2590
2591         rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
2592         if (rc)
2593                 return -EIO;
2594
2595         ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
2596         dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd,
2597                 ntb_topo_string(ndev->ntb.topo));
2598         if (ndev->ntb.topo == NTB_TOPO_NONE)
2599                 return -EINVAL;
2600
2601         if (ndev->ntb.topo != NTB_TOPO_SEC) {
2602                 ndev->bar4_split = xeon_ppd_bar4_split(ndev, ppd);
2603                 dev_dbg(&pdev->dev, "ppd %#x bar4_split %d\n",
2604                         ppd, ndev->bar4_split);
2605         } else {
2606                 /* This is a way for transparent BAR to figure out if we are
2607                  * doing split BAR or not. There is no way for the hw on the
2608                  * transparent side to know and set the PPD.
2609                  */
2610                 mem = pci_select_bars(pdev, IORESOURCE_MEM);
2611                 ndev->bar4_split = hweight32(mem) ==
2612                         HSX_SPLIT_BAR_MW_COUNT + 1;
2613                 dev_dbg(&pdev->dev, "mem %#x bar4_split %d\n",
2614                         mem, ndev->bar4_split);
2615         }
2616
2617         rc = xeon_init_ntb(ndev);
2618         if (rc)
2619                 return rc;
2620
2621         return xeon_init_isr(ndev);
2622 }
2623
2624 static void xeon_deinit_dev(struct intel_ntb_dev *ndev)
2625 {
2626         xeon_deinit_isr(ndev);
2627 }
2628
2629 static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
2630 {
2631         int rc;
2632
2633         pci_set_drvdata(pdev, ndev);
2634
2635         rc = pci_enable_device(pdev);
2636         if (rc)
2637                 goto err_pci_enable;
2638
2639         rc = pci_request_regions(pdev, NTB_NAME);
2640         if (rc)
2641                 goto err_pci_regions;
2642
2643         pci_set_master(pdev);
2644
2645         rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
2646         if (rc) {
2647                 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
2648                 if (rc)
2649                         goto err_dma_mask;
2650                 dev_warn(&pdev->dev, "Cannot DMA highmem\n");
2651         }
2652
2653         rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
2654         if (rc) {
2655                 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
2656                 if (rc)
2657                         goto err_dma_mask;
2658                 dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
2659         }
2660
2661         ndev->self_mmio = pci_iomap(pdev, 0, 0);
2662         if (!ndev->self_mmio) {
2663                 rc = -EIO;
2664                 goto err_mmio;
2665         }
2666         ndev->peer_mmio = ndev->self_mmio;
2667         ndev->peer_addr = pci_resource_start(pdev, 0);
2668
2669         return 0;
2670
2671 err_mmio:
2672 err_dma_mask:
2673         pci_clear_master(pdev);
2674         pci_release_regions(pdev);
2675 err_pci_regions:
2676         pci_disable_device(pdev);
2677 err_pci_enable:
2678         pci_set_drvdata(pdev, NULL);
2679         return rc;
2680 }
2681
2682 static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev)
2683 {
2684         struct pci_dev *pdev = ndev->ntb.pdev;
2685
2686         if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio)
2687                 pci_iounmap(pdev, ndev->peer_mmio);
2688         pci_iounmap(pdev, ndev->self_mmio);
2689
2690         pci_clear_master(pdev);
2691         pci_release_regions(pdev);
2692         pci_disable_device(pdev);
2693         pci_set_drvdata(pdev, NULL);
2694 }
2695
2696 static inline void ndev_init_struct(struct intel_ntb_dev *ndev,
2697                                     struct pci_dev *pdev)
2698 {
2699         ndev->ntb.pdev = pdev;
2700         ndev->ntb.topo = NTB_TOPO_NONE;
2701         ndev->ntb.ops = &intel_ntb_ops;
2702
2703         ndev->b2b_off = 0;
2704         ndev->b2b_idx = UINT_MAX;
2705
2706         ndev->bar4_split = 0;
2707
2708         ndev->mw_count = 0;
2709         ndev->spad_count = 0;
2710         ndev->db_count = 0;
2711         ndev->db_vec_count = 0;
2712         ndev->db_vec_shift = 0;
2713
2714         ndev->ntb_ctl = 0;
2715         ndev->lnk_sta = 0;
2716
2717         ndev->db_valid_mask = 0;
2718         ndev->db_link_mask = 0;
2719         ndev->db_mask = 0;
2720
2721         spin_lock_init(&ndev->db_mask_lock);
2722 }
2723
2724 static int intel_ntb_pci_probe(struct pci_dev *pdev,
2725                                const struct pci_device_id *id)
2726 {
2727         struct intel_ntb_dev *ndev;
2728         int rc, node;
2729
2730         node = dev_to_node(&pdev->dev);
2731
2732         if (pdev_is_atom(pdev)) {
2733                 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
2734                 if (!ndev) {
2735                         rc = -ENOMEM;
2736                         goto err_ndev;
2737                 }
2738
2739                 ndev_init_struct(ndev, pdev);
2740
2741                 rc = intel_ntb_init_pci(ndev, pdev);
2742                 if (rc)
2743                         goto err_init_pci;
2744
2745                 rc = atom_init_dev(ndev);
2746                 if (rc)
2747                         goto err_init_dev;
2748
2749         } else if (pdev_is_xeon(pdev)) {
2750                 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
2751                 if (!ndev) {
2752                         rc = -ENOMEM;
2753                         goto err_ndev;
2754                 }
2755
2756                 ndev_init_struct(ndev, pdev);
2757
2758                 rc = intel_ntb_init_pci(ndev, pdev);
2759                 if (rc)
2760                         goto err_init_pci;
2761
2762                 rc = xeon_init_dev(ndev);
2763                 if (rc)
2764                         goto err_init_dev;
2765
2766         } else if (pdev_is_skx_xeon(pdev)) {
2767                 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
2768                 if (!ndev) {
2769                         rc = -ENOMEM;
2770                         goto err_ndev;
2771                 }
2772
2773                 ndev_init_struct(ndev, pdev);
2774                 ndev->ntb.ops = &intel_ntb3_ops;
2775
2776                 rc = intel_ntb_init_pci(ndev, pdev);
2777                 if (rc)
2778                         goto err_init_pci;
2779
2780                 rc = skx_init_dev(ndev);
2781                 if (rc)
2782                         goto err_init_dev;
2783
2784         } else {
2785                 rc = -EINVAL;
2786                 goto err_ndev;
2787         }
2788
2789         ndev_reset_unsafe_flags(ndev);
2790
2791         ndev->reg->poll_link(ndev);
2792
2793         ndev_init_debugfs(ndev);
2794
2795         rc = ntb_register_device(&ndev->ntb);
2796         if (rc)
2797                 goto err_register;
2798
2799         dev_info(&pdev->dev, "NTB device registered.\n");
2800
2801         return 0;
2802
2803 err_register:
2804         ndev_deinit_debugfs(ndev);
2805         if (pdev_is_atom(pdev))
2806                 atom_deinit_dev(ndev);
2807         else if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev))
2808                 xeon_deinit_dev(ndev);
2809 err_init_dev:
2810         intel_ntb_deinit_pci(ndev);
2811 err_init_pci:
2812         kfree(ndev);
2813 err_ndev:
2814         return rc;
2815 }
2816
2817 static void intel_ntb_pci_remove(struct pci_dev *pdev)
2818 {
2819         struct intel_ntb_dev *ndev = pci_get_drvdata(pdev);
2820
2821         ntb_unregister_device(&ndev->ntb);
2822         ndev_deinit_debugfs(ndev);
2823         if (pdev_is_atom(pdev))
2824                 atom_deinit_dev(ndev);
2825         else if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev))
2826                 xeon_deinit_dev(ndev);
2827         intel_ntb_deinit_pci(ndev);
2828         kfree(ndev);
2829 }
2830
2831 static const struct intel_ntb_reg atom_reg = {
2832         .poll_link              = atom_poll_link,
2833         .link_is_up             = atom_link_is_up,
2834         .db_ioread              = atom_db_ioread,
2835         .db_iowrite             = atom_db_iowrite,
2836         .db_size                = sizeof(u64),
2837         .ntb_ctl                = ATOM_NTBCNTL_OFFSET,
2838         .mw_bar                 = {2, 4},
2839 };
2840
2841 static const struct intel_ntb_alt_reg atom_pri_reg = {
2842         .db_bell                = ATOM_PDOORBELL_OFFSET,
2843         .db_mask                = ATOM_PDBMSK_OFFSET,
2844         .spad                   = ATOM_SPAD_OFFSET,
2845 };
2846
2847 static const struct intel_ntb_alt_reg atom_b2b_reg = {
2848         .db_bell                = ATOM_B2B_DOORBELL_OFFSET,
2849         .spad                   = ATOM_B2B_SPAD_OFFSET,
2850 };
2851
2852 static const struct intel_ntb_xlat_reg atom_sec_xlat = {
2853         /* FIXME : .bar0_base   = ATOM_SBAR0BASE_OFFSET, */
2854         /* FIXME : .bar2_limit  = ATOM_SBAR2LMT_OFFSET, */
2855         .bar2_xlat              = ATOM_SBAR2XLAT_OFFSET,
2856 };
2857
2858 static const struct intel_ntb_reg xeon_reg = {
2859         .poll_link              = xeon_poll_link,
2860         .link_is_up             = xeon_link_is_up,
2861         .db_ioread              = xeon_db_ioread,
2862         .db_iowrite             = xeon_db_iowrite,
2863         .db_size                = sizeof(u32),
2864         .ntb_ctl                = XEON_NTBCNTL_OFFSET,
2865         .mw_bar                 = {2, 4, 5},
2866 };
2867
2868 static const struct intel_ntb_alt_reg xeon_pri_reg = {
2869         .db_bell                = XEON_PDOORBELL_OFFSET,
2870         .db_mask                = XEON_PDBMSK_OFFSET,
2871         .spad                   = XEON_SPAD_OFFSET,
2872 };
2873
2874 static const struct intel_ntb_alt_reg xeon_sec_reg = {
2875         .db_bell                = XEON_SDOORBELL_OFFSET,
2876         .db_mask                = XEON_SDBMSK_OFFSET,
2877         /* second half of the scratchpads */
2878         .spad                   = XEON_SPAD_OFFSET + (XEON_SPAD_COUNT << 1),
2879 };
2880
2881 static const struct intel_ntb_alt_reg xeon_b2b_reg = {
2882         .db_bell                = XEON_B2B_DOORBELL_OFFSET,
2883         .spad                   = XEON_B2B_SPAD_OFFSET,
2884 };
2885
2886 static const struct intel_ntb_xlat_reg xeon_pri_xlat = {
2887         /* Note: no primary .bar0_base visible to the secondary side.
2888          *
2889          * The secondary side cannot get the base address stored in primary
2890          * bars.  The base address is necessary to set the limit register to
2891          * any value other than zero, or unlimited.
2892          *
2893          * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the
2894          * window by setting the limit equal to base, nor can it limit the size
2895          * of the memory window by setting the limit to base + size.
2896          */
2897         .bar2_limit             = XEON_PBAR23LMT_OFFSET,
2898         .bar2_xlat              = XEON_PBAR23XLAT_OFFSET,
2899 };
2900
2901 static const struct intel_ntb_xlat_reg xeon_sec_xlat = {
2902         .bar0_base              = XEON_SBAR0BASE_OFFSET,
2903         .bar2_limit             = XEON_SBAR23LMT_OFFSET,
2904         .bar2_xlat              = XEON_SBAR23XLAT_OFFSET,
2905 };
2906
2907 static struct intel_b2b_addr xeon_b2b_usd_addr = {
2908         .bar2_addr64            = XEON_B2B_BAR2_ADDR64,
2909         .bar4_addr64            = XEON_B2B_BAR4_ADDR64,
2910         .bar4_addr32            = XEON_B2B_BAR4_ADDR32,
2911         .bar5_addr32            = XEON_B2B_BAR5_ADDR32,
2912 };
2913
2914 static struct intel_b2b_addr xeon_b2b_dsd_addr = {
2915         .bar2_addr64            = XEON_B2B_BAR2_ADDR64,
2916         .bar4_addr64            = XEON_B2B_BAR4_ADDR64,
2917         .bar4_addr32            = XEON_B2B_BAR4_ADDR32,
2918         .bar5_addr32            = XEON_B2B_BAR5_ADDR32,
2919 };
2920
2921 static const struct intel_ntb_reg skx_reg = {
2922         .poll_link              = skx_poll_link,
2923         .link_is_up             = xeon_link_is_up,
2924         .db_ioread              = skx_db_ioread,
2925         .db_iowrite             = skx_db_iowrite,
2926         .db_size                = sizeof(u32),
2927         .ntb_ctl                = SKX_NTBCNTL_OFFSET,
2928         .mw_bar                 = {2, 4},
2929 };
2930
2931 static const struct intel_ntb_alt_reg skx_pri_reg = {
2932         .db_bell                = SKX_EM_DOORBELL_OFFSET,
2933         .db_clear               = SKX_IM_INT_STATUS_OFFSET,
2934         .db_mask                = SKX_IM_INT_DISABLE_OFFSET,
2935         .spad                   = SKX_IM_SPAD_OFFSET,
2936 };
2937
2938 static const struct intel_ntb_alt_reg skx_b2b_reg = {
2939         .db_bell                = SKX_IM_DOORBELL_OFFSET,
2940         .db_clear               = SKX_EM_INT_STATUS_OFFSET,
2941         .db_mask                = SKX_EM_INT_DISABLE_OFFSET,
2942         .spad                   = SKX_B2B_SPAD_OFFSET,
2943 };
2944
2945 static const struct intel_ntb_xlat_reg skx_sec_xlat = {
2946 /*      .bar0_base              = SKX_EMBAR0_OFFSET, */
2947         .bar2_limit             = SKX_IMBAR1XLMT_OFFSET,
2948         .bar2_xlat              = SKX_IMBAR1XBASE_OFFSET,
2949 };
2950
2951 /* operations for primary side of local ntb */
2952 static const struct ntb_dev_ops intel_ntb_ops = {
2953         .mw_count               = intel_ntb_mw_count,
2954         .mw_get_align           = intel_ntb_mw_get_align,
2955         .mw_set_trans           = intel_ntb_mw_set_trans,
2956         .peer_mw_count          = intel_ntb_peer_mw_count,
2957         .peer_mw_get_addr       = intel_ntb_peer_mw_get_addr,
2958         .link_is_up             = intel_ntb_link_is_up,
2959         .link_enable            = intel_ntb_link_enable,
2960         .link_disable           = intel_ntb_link_disable,
2961         .db_is_unsafe           = intel_ntb_db_is_unsafe,
2962         .db_valid_mask          = intel_ntb_db_valid_mask,
2963         .db_vector_count        = intel_ntb_db_vector_count,
2964         .db_vector_mask         = intel_ntb_db_vector_mask,
2965         .db_read                = intel_ntb_db_read,
2966         .db_clear               = intel_ntb_db_clear,
2967         .db_set_mask            = intel_ntb_db_set_mask,
2968         .db_clear_mask          = intel_ntb_db_clear_mask,
2969         .peer_db_addr           = intel_ntb_peer_db_addr,
2970         .peer_db_set            = intel_ntb_peer_db_set,
2971         .spad_is_unsafe         = intel_ntb_spad_is_unsafe,
2972         .spad_count             = intel_ntb_spad_count,
2973         .spad_read              = intel_ntb_spad_read,
2974         .spad_write             = intel_ntb_spad_write,
2975         .peer_spad_addr         = intel_ntb_peer_spad_addr,
2976         .peer_spad_read         = intel_ntb_peer_spad_read,
2977         .peer_spad_write        = intel_ntb_peer_spad_write,
2978 };
2979
2980 static const struct ntb_dev_ops intel_ntb3_ops = {
2981         .mw_count               = intel_ntb_mw_count,
2982         .mw_get_align           = intel_ntb_mw_get_align,
2983         .mw_set_trans           = intel_ntb3_mw_set_trans,
2984         .peer_mw_count          = intel_ntb_peer_mw_count,
2985         .peer_mw_get_addr       = intel_ntb_peer_mw_get_addr,
2986         .link_is_up             = intel_ntb_link_is_up,
2987         .link_enable            = intel_ntb3_link_enable,
2988         .link_disable           = intel_ntb_link_disable,
2989         .db_valid_mask          = intel_ntb_db_valid_mask,
2990         .db_vector_count        = intel_ntb_db_vector_count,
2991         .db_vector_mask         = intel_ntb_db_vector_mask,
2992         .db_read                = intel_ntb3_db_read,
2993         .db_clear               = intel_ntb3_db_clear,
2994         .db_set_mask            = intel_ntb_db_set_mask,
2995         .db_clear_mask          = intel_ntb_db_clear_mask,
2996         .peer_db_addr           = intel_ntb_peer_db_addr,
2997         .peer_db_set            = intel_ntb3_peer_db_set,
2998         .spad_is_unsafe         = intel_ntb_spad_is_unsafe,
2999         .spad_count             = intel_ntb_spad_count,
3000         .spad_read              = intel_ntb_spad_read,
3001         .spad_write             = intel_ntb_spad_write,
3002         .peer_spad_addr         = intel_ntb_peer_spad_addr,
3003         .peer_spad_read         = intel_ntb_peer_spad_read,
3004         .peer_spad_write        = intel_ntb_peer_spad_write,
3005 };
3006
3007 static const struct file_operations intel_ntb_debugfs_info = {
3008         .owner = THIS_MODULE,
3009         .open = simple_open,
3010         .read = ndev_debugfs_read,
3011 };
3012
3013 static const struct pci_device_id intel_ntb_pci_tbl[] = {
3014         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
3015         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
3016         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
3017         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
3018         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
3019         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BDX)},
3020         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
3021         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
3022         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
3023         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
3024         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_BDX)},
3025         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
3026         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
3027         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
3028         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
3029         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_BDX)},
3030         {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)},
3031         {0}
3032 };
3033 MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl);
3034
3035 static struct pci_driver intel_ntb_pci_driver = {
3036         .name = KBUILD_MODNAME,
3037         .id_table = intel_ntb_pci_tbl,
3038         .probe = intel_ntb_pci_probe,
3039         .remove = intel_ntb_pci_remove,
3040 };
3041
3042 static int __init intel_ntb_pci_driver_init(void)
3043 {
3044         pr_info("%s %s\n", NTB_DESC, NTB_VER);
3045
3046         if (debugfs_initialized())
3047                 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
3048
3049         return pci_register_driver(&intel_ntb_pci_driver);
3050 }
3051 module_init(intel_ntb_pci_driver_init);
3052
3053 static void __exit intel_ntb_pci_driver_exit(void)
3054 {
3055         pci_unregister_driver(&intel_ntb_pci_driver);
3056
3057         debugfs_remove_recursive(debugfs_dir);
3058 }
3059 module_exit(intel_ntb_pci_driver_exit);