drivers/ntb/hw/mscc/ntb_hw_switchtec.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Microsemi Switchtec(tm) PCIe Management Driver
   4  * Copyright (c) 2017, Microsemi Corporation
   5  */
   6
   7 #include <linux/interrupt.h>
   8 #include <linux/io-64-nonatomic-lo-hi.h>
   9 #include <linux/delay.h>
  10 #include <linux/kthread.h>
  11 #include <linux/module.h>
  12 #include <linux/ntb.h>
  13 #include <linux/pci.h>
  14 #include <linux/switchtec.h>
  15
  16 MODULE_DESCRIPTION("Microsemi Switchtec(tm) NTB Driver");
  17 MODULE_VERSION("0.1");
  18 MODULE_LICENSE("GPL");
  19 MODULE_AUTHOR("Microsemi Corporation");
  20
  21 static ulong max_mw_size = SZ_2M;
  22 module_param(max_mw_size, ulong, 0644);
  23 MODULE_PARM_DESC(max_mw_size,
  24         "Max memory window size reported to the upper layer");
  25
  26 static bool use_lut_mws;
  27 module_param(use_lut_mws, bool, 0644);
  28 MODULE_PARM_DESC(use_lut_mws,
  29                  "Enable the use of the LUT based memory windows");
  30
  31 #define SWITCHTEC_NTB_MAGIC 0x45CC0001
  32 #define MAX_MWS     128
  33
  34 struct shared_mw {
  35         u32 magic;
  36         u32 link_sta;
  37         u32 partition_id;
  38         u64 mw_sizes[MAX_MWS];
  39         u32 spad[128];
  40 };
  41
  42 #define MAX_DIRECT_MW ARRAY_SIZE(((struct ntb_ctrl_regs *)(0))->bar_entry)
  43 #define LUT_SIZE SZ_64K
  44
  45 struct switchtec_ntb {
  46         struct ntb_dev ntb;
  47         struct switchtec_dev *stdev;
  48
  49         int self_partition;
  50         int peer_partition;
  51
  52         int doorbell_irq;
  53         int message_irq;
  54
  55         struct ntb_info_regs __iomem *mmio_ntb;
  56         struct ntb_ctrl_regs __iomem *mmio_ctrl;
  57         struct ntb_dbmsg_regs __iomem *mmio_dbmsg;
  58         struct ntb_ctrl_regs __iomem *mmio_self_ctrl;
  59         struct ntb_ctrl_regs __iomem *mmio_peer_ctrl;
  60         struct ntb_dbmsg_regs __iomem *mmio_self_dbmsg;
  61         struct ntb_dbmsg_regs __iomem *mmio_peer_dbmsg;
  62
  63         void __iomem *mmio_xlink_win;
  64
  65         struct shared_mw *self_shared;
  66         struct shared_mw __iomem *peer_shared;
  67         dma_addr_t self_shared_dma;
  68
  69         u64 db_mask;
  70         u64 db_valid_mask;
  71         int db_shift;
  72         int db_peer_shift;
  73
  74         /* synchronize rmw access of db_mask and hw reg */
  75         spinlock_t db_mask_lock;
  76
  77         int nr_direct_mw;
  78         int nr_lut_mw;
  79         int nr_rsvd_luts;
  80         int direct_mw_to_bar[MAX_DIRECT_MW];
  81
  82         int peer_nr_direct_mw;
  83         int peer_nr_lut_mw;
  84         int peer_direct_mw_to_bar[MAX_DIRECT_MW];
  85
  86         bool link_is_up;
  87         enum ntb_speed link_speed;
  88         enum ntb_width link_width;
  89         struct work_struct check_link_status_work;
  90         bool link_force_down;
  91 };
  92
  93 static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb)
  94 {
  95         return container_of(ntb, struct switchtec_ntb, ntb);
  96 }
  97
  98 static int switchtec_ntb_part_op(struct switchtec_ntb *sndev,
  99                                  struct ntb_ctrl_regs __iomem *ctl,
 100                                  u32 op, int wait_status)
 101 {
 102         static const char * const op_text[] = {
 103                 [NTB_CTRL_PART_OP_LOCK] = "lock",
 104                 [NTB_CTRL_PART_OP_CFG] = "configure",
 105                 [NTB_CTRL_PART_OP_RESET] = "reset",
 106         };
 107
 108         int i;
 109         u32 ps;
 110         int status;
 111
 112         switch (op) {
 113         case NTB_CTRL_PART_OP_LOCK:
 114                 status = NTB_CTRL_PART_STATUS_LOCKING;
 115                 break;
 116         case NTB_CTRL_PART_OP_CFG:
 117                 status = NTB_CTRL_PART_STATUS_CONFIGURING;
 118                 break;
 119         case NTB_CTRL_PART_OP_RESET:
 120                 status = NTB_CTRL_PART_STATUS_RESETTING;
 121                 break;
 122         default:
 123                 return -EINVAL;
 124         }
 125
 126         iowrite32(op, &ctl->partition_op);
 127
 128         for (i = 0; i < 1000; i++) {
 129                 if (msleep_interruptible(50) != 0) {
 130                         iowrite32(NTB_CTRL_PART_OP_RESET, &ctl->partition_op);
 131                         return -EINTR;
 132                 }
 133
 134                 ps = ioread32(&ctl->partition_status) & 0xFFFF;
 135
 136                 if (ps != status)
 137                         break;
 138         }
 139
 140         if (ps == wait_status)
 141                 return 0;
 142
 143         if (ps == status) {
 144                 dev_err(&sndev->stdev->dev,
 145                         "Timed out while performing %s (%d). (%08x)\n",
 146                         op_text[op], op,
 147                         ioread32(&ctl->partition_status));
 148
 149                 return -ETIMEDOUT;
 150         }
 151
 152         return -EIO;
 153 }
 154
 155 static int switchtec_ntb_send_msg(struct switchtec_ntb *sndev, int idx,
 156                                   u32 val)
 157 {
 158         if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_peer_dbmsg->omsg))
 159                 return -EINVAL;
 160
 161         iowrite32(val, &sndev->mmio_peer_dbmsg->omsg[idx].msg);
 162
 163         return 0;
 164 }
 165
 166 static int switchtec_ntb_mw_count(struct ntb_dev *ntb, int pidx)
 167 {
 168         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 169         int nr_direct_mw = sndev->peer_nr_direct_mw;
 170         int nr_lut_mw = sndev->peer_nr_lut_mw - sndev->nr_rsvd_luts;
 171
 172         if (pidx != NTB_DEF_PEER_IDX)
 173                 return -EINVAL;
 174
 175         if (!use_lut_mws)
 176                 nr_lut_mw = 0;
 177
 178         return nr_direct_mw + nr_lut_mw;
 179 }
 180
 181 static int lut_index(struct switchtec_ntb *sndev, int mw_idx)
 182 {
 183         return mw_idx - sndev->nr_direct_mw + sndev->nr_rsvd_luts;
 184 }
 185
 186 static int peer_lut_index(struct switchtec_ntb *sndev, int mw_idx)
 187 {
 188         return mw_idx - sndev->peer_nr_direct_mw + sndev->nr_rsvd_luts;
 189 }
 190
 191 static int switchtec_ntb_mw_get_align(struct ntb_dev *ntb, int pidx,
 192                                       int widx, resource_size_t *addr_align,
 193                                       resource_size_t *size_align,
 194                                       resource_size_t *size_max)
 195 {
 196         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 197         int lut;
 198         resource_size_t size;
 199
 200         if (pidx != NTB_DEF_PEER_IDX)
 201                 return -EINVAL;
 202
 203         lut = widx >= sndev->peer_nr_direct_mw;
 204         size = ioread64(&sndev->peer_shared->mw_sizes[widx]);
 205
 206         if (size == 0)
 207                 return -EINVAL;
 208
 209         if (addr_align)
 210                 *addr_align = lut ? size : SZ_4K;
 211
 212         if (size_align)
 213                 *size_align = lut ? size : SZ_4K;
 214
 215         if (size_max)
 216                 *size_max = size;
 217
 218         return 0;
 219 }
 220
 221 static void switchtec_ntb_mw_clr_direct(struct switchtec_ntb *sndev, int idx)
 222 {
 223         struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
 224         int bar = sndev->peer_direct_mw_to_bar[idx];
 225         u32 ctl_val;
 226
 227         ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
 228         ctl_val &= ~NTB_CTRL_BAR_DIR_WIN_EN;
 229         iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
 230         iowrite32(0, &ctl->bar_entry[bar].win_size);
 231         iowrite32(0, &ctl->bar_ext_entry[bar].win_size);
 232         iowrite64(sndev->self_partition, &ctl->bar_entry[bar].xlate_addr);
 233 }
 234
 235 static void switchtec_ntb_mw_clr_lut(struct switchtec_ntb *sndev, int idx)
 236 {
 237         struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
 238
 239         iowrite64(0, &ctl->lut_entry[peer_lut_index(sndev, idx)]);
 240 }
 241
 242 static void switchtec_ntb_mw_set_direct(struct switchtec_ntb *sndev, int idx,
 243                                         dma_addr_t addr, resource_size_t size)
 244 {
 245         int xlate_pos = ilog2(size);
 246         int bar = sndev->peer_direct_mw_to_bar[idx];
 247         struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
 248         u32 ctl_val;
 249
 250         ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
 251         ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN;
 252
 253         iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
 254         iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000),
 255                   &ctl->bar_entry[bar].win_size);
 256         iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size);
 257         iowrite64(sndev->self_partition | addr,
 258                   &ctl->bar_entry[bar].xlate_addr);
 259 }
 260
 261 static void switchtec_ntb_mw_set_lut(struct switchtec_ntb *sndev, int idx,
 262                                      dma_addr_t addr, resource_size_t size)
 263 {
 264         struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
 265
 266         iowrite64((NTB_CTRL_LUT_EN | (sndev->self_partition << 1) | addr),
 267                   &ctl->lut_entry[peer_lut_index(sndev, idx)]);
 268 }
 269
 270 static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
 271                                       dma_addr_t addr, resource_size_t size)
 272 {
 273         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 274         struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
 275         int xlate_pos = ilog2(size);
 276         int nr_direct_mw = sndev->peer_nr_direct_mw;
 277         int rc;
 278
 279         if (pidx != NTB_DEF_PEER_IDX)
 280                 return -EINVAL;
 281
 282         dev_dbg(&sndev->stdev->dev, "MW %d: part %d addr %pad size %pap\n",
 283                 widx, pidx, &addr, &size);
 284
 285         if (widx >= switchtec_ntb_mw_count(ntb, pidx))
 286                 return -EINVAL;
 287
 288         if (size != 0 && xlate_pos < 12)
 289                 return -EINVAL;
 290
 291         if (!IS_ALIGNED(addr, BIT_ULL(xlate_pos))) {
 292                 /*
 293                  * In certain circumstances we can get a buffer that is
 294                  * not aligned to its size. (Most of the time
 295                  * dma_alloc_coherent ensures this). This can happen when
 296                  * using large buffers allocated by the CMA
 297                  * (see CMA_CONFIG_ALIGNMENT)
 298                  */
 299                 dev_err(&sndev->stdev->dev,
 300                         "ERROR: Memory window address is not aligned to its size!\n");
 301                 return -EINVAL;
 302         }
 303
 304         rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
 305                                    NTB_CTRL_PART_STATUS_LOCKED);
 306         if (rc)
 307                 return rc;
 308
 309         if (size == 0) {
 310                 if (widx < nr_direct_mw)
 311                         switchtec_ntb_mw_clr_direct(sndev, widx);
 312                 else
 313                         switchtec_ntb_mw_clr_lut(sndev, widx);
 314         } else {
 315                 if (widx < nr_direct_mw)
 316                         switchtec_ntb_mw_set_direct(sndev, widx, addr, size);
 317                 else
 318                         switchtec_ntb_mw_set_lut(sndev, widx, addr, size);
 319         }
 320
 321         rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
 322                                    NTB_CTRL_PART_STATUS_NORMAL);
 323
 324         if (rc == -EIO) {
 325                 dev_err(&sndev->stdev->dev,
 326                         "Hardware reported an error configuring mw %d: %08x\n",
 327                         widx, ioread32(&ctl->bar_error));
 328
 329                 if (widx < nr_direct_mw)
 330                         switchtec_ntb_mw_clr_direct(sndev, widx);
 331                 else
 332                         switchtec_ntb_mw_clr_lut(sndev, widx);
 333
 334                 switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
 335                                       NTB_CTRL_PART_STATUS_NORMAL);
 336         }
 337
 338         return rc;
 339 }
 340
 341 static int switchtec_ntb_peer_mw_count(struct ntb_dev *ntb)
 342 {
 343         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 344         int nr_lut_mw = sndev->nr_lut_mw - sndev->nr_rsvd_luts;
 345
 346         return sndev->nr_direct_mw + (use_lut_mws ? nr_lut_mw : 0);
 347 }
 348
 349 static int switchtec_ntb_direct_get_addr(struct switchtec_ntb *sndev,
 350                                          int idx, phys_addr_t *base,
 351                                          resource_size_t *size)
 352 {
 353         int bar = sndev->direct_mw_to_bar[idx];
 354         size_t offset = 0;
 355
 356         if (bar < 0)
 357                 return -EINVAL;
 358
 359         if (idx == 0) {
 360                 /*
 361                  * This is the direct BAR shared with the LUTs
 362                  * which means the actual window will be offset
 363                  * by the size of all the LUT entries.
 364                  */
 365
 366                 offset = LUT_SIZE * sndev->nr_lut_mw;
 367         }
 368
 369         if (base)
 370                 *base = pci_resource_start(sndev->ntb.pdev, bar) + offset;
 371
 372         if (size) {
 373                 *size = pci_resource_len(sndev->ntb.pdev, bar) - offset;
 374                 if (offset && *size > offset)
 375                         *size = offset;
 376
 377                 if (*size > max_mw_size)
 378                         *size = max_mw_size;
 379         }
 380
 381         return 0;
 382 }
 383
 384 static int switchtec_ntb_lut_get_addr(struct switchtec_ntb *sndev,
 385                                       int idx, phys_addr_t *base,
 386                                       resource_size_t *size)
 387 {
 388         int bar = sndev->direct_mw_to_bar[0];
 389         int offset;
 390
 391         offset = LUT_SIZE * lut_index(sndev, idx);
 392
 393         if (base)
 394                 *base = pci_resource_start(sndev->ntb.pdev, bar) + offset;
 395
 396         if (size)
 397                 *size = LUT_SIZE;
 398
 399         return 0;
 400 }
 401
 402 static int switchtec_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
 403                                           phys_addr_t *base,
 404                                           resource_size_t *size)
 405 {
 406         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 407
 408         if (idx < sndev->nr_direct_mw)
 409                 return switchtec_ntb_direct_get_addr(sndev, idx, base, size);
 410         else if (idx < switchtec_ntb_peer_mw_count(ntb))
 411                 return switchtec_ntb_lut_get_addr(sndev, idx, base, size);
 412         else
 413                 return -EINVAL;
 414 }
 415
 416 static void switchtec_ntb_part_link_speed(struct switchtec_ntb *sndev,
 417                                           int partition,
 418                                           enum ntb_speed *speed,
 419                                           enum ntb_width *width)
 420 {
 421         struct switchtec_dev *stdev = sndev->stdev;
 422         struct part_cfg_regs __iomem *part_cfg =
 423                 &stdev->mmio_part_cfg_all[partition];
 424
 425         u32 pff = ioread32(&part_cfg->vep_pff_inst_id) & 0xFF;
 426         u32 linksta = ioread32(&stdev->mmio_pff_csr[pff].pci_cap_region[13]);
 427
 428         if (speed)
 429                 *speed = (linksta >> 16) & 0xF;
 430
 431         if (width)
 432                 *width = (linksta >> 20) & 0x3F;
 433 }
 434
 435 static void switchtec_ntb_set_link_speed(struct switchtec_ntb *sndev)
 436 {
 437         enum ntb_speed self_speed, peer_speed;
 438         enum ntb_width self_width, peer_width;
 439
 440         if (!sndev->link_is_up) {
 441                 sndev->link_speed = NTB_SPEED_NONE;
 442                 sndev->link_width = NTB_WIDTH_NONE;
 443                 return;
 444         }
 445
 446         switchtec_ntb_part_link_speed(sndev, sndev->self_partition,
 447                                       &self_speed, &self_width);
 448         switchtec_ntb_part_link_speed(sndev, sndev->peer_partition,
 449                                       &peer_speed, &peer_width);
 450
 451         sndev->link_speed = min(self_speed, peer_speed);
 452         sndev->link_width = min(self_width, peer_width);
 453 }
 454
 455 static int crosslink_is_enabled(struct switchtec_ntb *sndev)
 456 {
 457         struct ntb_info_regs __iomem *inf = sndev->mmio_ntb;
 458
 459         return ioread8(&inf->ntp_info[sndev->peer_partition].xlink_enabled);
 460 }
 461
 462 static void crosslink_init_dbmsgs(struct switchtec_ntb *sndev)
 463 {
 464         int i;
 465         u32 msg_map = 0;
 466
 467         if (!crosslink_is_enabled(sndev))
 468                 return;
 469
 470         for (i = 0; i < ARRAY_SIZE(sndev->mmio_peer_dbmsg->imsg); i++) {
 471                 int m = i | sndev->self_partition << 2;
 472
 473                 msg_map |= m << i * 8;
 474         }
 475
 476         iowrite32(msg_map, &sndev->mmio_peer_dbmsg->msg_map);
 477         iowrite64(sndev->db_valid_mask << sndev->db_peer_shift,
 478                   &sndev->mmio_peer_dbmsg->odb_mask);
 479 }
 480
 481 enum switchtec_msg {
 482         LINK_MESSAGE = 0,
 483         MSG_LINK_UP = 1,
 484         MSG_LINK_DOWN = 2,
 485         MSG_CHECK_LINK = 3,
 486         MSG_LINK_FORCE_DOWN = 4,
 487 };
 488
 489 static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev);
 490
 491 static void switchtec_ntb_link_status_update(struct switchtec_ntb *sndev)
 492 {
 493         int link_sta;
 494         int old = sndev->link_is_up;
 495
 496         link_sta = sndev->self_shared->link_sta;
 497         if (link_sta) {
 498                 u64 peer = ioread64(&sndev->peer_shared->magic);
 499
 500                 if ((peer & 0xFFFFFFFF) == SWITCHTEC_NTB_MAGIC)
 501                         link_sta = peer >> 32;
 502                 else
 503                         link_sta = 0;
 504         }
 505
 506         sndev->link_is_up = link_sta;
 507         switchtec_ntb_set_link_speed(sndev);
 508
 509         if (link_sta != old) {
 510                 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_CHECK_LINK);
 511                 ntb_link_event(&sndev->ntb);
 512                 dev_info(&sndev->stdev->dev, "ntb link %s\n",
 513                          link_sta ? "up" : "down");
 514
 515                 if (link_sta)
 516                         crosslink_init_dbmsgs(sndev);
 517         }
 518 }
 519
 520 static void check_link_status_work(struct work_struct *work)
 521 {
 522         struct switchtec_ntb *sndev;
 523
 524         sndev = container_of(work, struct switchtec_ntb,
 525                              check_link_status_work);
 526
 527         if (sndev->link_force_down) {
 528                 sndev->link_force_down = false;
 529                 switchtec_ntb_reinit_peer(sndev);
 530
 531                 if (sndev->link_is_up) {
 532                         sndev->link_is_up = 0;
 533                         ntb_link_event(&sndev->ntb);
 534                         dev_info(&sndev->stdev->dev, "ntb link forced down\n");
 535                 }
 536
 537                 return;
 538         }
 539
 540         switchtec_ntb_link_status_update(sndev);
 541 }
 542
 543 static void switchtec_ntb_check_link(struct switchtec_ntb *sndev,
 544                                       enum switchtec_msg msg)
 545 {
 546         if (msg == MSG_LINK_FORCE_DOWN)
 547                 sndev->link_force_down = true;
 548
 549         schedule_work(&sndev->check_link_status_work);
 550 }
 551
 552 static void switchtec_ntb_link_notification(struct switchtec_dev *stdev)
 553 {
 554         struct switchtec_ntb *sndev = stdev->sndev;
 555
 556         switchtec_ntb_check_link(sndev, MSG_CHECK_LINK);
 557 }
 558
 559 static u64 switchtec_ntb_link_is_up(struct ntb_dev *ntb,
 560                                     enum ntb_speed *speed,
 561                                     enum ntb_width *width)
 562 {
 563         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 564
 565         if (speed)
 566                 *speed = sndev->link_speed;
 567         if (width)
 568                 *width = sndev->link_width;
 569
 570         return sndev->link_is_up;
 571 }
 572
 573 static int switchtec_ntb_link_enable(struct ntb_dev *ntb,
 574                                      enum ntb_speed max_speed,
 575                                      enum ntb_width max_width)
 576 {
 577         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 578
 579         dev_dbg(&sndev->stdev->dev, "enabling link\n");
 580
 581         sndev->self_shared->link_sta = 1;
 582         switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP);
 583
 584         switchtec_ntb_link_status_update(sndev);
 585
 586         return 0;
 587 }
 588
 589 static int switchtec_ntb_link_disable(struct ntb_dev *ntb)
 590 {
 591         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 592
 593         dev_dbg(&sndev->stdev->dev, "disabling link\n");
 594
 595         sndev->self_shared->link_sta = 0;
 596         switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_DOWN);
 597
 598         switchtec_ntb_link_status_update(sndev);
 599
 600         return 0;
 601 }
 602
 603 static u64 switchtec_ntb_db_valid_mask(struct ntb_dev *ntb)
 604 {
 605         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 606
 607         return sndev->db_valid_mask;
 608 }
 609
 610 static int switchtec_ntb_db_vector_count(struct ntb_dev *ntb)
 611 {
 612         return 1;
 613 }
 614
 615 static u64 switchtec_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
 616 {
 617         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 618
 619         if (db_vector < 0 || db_vector > 1)
 620                 return 0;
 621
 622         return sndev->db_valid_mask;
 623 }
 624
 625 static u64 switchtec_ntb_db_read(struct ntb_dev *ntb)
 626 {
 627         u64 ret;
 628         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 629
 630         ret = ioread64(&sndev->mmio_self_dbmsg->idb) >> sndev->db_shift;
 631
 632         return ret & sndev->db_valid_mask;
 633 }
 634
 635 static int switchtec_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
 636 {
 637         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 638
 639         iowrite64(db_bits << sndev->db_shift, &sndev->mmio_self_dbmsg->idb);
 640
 641         return 0;
 642 }
 643
 644 static int switchtec_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
 645 {
 646         unsigned long irqflags;
 647         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 648
 649         if (db_bits & ~sndev->db_valid_mask)
 650                 return -EINVAL;
 651
 652         spin_lock_irqsave(&sndev->db_mask_lock, irqflags);
 653
 654         sndev->db_mask |= db_bits << sndev->db_shift;
 655         iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
 656
 657         spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags);
 658
 659         return 0;
 660 }
 661
 662 static int switchtec_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
 663 {
 664         unsigned long irqflags;
 665         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 666
 667         if (db_bits & ~sndev->db_valid_mask)
 668                 return -EINVAL;
 669
 670         spin_lock_irqsave(&sndev->db_mask_lock, irqflags);
 671
 672         sndev->db_mask &= ~(db_bits << sndev->db_shift);
 673         iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
 674
 675         spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags);
 676
 677         return 0;
 678 }
 679
 680 static u64 switchtec_ntb_db_read_mask(struct ntb_dev *ntb)
 681 {
 682         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 683
 684         return (sndev->db_mask >> sndev->db_shift) & sndev->db_valid_mask;
 685 }
 686
 687 static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb,
 688                                       phys_addr_t *db_addr,
 689                                       resource_size_t *db_size,
 690                                       u64 *db_data,
 691                                       int db_bit)
 692 {
 693         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 694         unsigned long offset;
 695
 696         if (unlikely(db_bit >= BITS_PER_LONG_LONG))
 697                 return -EINVAL;
 698
 699         offset = (unsigned long)sndev->mmio_peer_dbmsg->odb -
 700                 (unsigned long)sndev->stdev->mmio;
 701
 702         offset += sndev->db_shift / 8;
 703
 704         if (db_addr)
 705                 *db_addr = pci_resource_start(ntb->pdev, 0) + offset;
 706         if (db_size)
 707                 *db_size = sizeof(u32);
 708         if (db_data)
 709                 *db_data = BIT_ULL(db_bit) << sndev->db_peer_shift;
 710
 711         return 0;
 712 }
 713
 714 static int switchtec_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
 715 {
 716         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 717
 718         iowrite64(db_bits << sndev->db_peer_shift,
 719                   &sndev->mmio_peer_dbmsg->odb);
 720
 721         return 0;
 722 }
 723
 724 static int switchtec_ntb_spad_count(struct ntb_dev *ntb)
 725 {
 726         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 727
 728         return ARRAY_SIZE(sndev->self_shared->spad);
 729 }
 730
 731 static u32 switchtec_ntb_spad_read(struct ntb_dev *ntb, int idx)
 732 {
 733         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 734
 735         if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad))
 736                 return 0;
 737
 738         if (!sndev->self_shared)
 739                 return 0;
 740
 741         return sndev->self_shared->spad[idx];
 742 }
 743
 744 static int switchtec_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val)
 745 {
 746         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 747
 748         if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad))
 749                 return -EINVAL;
 750
 751         if (!sndev->self_shared)
 752                 return -EIO;
 753
 754         sndev->self_shared->spad[idx] = val;
 755
 756         return 0;
 757 }
 758
 759 static u32 switchtec_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx,
 760                                         int sidx)
 761 {
 762         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 763
 764         if (pidx != NTB_DEF_PEER_IDX)
 765                 return -EINVAL;
 766
 767         if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad))
 768                 return 0;
 769
 770         if (!sndev->peer_shared)
 771                 return 0;
 772
 773         return ioread32(&sndev->peer_shared->spad[sidx]);
 774 }
 775
 776 static int switchtec_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
 777                                          int sidx, u32 val)
 778 {
 779         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 780
 781         if (pidx != NTB_DEF_PEER_IDX)
 782                 return -EINVAL;
 783
 784         if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad))
 785                 return -EINVAL;
 786
 787         if (!sndev->peer_shared)
 788                 return -EIO;
 789
 790         iowrite32(val, &sndev->peer_shared->spad[sidx]);
 791
 792         return 0;
 793 }
 794
 795 static int switchtec_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx,
 796                                         int sidx, phys_addr_t *spad_addr)
 797 {
 798         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 799         unsigned long offset;
 800
 801         if (pidx != NTB_DEF_PEER_IDX)
 802                 return -EINVAL;
 803
 804         offset = (unsigned long)&sndev->peer_shared->spad[sidx] -
 805                 (unsigned long)sndev->stdev->mmio;
 806
 807         if (spad_addr)
 808                 *spad_addr = pci_resource_start(ntb->pdev, 0) + offset;
 809
 810         return 0;
 811 }
 812
 813 static const struct ntb_dev_ops switchtec_ntb_ops = {
 814         .mw_count               = switchtec_ntb_mw_count,
 815         .mw_get_align           = switchtec_ntb_mw_get_align,
 816         .mw_set_trans           = switchtec_ntb_mw_set_trans,
 817         .peer_mw_count          = switchtec_ntb_peer_mw_count,
 818         .peer_mw_get_addr       = switchtec_ntb_peer_mw_get_addr,
 819         .link_is_up             = switchtec_ntb_link_is_up,
 820         .link_enable            = switchtec_ntb_link_enable,
 821         .link_disable           = switchtec_ntb_link_disable,
 822         .db_valid_mask          = switchtec_ntb_db_valid_mask,
 823         .db_vector_count        = switchtec_ntb_db_vector_count,
 824         .db_vector_mask         = switchtec_ntb_db_vector_mask,
 825         .db_read                = switchtec_ntb_db_read,
 826         .db_clear               = switchtec_ntb_db_clear,
 827         .db_set_mask            = switchtec_ntb_db_set_mask,
 828         .db_clear_mask          = switchtec_ntb_db_clear_mask,
 829         .db_read_mask           = switchtec_ntb_db_read_mask,
 830         .peer_db_addr           = switchtec_ntb_peer_db_addr,
 831         .peer_db_set            = switchtec_ntb_peer_db_set,
 832         .spad_count             = switchtec_ntb_spad_count,
 833         .spad_read              = switchtec_ntb_spad_read,
 834         .spad_write             = switchtec_ntb_spad_write,
 835         .peer_spad_read         = switchtec_ntb_peer_spad_read,
 836         .peer_spad_write        = switchtec_ntb_peer_spad_write,
 837         .peer_spad_addr         = switchtec_ntb_peer_spad_addr,
 838 };
 839
 840 static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev)
 841 {
 842         u64 tpart_vec;
 843         int self;
 844         u64 part_map;
 845
 846         sndev->ntb.pdev = sndev->stdev->pdev;
 847         sndev->ntb.topo = NTB_TOPO_SWITCH;
 848         sndev->ntb.ops = &switchtec_ntb_ops;
 849
 850         INIT_WORK(&sndev->check_link_status_work, check_link_status_work);
 851         sndev->link_force_down = false;
 852
 853         sndev->self_partition = sndev->stdev->partition;
 854
 855         sndev->mmio_ntb = sndev->stdev->mmio_ntb;
 856
 857         self = sndev->self_partition;
 858         tpart_vec = ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_high);
 859         tpart_vec <<= 32;
 860         tpart_vec |= ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_low);
 861
 862         part_map = ioread64(&sndev->mmio_ntb->ep_map);
 863         tpart_vec &= part_map;
 864         part_map &= ~(1 << sndev->self_partition);
 865
 866         if (!tpart_vec) {
 867                 if (sndev->stdev->partition_count != 2) {
 868                         dev_err(&sndev->stdev->dev,
 869                                 "ntb target partition not defined\n");
 870                         return -ENODEV;
 871                 }
 872
 873                 if (!part_map) {
 874                         dev_err(&sndev->stdev->dev,
 875                                 "peer partition is not NT partition\n");
 876                         return -ENODEV;
 877                 }
 878
 879                 sndev->peer_partition = __ffs64(part_map);
 880         } else {
 881                 if (__ffs64(tpart_vec) != (fls64(tpart_vec) - 1)) {
 882                         dev_err(&sndev->stdev->dev,
 883                                 "ntb driver only supports 1 pair of 1-1 ntb mapping\n");
 884                         return -ENODEV;
 885                 }
 886
 887                 sndev->peer_partition = __ffs64(tpart_vec);
 888                 if (!(part_map & (1ULL << sndev->peer_partition))) {
 889                         dev_err(&sndev->stdev->dev,
 890                                 "ntb target partition is not NT partition\n");
 891                         return -ENODEV;
 892                 }
 893         }
 894
 895         dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d\n",
 896                 sndev->self_partition, sndev->stdev->partition_count);
 897
 898         sndev->mmio_ctrl = (void * __iomem)sndev->mmio_ntb +
 899                 SWITCHTEC_NTB_REG_CTRL_OFFSET;
 900         sndev->mmio_dbmsg = (void * __iomem)sndev->mmio_ntb +
 901                 SWITCHTEC_NTB_REG_DBMSG_OFFSET;
 902
 903         sndev->mmio_self_ctrl = &sndev->mmio_ctrl[sndev->self_partition];
 904         sndev->mmio_peer_ctrl = &sndev->mmio_ctrl[sndev->peer_partition];
 905         sndev->mmio_self_dbmsg = &sndev->mmio_dbmsg[sndev->self_partition];
 906         sndev->mmio_peer_dbmsg = sndev->mmio_self_dbmsg;
 907
 908         return 0;
 909 }
 910
 911 static int config_rsvd_lut_win(struct switchtec_ntb *sndev,
 912                                struct ntb_ctrl_regs __iomem *ctl,
 913                                int lut_idx, int partition, u64 addr)
 914 {
 915         int peer_bar = sndev->peer_direct_mw_to_bar[0];
 916         u32 ctl_val;
 917         int rc;
 918
 919         rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
 920                                    NTB_CTRL_PART_STATUS_LOCKED);
 921         if (rc)
 922                 return rc;
 923
 924         ctl_val = ioread32(&ctl->bar_entry[peer_bar].ctl);
 925         ctl_val &= 0xFF;
 926         ctl_val |= NTB_CTRL_BAR_LUT_WIN_EN;
 927         ctl_val |= ilog2(LUT_SIZE) << 8;
 928         ctl_val |= (sndev->nr_lut_mw - 1) << 14;
 929         iowrite32(ctl_val, &ctl->bar_entry[peer_bar].ctl);
 930
 931         iowrite64((NTB_CTRL_LUT_EN | (partition << 1) | addr),
 932                   &ctl->lut_entry[lut_idx]);
 933
 934         rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
 935                                    NTB_CTRL_PART_STATUS_NORMAL);
 936         if (rc) {
 937                 u32 bar_error, lut_error;
 938
 939                 bar_error = ioread32(&ctl->bar_error);
 940                 lut_error = ioread32(&ctl->lut_error);
 941                 dev_err(&sndev->stdev->dev,
 942                         "Error setting up reserved lut window: %08x / %08x\n",
 943                         bar_error, lut_error);
 944                 return rc;
 945         }
 946
 947         return 0;
 948 }
 949
 950 static int config_req_id_table(struct switchtec_ntb *sndev,
 951                                struct ntb_ctrl_regs __iomem *mmio_ctrl,
 952                                int *req_ids, int count)
 953 {
 954         int i, rc = 0;
 955         u32 error;
 956         u32 proxy_id;
 957
 958         if (ioread16(&mmio_ctrl->req_id_table_size) < count) {
 959                 dev_err(&sndev->stdev->dev,
 960                         "Not enough requester IDs available.\n");
 961                 return -EFAULT;
 962         }
 963
 964         rc = switchtec_ntb_part_op(sndev, mmio_ctrl,
 965                                    NTB_CTRL_PART_OP_LOCK,
 966                                    NTB_CTRL_PART_STATUS_LOCKED);
 967         if (rc)
 968                 return rc;
 969
 970         for (i = 0; i < count; i++) {
 971                 iowrite32(req_ids[i] << 16 | NTB_CTRL_REQ_ID_EN,
 972                           &mmio_ctrl->req_id_table[i]);
 973
 974                 proxy_id = ioread32(&mmio_ctrl->req_id_table[i]);
 975                 dev_dbg(&sndev->stdev->dev,
 976                         "Requester ID %02X:%02X.%X -> BB:%02X.%X\n",
 977                         req_ids[i] >> 8, (req_ids[i] >> 3) & 0x1F,
 978                         req_ids[i] & 0x7, (proxy_id >> 4) & 0x1F,
 979                         (proxy_id >> 1) & 0x7);
 980         }
 981
 982         rc = switchtec_ntb_part_op(sndev, mmio_ctrl,
 983                                    NTB_CTRL_PART_OP_CFG,
 984                                    NTB_CTRL_PART_STATUS_NORMAL);
 985
 986         if (rc == -EIO) {
 987                 error = ioread32(&mmio_ctrl->req_id_error);
 988                 dev_err(&sndev->stdev->dev,
 989                         "Error setting up the requester ID table: %08x\n",
 990                         error);
 991         }
 992
 993         return 0;
 994 }
 995
 996 static int crosslink_setup_mws(struct switchtec_ntb *sndev, int ntb_lut_idx,
 997                                u64 *mw_addrs, int mw_count)
 998 {
 999         int rc, i;
1000         struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_self_ctrl;
1001         u64 addr;
1002         size_t size, offset;
1003         int bar;
1004         int xlate_pos;
1005         u32 ctl_val;
1006
1007         rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
1008                                    NTB_CTRL_PART_STATUS_LOCKED);
1009         if (rc)
1010                 return rc;
1011
1012         for (i = 0; i < sndev->nr_lut_mw; i++) {
1013                 if (i == ntb_lut_idx)
1014                         continue;
1015
1016                 addr = mw_addrs[0] + LUT_SIZE * i;
1017
1018                 iowrite64((NTB_CTRL_LUT_EN | (sndev->peer_partition << 1) |
1019                            addr),
1020                           &ctl->lut_entry[i]);
1021         }
1022
1023         sndev->nr_direct_mw = min_t(int, sndev->nr_direct_mw, mw_count);
1024
1025         for (i = 0; i < sndev->nr_direct_mw; i++) {
1026                 bar = sndev->direct_mw_to_bar[i];
1027                 offset = (i == 0) ? LUT_SIZE * sndev->nr_lut_mw : 0;
1028                 addr = mw_addrs[i] + offset;
1029                 size = pci_resource_len(sndev->ntb.pdev, bar) - offset;
1030                 xlate_pos = ilog2(size);
1031
1032                 if (offset && size > offset)
1033                         size = offset;
1034
1035                 ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
1036                 ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN;
1037
1038                 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
1039                 iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000),
1040                           &ctl->bar_entry[bar].win_size);
1041                 iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size);
1042                 iowrite64(sndev->peer_partition | addr,
1043                           &ctl->bar_entry[bar].xlate_addr);
1044         }
1045
1046         rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
1047                                    NTB_CTRL_PART_STATUS_NORMAL);
1048         if (rc) {
1049                 u32 bar_error, lut_error;
1050
1051                 bar_error = ioread32(&ctl->bar_error);
1052                 lut_error = ioread32(&ctl->lut_error);
1053                 dev_err(&sndev->stdev->dev,
1054                         "Error setting up cross link windows: %08x / %08x\n",
1055                         bar_error, lut_error);
1056                 return rc;
1057         }
1058
1059         return 0;
1060 }
1061
1062 static int crosslink_setup_req_ids(struct switchtec_ntb *sndev,
1063         struct ntb_ctrl_regs __iomem *mmio_ctrl)
1064 {
1065         int req_ids[16];
1066         int i;
1067         u32 proxy_id;
1068
1069         for (i = 0; i < ARRAY_SIZE(req_ids); i++) {
1070                 proxy_id = ioread32(&sndev->mmio_self_ctrl->req_id_table[i]);
1071
1072                 if (!(proxy_id & NTB_CTRL_REQ_ID_EN))
1073                         break;
1074
1075                 req_ids[i] = ((proxy_id >> 1) & 0xFF);
1076         }
1077
1078         return config_req_id_table(sndev, mmio_ctrl, req_ids, i);
1079 }
1080
1081 /*
1082  * In crosslink configuration there is a virtual partition in the
1083  * middle of the two switches. The BARs in this partition have to be
1084  * enumerated and assigned addresses.
1085  */
1086 static int crosslink_enum_partition(struct switchtec_ntb *sndev,
1087                                     u64 *bar_addrs)
1088 {
1089         struct part_cfg_regs __iomem *part_cfg =
1090                 &sndev->stdev->mmio_part_cfg_all[sndev->peer_partition];
1091         u32 pff = ioread32(&part_cfg->vep_pff_inst_id) & 0xFF;
1092         struct pff_csr_regs __iomem *mmio_pff =
1093                 &sndev->stdev->mmio_pff_csr[pff];
1094         const u64 bar_space = 0x1000000000LL;
1095         u64 bar_addr;
1096         int bar_cnt = 0;
1097         int i;
1098
1099         iowrite16(0x6, &mmio_pff->pcicmd);
1100
1101         for (i = 0; i < ARRAY_SIZE(mmio_pff->pci_bar64); i++) {
1102                 iowrite64(bar_space * i, &mmio_pff->pci_bar64[i]);
1103                 bar_addr = ioread64(&mmio_pff->pci_bar64[i]);
1104                 bar_addr &= ~0xf;
1105
1106                 dev_dbg(&sndev->stdev->dev,
1107                         "Crosslink BAR%d addr: %llx\n",
1108                         i*2, bar_addr);
1109
1110                 if (bar_addr != bar_space * i)
1111                         continue;
1112
1113                 bar_addrs[bar_cnt++] = bar_addr;
1114         }
1115
1116         return bar_cnt;
1117 }
1118
1119 static int switchtec_ntb_init_crosslink(struct switchtec_ntb *sndev)
1120 {
1121         int rc;
1122         int bar = sndev->direct_mw_to_bar[0];
1123         const int ntb_lut_idx = 1;
1124         u64 bar_addrs[6];
1125         u64 addr;
1126         int offset;
1127         int bar_cnt;
1128
1129         if (!crosslink_is_enabled(sndev))
1130                 return 0;
1131
1132         dev_info(&sndev->stdev->dev, "Using crosslink configuration\n");
1133         sndev->ntb.topo = NTB_TOPO_CROSSLINK;
1134
1135         bar_cnt = crosslink_enum_partition(sndev, bar_addrs);
1136         if (bar_cnt < sndev->nr_direct_mw + 1) {
1137                 dev_err(&sndev->stdev->dev,
1138                         "Error enumerating crosslink partition\n");
1139                 return -EINVAL;
1140         }
1141
1142         addr = (bar_addrs[0] + SWITCHTEC_GAS_NTB_OFFSET +
1143                 SWITCHTEC_NTB_REG_DBMSG_OFFSET +
1144                 sizeof(struct ntb_dbmsg_regs) * sndev->peer_partition);
1145
1146         offset = addr & (LUT_SIZE - 1);
1147         addr -= offset;
1148
1149         rc = config_rsvd_lut_win(sndev, sndev->mmio_self_ctrl, ntb_lut_idx,
1150                                  sndev->peer_partition, addr);
1151         if (rc)
1152                 return rc;
1153
1154         rc = crosslink_setup_mws(sndev, ntb_lut_idx, &bar_addrs[1],
1155                                  bar_cnt - 1);
1156         if (rc)
1157                 return rc;
1158
1159         rc = crosslink_setup_req_ids(sndev, sndev->mmio_peer_ctrl);
1160         if (rc)
1161                 return rc;
1162
1163         sndev->mmio_xlink_win = pci_iomap_range(sndev->stdev->pdev, bar,
1164                                                 LUT_SIZE, LUT_SIZE);
1165         if (!sndev->mmio_xlink_win) {
1166                 rc = -ENOMEM;
1167                 return rc;
1168         }
1169
1170         sndev->mmio_peer_dbmsg = sndev->mmio_xlink_win + offset;
1171         sndev->nr_rsvd_luts++;
1172
1173         crosslink_init_dbmsgs(sndev);
1174
1175         return 0;
1176 }
1177
1178 static void switchtec_ntb_deinit_crosslink(struct switchtec_ntb *sndev)
1179 {
1180         if (sndev->mmio_xlink_win)
1181                 pci_iounmap(sndev->stdev->pdev, sndev->mmio_xlink_win);
1182 }
1183
1184 static int map_bars(int *map, struct ntb_ctrl_regs __iomem *ctrl)
1185 {
1186         int i;
1187         int cnt = 0;
1188
1189         for (i = 0; i < ARRAY_SIZE(ctrl->bar_entry); i++) {
1190                 u32 r = ioread32(&ctrl->bar_entry[i].ctl);
1191
1192                 if (r & NTB_CTRL_BAR_VALID)
1193                         map[cnt++] = i;
1194         }
1195
1196         return cnt;
1197 }
1198
1199 static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev)
1200 {
1201         sndev->nr_direct_mw = map_bars(sndev->direct_mw_to_bar,
1202                                        sndev->mmio_self_ctrl);
1203
1204         sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries);
1205         sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw);
1206
1207         dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n",
1208                 sndev->nr_direct_mw, sndev->nr_lut_mw);
1209
1210         sndev->peer_nr_direct_mw = map_bars(sndev->peer_direct_mw_to_bar,
1211                                             sndev->mmio_peer_ctrl);
1212
1213         sndev->peer_nr_lut_mw =
1214                 ioread16(&sndev->mmio_peer_ctrl->lut_table_entries);
1215         sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw);
1216
1217         dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n",
1218                 sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw);
1219
1220 }
1221
1222 /*
1223  * There are 64 doorbells in the switch hardware but this is
1224  * shared among all partitions. So we must split them in half
1225  * (32 for each partition). However, the message interrupts are
1226  * also shared with the top 4 doorbells so we just limit this to
1227  * 28 doorbells per partition.
1228  *
1229  * In crosslink mode, each side has it's own dbmsg register so
1230  * they can each use all 60 of the available doorbells.
1231  */
1232 static void switchtec_ntb_init_db(struct switchtec_ntb *sndev)
1233 {
1234         sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL;
1235
1236         if (sndev->mmio_peer_dbmsg != sndev->mmio_self_dbmsg) {
1237                 sndev->db_shift = 0;
1238                 sndev->db_peer_shift = 0;
1239                 sndev->db_valid_mask = sndev->db_mask;
1240         } else if (sndev->self_partition < sndev->peer_partition) {
1241                 sndev->db_shift = 0;
1242                 sndev->db_peer_shift = 32;
1243                 sndev->db_valid_mask = 0x0FFFFFFF;
1244         } else {
1245                 sndev->db_shift = 32;
1246                 sndev->db_peer_shift = 0;
1247                 sndev->db_valid_mask = 0x0FFFFFFF;
1248         }
1249
1250         iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
1251         iowrite64(sndev->db_valid_mask << sndev->db_peer_shift,
1252                   &sndev->mmio_peer_dbmsg->odb_mask);
1253
1254         dev_dbg(&sndev->stdev->dev, "dbs: shift %d/%d, mask %016llx\n",
1255                 sndev->db_shift, sndev->db_peer_shift, sndev->db_valid_mask);
1256 }
1257
1258 static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev)
1259 {
1260         int i;
1261         u32 msg_map = 0;
1262
1263         for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) {
1264                 int m = i | sndev->peer_partition << 2;
1265
1266                 msg_map |= m << i * 8;
1267         }
1268
1269         iowrite32(msg_map, &sndev->mmio_self_dbmsg->msg_map);
1270
1271         for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++)
1272                 iowrite64(NTB_DBMSG_IMSG_STATUS | NTB_DBMSG_IMSG_MASK,
1273                           &sndev->mmio_self_dbmsg->imsg[i]);
1274 }
1275
1276 static int
1277 switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev)
1278 {
1279         int req_ids[2];
1280
1281         /*
1282          * Root Complex Requester ID (which is 0:00.0)
1283          */
1284         req_ids[0] = 0;
1285
1286         /*
1287          * Host Bridge Requester ID (as read from the mmap address)
1288          */
1289         req_ids[1] = ioread16(&sndev->mmio_ntb->requester_id);
1290
1291         return config_req_id_table(sndev, sndev->mmio_self_ctrl, req_ids,
1292                                    ARRAY_SIZE(req_ids));
1293 }
1294
1295 static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev)
1296 {
1297         int i;
1298
1299         memset(sndev->self_shared, 0, LUT_SIZE);
1300         sndev->self_shared->magic = SWITCHTEC_NTB_MAGIC;
1301         sndev->self_shared->partition_id = sndev->stdev->partition;
1302
1303         for (i = 0; i < sndev->nr_direct_mw; i++) {
1304                 int bar = sndev->direct_mw_to_bar[i];
1305                 resource_size_t sz = pci_resource_len(sndev->stdev->pdev, bar);
1306
1307                 if (i == 0)
1308                         sz = min_t(resource_size_t, sz,
1309                                    LUT_SIZE * sndev->nr_lut_mw);
1310
1311                 sndev->self_shared->mw_sizes[i] = sz;
1312         }
1313
1314         for (i = 0; i < sndev->nr_lut_mw; i++) {
1315                 int idx = sndev->nr_direct_mw + i;
1316
1317                 sndev->self_shared->mw_sizes[idx] = LUT_SIZE;
1318         }
1319 }
1320
1321 static int switchtec_ntb_init_shared_mw(struct switchtec_ntb *sndev)
1322 {
1323         int self_bar = sndev->direct_mw_to_bar[0];
1324         int rc;
1325
1326         sndev->nr_rsvd_luts++;
1327         sndev->self_shared = dma_alloc_coherent(&sndev->stdev->pdev->dev,
1328                                                 LUT_SIZE,
1329                                                 &sndev->self_shared_dma,
1330                                                 GFP_KERNEL);
1331         if (!sndev->self_shared) {
1332                 dev_err(&sndev->stdev->dev,
1333                         "unable to allocate memory for shared mw\n");
1334                 return -ENOMEM;
1335         }
1336
1337         switchtec_ntb_init_shared(sndev);
1338
1339         rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0,
1340                                  sndev->self_partition,
1341                                  sndev->self_shared_dma);
1342         if (rc)
1343                 goto unalloc_and_exit;
1344
1345         sndev->peer_shared = pci_iomap(sndev->stdev->pdev, self_bar, LUT_SIZE);
1346         if (!sndev->peer_shared) {
1347                 rc = -ENOMEM;
1348                 goto unalloc_and_exit;
1349         }
1350
1351         dev_dbg(&sndev->stdev->dev, "Shared MW Ready\n");
1352         return 0;
1353
1354 unalloc_and_exit:
1355         dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE,
1356                           sndev->self_shared, sndev->self_shared_dma);
1357
1358         return rc;
1359 }
1360
1361 static void switchtec_ntb_deinit_shared_mw(struct switchtec_ntb *sndev)
1362 {
1363         if (sndev->peer_shared)
1364                 pci_iounmap(sndev->stdev->pdev, sndev->peer_shared);
1365
1366         if (sndev->self_shared)
1367                 dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE,
1368                                   sndev->self_shared,
1369                                   sndev->self_shared_dma);
1370         sndev->nr_rsvd_luts--;
1371 }
1372
1373 static irqreturn_t switchtec_ntb_doorbell_isr(int irq, void *dev)
1374 {
1375         struct switchtec_ntb *sndev = dev;
1376
1377         dev_dbg(&sndev->stdev->dev, "doorbell\n");
1378
1379         ntb_db_event(&sndev->ntb, 0);
1380
1381         return IRQ_HANDLED;
1382 }
1383
1384 static irqreturn_t switchtec_ntb_message_isr(int irq, void *dev)
1385 {
1386         int i;
1387         struct switchtec_ntb *sndev = dev;
1388
1389         for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) {
1390                 u64 msg = ioread64(&sndev->mmio_self_dbmsg->imsg[i]);
1391
1392                 if (msg & NTB_DBMSG_IMSG_STATUS) {
1393                         dev_dbg(&sndev->stdev->dev, "message: %d %08x\n",
1394                                 i, (u32)msg);
1395                         iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status);
1396
1397                         if (i == LINK_MESSAGE)
1398                                 switchtec_ntb_check_link(sndev, msg);
1399                 }
1400         }
1401
1402         return IRQ_HANDLED;
1403 }
1404
1405 static int switchtec_ntb_init_db_msg_irq(struct switchtec_ntb *sndev)
1406 {
1407         int i;
1408         int rc;
1409         int doorbell_irq = 0;
1410         int message_irq = 0;
1411         int event_irq;
1412         int idb_vecs = sizeof(sndev->mmio_self_dbmsg->idb_vec_map);
1413
1414         event_irq = ioread32(&sndev->stdev->mmio_part_cfg->vep_vector_number);
1415
1416         while (doorbell_irq == event_irq)
1417                 doorbell_irq++;
1418         while (message_irq == doorbell_irq ||
1419                message_irq == event_irq)
1420                 message_irq++;
1421
1422         dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d\n",
1423                 event_irq, doorbell_irq, message_irq);
1424
1425         for (i = 0; i < idb_vecs - 4; i++)
1426                 iowrite8(doorbell_irq,
1427                          &sndev->mmio_self_dbmsg->idb_vec_map[i]);
1428
1429         for (; i < idb_vecs; i++)
1430                 iowrite8(message_irq,
1431                          &sndev->mmio_self_dbmsg->idb_vec_map[i]);
1432
1433         sndev->doorbell_irq = pci_irq_vector(sndev->stdev->pdev, doorbell_irq);
1434         sndev->message_irq = pci_irq_vector(sndev->stdev->pdev, message_irq);
1435
1436         rc = request_irq(sndev->doorbell_irq,
1437                          switchtec_ntb_doorbell_isr, 0,
1438                          "switchtec_ntb_doorbell", sndev);
1439         if (rc)
1440                 return rc;
1441
1442         rc = request_irq(sndev->message_irq,
1443                          switchtec_ntb_message_isr, 0,
1444                          "switchtec_ntb_message", sndev);
1445         if (rc) {
1446                 free_irq(sndev->doorbell_irq, sndev);
1447                 return rc;
1448         }
1449
1450         return 0;
1451 }
1452
1453 static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev)
1454 {
1455         free_irq(sndev->doorbell_irq, sndev);
1456         free_irq(sndev->message_irq, sndev);
1457 }
1458
1459 static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev)
1460 {
1461         int rc;
1462
1463         if (crosslink_is_enabled(sndev))
1464                 return 0;
1465
1466         dev_info(&sndev->stdev->dev, "reinitialize shared memory window\n");
1467         rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0,
1468                                  sndev->self_partition,
1469                                  sndev->self_shared_dma);
1470         return rc;
1471 }
1472
1473 static int switchtec_ntb_add(struct device *dev)
1474 {
1475         struct switchtec_dev *stdev = to_stdev(dev);
1476         struct switchtec_ntb *sndev;
1477         int rc;
1478
1479         stdev->sndev = NULL;
1480
1481         if (stdev->pdev->class != (PCI_CLASS_BRIDGE_OTHER << 8))
1482                 return -ENODEV;
1483
1484         sndev = kzalloc_node(sizeof(*sndev), GFP_KERNEL, dev_to_node(dev));
1485         if (!sndev)
1486                 return -ENOMEM;
1487
1488         sndev->stdev = stdev;
1489         rc = switchtec_ntb_init_sndev(sndev);
1490         if (rc)
1491                 goto free_and_exit;
1492
1493         switchtec_ntb_init_mw(sndev);
1494
1495         rc = switchtec_ntb_init_req_id_table(sndev);
1496         if (rc)
1497                 goto free_and_exit;
1498
1499         rc = switchtec_ntb_init_crosslink(sndev);
1500         if (rc)
1501                 goto free_and_exit;
1502
1503         switchtec_ntb_init_db(sndev);
1504         switchtec_ntb_init_msgs(sndev);
1505
1506         rc = switchtec_ntb_init_shared_mw(sndev);
1507         if (rc)
1508                 goto deinit_crosslink;
1509
1510         rc = switchtec_ntb_init_db_msg_irq(sndev);
1511         if (rc)
1512                 goto deinit_shared_and_exit;
1513
1514         /*
1515          * If this host crashed, the other host may think the link is
1516          * still up. Tell them to force it down (it will go back up
1517          * once we register the ntb device).
1518          */
1519         switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_FORCE_DOWN);
1520
1521         rc = ntb_register_device(&sndev->ntb);
1522         if (rc)
1523                 goto deinit_and_exit;
1524
1525         stdev->sndev = sndev;
1526         stdev->link_notifier = switchtec_ntb_link_notification;
1527         dev_info(dev, "NTB device registered\n");
1528
1529         return 0;
1530
1531 deinit_and_exit:
1532         switchtec_ntb_deinit_db_msg_irq(sndev);
1533 deinit_shared_and_exit:
1534         switchtec_ntb_deinit_shared_mw(sndev);
1535 deinit_crosslink:
1536         switchtec_ntb_deinit_crosslink(sndev);
1537 free_and_exit:
1538         kfree(sndev);
1539         dev_err(dev, "failed to register ntb device: %d\n", rc);
1540         return rc;
1541 }
1542
1543 static void switchtec_ntb_remove(struct device *dev)
1544 {
1545         struct switchtec_dev *stdev = to_stdev(dev);
1546         struct switchtec_ntb *sndev = stdev->sndev;
1547
1548         if (!sndev)
1549                 return;
1550
1551         stdev->link_notifier = NULL;
1552         stdev->sndev = NULL;
1553         ntb_unregister_device(&sndev->ntb);
1554         switchtec_ntb_deinit_db_msg_irq(sndev);
1555         switchtec_ntb_deinit_shared_mw(sndev);
1556         switchtec_ntb_deinit_crosslink(sndev);
1557         kfree(sndev);
1558         dev_info(dev, "ntb device unregistered\n");
1559 }
1560
1561 static struct class_interface switchtec_interface  = {
1562         .add_dev = switchtec_ntb_add,
1563         .remove_dev = switchtec_ntb_remove,
1564 };
1565
1566 static int __init switchtec_ntb_init(void)
1567 {
1568         switchtec_interface.class = switchtec_class;
1569         return class_interface_register(&switchtec_interface);
1570 }
1571 module_init(switchtec_ntb_init);
1572
1573 static void __exit switchtec_ntb_exit(void)
1574 {
1575         class_interface_unregister(&switchtec_interface);
1576 }
1577 module_exit(switchtec_ntb_exit);