drivers/memory/tegra/mc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2014 NVIDIA CORPORATION.  All rights reserved.
   4  */
   5
   6 #include <linux/clk.h>
   7 #include <linux/delay.h>
   8 #include <linux/dma-mapping.h>
   9 #include <linux/export.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/kernel.h>
  12 #include <linux/module.h>
  13 #include <linux/of.h>
  14 #include <linux/of_device.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/slab.h>
  17 #include <linux/sort.h>
  18
  19 #include <soc/tegra/fuse.h>
  20
  21 #include "mc.h"
  22
  23 static const struct of_device_id tegra_mc_of_match[] = {
  24 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
  25         { .compatible = "nvidia,tegra20-mc-gart", .data = &tegra20_mc_soc },
  26 #endif
  27 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
  28         { .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc },
  29 #endif
  30 #ifdef CONFIG_ARCH_TEGRA_114_SOC
  31         { .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc },
  32 #endif
  33 #ifdef CONFIG_ARCH_TEGRA_124_SOC
  34         { .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc },
  35 #endif
  36 #ifdef CONFIG_ARCH_TEGRA_132_SOC
  37         { .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc },
  38 #endif
  39 #ifdef CONFIG_ARCH_TEGRA_210_SOC
  40         { .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc },
  41 #endif
  42         { }
  43 };
  44 MODULE_DEVICE_TABLE(of, tegra_mc_of_match);
  45
  46 static void tegra_mc_devm_action_put_device(void *data)
  47 {
  48         struct tegra_mc *mc = data;
  49
  50         put_device(mc->dev);
  51 }
  52
  53 /**
  54  * devm_tegra_memory_controller_get() - get Tegra Memory Controller handle
  55  * @dev: device pointer for the consumer device
  56  *
  57  * This function will search for the Memory Controller node in a device-tree
  58  * and retrieve the Memory Controller handle.
  59  *
  60  * Return: ERR_PTR() on error or a valid pointer to a struct tegra_mc.
  61  */
  62 struct tegra_mc *devm_tegra_memory_controller_get(struct device *dev)
  63 {
  64         struct platform_device *pdev;
  65         struct device_node *np;
  66         struct tegra_mc *mc;
  67         int err;
  68
  69         np = of_parse_phandle(dev->of_node, "nvidia,memory-controller", 0);
  70         if (!np)
  71                 return ERR_PTR(-ENOENT);
  72
  73         pdev = of_find_device_by_node(np);
  74         of_node_put(np);
  75         if (!pdev)
  76                 return ERR_PTR(-ENODEV);
  77
  78         mc = platform_get_drvdata(pdev);
  79         if (!mc) {
  80                 put_device(&pdev->dev);
  81                 return ERR_PTR(-EPROBE_DEFER);
  82         }
  83
  84         err = devm_add_action(dev, tegra_mc_devm_action_put_device, mc);
  85         if (err) {
  86                 put_device(mc->dev);
  87                 return ERR_PTR(err);
  88         }
  89
  90         return mc;
  91 }
  92 EXPORT_SYMBOL_GPL(devm_tegra_memory_controller_get);
  93
  94 static int tegra_mc_block_dma_common(struct tegra_mc *mc,
  95                                      const struct tegra_mc_reset *rst)
  96 {
  97         unsigned long flags;
  98         u32 value;
  99
 100         spin_lock_irqsave(&mc->lock, flags);
 101
 102         value = mc_readl(mc, rst->control) | BIT(rst->bit);
 103         mc_writel(mc, value, rst->control);
 104
 105         spin_unlock_irqrestore(&mc->lock, flags);
 106
 107         return 0;
 108 }
 109
 110 static bool tegra_mc_dma_idling_common(struct tegra_mc *mc,
 111                                        const struct tegra_mc_reset *rst)
 112 {
 113         return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0;
 114 }
 115
 116 static int tegra_mc_unblock_dma_common(struct tegra_mc *mc,
 117                                        const struct tegra_mc_reset *rst)
 118 {
 119         unsigned long flags;
 120         u32 value;
 121
 122         spin_lock_irqsave(&mc->lock, flags);
 123
 124         value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
 125         mc_writel(mc, value, rst->control);
 126
 127         spin_unlock_irqrestore(&mc->lock, flags);
 128
 129         return 0;
 130 }
 131
 132 static int tegra_mc_reset_status_common(struct tegra_mc *mc,
 133                                         const struct tegra_mc_reset *rst)
 134 {
 135         return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0;
 136 }
 137
 138 const struct tegra_mc_reset_ops tegra_mc_reset_ops_common = {
 139         .block_dma = tegra_mc_block_dma_common,
 140         .dma_idling = tegra_mc_dma_idling_common,
 141         .unblock_dma = tegra_mc_unblock_dma_common,
 142         .reset_status = tegra_mc_reset_status_common,
 143 };
 144
 145 static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev)
 146 {
 147         return container_of(rcdev, struct tegra_mc, reset);
 148 }
 149
 150 static const struct tegra_mc_reset *tegra_mc_reset_find(struct tegra_mc *mc,
 151                                                         unsigned long id)
 152 {
 153         unsigned int i;
 154
 155         for (i = 0; i < mc->soc->num_resets; i++)
 156                 if (mc->soc->resets[i].id == id)
 157                         return &mc->soc->resets[i];
 158
 159         return NULL;
 160 }
 161
 162 static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev,
 163                                     unsigned long id)
 164 {
 165         struct tegra_mc *mc = reset_to_mc(rcdev);
 166         const struct tegra_mc_reset_ops *rst_ops;
 167         const struct tegra_mc_reset *rst;
 168         int retries = 500;
 169         int err;
 170
 171         rst = tegra_mc_reset_find(mc, id);
 172         if (!rst)
 173                 return -ENODEV;
 174
 175         rst_ops = mc->soc->reset_ops;
 176         if (!rst_ops)
 177                 return -ENODEV;
 178
 179         /* DMA flushing will fail if reset is already asserted */
 180         if (rst_ops->reset_status) {
 181                 /* check whether reset is asserted */
 182                 if (rst_ops->reset_status(mc, rst))
 183                         return 0;
 184         }
 185
 186         if (rst_ops->block_dma) {
 187                 /* block clients DMA requests */
 188                 err = rst_ops->block_dma(mc, rst);
 189                 if (err) {
 190                         dev_err(mc->dev, "failed to block %s DMA: %d\n",
 191                                 rst->name, err);
 192                         return err;
 193                 }
 194         }
 195
 196         if (rst_ops->dma_idling) {
 197                 /* wait for completion of the outstanding DMA requests */
 198                 while (!rst_ops->dma_idling(mc, rst)) {
 199                         if (!retries--) {
 200                                 dev_err(mc->dev, "failed to flush %s DMA\n",
 201                                         rst->name);
 202                                 return -EBUSY;
 203                         }
 204
 205                         usleep_range(10, 100);
 206                 }
 207         }
 208
 209         if (rst_ops->hotreset_assert) {
 210                 /* clear clients DMA requests sitting before arbitration */
 211                 err = rst_ops->hotreset_assert(mc, rst);
 212                 if (err) {
 213                         dev_err(mc->dev, "failed to hot reset %s: %d\n",
 214                                 rst->name, err);
 215                         return err;
 216                 }
 217         }
 218
 219         return 0;
 220 }
 221
 222 static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev,
 223                                       unsigned long id)
 224 {
 225         struct tegra_mc *mc = reset_to_mc(rcdev);
 226         const struct tegra_mc_reset_ops *rst_ops;
 227         const struct tegra_mc_reset *rst;
 228         int err;
 229
 230         rst = tegra_mc_reset_find(mc, id);
 231         if (!rst)
 232                 return -ENODEV;
 233
 234         rst_ops = mc->soc->reset_ops;
 235         if (!rst_ops)
 236                 return -ENODEV;
 237
 238         if (rst_ops->hotreset_deassert) {
 239                 /* take out client from hot reset */
 240                 err = rst_ops->hotreset_deassert(mc, rst);
 241                 if (err) {
 242                         dev_err(mc->dev, "failed to deassert hot reset %s: %d\n",
 243                                 rst->name, err);
 244                         return err;
 245                 }
 246         }
 247
 248         if (rst_ops->unblock_dma) {
 249                 /* allow new DMA requests to proceed to arbitration */
 250                 err = rst_ops->unblock_dma(mc, rst);
 251                 if (err) {
 252                         dev_err(mc->dev, "failed to unblock %s DMA : %d\n",
 253                                 rst->name, err);
 254                         return err;
 255                 }
 256         }
 257
 258         return 0;
 259 }
 260
 261 static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev,
 262                                     unsigned long id)
 263 {
 264         struct tegra_mc *mc = reset_to_mc(rcdev);
 265         const struct tegra_mc_reset_ops *rst_ops;
 266         const struct tegra_mc_reset *rst;
 267
 268         rst = tegra_mc_reset_find(mc, id);
 269         if (!rst)
 270                 return -ENODEV;
 271
 272         rst_ops = mc->soc->reset_ops;
 273         if (!rst_ops)
 274                 return -ENODEV;
 275
 276         return rst_ops->reset_status(mc, rst);
 277 }
 278
 279 static const struct reset_control_ops tegra_mc_reset_ops = {
 280         .assert = tegra_mc_hotreset_assert,
 281         .deassert = tegra_mc_hotreset_deassert,
 282         .status = tegra_mc_hotreset_status,
 283 };
 284
 285 static int tegra_mc_reset_setup(struct tegra_mc *mc)
 286 {
 287         int err;
 288
 289         mc->reset.ops = &tegra_mc_reset_ops;
 290         mc->reset.owner = THIS_MODULE;
 291         mc->reset.of_node = mc->dev->of_node;
 292         mc->reset.of_reset_n_cells = 1;
 293         mc->reset.nr_resets = mc->soc->num_resets;
 294
 295         err = reset_controller_register(&mc->reset);
 296         if (err < 0)
 297                 return err;
 298
 299         return 0;
 300 }
 301
 302 static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc)
 303 {
 304         unsigned long long tick;
 305         unsigned int i;
 306         u32 value;
 307
 308         /* compute the number of MC clock cycles per tick */
 309         tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk);
 310         do_div(tick, NSEC_PER_SEC);
 311
 312         value = mc_readl(mc, MC_EMEM_ARB_CFG);
 313         value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK;
 314         value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick);
 315         mc_writel(mc, value, MC_EMEM_ARB_CFG);
 316
 317         /* write latency allowance defaults */
 318         for (i = 0; i < mc->soc->num_clients; i++) {
 319                 const struct tegra_mc_la *la = &mc->soc->clients[i].la;
 320                 u32 value;
 321
 322                 value = mc_readl(mc, la->reg);
 323                 value &= ~(la->mask << la->shift);
 324                 value |= (la->def & la->mask) << la->shift;
 325                 mc_writel(mc, value, la->reg);
 326         }
 327
 328         /* latch new values */
 329         mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL);
 330
 331         return 0;
 332 }
 333
 334 int tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate)
 335 {
 336         unsigned int i;
 337         struct tegra_mc_timing *timing = NULL;
 338
 339         for (i = 0; i < mc->num_timings; i++) {
 340                 if (mc->timings[i].rate == rate) {
 341                         timing = &mc->timings[i];
 342                         break;
 343                 }
 344         }
 345
 346         if (!timing) {
 347                 dev_err(mc->dev, "no memory timing registered for rate %lu\n",
 348                         rate);
 349                 return -EINVAL;
 350         }
 351
 352         for (i = 0; i < mc->soc->num_emem_regs; ++i)
 353                 mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]);
 354
 355         return 0;
 356 }
 357 EXPORT_SYMBOL_GPL(tegra_mc_write_emem_configuration);
 358
 359 unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc)
 360 {
 361         u8 dram_count;
 362
 363         dram_count = mc_readl(mc, MC_EMEM_ADR_CFG);
 364         dram_count &= MC_EMEM_ADR_CFG_EMEM_NUMDEV;
 365         dram_count++;
 366
 367         return dram_count;
 368 }
 369 EXPORT_SYMBOL_GPL(tegra_mc_get_emem_device_count);
 370
 371 static int load_one_timing(struct tegra_mc *mc,
 372                            struct tegra_mc_timing *timing,
 373                            struct device_node *node)
 374 {
 375         int err;
 376         u32 tmp;
 377
 378         err = of_property_read_u32(node, "clock-frequency", &tmp);
 379         if (err) {
 380                 dev_err(mc->dev,
 381                         "timing %pOFn: failed to read rate\n", node);
 382                 return err;
 383         }
 384
 385         timing->rate = tmp;
 386         timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs,
 387                                          sizeof(u32), GFP_KERNEL);
 388         if (!timing->emem_data)
 389                 return -ENOMEM;
 390
 391         err = of_property_read_u32_array(node, "nvidia,emem-configuration",
 392                                          timing->emem_data,
 393                                          mc->soc->num_emem_regs);
 394         if (err) {
 395                 dev_err(mc->dev,
 396                         "timing %pOFn: failed to read EMEM configuration\n",
 397                         node);
 398                 return err;
 399         }
 400
 401         return 0;
 402 }
 403
 404 static int load_timings(struct tegra_mc *mc, struct device_node *node)
 405 {
 406         struct device_node *child;
 407         struct tegra_mc_timing *timing;
 408         int child_count = of_get_child_count(node);
 409         int i = 0, err;
 410
 411         mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing),
 412                                    GFP_KERNEL);
 413         if (!mc->timings)
 414                 return -ENOMEM;
 415
 416         mc->num_timings = child_count;
 417
 418         for_each_child_of_node(node, child) {
 419                 timing = &mc->timings[i++];
 420
 421                 err = load_one_timing(mc, timing, child);
 422                 if (err) {
 423                         of_node_put(child);
 424                         return err;
 425                 }
 426         }
 427
 428         return 0;
 429 }
 430
 431 static int tegra_mc_setup_timings(struct tegra_mc *mc)
 432 {
 433         struct device_node *node;
 434         u32 ram_code, node_ram_code;
 435         int err;
 436
 437         ram_code = tegra_read_ram_code();
 438
 439         mc->num_timings = 0;
 440
 441         for_each_child_of_node(mc->dev->of_node, node) {
 442                 err = of_property_read_u32(node, "nvidia,ram-code",
 443                                            &node_ram_code);
 444                 if (err || (node_ram_code != ram_code))
 445                         continue;
 446
 447                 err = load_timings(mc, node);
 448                 of_node_put(node);
 449                 if (err)
 450                         return err;
 451                 break;
 452         }
 453
 454         if (mc->num_timings == 0)
 455                 dev_warn(mc->dev,
 456                          "no memory timings for RAM code %u registered\n",
 457                          ram_code);
 458
 459         return 0;
 460 }
 461
 462 static const char *const status_names[32] = {
 463         [ 1] = "External interrupt",
 464         [ 6] = "EMEM address decode error",
 465         [ 7] = "GART page fault",
 466         [ 8] = "Security violation",
 467         [ 9] = "EMEM arbitration error",
 468         [10] = "Page fault",
 469         [11] = "Invalid APB ASID update",
 470         [12] = "VPR violation",
 471         [13] = "Secure carveout violation",
 472         [16] = "MTS carveout violation",
 473 };
 474
 475 static const char *const error_names[8] = {
 476         [2] = "EMEM decode error",
 477         [3] = "TrustZone violation",
 478         [4] = "Carveout violation",
 479         [6] = "SMMU translation error",
 480 };
 481
 482 static irqreturn_t tegra_mc_irq(int irq, void *data)
 483 {
 484         struct tegra_mc *mc = data;
 485         unsigned long status;
 486         unsigned int bit;
 487
 488         /* mask all interrupts to avoid flooding */
 489         status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
 490         if (!status)
 491                 return IRQ_NONE;
 492
 493         for_each_set_bit(bit, &status, 32) {
 494                 const char *error = status_names[bit] ?: "unknown";
 495                 const char *client = "unknown", *desc;
 496                 const char *direction, *secure;
 497                 phys_addr_t addr = 0;
 498                 unsigned int i;
 499                 char perm[7];
 500                 u8 id, type;
 501                 u32 value;
 502
 503                 value = mc_readl(mc, MC_ERR_STATUS);
 504
 505 #ifdef CONFIG_PHYS_ADDR_T_64BIT
 506                 if (mc->soc->num_address_bits > 32) {
 507                         addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) &
 508                                 MC_ERR_STATUS_ADR_HI_MASK);
 509                         addr <<= 32;
 510                 }
 511 #endif
 512
 513                 if (value & MC_ERR_STATUS_RW)
 514                         direction = "write";
 515                 else
 516                         direction = "read";
 517
 518                 if (value & MC_ERR_STATUS_SECURITY)
 519                         secure = "secure ";
 520                 else
 521                         secure = "";
 522
 523                 id = value & mc->soc->client_id_mask;
 524
 525                 for (i = 0; i < mc->soc->num_clients; i++) {
 526                         if (mc->soc->clients[i].id == id) {
 527                                 client = mc->soc->clients[i].name;
 528                                 break;
 529                         }
 530                 }
 531
 532                 type = (value & MC_ERR_STATUS_TYPE_MASK) >>
 533                        MC_ERR_STATUS_TYPE_SHIFT;
 534                 desc = error_names[type];
 535
 536                 switch (value & MC_ERR_STATUS_TYPE_MASK) {
 537                 case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE:
 538                         perm[0] = ' ';
 539                         perm[1] = '[';
 540
 541                         if (value & MC_ERR_STATUS_READABLE)
 542                                 perm[2] = 'R';
 543                         else
 544                                 perm[2] = '-';
 545
 546                         if (value & MC_ERR_STATUS_WRITABLE)
 547                                 perm[3] = 'W';
 548                         else
 549                                 perm[3] = '-';
 550
 551                         if (value & MC_ERR_STATUS_NONSECURE)
 552                                 perm[4] = '-';
 553                         else
 554                                 perm[4] = 'S';
 555
 556                         perm[5] = ']';
 557                         perm[6] = '\0';
 558                         break;
 559
 560                 default:
 561                         perm[0] = '\0';
 562                         break;
 563                 }
 564
 565                 value = mc_readl(mc, MC_ERR_ADR);
 566                 addr |= value;
 567
 568                 dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n",
 569                                     client, secure, direction, &addr, error,
 570                                     desc, perm);
 571         }
 572
 573         /* clear interrupts */
 574         mc_writel(mc, status, MC_INTSTATUS);
 575
 576         return IRQ_HANDLED;
 577 }
 578
 579 static __maybe_unused irqreturn_t tegra20_mc_irq(int irq, void *data)
 580 {
 581         struct tegra_mc *mc = data;
 582         unsigned long status;
 583         unsigned int bit;
 584
 585         /* mask all interrupts to avoid flooding */
 586         status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
 587         if (!status)
 588                 return IRQ_NONE;
 589
 590         for_each_set_bit(bit, &status, 32) {
 591                 const char *direction = "read", *secure = "";
 592                 const char *error = status_names[bit];
 593                 const char *client, *desc;
 594                 phys_addr_t addr;
 595                 u32 value, reg;
 596                 u8 id, type;
 597
 598                 switch (BIT(bit)) {
 599                 case MC_INT_DECERR_EMEM:
 600                         reg = MC_DECERR_EMEM_OTHERS_STATUS;
 601                         value = mc_readl(mc, reg);
 602
 603                         id = value & mc->soc->client_id_mask;
 604                         desc = error_names[2];
 605
 606                         if (value & BIT(31))
 607                                 direction = "write";
 608                         break;
 609
 610                 case MC_INT_INVALID_GART_PAGE:
 611                         reg = MC_GART_ERROR_REQ;
 612                         value = mc_readl(mc, reg);
 613
 614                         id = (value >> 1) & mc->soc->client_id_mask;
 615                         desc = error_names[2];
 616
 617                         if (value & BIT(0))
 618                                 direction = "write";
 619                         break;
 620
 621                 case MC_INT_SECURITY_VIOLATION:
 622                         reg = MC_SECURITY_VIOLATION_STATUS;
 623                         value = mc_readl(mc, reg);
 624
 625                         id = value & mc->soc->client_id_mask;
 626                         type = (value & BIT(30)) ? 4 : 3;
 627                         desc = error_names[type];
 628                         secure = "secure ";
 629
 630                         if (value & BIT(31))
 631                                 direction = "write";
 632                         break;
 633
 634                 default:
 635                         continue;
 636                 }
 637
 638                 client = mc->soc->clients[id].name;
 639                 addr = mc_readl(mc, reg + sizeof(u32));
 640
 641                 dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s)\n",
 642                                     client, secure, direction, &addr, error,
 643                                     desc);
 644         }
 645
 646         /* clear interrupts */
 647         mc_writel(mc, status, MC_INTSTATUS);
 648
 649         return IRQ_HANDLED;
 650 }
 651
 652 /*
 653  * Memory Controller (MC) has few Memory Clients that are issuing memory
 654  * bandwidth allocation requests to the MC interconnect provider. The MC
 655  * provider aggregates the requests and then sends the aggregated request
 656  * up to the External Memory Controller (EMC) interconnect provider which
 657  * re-configures hardware interface to External Memory (EMEM) in accordance
 658  * to the required bandwidth. Each MC interconnect node represents an
 659  * individual Memory Client.
 660  *
 661  * Memory interconnect topology:
 662  *
 663  *               +----+
 664  * +--------+    |    |
 665  * | TEXSRD +--->+    |
 666  * +--------+    |    |
 667  *               |    |    +-----+    +------+
 668  *    ...        | MC +--->+ EMC +--->+ EMEM |
 669  *               |    |    +-----+    +------+
 670  * +--------+    |    |
 671  * | DISP.. +--->+    |
 672  * +--------+    |    |
 673  *               +----+
 674  */
 675 static int tegra_mc_interconnect_setup(struct tegra_mc *mc)
 676 {
 677         struct icc_node *node;
 678         unsigned int i;
 679         int err;
 680
 681         /* older device-trees don't have interconnect properties */
 682         if (!device_property_present(mc->dev, "#interconnect-cells") ||
 683             !mc->soc->icc_ops)
 684                 return 0;
 685
 686         mc->provider.dev = mc->dev;
 687         mc->provider.data = &mc->provider;
 688         mc->provider.set = mc->soc->icc_ops->set;
 689         mc->provider.aggregate = mc->soc->icc_ops->aggregate;
 690         mc->provider.xlate_extended = mc->soc->icc_ops->xlate_extended;
 691
 692         err = icc_provider_add(&mc->provider);
 693         if (err)
 694                 return err;
 695
 696         /* create Memory Controller node */
 697         node = icc_node_create(TEGRA_ICC_MC);
 698         if (IS_ERR(node)) {
 699                 err = PTR_ERR(node);
 700                 goto del_provider;
 701         }
 702
 703         node->name = "Memory Controller";
 704         icc_node_add(node, &mc->provider);
 705
 706         /* link Memory Controller to External Memory Controller */
 707         err = icc_link_create(node, TEGRA_ICC_EMC);
 708         if (err)
 709                 goto remove_nodes;
 710
 711         for (i = 0; i < mc->soc->num_clients; i++) {
 712                 /* create MC client node */
 713                 node = icc_node_create(mc->soc->clients[i].id);
 714                 if (IS_ERR(node)) {
 715                         err = PTR_ERR(node);
 716                         goto remove_nodes;
 717                 }
 718
 719                 node->name = mc->soc->clients[i].name;
 720                 icc_node_add(node, &mc->provider);
 721
 722                 /* link Memory Client to Memory Controller */
 723                 err = icc_link_create(node, TEGRA_ICC_MC);
 724                 if (err)
 725                         goto remove_nodes;
 726         }
 727
 728         /*
 729          * MC driver is registered too early, so early that generic driver
 730          * syncing doesn't work for the MC. But it doesn't really matter
 731          * since syncing works for the EMC drivers, hence we can sync the
 732          * MC driver by ourselves and then EMC will complete syncing of
 733          * the whole ICC state.
 734          */
 735         icc_sync_state(mc->dev);
 736
 737         return 0;
 738
 739 remove_nodes:
 740         icc_nodes_remove(&mc->provider);
 741 del_provider:
 742         icc_provider_del(&mc->provider);
 743
 744         return err;
 745 }
 746
 747 static int tegra_mc_probe(struct platform_device *pdev)
 748 {
 749         struct resource *res;
 750         struct tegra_mc *mc;
 751         void *isr;
 752         u64 mask;
 753         int err;
 754
 755         mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);
 756         if (!mc)
 757                 return -ENOMEM;
 758
 759         platform_set_drvdata(pdev, mc);
 760         spin_lock_init(&mc->lock);
 761         mc->soc = of_device_get_match_data(&pdev->dev);
 762         mc->dev = &pdev->dev;
 763
 764         mask = DMA_BIT_MASK(mc->soc->num_address_bits);
 765
 766         err = dma_coerce_mask_and_coherent(&pdev->dev, mask);
 767         if (err < 0) {
 768                 dev_err(&pdev->dev, "failed to set DMA mask: %d\n", err);
 769                 return err;
 770         }
 771
 772         /* length of MC tick in nanoseconds */
 773         mc->tick = 30;
 774
 775         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 776         mc->regs = devm_ioremap_resource(&pdev->dev, res);
 777         if (IS_ERR(mc->regs))
 778                 return PTR_ERR(mc->regs);
 779
 780         mc->clk = devm_clk_get(&pdev->dev, "mc");
 781         if (IS_ERR(mc->clk)) {
 782                 dev_err(&pdev->dev, "failed to get MC clock: %ld\n",
 783                         PTR_ERR(mc->clk));
 784                 return PTR_ERR(mc->clk);
 785         }
 786
 787 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
 788         if (mc->soc == &tegra20_mc_soc) {
 789                 isr = tegra20_mc_irq;
 790         } else
 791 #endif
 792         {
 793                 /* ensure that debug features are disabled */
 794                 mc_writel(mc, 0x00000000, MC_TIMING_CONTROL_DBG);
 795
 796                 err = tegra_mc_setup_latency_allowance(mc);
 797                 if (err < 0) {
 798                         dev_err(&pdev->dev,
 799                                 "failed to setup latency allowance: %d\n",
 800                                 err);
 801                         return err;
 802                 }
 803
 804                 isr = tegra_mc_irq;
 805
 806                 err = tegra_mc_setup_timings(mc);
 807                 if (err < 0) {
 808                         dev_err(&pdev->dev, "failed to setup timings: %d\n",
 809                                 err);
 810                         return err;
 811                 }
 812         }
 813
 814         mc->irq = platform_get_irq(pdev, 0);
 815         if (mc->irq < 0)
 816                 return mc->irq;
 817
 818         WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n");
 819
 820         mc_writel(mc, mc->soc->intmask, MC_INTMASK);
 821
 822         err = devm_request_irq(&pdev->dev, mc->irq, isr, 0,
 823                                dev_name(&pdev->dev), mc);
 824         if (err < 0) {
 825                 dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq,
 826                         err);
 827                 return err;
 828         }
 829
 830         mc->debugfs.root = debugfs_create_dir("mc", NULL);
 831
 832         if (mc->soc->init) {
 833                 err = mc->soc->init(mc);
 834                 if (err < 0)
 835                         dev_err(&pdev->dev, "failed to initialize SoC driver: %d\n",
 836                                 err);
 837         }
 838
 839         err = tegra_mc_reset_setup(mc);
 840         if (err < 0)
 841                 dev_err(&pdev->dev, "failed to register reset controller: %d\n",
 842                         err);
 843
 844         err = tegra_mc_interconnect_setup(mc);
 845         if (err < 0)
 846                 dev_err(&pdev->dev, "failed to initialize interconnect: %d\n",
 847                         err);
 848
 849         if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) {
 850                 mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);
 851                 if (IS_ERR(mc->smmu)) {
 852                         dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",
 853                                 PTR_ERR(mc->smmu));
 854                         mc->smmu = NULL;
 855                 }
 856         }
 857
 858         if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && !mc->soc->smmu) {
 859                 mc->gart = tegra_gart_probe(&pdev->dev, mc);
 860                 if (IS_ERR(mc->gart)) {
 861                         dev_err(&pdev->dev, "failed to probe GART: %ld\n",
 862                                 PTR_ERR(mc->gart));
 863                         mc->gart = NULL;
 864                 }
 865         }
 866
 867         return 0;
 868 }
 869
 870 static int tegra_mc_suspend(struct device *dev)
 871 {
 872         struct tegra_mc *mc = dev_get_drvdata(dev);
 873         int err;
 874
 875         if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) {
 876                 err = tegra_gart_suspend(mc->gart);
 877                 if (err)
 878                         return err;
 879         }
 880
 881         return 0;
 882 }
 883
 884 static int tegra_mc_resume(struct device *dev)
 885 {
 886         struct tegra_mc *mc = dev_get_drvdata(dev);
 887         int err;
 888
 889         if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) {
 890                 err = tegra_gart_resume(mc->gart);
 891                 if (err)
 892                         return err;
 893         }
 894
 895         return 0;
 896 }
 897
 898 static const struct dev_pm_ops tegra_mc_pm_ops = {
 899         .suspend = tegra_mc_suspend,
 900         .resume = tegra_mc_resume,
 901 };
 902
 903 static struct platform_driver tegra_mc_driver = {
 904         .driver = {
 905                 .name = "tegra-mc",
 906                 .of_match_table = tegra_mc_of_match,
 907                 .pm = &tegra_mc_pm_ops,
 908                 .suppress_bind_attrs = true,
 909         },
 910         .prevent_deferred_probe = true,
 911         .probe = tegra_mc_probe,
 912 };
 913
 914 static int tegra_mc_init(void)
 915 {
 916         return platform_driver_register(&tegra_mc_driver);
 917 }
 918 arch_initcall(tegra_mc_init);
 919
 920 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
 921 MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver");
 922 MODULE_LICENSE("GPL v2");