drivers/memory/tegra/mc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2014 NVIDIA CORPORATION.  All rights reserved.
   4  */
   5
   6 #include <linux/clk.h>
   7 #include <linux/delay.h>
   8 #include <linux/dma-mapping.h>
   9 #include <linux/export.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/kernel.h>
  12 #include <linux/module.h>
  13 #include <linux/of.h>
  14 #include <linux/of_device.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/slab.h>
  17 #include <linux/sort.h>
  18
  19 #include <soc/tegra/fuse.h>
  20
  21 #include "mc.h"
  22
  23 static const struct of_device_id tegra_mc_of_match[] = {
  24 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
  25         { .compatible = "nvidia,tegra20-mc-gart", .data = &tegra20_mc_soc },
  26 #endif
  27 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
  28         { .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc },
  29 #endif
  30 #ifdef CONFIG_ARCH_TEGRA_114_SOC
  31         { .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc },
  32 #endif
  33 #ifdef CONFIG_ARCH_TEGRA_124_SOC
  34         { .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc },
  35 #endif
  36 #ifdef CONFIG_ARCH_TEGRA_132_SOC
  37         { .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc },
  38 #endif
  39 #ifdef CONFIG_ARCH_TEGRA_210_SOC
  40         { .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc },
  41 #endif
  42 #ifdef CONFIG_ARCH_TEGRA_186_SOC
  43         { .compatible = "nvidia,tegra186-mc", .data = &tegra186_mc_soc },
  44 #endif
  45 #ifdef CONFIG_ARCH_TEGRA_194_SOC
  46         { .compatible = "nvidia,tegra194-mc", .data = &tegra194_mc_soc },
  47 #endif
  48         { /* sentinel */ }
  49 };
  50 MODULE_DEVICE_TABLE(of, tegra_mc_of_match);
  51
  52 static void tegra_mc_devm_action_put_device(void *data)
  53 {
  54         struct tegra_mc *mc = data;
  55
  56         put_device(mc->dev);
  57 }
  58
  59 /**
  60  * devm_tegra_memory_controller_get() - get Tegra Memory Controller handle
  61  * @dev: device pointer for the consumer device
  62  *
  63  * This function will search for the Memory Controller node in a device-tree
  64  * and retrieve the Memory Controller handle.
  65  *
  66  * Return: ERR_PTR() on error or a valid pointer to a struct tegra_mc.
  67  */
  68 struct tegra_mc *devm_tegra_memory_controller_get(struct device *dev)
  69 {
  70         struct platform_device *pdev;
  71         struct device_node *np;
  72         struct tegra_mc *mc;
  73         int err;
  74
  75         np = of_parse_phandle(dev->of_node, "nvidia,memory-controller", 0);
  76         if (!np)
  77                 return ERR_PTR(-ENOENT);
  78
  79         pdev = of_find_device_by_node(np);
  80         of_node_put(np);
  81         if (!pdev)
  82                 return ERR_PTR(-ENODEV);
  83
  84         mc = platform_get_drvdata(pdev);
  85         if (!mc) {
  86                 put_device(&pdev->dev);
  87                 return ERR_PTR(-EPROBE_DEFER);
  88         }
  89
  90         err = devm_add_action(dev, tegra_mc_devm_action_put_device, mc);
  91         if (err) {
  92                 put_device(mc->dev);
  93                 return ERR_PTR(err);
  94         }
  95
  96         return mc;
  97 }
  98 EXPORT_SYMBOL_GPL(devm_tegra_memory_controller_get);
  99
 100 int tegra_mc_probe_device(struct tegra_mc *mc, struct device *dev)
 101 {
 102         if (mc->soc->ops && mc->soc->ops->probe_device)
 103                 return mc->soc->ops->probe_device(mc, dev);
 104
 105         return 0;
 106 }
 107 EXPORT_SYMBOL_GPL(tegra_mc_probe_device);
 108
 109 static int tegra_mc_block_dma_common(struct tegra_mc *mc,
 110                                      const struct tegra_mc_reset *rst)
 111 {
 112         unsigned long flags;
 113         u32 value;
 114
 115         spin_lock_irqsave(&mc->lock, flags);
 116
 117         value = mc_readl(mc, rst->control) | BIT(rst->bit);
 118         mc_writel(mc, value, rst->control);
 119
 120         spin_unlock_irqrestore(&mc->lock, flags);
 121
 122         return 0;
 123 }
 124
 125 static bool tegra_mc_dma_idling_common(struct tegra_mc *mc,
 126                                        const struct tegra_mc_reset *rst)
 127 {
 128         return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0;
 129 }
 130
 131 static int tegra_mc_unblock_dma_common(struct tegra_mc *mc,
 132                                        const struct tegra_mc_reset *rst)
 133 {
 134         unsigned long flags;
 135         u32 value;
 136
 137         spin_lock_irqsave(&mc->lock, flags);
 138
 139         value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
 140         mc_writel(mc, value, rst->control);
 141
 142         spin_unlock_irqrestore(&mc->lock, flags);
 143
 144         return 0;
 145 }
 146
 147 static int tegra_mc_reset_status_common(struct tegra_mc *mc,
 148                                         const struct tegra_mc_reset *rst)
 149 {
 150         return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0;
 151 }
 152
 153 const struct tegra_mc_reset_ops tegra_mc_reset_ops_common = {
 154         .block_dma = tegra_mc_block_dma_common,
 155         .dma_idling = tegra_mc_dma_idling_common,
 156         .unblock_dma = tegra_mc_unblock_dma_common,
 157         .reset_status = tegra_mc_reset_status_common,
 158 };
 159
 160 static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev)
 161 {
 162         return container_of(rcdev, struct tegra_mc, reset);
 163 }
 164
 165 static const struct tegra_mc_reset *tegra_mc_reset_find(struct tegra_mc *mc,
 166                                                         unsigned long id)
 167 {
 168         unsigned int i;
 169
 170         for (i = 0; i < mc->soc->num_resets; i++)
 171                 if (mc->soc->resets[i].id == id)
 172                         return &mc->soc->resets[i];
 173
 174         return NULL;
 175 }
 176
 177 static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev,
 178                                     unsigned long id)
 179 {
 180         struct tegra_mc *mc = reset_to_mc(rcdev);
 181         const struct tegra_mc_reset_ops *rst_ops;
 182         const struct tegra_mc_reset *rst;
 183         int retries = 500;
 184         int err;
 185
 186         rst = tegra_mc_reset_find(mc, id);
 187         if (!rst)
 188                 return -ENODEV;
 189
 190         rst_ops = mc->soc->reset_ops;
 191         if (!rst_ops)
 192                 return -ENODEV;
 193
 194         /* DMA flushing will fail if reset is already asserted */
 195         if (rst_ops->reset_status) {
 196                 /* check whether reset is asserted */
 197                 if (rst_ops->reset_status(mc, rst))
 198                         return 0;
 199         }
 200
 201         if (rst_ops->block_dma) {
 202                 /* block clients DMA requests */
 203                 err = rst_ops->block_dma(mc, rst);
 204                 if (err) {
 205                         dev_err(mc->dev, "failed to block %s DMA: %d\n",
 206                                 rst->name, err);
 207                         return err;
 208                 }
 209         }
 210
 211         if (rst_ops->dma_idling) {
 212                 /* wait for completion of the outstanding DMA requests */
 213                 while (!rst_ops->dma_idling(mc, rst)) {
 214                         if (!retries--) {
 215                                 dev_err(mc->dev, "failed to flush %s DMA\n",
 216                                         rst->name);
 217                                 return -EBUSY;
 218                         }
 219
 220                         usleep_range(10, 100);
 221                 }
 222         }
 223
 224         if (rst_ops->hotreset_assert) {
 225                 /* clear clients DMA requests sitting before arbitration */
 226                 err = rst_ops->hotreset_assert(mc, rst);
 227                 if (err) {
 228                         dev_err(mc->dev, "failed to hot reset %s: %d\n",
 229                                 rst->name, err);
 230                         return err;
 231                 }
 232         }
 233
 234         return 0;
 235 }
 236
 237 static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev,
 238                                       unsigned long id)
 239 {
 240         struct tegra_mc *mc = reset_to_mc(rcdev);
 241         const struct tegra_mc_reset_ops *rst_ops;
 242         const struct tegra_mc_reset *rst;
 243         int err;
 244
 245         rst = tegra_mc_reset_find(mc, id);
 246         if (!rst)
 247                 return -ENODEV;
 248
 249         rst_ops = mc->soc->reset_ops;
 250         if (!rst_ops)
 251                 return -ENODEV;
 252
 253         if (rst_ops->hotreset_deassert) {
 254                 /* take out client from hot reset */
 255                 err = rst_ops->hotreset_deassert(mc, rst);
 256                 if (err) {
 257                         dev_err(mc->dev, "failed to deassert hot reset %s: %d\n",
 258                                 rst->name, err);
 259                         return err;
 260                 }
 261         }
 262
 263         if (rst_ops->unblock_dma) {
 264                 /* allow new DMA requests to proceed to arbitration */
 265                 err = rst_ops->unblock_dma(mc, rst);
 266                 if (err) {
 267                         dev_err(mc->dev, "failed to unblock %s DMA : %d\n",
 268                                 rst->name, err);
 269                         return err;
 270                 }
 271         }
 272
 273         return 0;
 274 }
 275
 276 static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev,
 277                                     unsigned long id)
 278 {
 279         struct tegra_mc *mc = reset_to_mc(rcdev);
 280         const struct tegra_mc_reset_ops *rst_ops;
 281         const struct tegra_mc_reset *rst;
 282
 283         rst = tegra_mc_reset_find(mc, id);
 284         if (!rst)
 285                 return -ENODEV;
 286
 287         rst_ops = mc->soc->reset_ops;
 288         if (!rst_ops)
 289                 return -ENODEV;
 290
 291         return rst_ops->reset_status(mc, rst);
 292 }
 293
 294 static const struct reset_control_ops tegra_mc_reset_ops = {
 295         .assert = tegra_mc_hotreset_assert,
 296         .deassert = tegra_mc_hotreset_deassert,
 297         .status = tegra_mc_hotreset_status,
 298 };
 299
 300 static int tegra_mc_reset_setup(struct tegra_mc *mc)
 301 {
 302         int err;
 303
 304         mc->reset.ops = &tegra_mc_reset_ops;
 305         mc->reset.owner = THIS_MODULE;
 306         mc->reset.of_node = mc->dev->of_node;
 307         mc->reset.of_reset_n_cells = 1;
 308         mc->reset.nr_resets = mc->soc->num_resets;
 309
 310         err = reset_controller_register(&mc->reset);
 311         if (err < 0)
 312                 return err;
 313
 314         return 0;
 315 }
 316
 317 int tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate)
 318 {
 319         unsigned int i;
 320         struct tegra_mc_timing *timing = NULL;
 321
 322         for (i = 0; i < mc->num_timings; i++) {
 323                 if (mc->timings[i].rate == rate) {
 324                         timing = &mc->timings[i];
 325                         break;
 326                 }
 327         }
 328
 329         if (!timing) {
 330                 dev_err(mc->dev, "no memory timing registered for rate %lu\n",
 331                         rate);
 332                 return -EINVAL;
 333         }
 334
 335         for (i = 0; i < mc->soc->num_emem_regs; ++i)
 336                 mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]);
 337
 338         return 0;
 339 }
 340 EXPORT_SYMBOL_GPL(tegra_mc_write_emem_configuration);
 341
 342 unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc)
 343 {
 344         u8 dram_count;
 345
 346         dram_count = mc_readl(mc, MC_EMEM_ADR_CFG);
 347         dram_count &= MC_EMEM_ADR_CFG_EMEM_NUMDEV;
 348         dram_count++;
 349
 350         return dram_count;
 351 }
 352 EXPORT_SYMBOL_GPL(tegra_mc_get_emem_device_count);
 353
 354 #if defined(CONFIG_ARCH_TEGRA_3x_SOC) || \
 355     defined(CONFIG_ARCH_TEGRA_114_SOC) || \
 356     defined(CONFIG_ARCH_TEGRA_124_SOC) || \
 357     defined(CONFIG_ARCH_TEGRA_132_SOC) || \
 358     defined(CONFIG_ARCH_TEGRA_210_SOC)
 359 static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc)
 360 {
 361         unsigned long long tick;
 362         unsigned int i;
 363         u32 value;
 364
 365         /* compute the number of MC clock cycles per tick */
 366         tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk);
 367         do_div(tick, NSEC_PER_SEC);
 368
 369         value = mc_readl(mc, MC_EMEM_ARB_CFG);
 370         value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK;
 371         value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick);
 372         mc_writel(mc, value, MC_EMEM_ARB_CFG);
 373
 374         /* write latency allowance defaults */
 375         for (i = 0; i < mc->soc->num_clients; i++) {
 376                 const struct tegra_mc_client *client = &mc->soc->clients[i];
 377                 u32 value;
 378
 379                 value = mc_readl(mc, client->regs.la.reg);
 380                 value &= ~(client->regs.la.mask << client->regs.la.shift);
 381                 value |= (client->regs.la.def & client->regs.la.mask) << client->regs.la.shift;
 382                 mc_writel(mc, value, client->regs.la.reg);
 383         }
 384
 385         /* latch new values */
 386         mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL);
 387
 388         return 0;
 389 }
 390
 391 static int load_one_timing(struct tegra_mc *mc,
 392                            struct tegra_mc_timing *timing,
 393                            struct device_node *node)
 394 {
 395         int err;
 396         u32 tmp;
 397
 398         err = of_property_read_u32(node, "clock-frequency", &tmp);
 399         if (err) {
 400                 dev_err(mc->dev,
 401                         "timing %pOFn: failed to read rate\n", node);
 402                 return err;
 403         }
 404
 405         timing->rate = tmp;
 406         timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs,
 407                                          sizeof(u32), GFP_KERNEL);
 408         if (!timing->emem_data)
 409                 return -ENOMEM;
 410
 411         err = of_property_read_u32_array(node, "nvidia,emem-configuration",
 412                                          timing->emem_data,
 413                                          mc->soc->num_emem_regs);
 414         if (err) {
 415                 dev_err(mc->dev,
 416                         "timing %pOFn: failed to read EMEM configuration\n",
 417                         node);
 418                 return err;
 419         }
 420
 421         return 0;
 422 }
 423
 424 static int load_timings(struct tegra_mc *mc, struct device_node *node)
 425 {
 426         struct device_node *child;
 427         struct tegra_mc_timing *timing;
 428         int child_count = of_get_child_count(node);
 429         int i = 0, err;
 430
 431         mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing),
 432                                    GFP_KERNEL);
 433         if (!mc->timings)
 434                 return -ENOMEM;
 435
 436         mc->num_timings = child_count;
 437
 438         for_each_child_of_node(node, child) {
 439                 timing = &mc->timings[i++];
 440
 441                 err = load_one_timing(mc, timing, child);
 442                 if (err) {
 443                         of_node_put(child);
 444                         return err;
 445                 }
 446         }
 447
 448         return 0;
 449 }
 450
 451 static int tegra_mc_setup_timings(struct tegra_mc *mc)
 452 {
 453         struct device_node *node;
 454         u32 ram_code, node_ram_code;
 455         int err;
 456
 457         ram_code = tegra_read_ram_code();
 458
 459         mc->num_timings = 0;
 460
 461         for_each_child_of_node(mc->dev->of_node, node) {
 462                 err = of_property_read_u32(node, "nvidia,ram-code",
 463                                            &node_ram_code);
 464                 if (err || (node_ram_code != ram_code))
 465                         continue;
 466
 467                 err = load_timings(mc, node);
 468                 of_node_put(node);
 469                 if (err)
 470                         return err;
 471                 break;
 472         }
 473
 474         if (mc->num_timings == 0)
 475                 dev_warn(mc->dev,
 476                          "no memory timings for RAM code %u registered\n",
 477                          ram_code);
 478
 479         return 0;
 480 }
 481
 482 int tegra30_mc_probe(struct tegra_mc *mc)
 483 {
 484         int err;
 485
 486         mc->clk = devm_clk_get_optional(mc->dev, "mc");
 487         if (IS_ERR(mc->clk)) {
 488                 dev_err(mc->dev, "failed to get MC clock: %ld\n", PTR_ERR(mc->clk));
 489                 return PTR_ERR(mc->clk);
 490         }
 491
 492         /* ensure that debug features are disabled */
 493         mc_writel(mc, 0x00000000, MC_TIMING_CONTROL_DBG);
 494
 495         err = tegra_mc_setup_latency_allowance(mc);
 496         if (err < 0) {
 497                 dev_err(mc->dev, "failed to setup latency allowance: %d\n", err);
 498                 return err;
 499         }
 500
 501         err = tegra_mc_setup_timings(mc);
 502         if (err < 0) {
 503                 dev_err(mc->dev, "failed to setup timings: %d\n", err);
 504                 return err;
 505         }
 506
 507         return 0;
 508 }
 509
 510 static irqreturn_t tegra30_mc_handle_irq(int irq, void *data)
 511 {
 512         struct tegra_mc *mc = data;
 513         unsigned long status;
 514         unsigned int bit;
 515
 516         /* mask all interrupts to avoid flooding */
 517         status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
 518         if (!status)
 519                 return IRQ_NONE;
 520
 521         for_each_set_bit(bit, &status, 32) {
 522                 const char *error = tegra_mc_status_names[bit] ?: "unknown";
 523                 const char *client = "unknown", *desc;
 524                 const char *direction, *secure;
 525                 phys_addr_t addr = 0;
 526                 unsigned int i;
 527                 char perm[7];
 528                 u8 id, type;
 529                 u32 value;
 530
 531                 value = mc_readl(mc, MC_ERR_STATUS);
 532
 533 #ifdef CONFIG_PHYS_ADDR_T_64BIT
 534                 if (mc->soc->num_address_bits > 32) {
 535                         addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) &
 536                                 MC_ERR_STATUS_ADR_HI_MASK);
 537                         addr <<= 32;
 538                 }
 539 #endif
 540
 541                 if (value & MC_ERR_STATUS_RW)
 542                         direction = "write";
 543                 else
 544                         direction = "read";
 545
 546                 if (value & MC_ERR_STATUS_SECURITY)
 547                         secure = "secure ";
 548                 else
 549                         secure = "";
 550
 551                 id = value & mc->soc->client_id_mask;
 552
 553                 for (i = 0; i < mc->soc->num_clients; i++) {
 554                         if (mc->soc->clients[i].id == id) {
 555                                 client = mc->soc->clients[i].name;
 556                                 break;
 557                         }
 558                 }
 559
 560                 type = (value & MC_ERR_STATUS_TYPE_MASK) >>
 561                        MC_ERR_STATUS_TYPE_SHIFT;
 562                 desc = tegra_mc_error_names[type];
 563
 564                 switch (value & MC_ERR_STATUS_TYPE_MASK) {
 565                 case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE:
 566                         perm[0] = ' ';
 567                         perm[1] = '[';
 568
 569                         if (value & MC_ERR_STATUS_READABLE)
 570                                 perm[2] = 'R';
 571                         else
 572                                 perm[2] = '-';
 573
 574                         if (value & MC_ERR_STATUS_WRITABLE)
 575                                 perm[3] = 'W';
 576                         else
 577                                 perm[3] = '-';
 578
 579                         if (value & MC_ERR_STATUS_NONSECURE)
 580                                 perm[4] = '-';
 581                         else
 582                                 perm[4] = 'S';
 583
 584                         perm[5] = ']';
 585                         perm[6] = '\0';
 586                         break;
 587
 588                 default:
 589                         perm[0] = '\0';
 590                         break;
 591                 }
 592
 593                 value = mc_readl(mc, MC_ERR_ADR);
 594                 addr |= value;
 595
 596                 dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n",
 597                                     client, secure, direction, &addr, error,
 598                                     desc, perm);
 599         }
 600
 601         /* clear interrupts */
 602         mc_writel(mc, status, MC_INTSTATUS);
 603
 604         return IRQ_HANDLED;
 605 }
 606
 607 const struct tegra_mc_ops tegra30_mc_ops = {
 608         .probe = tegra30_mc_probe,
 609         .handle_irq = tegra30_mc_handle_irq,
 610 };
 611 #endif
 612
 613 const char *const tegra_mc_status_names[32] = {
 614         [ 1] = "External interrupt",
 615         [ 6] = "EMEM address decode error",
 616         [ 7] = "GART page fault",
 617         [ 8] = "Security violation",
 618         [ 9] = "EMEM arbitration error",
 619         [10] = "Page fault",
 620         [11] = "Invalid APB ASID update",
 621         [12] = "VPR violation",
 622         [13] = "Secure carveout violation",
 623         [16] = "MTS carveout violation",
 624 };
 625
 626 const char *const tegra_mc_error_names[8] = {
 627         [2] = "EMEM decode error",
 628         [3] = "TrustZone violation",
 629         [4] = "Carveout violation",
 630         [6] = "SMMU translation error",
 631 };
 632
 633 /*
 634  * Memory Controller (MC) has few Memory Clients that are issuing memory
 635  * bandwidth allocation requests to the MC interconnect provider. The MC
 636  * provider aggregates the requests and then sends the aggregated request
 637  * up to the External Memory Controller (EMC) interconnect provider which
 638  * re-configures hardware interface to External Memory (EMEM) in accordance
 639  * to the required bandwidth. Each MC interconnect node represents an
 640  * individual Memory Client.
 641  *
 642  * Memory interconnect topology:
 643  *
 644  *               +----+
 645  * +--------+    |    |
 646  * | TEXSRD +--->+    |
 647  * +--------+    |    |
 648  *               |    |    +-----+    +------+
 649  *    ...        | MC +--->+ EMC +--->+ EMEM |
 650  *               |    |    +-----+    +------+
 651  * +--------+    |    |
 652  * | DISP.. +--->+    |
 653  * +--------+    |    |
 654  *               +----+
 655  */
 656 static int tegra_mc_interconnect_setup(struct tegra_mc *mc)
 657 {
 658         struct icc_node *node;
 659         unsigned int i;
 660         int err;
 661
 662         /* older device-trees don't have interconnect properties */
 663         if (!device_property_present(mc->dev, "#interconnect-cells") ||
 664             !mc->soc->icc_ops)
 665                 return 0;
 666
 667         mc->provider.dev = mc->dev;
 668         mc->provider.data = &mc->provider;
 669         mc->provider.set = mc->soc->icc_ops->set;
 670         mc->provider.aggregate = mc->soc->icc_ops->aggregate;
 671         mc->provider.xlate_extended = mc->soc->icc_ops->xlate_extended;
 672
 673         err = icc_provider_add(&mc->provider);
 674         if (err)
 675                 return err;
 676
 677         /* create Memory Controller node */
 678         node = icc_node_create(TEGRA_ICC_MC);
 679         if (IS_ERR(node)) {
 680                 err = PTR_ERR(node);
 681                 goto del_provider;
 682         }
 683
 684         node->name = "Memory Controller";
 685         icc_node_add(node, &mc->provider);
 686
 687         /* link Memory Controller to External Memory Controller */
 688         err = icc_link_create(node, TEGRA_ICC_EMC);
 689         if (err)
 690                 goto remove_nodes;
 691
 692         for (i = 0; i < mc->soc->num_clients; i++) {
 693                 /* create MC client node */
 694                 node = icc_node_create(mc->soc->clients[i].id);
 695                 if (IS_ERR(node)) {
 696                         err = PTR_ERR(node);
 697                         goto remove_nodes;
 698                 }
 699
 700                 node->name = mc->soc->clients[i].name;
 701                 icc_node_add(node, &mc->provider);
 702
 703                 /* link Memory Client to Memory Controller */
 704                 err = icc_link_create(node, TEGRA_ICC_MC);
 705                 if (err)
 706                         goto remove_nodes;
 707         }
 708
 709         /*
 710          * MC driver is registered too early, so early that generic driver
 711          * syncing doesn't work for the MC. But it doesn't really matter
 712          * since syncing works for the EMC drivers, hence we can sync the
 713          * MC driver by ourselves and then EMC will complete syncing of
 714          * the whole ICC state.
 715          */
 716         icc_sync_state(mc->dev);
 717
 718         return 0;
 719
 720 remove_nodes:
 721         icc_nodes_remove(&mc->provider);
 722 del_provider:
 723         icc_provider_del(&mc->provider);
 724
 725         return err;
 726 }
 727
 728 static int tegra_mc_probe(struct platform_device *pdev)
 729 {
 730         struct resource *res;
 731         struct tegra_mc *mc;
 732         u64 mask;
 733         int err;
 734
 735         mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);
 736         if (!mc)
 737                 return -ENOMEM;
 738
 739         platform_set_drvdata(pdev, mc);
 740         spin_lock_init(&mc->lock);
 741         mc->soc = of_device_get_match_data(&pdev->dev);
 742         mc->dev = &pdev->dev;
 743
 744         mask = DMA_BIT_MASK(mc->soc->num_address_bits);
 745
 746         err = dma_coerce_mask_and_coherent(&pdev->dev, mask);
 747         if (err < 0) {
 748                 dev_err(&pdev->dev, "failed to set DMA mask: %d\n", err);
 749                 return err;
 750         }
 751
 752         /* length of MC tick in nanoseconds */
 753         mc->tick = 30;
 754
 755         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 756         mc->regs = devm_ioremap_resource(&pdev->dev, res);
 757         if (IS_ERR(mc->regs))
 758                 return PTR_ERR(mc->regs);
 759
 760         mc->debugfs.root = debugfs_create_dir("mc", NULL);
 761
 762         if (mc->soc->ops && mc->soc->ops->probe) {
 763                 err = mc->soc->ops->probe(mc);
 764                 if (err < 0)
 765                         return err;
 766         }
 767
 768         if (mc->soc->ops && mc->soc->ops->handle_irq) {
 769                 mc->irq = platform_get_irq(pdev, 0);
 770                 if (mc->irq < 0)
 771                         return mc->irq;
 772
 773                 WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n");
 774
 775                 mc_writel(mc, mc->soc->intmask, MC_INTMASK);
 776
 777                 err = devm_request_irq(&pdev->dev, mc->irq, mc->soc->ops->handle_irq, 0,
 778                                        dev_name(&pdev->dev), mc);
 779                 if (err < 0) {
 780                         dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq,
 781                                 err);
 782                         return err;
 783                 }
 784         }
 785
 786         if (mc->soc->reset_ops) {
 787                 err = tegra_mc_reset_setup(mc);
 788                 if (err < 0)
 789                         dev_err(&pdev->dev, "failed to register reset controller: %d\n", err);
 790         }
 791
 792         err = tegra_mc_interconnect_setup(mc);
 793         if (err < 0)
 794                 dev_err(&pdev->dev, "failed to initialize interconnect: %d\n",
 795                         err);
 796
 797         if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) {
 798                 mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);
 799                 if (IS_ERR(mc->smmu)) {
 800                         dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",
 801                                 PTR_ERR(mc->smmu));
 802                         mc->smmu = NULL;
 803                 }
 804         }
 805
 806         if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && !mc->soc->smmu) {
 807                 mc->gart = tegra_gart_probe(&pdev->dev, mc);
 808                 if (IS_ERR(mc->gart)) {
 809                         dev_err(&pdev->dev, "failed to probe GART: %ld\n",
 810                                 PTR_ERR(mc->gart));
 811                         mc->gart = NULL;
 812                 }
 813         }
 814
 815         return 0;
 816 }
 817
 818 static int __maybe_unused tegra_mc_suspend(struct device *dev)
 819 {
 820         struct tegra_mc *mc = dev_get_drvdata(dev);
 821
 822         if (mc->soc->ops && mc->soc->ops->suspend)
 823                 return mc->soc->ops->suspend(mc);
 824
 825         return 0;
 826 }
 827
 828 static int __maybe_unused tegra_mc_resume(struct device *dev)
 829 {
 830         struct tegra_mc *mc = dev_get_drvdata(dev);
 831
 832         if (mc->soc->ops && mc->soc->ops->resume)
 833                 return mc->soc->ops->resume(mc);
 834
 835         return 0;
 836 }
 837
 838 static const struct dev_pm_ops tegra_mc_pm_ops = {
 839         SET_SYSTEM_SLEEP_PM_OPS(tegra_mc_suspend, tegra_mc_resume)
 840 };
 841
 842 static struct platform_driver tegra_mc_driver = {
 843         .driver = {
 844                 .name = "tegra-mc",
 845                 .of_match_table = tegra_mc_of_match,
 846                 .pm = &tegra_mc_pm_ops,
 847                 .suppress_bind_attrs = true,
 848         },
 849         .prevent_deferred_probe = true,
 850         .probe = tegra_mc_probe,
 851 };
 852
 853 static int tegra_mc_init(void)
 854 {
 855         return platform_driver_register(&tegra_mc_driver);
 856 }
 857 arch_initcall(tegra_mc_init);
 858
 859 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
 860 MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver");
 861 MODULE_LICENSE("GPL v2");