drivers/memory/tegra/mc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2014 NVIDIA CORPORATION.  All rights reserved.
   4  */
   5
   6 #include <linux/clk.h>
   7 #include <linux/delay.h>
   8 #include <linux/dma-mapping.h>
   9 #include <linux/export.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/kernel.h>
  12 #include <linux/module.h>
  13 #include <linux/of.h>
  14 #include <linux/of_device.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/slab.h>
  17 #include <linux/sort.h>
  18
  19 #include <soc/tegra/fuse.h>
  20
  21 #include "mc.h"
  22
  23 static const struct of_device_id tegra_mc_of_match[] = {
  24 #ifdef CONFIG_ARCH_TEGRA_2x_SOC
  25         { .compatible = "nvidia,tegra20-mc-gart", .data = &tegra20_mc_soc },
  26 #endif
  27 #ifdef CONFIG_ARCH_TEGRA_3x_SOC
  28         { .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc },
  29 #endif
  30 #ifdef CONFIG_ARCH_TEGRA_114_SOC
  31         { .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc },
  32 #endif
  33 #ifdef CONFIG_ARCH_TEGRA_124_SOC
  34         { .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc },
  35 #endif
  36 #ifdef CONFIG_ARCH_TEGRA_132_SOC
  37         { .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc },
  38 #endif
  39 #ifdef CONFIG_ARCH_TEGRA_210_SOC
  40         { .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc },
  41 #endif
  42 #ifdef CONFIG_ARCH_TEGRA_186_SOC
  43         { .compatible = "nvidia,tegra186-mc", .data = &tegra186_mc_soc },
  44 #endif
  45 #ifdef CONFIG_ARCH_TEGRA_194_SOC
  46         { .compatible = "nvidia,tegra194-mc", .data = &tegra194_mc_soc },
  47 #endif
  48 #ifdef CONFIG_ARCH_TEGRA_234_SOC
  49         { .compatible = "nvidia,tegra234-mc", .data = &tegra234_mc_soc },
  50 #endif
  51         { /* sentinel */ }
  52 };
  53 MODULE_DEVICE_TABLE(of, tegra_mc_of_match);
  54
  55 static void tegra_mc_devm_action_put_device(void *data)
  56 {
  57         struct tegra_mc *mc = data;
  58
  59         put_device(mc->dev);
  60 }
  61
  62 /**
  63  * devm_tegra_memory_controller_get() - get Tegra Memory Controller handle
  64  * @dev: device pointer for the consumer device
  65  *
  66  * This function will search for the Memory Controller node in a device-tree
  67  * and retrieve the Memory Controller handle.
  68  *
  69  * Return: ERR_PTR() on error or a valid pointer to a struct tegra_mc.
  70  */
  71 struct tegra_mc *devm_tegra_memory_controller_get(struct device *dev)
  72 {
  73         struct platform_device *pdev;
  74         struct device_node *np;
  75         struct tegra_mc *mc;
  76         int err;
  77
  78         np = of_parse_phandle(dev->of_node, "nvidia,memory-controller", 0);
  79         if (!np)
  80                 return ERR_PTR(-ENOENT);
  81
  82         pdev = of_find_device_by_node(np);
  83         of_node_put(np);
  84         if (!pdev)
  85                 return ERR_PTR(-ENODEV);
  86
  87         mc = platform_get_drvdata(pdev);
  88         if (!mc) {
  89                 put_device(&pdev->dev);
  90                 return ERR_PTR(-EPROBE_DEFER);
  91         }
  92
  93         err = devm_add_action_or_reset(dev, tegra_mc_devm_action_put_device, mc);
  94         if (err)
  95                 return ERR_PTR(err);
  96
  97         return mc;
  98 }
  99 EXPORT_SYMBOL_GPL(devm_tegra_memory_controller_get);
 100
 101 int tegra_mc_probe_device(struct tegra_mc *mc, struct device *dev)
 102 {
 103         if (mc->soc->ops && mc->soc->ops->probe_device)
 104                 return mc->soc->ops->probe_device(mc, dev);
 105
 106         return 0;
 107 }
 108 EXPORT_SYMBOL_GPL(tegra_mc_probe_device);
 109
 110 static int tegra_mc_block_dma_common(struct tegra_mc *mc,
 111                                      const struct tegra_mc_reset *rst)
 112 {
 113         unsigned long flags;
 114         u32 value;
 115
 116         spin_lock_irqsave(&mc->lock, flags);
 117
 118         value = mc_readl(mc, rst->control) | BIT(rst->bit);
 119         mc_writel(mc, value, rst->control);
 120
 121         spin_unlock_irqrestore(&mc->lock, flags);
 122
 123         return 0;
 124 }
 125
 126 static bool tegra_mc_dma_idling_common(struct tegra_mc *mc,
 127                                        const struct tegra_mc_reset *rst)
 128 {
 129         return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0;
 130 }
 131
 132 static int tegra_mc_unblock_dma_common(struct tegra_mc *mc,
 133                                        const struct tegra_mc_reset *rst)
 134 {
 135         unsigned long flags;
 136         u32 value;
 137
 138         spin_lock_irqsave(&mc->lock, flags);
 139
 140         value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
 141         mc_writel(mc, value, rst->control);
 142
 143         spin_unlock_irqrestore(&mc->lock, flags);
 144
 145         return 0;
 146 }
 147
 148 static int tegra_mc_reset_status_common(struct tegra_mc *mc,
 149                                         const struct tegra_mc_reset *rst)
 150 {
 151         return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0;
 152 }
 153
 154 const struct tegra_mc_reset_ops tegra_mc_reset_ops_common = {
 155         .block_dma = tegra_mc_block_dma_common,
 156         .dma_idling = tegra_mc_dma_idling_common,
 157         .unblock_dma = tegra_mc_unblock_dma_common,
 158         .reset_status = tegra_mc_reset_status_common,
 159 };
 160
 161 static inline struct tegra_mc *reset_to_mc(struct reset_controller_dev *rcdev)
 162 {
 163         return container_of(rcdev, struct tegra_mc, reset);
 164 }
 165
 166 static const struct tegra_mc_reset *tegra_mc_reset_find(struct tegra_mc *mc,
 167                                                         unsigned long id)
 168 {
 169         unsigned int i;
 170
 171         for (i = 0; i < mc->soc->num_resets; i++)
 172                 if (mc->soc->resets[i].id == id)
 173                         return &mc->soc->resets[i];
 174
 175         return NULL;
 176 }
 177
 178 static int tegra_mc_hotreset_assert(struct reset_controller_dev *rcdev,
 179                                     unsigned long id)
 180 {
 181         struct tegra_mc *mc = reset_to_mc(rcdev);
 182         const struct tegra_mc_reset_ops *rst_ops;
 183         const struct tegra_mc_reset *rst;
 184         int retries = 500;
 185         int err;
 186
 187         rst = tegra_mc_reset_find(mc, id);
 188         if (!rst)
 189                 return -ENODEV;
 190
 191         rst_ops = mc->soc->reset_ops;
 192         if (!rst_ops)
 193                 return -ENODEV;
 194
 195         /* DMA flushing will fail if reset is already asserted */
 196         if (rst_ops->reset_status) {
 197                 /* check whether reset is asserted */
 198                 if (rst_ops->reset_status(mc, rst))
 199                         return 0;
 200         }
 201
 202         if (rst_ops->block_dma) {
 203                 /* block clients DMA requests */
 204                 err = rst_ops->block_dma(mc, rst);
 205                 if (err) {
 206                         dev_err(mc->dev, "failed to block %s DMA: %d\n",
 207                                 rst->name, err);
 208                         return err;
 209                 }
 210         }
 211
 212         if (rst_ops->dma_idling) {
 213                 /* wait for completion of the outstanding DMA requests */
 214                 while (!rst_ops->dma_idling(mc, rst)) {
 215                         if (!retries--) {
 216                                 dev_err(mc->dev, "failed to flush %s DMA\n",
 217                                         rst->name);
 218                                 return -EBUSY;
 219                         }
 220
 221                         usleep_range(10, 100);
 222                 }
 223         }
 224
 225         if (rst_ops->hotreset_assert) {
 226                 /* clear clients DMA requests sitting before arbitration */
 227                 err = rst_ops->hotreset_assert(mc, rst);
 228                 if (err) {
 229                         dev_err(mc->dev, "failed to hot reset %s: %d\n",
 230                                 rst->name, err);
 231                         return err;
 232                 }
 233         }
 234
 235         return 0;
 236 }
 237
 238 static int tegra_mc_hotreset_deassert(struct reset_controller_dev *rcdev,
 239                                       unsigned long id)
 240 {
 241         struct tegra_mc *mc = reset_to_mc(rcdev);
 242         const struct tegra_mc_reset_ops *rst_ops;
 243         const struct tegra_mc_reset *rst;
 244         int err;
 245
 246         rst = tegra_mc_reset_find(mc, id);
 247         if (!rst)
 248                 return -ENODEV;
 249
 250         rst_ops = mc->soc->reset_ops;
 251         if (!rst_ops)
 252                 return -ENODEV;
 253
 254         if (rst_ops->hotreset_deassert) {
 255                 /* take out client from hot reset */
 256                 err = rst_ops->hotreset_deassert(mc, rst);
 257                 if (err) {
 258                         dev_err(mc->dev, "failed to deassert hot reset %s: %d\n",
 259                                 rst->name, err);
 260                         return err;
 261                 }
 262         }
 263
 264         if (rst_ops->unblock_dma) {
 265                 /* allow new DMA requests to proceed to arbitration */
 266                 err = rst_ops->unblock_dma(mc, rst);
 267                 if (err) {
 268                         dev_err(mc->dev, "failed to unblock %s DMA : %d\n",
 269                                 rst->name, err);
 270                         return err;
 271                 }
 272         }
 273
 274         return 0;
 275 }
 276
 277 static int tegra_mc_hotreset_status(struct reset_controller_dev *rcdev,
 278                                     unsigned long id)
 279 {
 280         struct tegra_mc *mc = reset_to_mc(rcdev);
 281         const struct tegra_mc_reset_ops *rst_ops;
 282         const struct tegra_mc_reset *rst;
 283
 284         rst = tegra_mc_reset_find(mc, id);
 285         if (!rst)
 286                 return -ENODEV;
 287
 288         rst_ops = mc->soc->reset_ops;
 289         if (!rst_ops)
 290                 return -ENODEV;
 291
 292         return rst_ops->reset_status(mc, rst);
 293 }
 294
 295 static const struct reset_control_ops tegra_mc_reset_ops = {
 296         .assert = tegra_mc_hotreset_assert,
 297         .deassert = tegra_mc_hotreset_deassert,
 298         .status = tegra_mc_hotreset_status,
 299 };
 300
 301 static int tegra_mc_reset_setup(struct tegra_mc *mc)
 302 {
 303         int err;
 304
 305         mc->reset.ops = &tegra_mc_reset_ops;
 306         mc->reset.owner = THIS_MODULE;
 307         mc->reset.of_node = mc->dev->of_node;
 308         mc->reset.of_reset_n_cells = 1;
 309         mc->reset.nr_resets = mc->soc->num_resets;
 310
 311         err = reset_controller_register(&mc->reset);
 312         if (err < 0)
 313                 return err;
 314
 315         return 0;
 316 }
 317
 318 int tegra_mc_write_emem_configuration(struct tegra_mc *mc, unsigned long rate)
 319 {
 320         unsigned int i;
 321         struct tegra_mc_timing *timing = NULL;
 322
 323         for (i = 0; i < mc->num_timings; i++) {
 324                 if (mc->timings[i].rate == rate) {
 325                         timing = &mc->timings[i];
 326                         break;
 327                 }
 328         }
 329
 330         if (!timing) {
 331                 dev_err(mc->dev, "no memory timing registered for rate %lu\n",
 332                         rate);
 333                 return -EINVAL;
 334         }
 335
 336         for (i = 0; i < mc->soc->num_emem_regs; ++i)
 337                 mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]);
 338
 339         return 0;
 340 }
 341 EXPORT_SYMBOL_GPL(tegra_mc_write_emem_configuration);
 342
 343 unsigned int tegra_mc_get_emem_device_count(struct tegra_mc *mc)
 344 {
 345         u8 dram_count;
 346
 347         dram_count = mc_readl(mc, MC_EMEM_ADR_CFG);
 348         dram_count &= MC_EMEM_ADR_CFG_EMEM_NUMDEV;
 349         dram_count++;
 350
 351         return dram_count;
 352 }
 353 EXPORT_SYMBOL_GPL(tegra_mc_get_emem_device_count);
 354
 355 #if defined(CONFIG_ARCH_TEGRA_3x_SOC) || \
 356     defined(CONFIG_ARCH_TEGRA_114_SOC) || \
 357     defined(CONFIG_ARCH_TEGRA_124_SOC) || \
 358     defined(CONFIG_ARCH_TEGRA_132_SOC) || \
 359     defined(CONFIG_ARCH_TEGRA_210_SOC)
 360 static int tegra_mc_setup_latency_allowance(struct tegra_mc *mc)
 361 {
 362         unsigned long long tick;
 363         unsigned int i;
 364         u32 value;
 365
 366         /* compute the number of MC clock cycles per tick */
 367         tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk);
 368         do_div(tick, NSEC_PER_SEC);
 369
 370         value = mc_readl(mc, MC_EMEM_ARB_CFG);
 371         value &= ~MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE_MASK;
 372         value |= MC_EMEM_ARB_CFG_CYCLES_PER_UPDATE(tick);
 373         mc_writel(mc, value, MC_EMEM_ARB_CFG);
 374
 375         /* write latency allowance defaults */
 376         for (i = 0; i < mc->soc->num_clients; i++) {
 377                 const struct tegra_mc_client *client = &mc->soc->clients[i];
 378                 u32 value;
 379
 380                 value = mc_readl(mc, client->regs.la.reg);
 381                 value &= ~(client->regs.la.mask << client->regs.la.shift);
 382                 value |= (client->regs.la.def & client->regs.la.mask) << client->regs.la.shift;
 383                 mc_writel(mc, value, client->regs.la.reg);
 384         }
 385
 386         /* latch new values */
 387         mc_writel(mc, MC_TIMING_UPDATE, MC_TIMING_CONTROL);
 388
 389         return 0;
 390 }
 391
 392 static int load_one_timing(struct tegra_mc *mc,
 393                            struct tegra_mc_timing *timing,
 394                            struct device_node *node)
 395 {
 396         int err;
 397         u32 tmp;
 398
 399         err = of_property_read_u32(node, "clock-frequency", &tmp);
 400         if (err) {
 401                 dev_err(mc->dev,
 402                         "timing %pOFn: failed to read rate\n", node);
 403                 return err;
 404         }
 405
 406         timing->rate = tmp;
 407         timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs,
 408                                          sizeof(u32), GFP_KERNEL);
 409         if (!timing->emem_data)
 410                 return -ENOMEM;
 411
 412         err = of_property_read_u32_array(node, "nvidia,emem-configuration",
 413                                          timing->emem_data,
 414                                          mc->soc->num_emem_regs);
 415         if (err) {
 416                 dev_err(mc->dev,
 417                         "timing %pOFn: failed to read EMEM configuration\n",
 418                         node);
 419                 return err;
 420         }
 421
 422         return 0;
 423 }
 424
 425 static int load_timings(struct tegra_mc *mc, struct device_node *node)
 426 {
 427         struct device_node *child;
 428         struct tegra_mc_timing *timing;
 429         int child_count = of_get_child_count(node);
 430         int i = 0, err;
 431
 432         mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing),
 433                                    GFP_KERNEL);
 434         if (!mc->timings)
 435                 return -ENOMEM;
 436
 437         mc->num_timings = child_count;
 438
 439         for_each_child_of_node(node, child) {
 440                 timing = &mc->timings[i++];
 441
 442                 err = load_one_timing(mc, timing, child);
 443                 if (err) {
 444                         of_node_put(child);
 445                         return err;
 446                 }
 447         }
 448
 449         return 0;
 450 }
 451
 452 static int tegra_mc_setup_timings(struct tegra_mc *mc)
 453 {
 454         struct device_node *node;
 455         u32 ram_code, node_ram_code;
 456         int err;
 457
 458         ram_code = tegra_read_ram_code();
 459
 460         mc->num_timings = 0;
 461
 462         for_each_child_of_node(mc->dev->of_node, node) {
 463                 err = of_property_read_u32(node, "nvidia,ram-code",
 464                                            &node_ram_code);
 465                 if (err || (node_ram_code != ram_code))
 466                         continue;
 467
 468                 err = load_timings(mc, node);
 469                 of_node_put(node);
 470                 if (err)
 471                         return err;
 472                 break;
 473         }
 474
 475         if (mc->num_timings == 0)
 476                 dev_warn(mc->dev,
 477                          "no memory timings for RAM code %u registered\n",
 478                          ram_code);
 479
 480         return 0;
 481 }
 482
 483 int tegra30_mc_probe(struct tegra_mc *mc)
 484 {
 485         int err;
 486
 487         mc->clk = devm_clk_get_optional(mc->dev, "mc");
 488         if (IS_ERR(mc->clk)) {
 489                 dev_err(mc->dev, "failed to get MC clock: %ld\n", PTR_ERR(mc->clk));
 490                 return PTR_ERR(mc->clk);
 491         }
 492
 493         /* ensure that debug features are disabled */
 494         mc_writel(mc, 0x00000000, MC_TIMING_CONTROL_DBG);
 495
 496         err = tegra_mc_setup_latency_allowance(mc);
 497         if (err < 0) {
 498                 dev_err(mc->dev, "failed to setup latency allowance: %d\n", err);
 499                 return err;
 500         }
 501
 502         err = tegra_mc_setup_timings(mc);
 503         if (err < 0) {
 504                 dev_err(mc->dev, "failed to setup timings: %d\n", err);
 505                 return err;
 506         }
 507
 508         return 0;
 509 }
 510
 511 const struct tegra_mc_ops tegra30_mc_ops = {
 512         .probe = tegra30_mc_probe,
 513         .handle_irq = tegra30_mc_handle_irq,
 514 };
 515 #endif
 516
 517 static int mc_global_intstatus_to_channel(const struct tegra_mc *mc, u32 status,
 518                                           unsigned int *mc_channel)
 519 {
 520         if ((status & mc->soc->ch_intmask) == 0)
 521                 return -EINVAL;
 522
 523         *mc_channel = __ffs((status & mc->soc->ch_intmask) >>
 524                             mc->soc->global_intstatus_channel_shift);
 525
 526         return 0;
 527 }
 528
 529 static u32 mc_channel_to_global_intstatus(const struct tegra_mc *mc,
 530                                           unsigned int channel)
 531 {
 532         return BIT(channel) << mc->soc->global_intstatus_channel_shift;
 533 }
 534
 535 irqreturn_t tegra30_mc_handle_irq(int irq, void *data)
 536 {
 537         struct tegra_mc *mc = data;
 538         unsigned int bit, channel;
 539         unsigned long status;
 540
 541         if (mc->soc->num_channels) {
 542                 u32 global_status;
 543                 int err;
 544
 545                 global_status = mc_ch_readl(mc, MC_BROADCAST_CHANNEL, MC_GLOBAL_INTSTATUS);
 546                 err = mc_global_intstatus_to_channel(mc, global_status, &channel);
 547                 if (err < 0) {
 548                         dev_err_ratelimited(mc->dev, "unknown interrupt channel 0x%08x\n",
 549                                             global_status);
 550                         return IRQ_NONE;
 551                 }
 552
 553                 /* mask all interrupts to avoid flooding */
 554                 status = mc_ch_readl(mc, channel, MC_INTSTATUS) & mc->soc->intmask;
 555         } else {
 556                 status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
 557         }
 558
 559         if (!status)
 560                 return IRQ_NONE;
 561
 562         for_each_set_bit(bit, &status, 32) {
 563                 const char *error = tegra_mc_status_names[bit] ?: "unknown";
 564                 const char *client = "unknown", *desc;
 565                 const char *direction, *secure;
 566                 u32 status_reg, addr_reg;
 567                 u32 intmask = BIT(bit);
 568                 phys_addr_t addr = 0;
 569 #ifdef CONFIG_PHYS_ADDR_T_64BIT
 570                 u32 addr_hi_reg = 0;
 571 #endif
 572                 unsigned int i;
 573                 char perm[7];
 574                 u8 id, type;
 575                 u32 value;
 576
 577                 switch (intmask) {
 578                 case MC_INT_DECERR_VPR:
 579                         status_reg = MC_ERR_VPR_STATUS;
 580                         addr_reg = MC_ERR_VPR_ADR;
 581                         break;
 582
 583                 case MC_INT_SECERR_SEC:
 584                         status_reg = MC_ERR_SEC_STATUS;
 585                         addr_reg = MC_ERR_SEC_ADR;
 586                         break;
 587
 588                 case MC_INT_DECERR_MTS:
 589                         status_reg = MC_ERR_MTS_STATUS;
 590                         addr_reg = MC_ERR_MTS_ADR;
 591                         break;
 592
 593                 case MC_INT_DECERR_GENERALIZED_CARVEOUT:
 594                         status_reg = MC_ERR_GENERALIZED_CARVEOUT_STATUS;
 595                         addr_reg = MC_ERR_GENERALIZED_CARVEOUT_ADR;
 596                         break;
 597
 598                 case MC_INT_DECERR_ROUTE_SANITY:
 599                         status_reg = MC_ERR_ROUTE_SANITY_STATUS;
 600                         addr_reg = MC_ERR_ROUTE_SANITY_ADR;
 601                         break;
 602
 603                 default:
 604                         status_reg = MC_ERR_STATUS;
 605                         addr_reg = MC_ERR_ADR;
 606
 607 #ifdef CONFIG_PHYS_ADDR_T_64BIT
 608                         if (mc->soc->has_addr_hi_reg)
 609                                 addr_hi_reg = MC_ERR_ADR_HI;
 610 #endif
 611                         break;
 612                 }
 613
 614                 if (mc->soc->num_channels)
 615                         value = mc_ch_readl(mc, channel, status_reg);
 616                 else
 617                         value = mc_readl(mc, status_reg);
 618
 619 #ifdef CONFIG_PHYS_ADDR_T_64BIT
 620                 if (mc->soc->num_address_bits > 32) {
 621                         if (addr_hi_reg) {
 622                                 if (mc->soc->num_channels)
 623                                         addr = mc_ch_readl(mc, channel, addr_hi_reg);
 624                                 else
 625                                         addr = mc_readl(mc, addr_hi_reg);
 626                         } else {
 627                                 addr = ((value >> MC_ERR_STATUS_ADR_HI_SHIFT) &
 628                                         MC_ERR_STATUS_ADR_HI_MASK);
 629                         }
 630                         addr <<= 32;
 631                 }
 632 #endif
 633
 634                 if (value & MC_ERR_STATUS_RW)
 635                         direction = "write";
 636                 else
 637                         direction = "read";
 638
 639                 if (value & MC_ERR_STATUS_SECURITY)
 640                         secure = "secure ";
 641                 else
 642                         secure = "";
 643
 644                 id = value & mc->soc->client_id_mask;
 645
 646                 for (i = 0; i < mc->soc->num_clients; i++) {
 647                         if (mc->soc->clients[i].id == id) {
 648                                 client = mc->soc->clients[i].name;
 649                                 break;
 650                         }
 651                 }
 652
 653                 type = (value & MC_ERR_STATUS_TYPE_MASK) >>
 654                        MC_ERR_STATUS_TYPE_SHIFT;
 655                 desc = tegra_mc_error_names[type];
 656
 657                 switch (value & MC_ERR_STATUS_TYPE_MASK) {
 658                 case MC_ERR_STATUS_TYPE_INVALID_SMMU_PAGE:
 659                         perm[0] = ' ';
 660                         perm[1] = '[';
 661
 662                         if (value & MC_ERR_STATUS_READABLE)
 663                                 perm[2] = 'R';
 664                         else
 665                                 perm[2] = '-';
 666
 667                         if (value & MC_ERR_STATUS_WRITABLE)
 668                                 perm[3] = 'W';
 669                         else
 670                                 perm[3] = '-';
 671
 672                         if (value & MC_ERR_STATUS_NONSECURE)
 673                                 perm[4] = '-';
 674                         else
 675                                 perm[4] = 'S';
 676
 677                         perm[5] = ']';
 678                         perm[6] = '\0';
 679                         break;
 680
 681                 default:
 682                         perm[0] = '\0';
 683                         break;
 684                 }
 685
 686                 if (mc->soc->num_channels)
 687                         value = mc_ch_readl(mc, channel, addr_reg);
 688                 else
 689                         value = mc_readl(mc, addr_reg);
 690                 addr |= value;
 691
 692                 dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n",
 693                                     client, secure, direction, &addr, error,
 694                                     desc, perm);
 695         }
 696
 697         /* clear interrupts */
 698         if (mc->soc->num_channels) {
 699                 mc_ch_writel(mc, channel, status, MC_INTSTATUS);
 700                 mc_ch_writel(mc, MC_BROADCAST_CHANNEL,
 701                              mc_channel_to_global_intstatus(mc, channel),
 702                              MC_GLOBAL_INTSTATUS);
 703         } else {
 704                 mc_writel(mc, status, MC_INTSTATUS);
 705         }
 706
 707         return IRQ_HANDLED;
 708 }
 709
 710 const char *const tegra_mc_status_names[32] = {
 711         [ 1] = "External interrupt",
 712         [ 6] = "EMEM address decode error",
 713         [ 7] = "GART page fault",
 714         [ 8] = "Security violation",
 715         [ 9] = "EMEM arbitration error",
 716         [10] = "Page fault",
 717         [11] = "Invalid APB ASID update",
 718         [12] = "VPR violation",
 719         [13] = "Secure carveout violation",
 720         [16] = "MTS carveout violation",
 721         [17] = "Generalized carveout violation",
 722         [20] = "Route Sanity error",
 723 };
 724
 725 const char *const tegra_mc_error_names[8] = {
 726         [2] = "EMEM decode error",
 727         [3] = "TrustZone violation",
 728         [4] = "Carveout violation",
 729         [6] = "SMMU translation error",
 730 };
 731
 732 /*
 733  * Memory Controller (MC) has few Memory Clients that are issuing memory
 734  * bandwidth allocation requests to the MC interconnect provider. The MC
 735  * provider aggregates the requests and then sends the aggregated request
 736  * up to the External Memory Controller (EMC) interconnect provider which
 737  * re-configures hardware interface to External Memory (EMEM) in accordance
 738  * to the required bandwidth. Each MC interconnect node represents an
 739  * individual Memory Client.
 740  *
 741  * Memory interconnect topology:
 742  *
 743  *               +----+
 744  * +--------+    |    |
 745  * | TEXSRD +--->+    |
 746  * +--------+    |    |
 747  *               |    |    +-----+    +------+
 748  *    ...        | MC +--->+ EMC +--->+ EMEM |
 749  *               |    |    +-----+    +------+
 750  * +--------+    |    |
 751  * | DISP.. +--->+    |
 752  * +--------+    |    |
 753  *               +----+
 754  */
 755 static int tegra_mc_interconnect_setup(struct tegra_mc *mc)
 756 {
 757         struct icc_node *node;
 758         unsigned int i;
 759         int err;
 760
 761         /* older device-trees don't have interconnect properties */
 762         if (!device_property_present(mc->dev, "#interconnect-cells") ||
 763             !mc->soc->icc_ops)
 764                 return 0;
 765
 766         mc->provider.dev = mc->dev;
 767         mc->provider.data = &mc->provider;
 768         mc->provider.set = mc->soc->icc_ops->set;
 769         mc->provider.aggregate = mc->soc->icc_ops->aggregate;
 770         mc->provider.xlate_extended = mc->soc->icc_ops->xlate_extended;
 771
 772         err = icc_provider_add(&mc->provider);
 773         if (err)
 774                 return err;
 775
 776         /* create Memory Controller node */
 777         node = icc_node_create(TEGRA_ICC_MC);
 778         if (IS_ERR(node)) {
 779                 err = PTR_ERR(node);
 780                 goto del_provider;
 781         }
 782
 783         node->name = "Memory Controller";
 784         icc_node_add(node, &mc->provider);
 785
 786         /* link Memory Controller to External Memory Controller */
 787         err = icc_link_create(node, TEGRA_ICC_EMC);
 788         if (err)
 789                 goto remove_nodes;
 790
 791         for (i = 0; i < mc->soc->num_clients; i++) {
 792                 /* create MC client node */
 793                 node = icc_node_create(mc->soc->clients[i].id);
 794                 if (IS_ERR(node)) {
 795                         err = PTR_ERR(node);
 796                         goto remove_nodes;
 797                 }
 798
 799                 node->name = mc->soc->clients[i].name;
 800                 icc_node_add(node, &mc->provider);
 801
 802                 /* link Memory Client to Memory Controller */
 803                 err = icc_link_create(node, TEGRA_ICC_MC);
 804                 if (err)
 805                         goto remove_nodes;
 806         }
 807
 808         return 0;
 809
 810 remove_nodes:
 811         icc_nodes_remove(&mc->provider);
 812 del_provider:
 813         icc_provider_del(&mc->provider);
 814
 815         return err;
 816 }
 817
 818 static int tegra_mc_probe(struct platform_device *pdev)
 819 {
 820         struct tegra_mc *mc;
 821         u64 mask;
 822         int err;
 823
 824         mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);
 825         if (!mc)
 826                 return -ENOMEM;
 827
 828         platform_set_drvdata(pdev, mc);
 829         spin_lock_init(&mc->lock);
 830         mc->soc = of_device_get_match_data(&pdev->dev);
 831         mc->dev = &pdev->dev;
 832
 833         mask = DMA_BIT_MASK(mc->soc->num_address_bits);
 834
 835         err = dma_coerce_mask_and_coherent(&pdev->dev, mask);
 836         if (err < 0) {
 837                 dev_err(&pdev->dev, "failed to set DMA mask: %d\n", err);
 838                 return err;
 839         }
 840
 841         /* length of MC tick in nanoseconds */
 842         mc->tick = 30;
 843
 844         mc->regs = devm_platform_ioremap_resource(pdev, 0);
 845         if (IS_ERR(mc->regs))
 846                 return PTR_ERR(mc->regs);
 847
 848         mc->debugfs.root = debugfs_create_dir("mc", NULL);
 849
 850         if (mc->soc->ops && mc->soc->ops->probe) {
 851                 err = mc->soc->ops->probe(mc);
 852                 if (err < 0)
 853                         return err;
 854         }
 855
 856         if (mc->soc->ops && mc->soc->ops->handle_irq) {
 857                 mc->irq = platform_get_irq(pdev, 0);
 858                 if (mc->irq < 0)
 859                         return mc->irq;
 860
 861                 WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n");
 862
 863                 if (mc->soc->num_channels)
 864                         mc_ch_writel(mc, MC_BROADCAST_CHANNEL, mc->soc->intmask,
 865                                      MC_INTMASK);
 866                 else
 867                         mc_writel(mc, mc->soc->intmask, MC_INTMASK);
 868
 869                 err = devm_request_irq(&pdev->dev, mc->irq, mc->soc->ops->handle_irq, 0,
 870                                        dev_name(&pdev->dev), mc);
 871                 if (err < 0) {
 872                         dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq,
 873                                 err);
 874                         return err;
 875                 }
 876         }
 877
 878         if (mc->soc->reset_ops) {
 879                 err = tegra_mc_reset_setup(mc);
 880                 if (err < 0)
 881                         dev_err(&pdev->dev, "failed to register reset controller: %d\n", err);
 882         }
 883
 884         err = tegra_mc_interconnect_setup(mc);
 885         if (err < 0)
 886                 dev_err(&pdev->dev, "failed to initialize interconnect: %d\n",
 887                         err);
 888
 889         if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) {
 890                 mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);
 891                 if (IS_ERR(mc->smmu)) {
 892                         dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",
 893                                 PTR_ERR(mc->smmu));
 894                         mc->smmu = NULL;
 895                 }
 896         }
 897
 898         if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && !mc->soc->smmu) {
 899                 mc->gart = tegra_gart_probe(&pdev->dev, mc);
 900                 if (IS_ERR(mc->gart)) {
 901                         dev_err(&pdev->dev, "failed to probe GART: %ld\n",
 902                                 PTR_ERR(mc->gart));
 903                         mc->gart = NULL;
 904                 }
 905         }
 906
 907         return 0;
 908 }
 909
 910 static int __maybe_unused tegra_mc_suspend(struct device *dev)
 911 {
 912         struct tegra_mc *mc = dev_get_drvdata(dev);
 913
 914         if (mc->soc->ops && mc->soc->ops->suspend)
 915                 return mc->soc->ops->suspend(mc);
 916
 917         return 0;
 918 }
 919
 920 static int __maybe_unused tegra_mc_resume(struct device *dev)
 921 {
 922         struct tegra_mc *mc = dev_get_drvdata(dev);
 923
 924         if (mc->soc->ops && mc->soc->ops->resume)
 925                 return mc->soc->ops->resume(mc);
 926
 927         return 0;
 928 }
 929
 930 static void tegra_mc_sync_state(struct device *dev)
 931 {
 932         struct tegra_mc *mc = dev_get_drvdata(dev);
 933
 934         /* check whether ICC provider is registered */
 935         if (mc->provider.dev == dev)
 936                 icc_sync_state(dev);
 937 }
 938
 939 static const struct dev_pm_ops tegra_mc_pm_ops = {
 940         SET_SYSTEM_SLEEP_PM_OPS(tegra_mc_suspend, tegra_mc_resume)
 941 };
 942
 943 static struct platform_driver tegra_mc_driver = {
 944         .driver = {
 945                 .name = "tegra-mc",
 946                 .of_match_table = tegra_mc_of_match,
 947                 .pm = &tegra_mc_pm_ops,
 948                 .suppress_bind_attrs = true,
 949                 .sync_state = tegra_mc_sync_state,
 950         },
 951         .prevent_deferred_probe = true,
 952         .probe = tegra_mc_probe,
 953 };
 954
 955 static int tegra_mc_init(void)
 956 {
 957         return platform_driver_register(&tegra_mc_driver);
 958 }
 959 arch_initcall(tegra_mc_init);
 960
 961 MODULE_AUTHOR("Thierry Reding <treding@nvidia.com>");
 962 MODULE_DESCRIPTION("NVIDIA Tegra Memory Controller driver");
 963 MODULE_LICENSE("GPL v2");