1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2012 NVIDIA CORPORATION. All rights reserved.
6 #include <linux/bitfield.h>
7 #include <linux/delay.h>
8 #include <linux/mutex.h>
9 #include <linux/of_device.h>
10 #include <linux/slab.h>
11 #include <linux/string.h>
13 #include <dt-bindings/memory/tegra20-mc.h>
17 #define MC_STAT_CONTROL 0x90
18 #define MC_STAT_EMC_CLOCK_LIMIT 0xa0
19 #define MC_STAT_EMC_CLOCKS 0xa4
20 #define MC_STAT_EMC_CONTROL_0 0xa8
21 #define MC_STAT_EMC_CONTROL_1 0xac
22 #define MC_STAT_EMC_COUNT_0 0xb8
23 #define MC_STAT_EMC_COUNT_1 0xbc
25 #define MC_STAT_CONTROL_CLIENT_ID GENMASK(13, 8)
26 #define MC_STAT_CONTROL_EVENT GENMASK(23, 16)
27 #define MC_STAT_CONTROL_PRI_EVENT GENMASK(25, 24)
28 #define MC_STAT_CONTROL_FILTER_CLIENT_ENABLE GENMASK(26, 26)
29 #define MC_STAT_CONTROL_FILTER_PRI GENMASK(29, 28)
31 #define MC_STAT_CONTROL_PRI_EVENT_HP 0
32 #define MC_STAT_CONTROL_PRI_EVENT_TM 1
33 #define MC_STAT_CONTROL_PRI_EVENT_BW 2
35 #define MC_STAT_CONTROL_FILTER_PRI_DISABLE 0
36 #define MC_STAT_CONTROL_FILTER_PRI_NO 1
37 #define MC_STAT_CONTROL_FILTER_PRI_YES 2
39 #define MC_STAT_CONTROL_EVENT_QUALIFIED 0
40 #define MC_STAT_CONTROL_EVENT_ANY_READ 1
41 #define MC_STAT_CONTROL_EVENT_ANY_WRITE 2
42 #define MC_STAT_CONTROL_EVENT_RD_WR_CHANGE 3
43 #define MC_STAT_CONTROL_EVENT_SUCCESSIVE 4
44 #define MC_STAT_CONTROL_EVENT_ARB_BANK_AA 5
45 #define MC_STAT_CONTROL_EVENT_ARB_BANK_BB 6
46 #define MC_STAT_CONTROL_EVENT_PAGE_MISS 7
47 #define MC_STAT_CONTROL_EVENT_AUTO_PRECHARGE 8
49 #define EMC_GATHER_RST (0 << 8)
50 #define EMC_GATHER_CLEAR (1 << 8)
51 #define EMC_GATHER_DISABLE (2 << 8)
52 #define EMC_GATHER_ENABLE (3 << 8)
54 #define MC_STAT_SAMPLE_TIME_USEC 16000
56 /* we store collected statistics as a fixed point values */
57 #define MC_FX_FRAC_SCALE 100
59 static DEFINE_MUTEX(tegra20_mc_stat_lock);
61 struct tegra20_mc_stat_gather {
62 unsigned int pri_filter;
63 unsigned int pri_event;
70 struct tegra20_mc_stat {
71 struct tegra20_mc_stat_gather gather0;
72 struct tegra20_mc_stat_gather gather1;
73 unsigned int sample_time_usec;
74 const struct tegra_mc *mc;
77 struct tegra20_mc_client_stat {
79 unsigned int arb_high_prio;
80 unsigned int arb_timeout;
81 unsigned int arb_bandwidth;
82 unsigned int rd_wr_change;
83 unsigned int successive;
84 unsigned int page_miss;
85 unsigned int auto_precharge;
86 unsigned int arb_bank_aa;
87 unsigned int arb_bank_bb;
90 static const struct tegra_mc_client tegra20_mc_clients[] = {
102 .name = "display0bb",
108 .name = "display0cb",
114 .name = "display1bb",
138 .name = "displayhcb",
147 .name = "host1xdmar",
168 .name = "ppcsahbdmar",
171 .name = "ppcsahbslvr",
234 .name = "ppcsahbdmaw",
237 .name = "ppcsahbslvw",
250 #define TEGRA20_MC_RESET(_name, _control, _status, _reset, _bit) \
253 .id = TEGRA20_MC_RESET_##_name, \
254 .control = _control, \
260 static const struct tegra_mc_reset tegra20_mc_resets[] = {
261 TEGRA20_MC_RESET(AVPC, 0x100, 0x140, 0x104, 0),
262 TEGRA20_MC_RESET(DC, 0x100, 0x144, 0x104, 1),
263 TEGRA20_MC_RESET(DCB, 0x100, 0x148, 0x104, 2),
264 TEGRA20_MC_RESET(EPP, 0x100, 0x14c, 0x104, 3),
265 TEGRA20_MC_RESET(2D, 0x100, 0x150, 0x104, 4),
266 TEGRA20_MC_RESET(HC, 0x100, 0x154, 0x104, 5),
267 TEGRA20_MC_RESET(ISP, 0x100, 0x158, 0x104, 6),
268 TEGRA20_MC_RESET(MPCORE, 0x100, 0x15c, 0x104, 7),
269 TEGRA20_MC_RESET(MPEA, 0x100, 0x160, 0x104, 8),
270 TEGRA20_MC_RESET(MPEB, 0x100, 0x164, 0x104, 9),
271 TEGRA20_MC_RESET(MPEC, 0x100, 0x168, 0x104, 10),
272 TEGRA20_MC_RESET(3D, 0x100, 0x16c, 0x104, 11),
273 TEGRA20_MC_RESET(PPCS, 0x100, 0x170, 0x104, 12),
274 TEGRA20_MC_RESET(VDE, 0x100, 0x174, 0x104, 13),
275 TEGRA20_MC_RESET(VI, 0x100, 0x178, 0x104, 14),
278 static int tegra20_mc_hotreset_assert(struct tegra_mc *mc,
279 const struct tegra_mc_reset *rst)
284 spin_lock_irqsave(&mc->lock, flags);
286 value = mc_readl(mc, rst->reset);
287 mc_writel(mc, value & ~BIT(rst->bit), rst->reset);
289 spin_unlock_irqrestore(&mc->lock, flags);
294 static int tegra20_mc_hotreset_deassert(struct tegra_mc *mc,
295 const struct tegra_mc_reset *rst)
300 spin_lock_irqsave(&mc->lock, flags);
302 value = mc_readl(mc, rst->reset);
303 mc_writel(mc, value | BIT(rst->bit), rst->reset);
305 spin_unlock_irqrestore(&mc->lock, flags);
310 static int tegra20_mc_block_dma(struct tegra_mc *mc,
311 const struct tegra_mc_reset *rst)
316 spin_lock_irqsave(&mc->lock, flags);
318 value = mc_readl(mc, rst->control) & ~BIT(rst->bit);
319 mc_writel(mc, value, rst->control);
321 spin_unlock_irqrestore(&mc->lock, flags);
326 static bool tegra20_mc_dma_idling(struct tegra_mc *mc,
327 const struct tegra_mc_reset *rst)
329 return mc_readl(mc, rst->status) == 0;
332 static int tegra20_mc_reset_status(struct tegra_mc *mc,
333 const struct tegra_mc_reset *rst)
335 return (mc_readl(mc, rst->reset) & BIT(rst->bit)) == 0;
338 static int tegra20_mc_unblock_dma(struct tegra_mc *mc,
339 const struct tegra_mc_reset *rst)
344 spin_lock_irqsave(&mc->lock, flags);
346 value = mc_readl(mc, rst->control) | BIT(rst->bit);
347 mc_writel(mc, value, rst->control);
349 spin_unlock_irqrestore(&mc->lock, flags);
354 static const struct tegra_mc_reset_ops tegra20_mc_reset_ops = {
355 .hotreset_assert = tegra20_mc_hotreset_assert,
356 .hotreset_deassert = tegra20_mc_hotreset_deassert,
357 .block_dma = tegra20_mc_block_dma,
358 .dma_idling = tegra20_mc_dma_idling,
359 .unblock_dma = tegra20_mc_unblock_dma,
360 .reset_status = tegra20_mc_reset_status,
363 static int tegra20_mc_icc_set(struct icc_node *src, struct icc_node *dst)
366 * It should be possible to tune arbitration knobs here, but the
367 * default values are known to work well on all devices. Hence
368 * nothing to do here so far.
373 static int tegra20_mc_icc_aggreate(struct icc_node *node, u32 tag, u32 avg_bw,
374 u32 peak_bw, u32 *agg_avg, u32 *agg_peak)
377 * ISO clients need to reserve extra bandwidth up-front because
378 * there could be high bandwidth pressure during initial filling
379 * of the client's FIFO buffers. Secondly, we need to take into
380 * account impurities of the memory subsystem.
382 if (tag & TEGRA_MC_ICC_TAG_ISO)
383 peak_bw = tegra_mc_scale_percents(peak_bw, 300);
386 *agg_peak = max(*agg_peak, peak_bw);
391 static struct icc_node_data *
392 tegra20_mc_of_icc_xlate_extended(struct of_phandle_args *spec, void *data)
394 struct tegra_mc *mc = icc_provider_to_tegra_mc(data);
395 unsigned int i, idx = spec->args[0];
396 struct icc_node_data *ndata;
397 struct icc_node *node;
399 list_for_each_entry(node, &mc->provider.nodes, node_list) {
403 ndata = kzalloc(sizeof(*ndata), GFP_KERNEL);
405 return ERR_PTR(-ENOMEM);
409 /* these clients are isochronous by default */
410 if (strstarts(node->name, "display") ||
411 strstarts(node->name, "vi"))
412 ndata->tag = TEGRA_MC_ICC_TAG_ISO;
414 ndata->tag = TEGRA_MC_ICC_TAG_DEFAULT;
419 for (i = 0; i < mc->soc->num_clients; i++) {
420 if (mc->soc->clients[i].id == idx)
421 return ERR_PTR(-EPROBE_DEFER);
424 dev_err(mc->dev, "invalid ICC client ID %u\n", idx);
426 return ERR_PTR(-EINVAL);
429 static const struct tegra_mc_icc_ops tegra20_mc_icc_ops = {
430 .xlate_extended = tegra20_mc_of_icc_xlate_extended,
431 .aggregate = tegra20_mc_icc_aggreate,
432 .set = tegra20_mc_icc_set,
435 static u32 tegra20_mc_stat_gather_control(const struct tegra20_mc_stat_gather *g)
439 control = FIELD_PREP(MC_STAT_CONTROL_EVENT, g->event);
440 control |= FIELD_PREP(MC_STAT_CONTROL_CLIENT_ID, g->client);
441 control |= FIELD_PREP(MC_STAT_CONTROL_PRI_EVENT, g->pri_event);
442 control |= FIELD_PREP(MC_STAT_CONTROL_FILTER_PRI, g->pri_filter);
443 control |= FIELD_PREP(MC_STAT_CONTROL_FILTER_CLIENT_ENABLE, g->client_enb);
448 static void tegra20_mc_stat_gather(struct tegra20_mc_stat *stat)
450 u32 clocks, count0, count1, control_0, control_1;
451 const struct tegra_mc *mc = stat->mc;
453 control_0 = tegra20_mc_stat_gather_control(&stat->gather0);
454 control_1 = tegra20_mc_stat_gather_control(&stat->gather1);
457 * Reset statistic gathers state, select statistics collection mode
458 * and set clocks counter saturation limit to maximum.
460 mc_writel(mc, 0x00000000, MC_STAT_CONTROL);
461 mc_writel(mc, control_0, MC_STAT_EMC_CONTROL_0);
462 mc_writel(mc, control_1, MC_STAT_EMC_CONTROL_1);
463 mc_writel(mc, 0xffffffff, MC_STAT_EMC_CLOCK_LIMIT);
465 mc_writel(mc, EMC_GATHER_ENABLE, MC_STAT_CONTROL);
466 fsleep(stat->sample_time_usec);
467 mc_writel(mc, EMC_GATHER_DISABLE, MC_STAT_CONTROL);
469 count0 = mc_readl(mc, MC_STAT_EMC_COUNT_0);
470 count1 = mc_readl(mc, MC_STAT_EMC_COUNT_1);
471 clocks = mc_readl(mc, MC_STAT_EMC_CLOCKS);
472 clocks = max(clocks / 100 / MC_FX_FRAC_SCALE, 1u);
474 stat->gather0.result = DIV_ROUND_UP(count0, clocks);
475 stat->gather1.result = DIV_ROUND_UP(count1, clocks);
478 static void tegra20_mc_stat_events(const struct tegra_mc *mc,
479 const struct tegra_mc_client *client0,
480 const struct tegra_mc_client *client1,
481 unsigned int pri_filter,
482 unsigned int pri_event,
484 unsigned int *result0,
485 unsigned int *result1)
487 struct tegra20_mc_stat stat = {};
489 stat.gather0.client = client0 ? client0->id : 0;
490 stat.gather0.pri_filter = pri_filter;
491 stat.gather0.client_enb = !!client0;
492 stat.gather0.pri_event = pri_event;
493 stat.gather0.event = event;
495 stat.gather1.client = client1 ? client1->id : 0;
496 stat.gather1.pri_filter = pri_filter;
497 stat.gather1.client_enb = !!client1;
498 stat.gather1.pri_event = pri_event;
499 stat.gather1.event = event;
501 stat.sample_time_usec = MC_STAT_SAMPLE_TIME_USEC;
504 tegra20_mc_stat_gather(&stat);
506 *result0 = stat.gather0.result;
507 *result1 = stat.gather1.result;
510 static void tegra20_mc_collect_stats(const struct tegra_mc *mc,
511 struct tegra20_mc_client_stat *stats)
513 const struct tegra_mc_client *client0, *client1;
516 /* collect memory controller utilization percent for each client */
517 for (i = 0; i < mc->soc->num_clients; i += 2) {
518 client0 = &mc->soc->clients[i];
519 client1 = &mc->soc->clients[i + 1];
521 if (i + 1 == mc->soc->num_clients)
524 tegra20_mc_stat_events(mc, client0, client1,
525 MC_STAT_CONTROL_FILTER_PRI_DISABLE,
526 MC_STAT_CONTROL_PRI_EVENT_HP,
527 MC_STAT_CONTROL_EVENT_QUALIFIED,
528 &stats[i + 0].events,
529 &stats[i + 1].events);
532 /* collect more info from active clients */
533 for (i = 0; i < mc->soc->num_clients; i++) {
534 unsigned int clienta, clientb = mc->soc->num_clients;
536 for (client0 = NULL; i < mc->soc->num_clients; i++) {
537 if (stats[i].events) {
538 client0 = &mc->soc->clients[i];
544 for (client1 = NULL; i < mc->soc->num_clients; i++) {
545 if (stats[i].events) {
546 client1 = &mc->soc->clients[i];
552 if (!client0 && !client1)
555 tegra20_mc_stat_events(mc, client0, client1,
556 MC_STAT_CONTROL_FILTER_PRI_YES,
557 MC_STAT_CONTROL_PRI_EVENT_HP,
558 MC_STAT_CONTROL_EVENT_QUALIFIED,
559 &stats[clienta].arb_high_prio,
560 &stats[clientb].arb_high_prio);
562 tegra20_mc_stat_events(mc, client0, client1,
563 MC_STAT_CONTROL_FILTER_PRI_YES,
564 MC_STAT_CONTROL_PRI_EVENT_TM,
565 MC_STAT_CONTROL_EVENT_QUALIFIED,
566 &stats[clienta].arb_timeout,
567 &stats[clientb].arb_timeout);
569 tegra20_mc_stat_events(mc, client0, client1,
570 MC_STAT_CONTROL_FILTER_PRI_YES,
571 MC_STAT_CONTROL_PRI_EVENT_BW,
572 MC_STAT_CONTROL_EVENT_QUALIFIED,
573 &stats[clienta].arb_bandwidth,
574 &stats[clientb].arb_bandwidth);
576 tegra20_mc_stat_events(mc, client0, client1,
577 MC_STAT_CONTROL_FILTER_PRI_DISABLE,
578 MC_STAT_CONTROL_PRI_EVENT_HP,
579 MC_STAT_CONTROL_EVENT_RD_WR_CHANGE,
580 &stats[clienta].rd_wr_change,
581 &stats[clientb].rd_wr_change);
583 tegra20_mc_stat_events(mc, client0, client1,
584 MC_STAT_CONTROL_FILTER_PRI_DISABLE,
585 MC_STAT_CONTROL_PRI_EVENT_HP,
586 MC_STAT_CONTROL_EVENT_SUCCESSIVE,
587 &stats[clienta].successive,
588 &stats[clientb].successive);
590 tegra20_mc_stat_events(mc, client0, client1,
591 MC_STAT_CONTROL_FILTER_PRI_DISABLE,
592 MC_STAT_CONTROL_PRI_EVENT_HP,
593 MC_STAT_CONTROL_EVENT_PAGE_MISS,
594 &stats[clienta].page_miss,
595 &stats[clientb].page_miss);
599 static void tegra20_mc_printf_percents(struct seq_file *s,
601 unsigned int percents_fx)
603 char percents_str[8];
605 snprintf(percents_str, ARRAY_SIZE(percents_str), "%3u.%02u%%",
606 percents_fx / MC_FX_FRAC_SCALE, percents_fx % MC_FX_FRAC_SCALE);
608 seq_printf(s, fmt, percents_str);
611 static int tegra20_mc_stats_show(struct seq_file *s, void *unused)
613 const struct tegra_mc *mc = dev_get_drvdata(s->private);
614 struct tegra20_mc_client_stat *stats;
617 stats = kcalloc(mc->soc->num_clients + 1, sizeof(*stats), GFP_KERNEL);
621 mutex_lock(&tegra20_mc_stat_lock);
623 tegra20_mc_collect_stats(mc, stats);
625 mutex_unlock(&tegra20_mc_stat_lock);
627 seq_puts(s, "Memory client Events Timeout High priority Bandwidth ARB RW change Successive Page miss\n");
628 seq_puts(s, "-----------------------------------------------------------------------------------------------------\n");
630 for (i = 0; i < mc->soc->num_clients; i++) {
631 seq_printf(s, "%-14s ", mc->soc->clients[i].name);
633 /* An event is generated when client performs R/W request. */
634 tegra20_mc_printf_percents(s, "%-9s", stats[i].events);
637 * An event is generated based on the timeout (TM) signal
638 * accompanying a request for arbitration.
640 tegra20_mc_printf_percents(s, "%-10s", stats[i].arb_timeout);
643 * An event is generated based on the high-priority (HP) signal
644 * accompanying a request for arbitration.
646 tegra20_mc_printf_percents(s, "%-16s", stats[i].arb_high_prio);
649 * An event is generated based on the bandwidth (BW) signal
650 * accompanying a request for arbitration.
652 tegra20_mc_printf_percents(s, "%-16s", stats[i].arb_bandwidth);
655 * An event is generated when the memory controller switches
656 * between making a read request to making a write request.
658 tegra20_mc_printf_percents(s, "%-12s", stats[i].rd_wr_change);
661 * An even generated when the chosen client has wins arbitration
662 * when it was also the winner at the previous request. If a
663 * client makes N requests in a row that are honored, SUCCESSIVE
664 * will be counted (N-1) times. Large values for this event
665 * imply that if we were patient enough, all of those requests
666 * could have been coalesced.
668 tegra20_mc_printf_percents(s, "%-13s", stats[i].successive);
671 * An event is generated when the memory controller detects a
672 * page miss for the current request.
674 tegra20_mc_printf_percents(s, "%-12s\n", stats[i].page_miss);
682 static int tegra20_mc_probe(struct tegra_mc *mc)
684 debugfs_create_devm_seqfile(mc->dev, "stats", mc->debugfs.root,
685 tegra20_mc_stats_show);
690 static int tegra20_mc_suspend(struct tegra_mc *mc)
694 if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) {
695 err = tegra_gart_suspend(mc->gart);
703 static int tegra20_mc_resume(struct tegra_mc *mc)
707 if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && mc->gart) {
708 err = tegra_gart_resume(mc->gart);
716 static irqreturn_t tegra20_mc_handle_irq(int irq, void *data)
718 struct tegra_mc *mc = data;
719 unsigned long status;
722 /* mask all interrupts to avoid flooding */
723 status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;
727 for_each_set_bit(bit, &status, 32) {
728 const char *error = tegra_mc_status_names[bit];
729 const char *direction = "read", *secure = "";
730 const char *client, *desc;
736 case MC_INT_DECERR_EMEM:
737 reg = MC_DECERR_EMEM_OTHERS_STATUS;
738 value = mc_readl(mc, reg);
740 id = value & mc->soc->client_id_mask;
741 desc = tegra_mc_error_names[2];
747 case MC_INT_INVALID_GART_PAGE:
748 reg = MC_GART_ERROR_REQ;
749 value = mc_readl(mc, reg);
751 id = (value >> 1) & mc->soc->client_id_mask;
752 desc = tegra_mc_error_names[2];
758 case MC_INT_SECURITY_VIOLATION:
759 reg = MC_SECURITY_VIOLATION_STATUS;
760 value = mc_readl(mc, reg);
762 id = value & mc->soc->client_id_mask;
763 type = (value & BIT(30)) ? 4 : 3;
764 desc = tegra_mc_error_names[type];
775 client = mc->soc->clients[id].name;
776 addr = mc_readl(mc, reg + sizeof(u32));
778 dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s)\n",
779 client, secure, direction, &addr, error,
783 /* clear interrupts */
784 mc_writel(mc, status, MC_INTSTATUS);
789 static const struct tegra_mc_ops tegra20_mc_ops = {
790 .probe = tegra20_mc_probe,
791 .suspend = tegra20_mc_suspend,
792 .resume = tegra20_mc_resume,
793 .handle_irq = tegra20_mc_handle_irq,
796 const struct tegra_mc_soc tegra20_mc_soc = {
797 .clients = tegra20_mc_clients,
798 .num_clients = ARRAY_SIZE(tegra20_mc_clients),
799 .num_address_bits = 32,
800 .client_id_mask = 0x3f,
801 .intmask = MC_INT_SECURITY_VIOLATION | MC_INT_INVALID_GART_PAGE |
803 .reset_ops = &tegra20_mc_reset_ops,
804 .resets = tegra20_mc_resets,
805 .num_resets = ARRAY_SIZE(tegra20_mc_resets),
806 .icc_ops = &tegra20_mc_icc_ops,
807 .ops = &tegra20_mc_ops,