drivers/perf/fsl_imx9_ddr_perf.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright 2023 NXP
   3
   4 #include <linux/bitfield.h>
   5 #include <linux/init.h>
   6 #include <linux/interrupt.h>
   7 #include <linux/io.h>
   8 #include <linux/module.h>
   9 #include <linux/of.h>
  10 #include <linux/platform_device.h>
  11 #include <linux/perf_event.h>
  12
  13 /* Performance monitor configuration */
  14 #define PMCFG1                          0x00
  15 #define PMCFG1_RD_TRANS_FILT_EN         BIT(31)
  16 #define PMCFG1_WR_TRANS_FILT_EN         BIT(30)
  17 #define PMCFG1_RD_BT_FILT_EN            BIT(29)
  18 #define PMCFG1_ID_MASK                  GENMASK(17, 0)
  19
  20 #define PMCFG2                          0x04
  21 #define PMCFG2_ID                       GENMASK(17, 0)
  22
  23 /* Global control register affects all counters and takes priority over local control registers */
  24 #define PMGC0           0x40
  25 /* Global control register bits */
  26 #define PMGC0_FAC       BIT(31)
  27 #define PMGC0_PMIE      BIT(30)
  28 #define PMGC0_FCECE     BIT(29)
  29
  30 /*
  31  * 64bit counter0 exclusively dedicated to counting cycles
  32  * 32bit counters monitor counter-specific events in addition to counting reference events
  33  */
  34 #define PMLCA(n)        (0x40 + 0x10 + (0x10 * n))
  35 #define PMLCB(n)        (0x40 + 0x14 + (0x10 * n))
  36 #define PMC(n)          (0x40 + 0x18 + (0x10 * n))
  37 /* Local control register bits */
  38 #define PMLCA_FC        BIT(31)
  39 #define PMLCA_CE        BIT(26)
  40 #define PMLCA_EVENT     GENMASK(22, 16)
  41
  42 #define NUM_COUNTERS            11
  43 #define CYCLES_COUNTER          0
  44
  45 #define to_ddr_pmu(p)           container_of(p, struct ddr_pmu, pmu)
  46
  47 #define DDR_PERF_DEV_NAME       "imx9_ddr"
  48 #define DDR_CPUHP_CB_NAME       DDR_PERF_DEV_NAME "_perf_pmu"
  49
  50 static DEFINE_IDA(ddr_ida);
  51
  52 struct imx_ddr_devtype_data {
  53         const char *identifier;         /* system PMU identifier for userspace */
  54 };
  55
  56 struct ddr_pmu {
  57         struct pmu pmu;
  58         void __iomem *base;
  59         unsigned int cpu;
  60         struct hlist_node node;
  61         struct device *dev;
  62         struct perf_event *events[NUM_COUNTERS];
  63         int active_events;
  64         enum cpuhp_state cpuhp_state;
  65         const struct imx_ddr_devtype_data *devtype_data;
  66         int irq;
  67         int id;
  68 };
  69
  70 static const struct imx_ddr_devtype_data imx93_devtype_data = {
  71         .identifier = "imx93",
  72 };
  73
  74 static const struct of_device_id imx_ddr_pmu_dt_ids[] = {
  75         {.compatible = "fsl,imx93-ddr-pmu", .data = &imx93_devtype_data},
  76         { /* sentinel */ }
  77 };
  78 MODULE_DEVICE_TABLE(of, imx_ddr_pmu_dt_ids);
  79
  80 static ssize_t ddr_perf_identifier_show(struct device *dev,
  81                                         struct device_attribute *attr,
  82                                         char *page)
  83 {
  84         struct ddr_pmu *pmu = dev_get_drvdata(dev);
  85
  86         return sysfs_emit(page, "%s\n", pmu->devtype_data->identifier);
  87 }
  88
  89 static struct device_attribute ddr_perf_identifier_attr =
  90         __ATTR(identifier, 0444, ddr_perf_identifier_show, NULL);
  91
  92 static struct attribute *ddr_perf_identifier_attrs[] = {
  93         &ddr_perf_identifier_attr.attr,
  94         NULL,
  95 };
  96
  97 static struct attribute_group ddr_perf_identifier_attr_group = {
  98         .attrs = ddr_perf_identifier_attrs,
  99 };
 100
 101 static ssize_t ddr_perf_cpumask_show(struct device *dev,
 102                                      struct device_attribute *attr, char *buf)
 103 {
 104         struct ddr_pmu *pmu = dev_get_drvdata(dev);
 105
 106         return cpumap_print_to_pagebuf(true, buf, cpumask_of(pmu->cpu));
 107 }
 108
 109 static struct device_attribute ddr_perf_cpumask_attr =
 110         __ATTR(cpumask, 0444, ddr_perf_cpumask_show, NULL);
 111
 112 static struct attribute *ddr_perf_cpumask_attrs[] = {
 113         &ddr_perf_cpumask_attr.attr,
 114         NULL,
 115 };
 116
 117 static const struct attribute_group ddr_perf_cpumask_attr_group = {
 118         .attrs = ddr_perf_cpumask_attrs,
 119 };
 120
 121 static ssize_t ddr_pmu_event_show(struct device *dev,
 122                                   struct device_attribute *attr, char *page)
 123 {
 124         struct perf_pmu_events_attr *pmu_attr;
 125
 126         pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
 127         return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
 128 }
 129
 130 #define IMX9_DDR_PMU_EVENT_ATTR(_name, _id)                             \
 131         (&((struct perf_pmu_events_attr[]) {                            \
 132                 { .attr = __ATTR(_name, 0444, ddr_pmu_event_show, NULL),\
 133                   .id = _id, }                                          \
 134         })[0].attr.attr)
 135
 136 static struct attribute *ddr_perf_events_attrs[] = {
 137         /* counter0 cycles event */
 138         IMX9_DDR_PMU_EVENT_ATTR(cycles, 0),
 139
 140         /* reference events for all normal counters, need assert DEBUG19[21] bit */
 141         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ddrc1_rmw_for_ecc, 12),
 142         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_rreorder, 13),
 143         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_wreorder, 14),
 144         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_0, 15),
 145         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_1, 16),
 146         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_2, 17),
 147         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_3, 18),
 148         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_4, 19),
 149         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_5, 22),
 150         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_6, 23),
 151         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_7, 24),
 152         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_8, 25),
 153         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_9, 26),
 154         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_10, 27),
 155         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_11, 28),
 156         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_12, 31),
 157         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_13, 59),
 158         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_15, 61),
 159         IMX9_DDR_PMU_EVENT_ATTR(ddrc_pm_29, 63),
 160
 161         /* counter1 specific events */
 162         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_0, 64),
 163         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_1, 65),
 164         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_2, 66),
 165         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_3, 67),
 166         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_4, 68),
 167         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_5, 69),
 168         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_6, 70),
 169         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_riq_7, 71),
 170
 171         /* counter2 specific events */
 172         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_0, 64),
 173         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_1, 65),
 174         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_2, 66),
 175         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_3, 67),
 176         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_4, 68),
 177         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_5, 69),
 178         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_6, 70),
 179         IMX9_DDR_PMU_EVENT_ATTR(ddrc_ld_wiq_7, 71),
 180         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_empty, 72),
 181         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pm_rd_trans_filt, 73),
 182
 183         /* counter3 specific events */
 184         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_0, 64),
 185         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_1, 65),
 186         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_2, 66),
 187         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_3, 67),
 188         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_4, 68),
 189         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_5, 69),
 190         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_6, 70),
 191         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_collision_7, 71),
 192         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_full, 72),
 193         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pm_wr_trans_filt, 73),
 194
 195         /* counter4 specific events */
 196         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_0, 64),
 197         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_1, 65),
 198         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_2, 66),
 199         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_3, 67),
 200         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_4, 68),
 201         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_5, 69),
 202         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_6, 70),
 203         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_row_open_7, 71),
 204         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq2_rmw, 72),
 205         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pm_rd_beat_filt, 73),
 206
 207         /* counter5 specific events */
 208         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_0, 64),
 209         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_1, 65),
 210         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_2, 66),
 211         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_3, 67),
 212         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_4, 68),
 213         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_5, 69),
 214         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_6, 70),
 215         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_start_7, 71),
 216         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq1, 72),
 217
 218         /* counter6 specific events */
 219         IMX9_DDR_PMU_EVENT_ATTR(ddrc_qx_valid_end_0, 64),
 220         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq2, 72),
 221
 222         /* counter7 specific events */
 223         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_1_2_full, 64),
 224         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_wrq0, 65),
 225
 226         /* counter8 specific events */
 227         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_bias_switched, 64),
 228         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_1_4_full, 65),
 229
 230         /* counter9 specific events */
 231         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_wrq1, 65),
 232         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_3_4_full, 66),
 233
 234         /* counter10 specific events */
 235         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_misc_mrk, 65),
 236         IMX9_DDR_PMU_EVENT_ATTR(eddrtq_pmon_ld_rdq0, 66),
 237         NULL,
 238 };
 239
 240 static const struct attribute_group ddr_perf_events_attr_group = {
 241         .name = "events",
 242         .attrs = ddr_perf_events_attrs,
 243 };
 244
 245 PMU_FORMAT_ATTR(event, "config:0-7");
 246 PMU_FORMAT_ATTR(counter, "config:8-15");
 247 PMU_FORMAT_ATTR(axi_id, "config1:0-17");
 248 PMU_FORMAT_ATTR(axi_mask, "config2:0-17");
 249
 250 static struct attribute *ddr_perf_format_attrs[] = {
 251         &format_attr_event.attr,
 252         &format_attr_counter.attr,
 253         &format_attr_axi_id.attr,
 254         &format_attr_axi_mask.attr,
 255         NULL,
 256 };
 257
 258 static const struct attribute_group ddr_perf_format_attr_group = {
 259         .name = "format",
 260         .attrs = ddr_perf_format_attrs,
 261 };
 262
 263 static const struct attribute_group *attr_groups[] = {
 264         &ddr_perf_identifier_attr_group,
 265         &ddr_perf_cpumask_attr_group,
 266         &ddr_perf_events_attr_group,
 267         &ddr_perf_format_attr_group,
 268         NULL,
 269 };
 270
 271 static void ddr_perf_clear_counter(struct ddr_pmu *pmu, int counter)
 272 {
 273         if (counter == CYCLES_COUNTER) {
 274                 writel(0, pmu->base + PMC(counter) + 0x4);
 275                 writel(0, pmu->base + PMC(counter));
 276         } else {
 277                 writel(0, pmu->base + PMC(counter));
 278         }
 279 }
 280
 281 static u64 ddr_perf_read_counter(struct ddr_pmu *pmu, int counter)
 282 {
 283         u32 val_lower, val_upper;
 284         u64 val;
 285
 286         if (counter != CYCLES_COUNTER) {
 287                 val = readl_relaxed(pmu->base + PMC(counter));
 288                 goto out;
 289         }
 290
 291         /* special handling for reading 64bit cycle counter */
 292         do {
 293                 val_upper = readl_relaxed(pmu->base + PMC(counter) + 0x4);
 294                 val_lower = readl_relaxed(pmu->base + PMC(counter));
 295         } while (val_upper != readl_relaxed(pmu->base + PMC(counter) + 0x4));
 296
 297         val = val_upper;
 298         val = (val << 32);
 299         val |= val_lower;
 300 out:
 301         return val;
 302 }
 303
 304 static void ddr_perf_counter_global_config(struct ddr_pmu *pmu, bool enable)
 305 {
 306         u32 ctrl;
 307
 308         ctrl = readl_relaxed(pmu->base + PMGC0);
 309
 310         if (enable) {
 311                 /*
 312                  * The performance monitor must be reset before event counting
 313                  * sequences. The performance monitor can be reset by first freezing
 314                  * one or more counters and then clearing the freeze condition to
 315                  * allow the counters to count according to the settings in the
 316                  * performance monitor registers. Counters can be frozen individually
 317                  * by setting PMLCAn[FC] bits, or simultaneously by setting PMGC0[FAC].
 318                  * Simply clearing these freeze bits will then allow the performance
 319                  * monitor to begin counting based on the register settings.
 320                  */
 321                 ctrl |= PMGC0_FAC;
 322                 writel(ctrl, pmu->base + PMGC0);
 323
 324                 /*
 325                  * Freeze all counters disabled, interrupt enabled, and freeze
 326                  * counters on condition enabled.
 327                  */
 328                 ctrl &= ~PMGC0_FAC;
 329                 ctrl |= PMGC0_PMIE | PMGC0_FCECE;
 330                 writel(ctrl, pmu->base + PMGC0);
 331         } else {
 332                 ctrl |= PMGC0_FAC;
 333                 ctrl &= ~(PMGC0_PMIE | PMGC0_FCECE);
 334                 writel(ctrl, pmu->base + PMGC0);
 335         }
 336 }
 337
 338 static void ddr_perf_counter_local_config(struct ddr_pmu *pmu, int config,
 339                                     int counter, bool enable)
 340 {
 341         u32 ctrl_a;
 342
 343         ctrl_a = readl_relaxed(pmu->base + PMLCA(counter));
 344
 345         if (enable) {
 346                 ctrl_a |= PMLCA_FC;
 347                 writel(ctrl_a, pmu->base + PMLCA(counter));
 348
 349                 ddr_perf_clear_counter(pmu, counter);
 350
 351                 /* Freeze counter disabled, condition enabled, and program event.*/
 352                 ctrl_a &= ~PMLCA_FC;
 353                 ctrl_a |= PMLCA_CE;
 354                 ctrl_a &= ~FIELD_PREP(PMLCA_EVENT, 0x7F);
 355                 ctrl_a |= FIELD_PREP(PMLCA_EVENT, (config & 0x000000FF));
 356                 writel(ctrl_a, pmu->base + PMLCA(counter));
 357         } else {
 358                 /* Freeze counter. */
 359                 ctrl_a |= PMLCA_FC;
 360                 writel(ctrl_a, pmu->base + PMLCA(counter));
 361         }
 362 }
 363
 364 static void ddr_perf_monitor_config(struct ddr_pmu *pmu, int cfg, int cfg1, int cfg2)
 365 {
 366         u32 pmcfg1, pmcfg2;
 367         int event, counter;
 368
 369         event = cfg & 0x000000FF;
 370         counter = (cfg & 0x0000FF00) >> 8;
 371
 372         pmcfg1 = readl_relaxed(pmu->base + PMCFG1);
 373
 374         if (counter == 2 && event == 73)
 375                 pmcfg1 |= PMCFG1_RD_TRANS_FILT_EN;
 376         else if (counter == 2 && event != 73)
 377                 pmcfg1 &= ~PMCFG1_RD_TRANS_FILT_EN;
 378
 379         if (counter == 3 && event == 73)
 380                 pmcfg1 |= PMCFG1_WR_TRANS_FILT_EN;
 381         else if (counter == 3 && event != 73)
 382                 pmcfg1 &= ~PMCFG1_WR_TRANS_FILT_EN;
 383
 384         if (counter == 4 && event == 73)
 385                 pmcfg1 |= PMCFG1_RD_BT_FILT_EN;
 386         else if (counter == 4 && event != 73)
 387                 pmcfg1 &= ~PMCFG1_RD_BT_FILT_EN;
 388
 389         pmcfg1 &= ~FIELD_PREP(PMCFG1_ID_MASK, 0x3FFFF);
 390         pmcfg1 |= FIELD_PREP(PMCFG1_ID_MASK, cfg2);
 391         writel(pmcfg1, pmu->base + PMCFG1);
 392
 393         pmcfg2 = readl_relaxed(pmu->base + PMCFG2);
 394         pmcfg2 &= ~FIELD_PREP(PMCFG2_ID, 0x3FFFF);
 395         pmcfg2 |= FIELD_PREP(PMCFG2_ID, cfg1);
 396         writel(pmcfg2, pmu->base + PMCFG2);
 397 }
 398
 399 static void ddr_perf_event_update(struct perf_event *event)
 400 {
 401         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 402         struct hw_perf_event *hwc = &event->hw;
 403         int counter = hwc->idx;
 404         u64 new_raw_count;
 405
 406         new_raw_count = ddr_perf_read_counter(pmu, counter);
 407         local64_add(new_raw_count, &event->count);
 408
 409         /* clear counter's value every time */
 410         ddr_perf_clear_counter(pmu, counter);
 411 }
 412
 413 static int ddr_perf_event_init(struct perf_event *event)
 414 {
 415         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 416         struct hw_perf_event *hwc = &event->hw;
 417         struct perf_event *sibling;
 418
 419         if (event->attr.type != event->pmu->type)
 420                 return -ENOENT;
 421
 422         if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
 423                 return -EOPNOTSUPP;
 424
 425         if (event->cpu < 0) {
 426                 dev_warn(pmu->dev, "Can't provide per-task data!\n");
 427                 return -EOPNOTSUPP;
 428         }
 429
 430         /*
 431          * We must NOT create groups containing mixed PMUs, although software
 432          * events are acceptable (for example to create a CCN group
 433          * periodically read when a hrtimer aka cpu-clock leader triggers).
 434          */
 435         if (event->group_leader->pmu != event->pmu &&
 436                         !is_software_event(event->group_leader))
 437                 return -EINVAL;
 438
 439         for_each_sibling_event(sibling, event->group_leader) {
 440                 if (sibling->pmu != event->pmu &&
 441                                 !is_software_event(sibling))
 442                         return -EINVAL;
 443         }
 444
 445         event->cpu = pmu->cpu;
 446         hwc->idx = -1;
 447
 448         return 0;
 449 }
 450
 451 static void ddr_perf_event_start(struct perf_event *event, int flags)
 452 {
 453         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 454         struct hw_perf_event *hwc = &event->hw;
 455         int counter = hwc->idx;
 456
 457         local64_set(&hwc->prev_count, 0);
 458
 459         ddr_perf_counter_local_config(pmu, event->attr.config, counter, true);
 460         hwc->state = 0;
 461 }
 462
 463 static int ddr_perf_event_add(struct perf_event *event, int flags)
 464 {
 465         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 466         struct hw_perf_event *hwc = &event->hw;
 467         int cfg = event->attr.config;
 468         int cfg1 = event->attr.config1;
 469         int cfg2 = event->attr.config2;
 470         int counter;
 471
 472         counter = (cfg & 0x0000FF00) >> 8;
 473
 474         pmu->events[counter] = event;
 475         pmu->active_events++;
 476         hwc->idx = counter;
 477         hwc->state |= PERF_HES_STOPPED;
 478
 479         if (flags & PERF_EF_START)
 480                 ddr_perf_event_start(event, flags);
 481
 482         /* read trans, write trans, read beat */
 483         ddr_perf_monitor_config(pmu, cfg, cfg1, cfg2);
 484
 485         return 0;
 486 }
 487
 488 static void ddr_perf_event_stop(struct perf_event *event, int flags)
 489 {
 490         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 491         struct hw_perf_event *hwc = &event->hw;
 492         int counter = hwc->idx;
 493
 494         ddr_perf_counter_local_config(pmu, event->attr.config, counter, false);
 495         ddr_perf_event_update(event);
 496
 497         hwc->state |= PERF_HES_STOPPED;
 498 }
 499
 500 static void ddr_perf_event_del(struct perf_event *event, int flags)
 501 {
 502         struct ddr_pmu *pmu = to_ddr_pmu(event->pmu);
 503         struct hw_perf_event *hwc = &event->hw;
 504
 505         ddr_perf_event_stop(event, PERF_EF_UPDATE);
 506
 507         pmu->active_events--;
 508         hwc->idx = -1;
 509 }
 510
 511 static void ddr_perf_pmu_enable(struct pmu *pmu)
 512 {
 513         struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);
 514
 515         ddr_perf_counter_global_config(ddr_pmu, true);
 516 }
 517
 518 static void ddr_perf_pmu_disable(struct pmu *pmu)
 519 {
 520         struct ddr_pmu *ddr_pmu = to_ddr_pmu(pmu);
 521
 522         ddr_perf_counter_global_config(ddr_pmu, false);
 523 }
 524
 525 static void ddr_perf_init(struct ddr_pmu *pmu, void __iomem *base,
 526                          struct device *dev)
 527 {
 528         *pmu = (struct ddr_pmu) {
 529                 .pmu = (struct pmu) {
 530                         .module       = THIS_MODULE,
 531                         .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
 532                         .task_ctx_nr  = perf_invalid_context,
 533                         .attr_groups  = attr_groups,
 534                         .event_init   = ddr_perf_event_init,
 535                         .add          = ddr_perf_event_add,
 536                         .del          = ddr_perf_event_del,
 537                         .start        = ddr_perf_event_start,
 538                         .stop         = ddr_perf_event_stop,
 539                         .read         = ddr_perf_event_update,
 540                         .pmu_enable   = ddr_perf_pmu_enable,
 541                         .pmu_disable  = ddr_perf_pmu_disable,
 542                 },
 543                 .base = base,
 544                 .dev = dev,
 545         };
 546 }
 547
 548 static irqreturn_t ddr_perf_irq_handler(int irq, void *p)
 549 {
 550         struct ddr_pmu *pmu = (struct ddr_pmu *)p;
 551         struct perf_event *event;
 552         int i;
 553
 554         /*
 555          * Counters can generate an interrupt on an overflow when msb of a
 556          * counter changes from 0 to 1. For the interrupt to be signalled,
 557          * below condition mush be satisfied:
 558          * PMGC0[PMIE] = 1, PMGC0[FCECE] = 1, PMLCAn[CE] = 1
 559          * When an interrupt is signalled, PMGC0[FAC] is set by hardware and
 560          * all of the registers are frozen.
 561          * Software can clear the interrupt condition by resetting the performance
 562          * monitor and clearing the most significant bit of the counter that
 563          * generate the overflow.
 564          */
 565         for (i = 0; i < NUM_COUNTERS; i++) {
 566                 if (!pmu->events[i])
 567                         continue;
 568
 569                 event = pmu->events[i];
 570
 571                 ddr_perf_event_update(event);
 572         }
 573
 574         ddr_perf_counter_global_config(pmu, true);
 575
 576         return IRQ_HANDLED;
 577 }
 578
 579 static int ddr_perf_offline_cpu(unsigned int cpu, struct hlist_node *node)
 580 {
 581         struct ddr_pmu *pmu = hlist_entry_safe(node, struct ddr_pmu, node);
 582         int target;
 583
 584         if (cpu != pmu->cpu)
 585                 return 0;
 586
 587         target = cpumask_any_but(cpu_online_mask, cpu);
 588         if (target >= nr_cpu_ids)
 589                 return 0;
 590
 591         perf_pmu_migrate_context(&pmu->pmu, cpu, target);
 592         pmu->cpu = target;
 593
 594         WARN_ON(irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu)));
 595
 596         return 0;
 597 }
 598
 599 static int ddr_perf_probe(struct platform_device *pdev)
 600 {
 601         struct ddr_pmu *pmu;
 602         void __iomem *base;
 603         int ret, irq;
 604         char *name;
 605
 606         base = devm_platform_ioremap_resource(pdev, 0);
 607         if (IS_ERR(base))
 608                 return PTR_ERR(base);
 609
 610         pmu = devm_kzalloc(&pdev->dev, sizeof(*pmu), GFP_KERNEL);
 611         if (!pmu)
 612                 return -ENOMEM;
 613
 614         ddr_perf_init(pmu, base, &pdev->dev);
 615
 616         pmu->devtype_data = of_device_get_match_data(&pdev->dev);
 617
 618         platform_set_drvdata(pdev, pmu);
 619
 620         pmu->id = ida_simple_get(&ddr_ida, 0, 0, GFP_KERNEL);
 621         name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME "%d", pmu->id);
 622         if (!name) {
 623                 ret = -ENOMEM;
 624                 goto format_string_err;
 625         }
 626
 627         pmu->cpu = raw_smp_processor_id();
 628         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DDR_CPUHP_CB_NAME,
 629                                       NULL, ddr_perf_offline_cpu);
 630         if (ret < 0) {
 631                 dev_err(&pdev->dev, "Failed to add callbacks for multi state\n");
 632                 goto cpuhp_state_err;
 633         }
 634         pmu->cpuhp_state = ret;
 635
 636         /* Register the pmu instance for cpu hotplug */
 637         ret = cpuhp_state_add_instance_nocalls(pmu->cpuhp_state, &pmu->node);
 638         if (ret) {
 639                 dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);
 640                 goto cpuhp_instance_err;
 641         }
 642
 643         /* Request irq */
 644         irq = platform_get_irq(pdev, 0);
 645         if (irq < 0) {
 646                 ret = irq;
 647                 goto ddr_perf_err;
 648         }
 649
 650         ret = devm_request_irq(&pdev->dev, irq, ddr_perf_irq_handler,
 651                                IRQF_NOBALANCING | IRQF_NO_THREAD,
 652                                DDR_CPUHP_CB_NAME, pmu);
 653         if (ret < 0) {
 654                 dev_err(&pdev->dev, "Request irq failed: %d", ret);
 655                 goto ddr_perf_err;
 656         }
 657
 658         pmu->irq = irq;
 659         ret = irq_set_affinity(pmu->irq, cpumask_of(pmu->cpu));
 660         if (ret) {
 661                 dev_err(pmu->dev, "Failed to set interrupt affinity\n");
 662                 goto ddr_perf_err;
 663         }
 664
 665         ret = perf_pmu_register(&pmu->pmu, name, -1);
 666         if (ret)
 667                 goto ddr_perf_err;
 668
 669         return 0;
 670
 671 ddr_perf_err:
 672         cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
 673 cpuhp_instance_err:
 674         cpuhp_remove_multi_state(pmu->cpuhp_state);
 675 cpuhp_state_err:
 676 format_string_err:
 677         ida_simple_remove(&ddr_ida, pmu->id);
 678         dev_warn(&pdev->dev, "i.MX9 DDR Perf PMU failed (%d), disabled\n", ret);
 679         return ret;
 680 }
 681
 682 static int ddr_perf_remove(struct platform_device *pdev)
 683 {
 684         struct ddr_pmu *pmu = platform_get_drvdata(pdev);
 685
 686         cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node);
 687         cpuhp_remove_multi_state(pmu->cpuhp_state);
 688
 689         perf_pmu_unregister(&pmu->pmu);
 690
 691         ida_simple_remove(&ddr_ida, pmu->id);
 692
 693         return 0;
 694 }
 695
 696 static struct platform_driver imx_ddr_pmu_driver = {
 697         .driver         = {
 698                 .name                = "imx9-ddr-pmu",
 699                 .of_match_table      = imx_ddr_pmu_dt_ids,
 700                 .suppress_bind_attrs = true,
 701         },
 702         .probe          = ddr_perf_probe,
 703         .remove         = ddr_perf_remove,
 704 };
 705 module_platform_driver(imx_ddr_pmu_driver);
 706
 707 MODULE_AUTHOR("Xu Yang <xu.yang_2@nxp.com>");
 708 MODULE_LICENSE("GPL v2");
 709 MODULE_DESCRIPTION("DDRC PerfMon for i.MX9 SoCs");