drivers/perf/arm_cspmu/arm_cspmu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * ARM CoreSight Architecture PMU driver.
   4  *
   5  * This driver adds support for uncore PMU based on ARM CoreSight Performance
   6  * Monitoring Unit Architecture. The PMU is accessible via MMIO registers and
   7  * like other uncore PMUs, it does not support process specific events and
   8  * cannot be used in sampling mode.
   9  *
  10  * This code is based on other uncore PMUs like ARM DSU PMU. It provides a
  11  * generic implementation to operate the PMU according to CoreSight PMU
  12  * architecture and ACPI ARM PMU table (APMT) documents below:
  13  *   - ARM CoreSight PMU architecture document number: ARM IHI 0091 A.a-00bet0.
  14  *   - APMT document number: ARM DEN0117.
  15  *
  16  * The user should refer to the vendor technical documentation to get details
  17  * about the supported events.
  18  *
  19  * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
  20  *
  21  */
  22
  23 #include <linux/acpi.h>
  24 #include <linux/cacheinfo.h>
  25 #include <linux/ctype.h>
  26 #include <linux/interrupt.h>
  27 #include <linux/io-64-nonatomic-lo-hi.h>
  28 #include <linux/module.h>
  29 #include <linux/mutex.h>
  30 #include <linux/perf_event.h>
  31 #include <linux/platform_device.h>
  32
  33 #include "arm_cspmu.h"
  34
  35 #define PMUNAME "arm_cspmu"
  36 #define DRVNAME "arm-cs-arch-pmu"
  37
  38 #define ARM_CSPMU_CPUMASK_ATTR(_name, _config)                  \
  39         ARM_CSPMU_EXT_ATTR(_name, arm_cspmu_cpumask_show,       \
  40                                 (unsigned long)_config)
  41
  42 /*
  43  * CoreSight PMU Arch register offsets.
  44  */
  45 #define PMEVCNTR_LO                                     0x0
  46 #define PMEVCNTR_HI                                     0x4
  47 #define PMEVTYPER                                       0x400
  48 #define PMCCFILTR                                       0x47C
  49 #define PMEVFILTR                                       0xA00
  50 #define PMCNTENSET                                      0xC00
  51 #define PMCNTENCLR                                      0xC20
  52 #define PMINTENSET                                      0xC40
  53 #define PMINTENCLR                                      0xC60
  54 #define PMOVSCLR                                        0xC80
  55 #define PMOVSSET                                        0xCC0
  56 #define PMCFGR                                          0xE00
  57 #define PMCR                                            0xE04
  58 #define PMIIDR                                          0xE08
  59
  60 /* PMCFGR register field */
  61 #define PMCFGR_NCG                                      GENMASK(31, 28)
  62 #define PMCFGR_HDBG                                     BIT(24)
  63 #define PMCFGR_TRO                                      BIT(23)
  64 #define PMCFGR_SS                                       BIT(22)
  65 #define PMCFGR_FZO                                      BIT(21)
  66 #define PMCFGR_MSI                                      BIT(20)
  67 #define PMCFGR_UEN                                      BIT(19)
  68 #define PMCFGR_NA                                       BIT(17)
  69 #define PMCFGR_EX                                       BIT(16)
  70 #define PMCFGR_CCD                                      BIT(15)
  71 #define PMCFGR_CC                                       BIT(14)
  72 #define PMCFGR_SIZE                                     GENMASK(13, 8)
  73 #define PMCFGR_N                                        GENMASK(7, 0)
  74
  75 /* PMCR register field */
  76 #define PMCR_TRO                                        BIT(11)
  77 #define PMCR_HDBG                                       BIT(10)
  78 #define PMCR_FZO                                        BIT(9)
  79 #define PMCR_NA                                         BIT(8)
  80 #define PMCR_DP                                         BIT(5)
  81 #define PMCR_X                                          BIT(4)
  82 #define PMCR_D                                          BIT(3)
  83 #define PMCR_C                                          BIT(2)
  84 #define PMCR_P                                          BIT(1)
  85 #define PMCR_E                                          BIT(0)
  86
  87 /* Each SET/CLR register supports up to 32 counters. */
  88 #define ARM_CSPMU_SET_CLR_COUNTER_SHIFT         5
  89 #define ARM_CSPMU_SET_CLR_COUNTER_NUM           \
  90         (1 << ARM_CSPMU_SET_CLR_COUNTER_SHIFT)
  91
  92 /* Convert counter idx into SET/CLR register number. */
  93 #define COUNTER_TO_SET_CLR_ID(idx)                      \
  94         (idx >> ARM_CSPMU_SET_CLR_COUNTER_SHIFT)
  95
  96 /* Convert counter idx into SET/CLR register bit. */
  97 #define COUNTER_TO_SET_CLR_BIT(idx)                     \
  98         (idx & (ARM_CSPMU_SET_CLR_COUNTER_NUM - 1))
  99
 100 #define ARM_CSPMU_ACTIVE_CPU_MASK               0x0
 101 #define ARM_CSPMU_ASSOCIATED_CPU_MASK           0x1
 102
 103 /* Check and use default if implementer doesn't provide attribute callback */
 104 #define CHECK_DEFAULT_IMPL_OPS(ops, callback)                   \
 105         do {                                                    \
 106                 if (!ops->callback)                             \
 107                         ops->callback = arm_cspmu_ ## callback; \
 108         } while (0)
 109
 110 /*
 111  * Maximum poll count for reading counter value using high-low-high sequence.
 112  */
 113 #define HILOHI_MAX_POLL 1000
 114
 115 static unsigned long arm_cspmu_cpuhp_state;
 116
 117 static DEFINE_MUTEX(arm_cspmu_lock);
 118
 119 static void arm_cspmu_set_ev_filter(struct arm_cspmu *cspmu,
 120                                     struct hw_perf_event *hwc, u32 filter);
 121
 122 static struct acpi_apmt_node *arm_cspmu_apmt_node(struct device *dev)
 123 {
 124         return *(struct acpi_apmt_node **)dev_get_platdata(dev);
 125 }
 126
 127 /*
 128  * In CoreSight PMU architecture, all of the MMIO registers are 32-bit except
 129  * counter register. The counter register can be implemented as 32-bit or 64-bit
 130  * register depending on the value of PMCFGR.SIZE field. For 64-bit access,
 131  * single-copy 64-bit atomic support is implementation defined. APMT node flag
 132  * is used to identify if the PMU supports 64-bit single copy atomic. If 64-bit
 133  * single copy atomic is not supported, the driver treats the register as a pair
 134  * of 32-bit register.
 135  */
 136
 137 /*
 138  * Read 64-bit register as a pair of 32-bit registers using hi-lo-hi sequence.
 139  */
 140 static u64 read_reg64_hilohi(const void __iomem *addr, u32 max_poll_count)
 141 {
 142         u32 val_lo, val_hi;
 143         u64 val;
 144
 145         /* Use high-low-high sequence to avoid tearing */
 146         do {
 147                 if (max_poll_count-- == 0) {
 148                         pr_err("ARM CSPMU: timeout hi-low-high sequence\n");
 149                         return 0;
 150                 }
 151
 152                 val_hi = readl(addr + 4);
 153                 val_lo = readl(addr);
 154         } while (val_hi != readl(addr + 4));
 155
 156         val = (((u64)val_hi << 32) | val_lo);
 157
 158         return val;
 159 }
 160
 161 /* Check if cycle counter is supported. */
 162 static inline bool supports_cycle_counter(const struct arm_cspmu *cspmu)
 163 {
 164         return (cspmu->pmcfgr & PMCFGR_CC);
 165 }
 166
 167 /* Get counter size, which is (PMCFGR_SIZE + 1). */
 168 static inline u32 counter_size(const struct arm_cspmu *cspmu)
 169 {
 170         return FIELD_GET(PMCFGR_SIZE, cspmu->pmcfgr) + 1;
 171 }
 172
 173 /* Get counter mask. */
 174 static inline u64 counter_mask(const struct arm_cspmu *cspmu)
 175 {
 176         return GENMASK_ULL(counter_size(cspmu) - 1, 0);
 177 }
 178
 179 /* Check if counter is implemented as 64-bit register. */
 180 static inline bool use_64b_counter_reg(const struct arm_cspmu *cspmu)
 181 {
 182         return (counter_size(cspmu) > 32);
 183 }
 184
 185 ssize_t arm_cspmu_sysfs_event_show(struct device *dev,
 186                                 struct device_attribute *attr, char *buf)
 187 {
 188         struct perf_pmu_events_attr *pmu_attr;
 189
 190         pmu_attr = container_of(attr, typeof(*pmu_attr), attr);
 191         return sysfs_emit(buf, "event=0x%llx\n", pmu_attr->id);
 192 }
 193 EXPORT_SYMBOL_GPL(arm_cspmu_sysfs_event_show);
 194
 195 /* Default event list. */
 196 static struct attribute *arm_cspmu_event_attrs[] = {
 197         ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
 198         NULL,
 199 };
 200
 201 static struct attribute **
 202 arm_cspmu_get_event_attrs(const struct arm_cspmu *cspmu)
 203 {
 204         struct attribute **attrs;
 205
 206         attrs = devm_kmemdup(cspmu->dev, arm_cspmu_event_attrs,
 207                 sizeof(arm_cspmu_event_attrs), GFP_KERNEL);
 208
 209         return attrs;
 210 }
 211
 212 static umode_t
 213 arm_cspmu_event_attr_is_visible(struct kobject *kobj,
 214                                 struct attribute *attr, int unused)
 215 {
 216         struct device *dev = kobj_to_dev(kobj);
 217         struct arm_cspmu *cspmu = to_arm_cspmu(dev_get_drvdata(dev));
 218         struct perf_pmu_events_attr *eattr;
 219
 220         eattr = container_of(attr, typeof(*eattr), attr.attr);
 221
 222         /* Hide cycle event if not supported */
 223         if (!supports_cycle_counter(cspmu) &&
 224             eattr->id == ARM_CSPMU_EVT_CYCLES_DEFAULT)
 225                 return 0;
 226
 227         return attr->mode;
 228 }
 229
 230 ssize_t arm_cspmu_sysfs_format_show(struct device *dev,
 231                                 struct device_attribute *attr,
 232                                 char *buf)
 233 {
 234         struct dev_ext_attribute *eattr =
 235                 container_of(attr, struct dev_ext_attribute, attr);
 236         return sysfs_emit(buf, "%s\n", (char *)eattr->var);
 237 }
 238 EXPORT_SYMBOL_GPL(arm_cspmu_sysfs_format_show);
 239
 240 static struct attribute *arm_cspmu_format_attrs[] = {
 241         ARM_CSPMU_FORMAT_EVENT_ATTR,
 242         ARM_CSPMU_FORMAT_FILTER_ATTR,
 243         NULL,
 244 };
 245
 246 static struct attribute **
 247 arm_cspmu_get_format_attrs(const struct arm_cspmu *cspmu)
 248 {
 249         struct attribute **attrs;
 250
 251         attrs = devm_kmemdup(cspmu->dev, arm_cspmu_format_attrs,
 252                 sizeof(arm_cspmu_format_attrs), GFP_KERNEL);
 253
 254         return attrs;
 255 }
 256
 257 static u32 arm_cspmu_event_type(const struct perf_event *event)
 258 {
 259         return event->attr.config & ARM_CSPMU_EVENT_MASK;
 260 }
 261
 262 static bool arm_cspmu_is_cycle_counter_event(const struct perf_event *event)
 263 {
 264         return (event->attr.config == ARM_CSPMU_EVT_CYCLES_DEFAULT);
 265 }
 266
 267 static u32 arm_cspmu_event_filter(const struct perf_event *event)
 268 {
 269         return event->attr.config1 & ARM_CSPMU_FILTER_MASK;
 270 }
 271
 272 static ssize_t arm_cspmu_identifier_show(struct device *dev,
 273                                          struct device_attribute *attr,
 274                                          char *page)
 275 {
 276         struct arm_cspmu *cspmu = to_arm_cspmu(dev_get_drvdata(dev));
 277
 278         return sysfs_emit(page, "%s\n", cspmu->identifier);
 279 }
 280
 281 static struct device_attribute arm_cspmu_identifier_attr =
 282         __ATTR(identifier, 0444, arm_cspmu_identifier_show, NULL);
 283
 284 static struct attribute *arm_cspmu_identifier_attrs[] = {
 285         &arm_cspmu_identifier_attr.attr,
 286         NULL,
 287 };
 288
 289 static struct attribute_group arm_cspmu_identifier_attr_group = {
 290         .attrs = arm_cspmu_identifier_attrs,
 291 };
 292
 293 static const char *arm_cspmu_get_identifier(const struct arm_cspmu *cspmu)
 294 {
 295         const char *identifier =
 296                 devm_kasprintf(cspmu->dev, GFP_KERNEL, "%x",
 297                                cspmu->impl.pmiidr);
 298         return identifier;
 299 }
 300
 301 static const char *arm_cspmu_type_str[ACPI_APMT_NODE_TYPE_COUNT] = {
 302         "mc",
 303         "smmu",
 304         "pcie",
 305         "acpi",
 306         "cache",
 307 };
 308
 309 static const char *arm_cspmu_get_name(const struct arm_cspmu *cspmu)
 310 {
 311         struct device *dev;
 312         struct acpi_apmt_node *apmt_node;
 313         u8 pmu_type;
 314         char *name;
 315         char acpi_hid_string[ACPI_ID_LEN] = { 0 };
 316         static atomic_t pmu_idx[ACPI_APMT_NODE_TYPE_COUNT] = { 0 };
 317
 318         dev = cspmu->dev;
 319         apmt_node = arm_cspmu_apmt_node(dev);
 320         pmu_type = apmt_node->type;
 321
 322         if (pmu_type >= ACPI_APMT_NODE_TYPE_COUNT) {
 323                 dev_err(dev, "unsupported PMU type-%u\n", pmu_type);
 324                 return NULL;
 325         }
 326
 327         if (pmu_type == ACPI_APMT_NODE_TYPE_ACPI) {
 328                 memcpy(acpi_hid_string,
 329                         &apmt_node->inst_primary,
 330                         sizeof(apmt_node->inst_primary));
 331                 name = devm_kasprintf(dev, GFP_KERNEL, "%s_%s_%s_%u", PMUNAME,
 332                                       arm_cspmu_type_str[pmu_type],
 333                                       acpi_hid_string,
 334                                       apmt_node->inst_secondary);
 335         } else {
 336                 name = devm_kasprintf(dev, GFP_KERNEL, "%s_%s_%d", PMUNAME,
 337                                       arm_cspmu_type_str[pmu_type],
 338                                       atomic_fetch_inc(&pmu_idx[pmu_type]));
 339         }
 340
 341         return name;
 342 }
 343
 344 static ssize_t arm_cspmu_cpumask_show(struct device *dev,
 345                                       struct device_attribute *attr,
 346                                       char *buf)
 347 {
 348         struct pmu *pmu = dev_get_drvdata(dev);
 349         struct arm_cspmu *cspmu = to_arm_cspmu(pmu);
 350         struct dev_ext_attribute *eattr =
 351                 container_of(attr, struct dev_ext_attribute, attr);
 352         unsigned long mask_id = (unsigned long)eattr->var;
 353         const cpumask_t *cpumask;
 354
 355         switch (mask_id) {
 356         case ARM_CSPMU_ACTIVE_CPU_MASK:
 357                 cpumask = &cspmu->active_cpu;
 358                 break;
 359         case ARM_CSPMU_ASSOCIATED_CPU_MASK:
 360                 cpumask = &cspmu->associated_cpus;
 361                 break;
 362         default:
 363                 return 0;
 364         }
 365         return cpumap_print_to_pagebuf(true, buf, cpumask);
 366 }
 367
 368 static struct attribute *arm_cspmu_cpumask_attrs[] = {
 369         ARM_CSPMU_CPUMASK_ATTR(cpumask, ARM_CSPMU_ACTIVE_CPU_MASK),
 370         ARM_CSPMU_CPUMASK_ATTR(associated_cpus, ARM_CSPMU_ASSOCIATED_CPU_MASK),
 371         NULL,
 372 };
 373
 374 static struct attribute_group arm_cspmu_cpumask_attr_group = {
 375         .attrs = arm_cspmu_cpumask_attrs,
 376 };
 377
 378 static struct arm_cspmu_impl_match impl_match[] = {
 379         {
 380                 .module_name    = "nvidia_cspmu",
 381                 .pmiidr_val     = ARM_CSPMU_IMPL_ID_NVIDIA,
 382                 .pmiidr_mask    = ARM_CSPMU_PMIIDR_IMPLEMENTER,
 383                 .module         = NULL,
 384                 .impl_init_ops  = NULL,
 385         },
 386         {
 387                 .module_name    = "ampere_cspmu",
 388                 .pmiidr_val     = ARM_CSPMU_IMPL_ID_AMPERE,
 389                 .pmiidr_mask    = ARM_CSPMU_PMIIDR_IMPLEMENTER,
 390                 .module         = NULL,
 391                 .impl_init_ops  = NULL,
 392         },
 393
 394         {0}
 395 };
 396
 397 static struct arm_cspmu_impl_match *arm_cspmu_impl_match_get(u32 pmiidr)
 398 {
 399         struct arm_cspmu_impl_match *match = impl_match;
 400
 401         for (; match->pmiidr_val; match++) {
 402                 u32 mask = match->pmiidr_mask;
 403
 404                 if ((match->pmiidr_val & mask) == (pmiidr & mask))
 405                         return match;
 406         }
 407
 408         return NULL;
 409 }
 410
 411 static int arm_cspmu_init_impl_ops(struct arm_cspmu *cspmu)
 412 {
 413         int ret = 0;
 414         struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
 415         struct acpi_apmt_node *apmt_node = arm_cspmu_apmt_node(cspmu->dev);
 416         struct arm_cspmu_impl_match *match;
 417
 418         /*
 419          * Get PMU implementer and product id from APMT node.
 420          * If APMT node doesn't have implementer/product id, try get it
 421          * from PMIIDR.
 422          */
 423         cspmu->impl.pmiidr =
 424                 (apmt_node->impl_id) ? apmt_node->impl_id :
 425                                        readl(cspmu->base0 + PMIIDR);
 426
 427         /* Find implementer specific attribute ops. */
 428         match = arm_cspmu_impl_match_get(cspmu->impl.pmiidr);
 429
 430         /* Load implementer module and initialize the callbacks. */
 431         if (match) {
 432                 mutex_lock(&arm_cspmu_lock);
 433
 434                 if (match->impl_init_ops) {
 435                         /* Prevent unload until PMU registration is done. */
 436                         if (try_module_get(match->module)) {
 437                                 cspmu->impl.module = match->module;
 438                                 cspmu->impl.match = match;
 439                                 ret = match->impl_init_ops(cspmu);
 440                                 if (ret)
 441                                         module_put(match->module);
 442                         } else {
 443                                 WARN(1, "arm_cspmu failed to get module: %s\n",
 444                                         match->module_name);
 445                                 ret = -EINVAL;
 446                         }
 447                 } else {
 448                         request_module_nowait(match->module_name);
 449                         ret = -EPROBE_DEFER;
 450                 }
 451
 452                 mutex_unlock(&arm_cspmu_lock);
 453
 454                 if (ret)
 455                         return ret;
 456         } else
 457                 cspmu->impl.module = THIS_MODULE;
 458
 459         /* Use default callbacks if implementer doesn't provide one. */
 460         CHECK_DEFAULT_IMPL_OPS(impl_ops, get_event_attrs);
 461         CHECK_DEFAULT_IMPL_OPS(impl_ops, get_format_attrs);
 462         CHECK_DEFAULT_IMPL_OPS(impl_ops, get_identifier);
 463         CHECK_DEFAULT_IMPL_OPS(impl_ops, get_name);
 464         CHECK_DEFAULT_IMPL_OPS(impl_ops, is_cycle_counter_event);
 465         CHECK_DEFAULT_IMPL_OPS(impl_ops, event_type);
 466         CHECK_DEFAULT_IMPL_OPS(impl_ops, event_filter);
 467         CHECK_DEFAULT_IMPL_OPS(impl_ops, event_attr_is_visible);
 468         CHECK_DEFAULT_IMPL_OPS(impl_ops, set_ev_filter);
 469
 470         return 0;
 471 }
 472
 473 static struct attribute_group *
 474 arm_cspmu_alloc_event_attr_group(struct arm_cspmu *cspmu)
 475 {
 476         struct attribute_group *event_group;
 477         struct device *dev = cspmu->dev;
 478         const struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
 479
 480         event_group =
 481                 devm_kzalloc(dev, sizeof(struct attribute_group), GFP_KERNEL);
 482         if (!event_group)
 483                 return NULL;
 484
 485         event_group->name = "events";
 486         event_group->is_visible = impl_ops->event_attr_is_visible;
 487         event_group->attrs = impl_ops->get_event_attrs(cspmu);
 488
 489         if (!event_group->attrs)
 490                 return NULL;
 491
 492         return event_group;
 493 }
 494
 495 static struct attribute_group *
 496 arm_cspmu_alloc_format_attr_group(struct arm_cspmu *cspmu)
 497 {
 498         struct attribute_group *format_group;
 499         struct device *dev = cspmu->dev;
 500
 501         format_group =
 502                 devm_kzalloc(dev, sizeof(struct attribute_group), GFP_KERNEL);
 503         if (!format_group)
 504                 return NULL;
 505
 506         format_group->name = "format";
 507         format_group->attrs = cspmu->impl.ops.get_format_attrs(cspmu);
 508
 509         if (!format_group->attrs)
 510                 return NULL;
 511
 512         return format_group;
 513 }
 514
 515 static struct attribute_group **
 516 arm_cspmu_alloc_attr_group(struct arm_cspmu *cspmu)
 517 {
 518         struct attribute_group **attr_groups = NULL;
 519         struct device *dev = cspmu->dev;
 520         const struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
 521
 522         cspmu->identifier = impl_ops->get_identifier(cspmu);
 523         cspmu->name = impl_ops->get_name(cspmu);
 524
 525         if (!cspmu->identifier || !cspmu->name)
 526                 return NULL;
 527
 528         attr_groups = devm_kcalloc(dev, 5, sizeof(struct attribute_group *),
 529                                    GFP_KERNEL);
 530         if (!attr_groups)
 531                 return NULL;
 532
 533         attr_groups[0] = arm_cspmu_alloc_event_attr_group(cspmu);
 534         attr_groups[1] = arm_cspmu_alloc_format_attr_group(cspmu);
 535         attr_groups[2] = &arm_cspmu_identifier_attr_group;
 536         attr_groups[3] = &arm_cspmu_cpumask_attr_group;
 537
 538         if (!attr_groups[0] || !attr_groups[1])
 539                 return NULL;
 540
 541         return attr_groups;
 542 }
 543
 544 static inline void arm_cspmu_reset_counters(struct arm_cspmu *cspmu)
 545 {
 546         u32 pmcr = 0;
 547
 548         pmcr |= PMCR_P;
 549         pmcr |= PMCR_C;
 550         writel(pmcr, cspmu->base0 + PMCR);
 551 }
 552
 553 static inline void arm_cspmu_start_counters(struct arm_cspmu *cspmu)
 554 {
 555         writel(PMCR_E, cspmu->base0 + PMCR);
 556 }
 557
 558 static inline void arm_cspmu_stop_counters(struct arm_cspmu *cspmu)
 559 {
 560         writel(0, cspmu->base0 + PMCR);
 561 }
 562
 563 static void arm_cspmu_enable(struct pmu *pmu)
 564 {
 565         bool disabled;
 566         struct arm_cspmu *cspmu = to_arm_cspmu(pmu);
 567
 568         disabled = bitmap_empty(cspmu->hw_events.used_ctrs,
 569                                 cspmu->num_logical_ctrs);
 570
 571         if (disabled)
 572                 return;
 573
 574         arm_cspmu_start_counters(cspmu);
 575 }
 576
 577 static void arm_cspmu_disable(struct pmu *pmu)
 578 {
 579         struct arm_cspmu *cspmu = to_arm_cspmu(pmu);
 580
 581         arm_cspmu_stop_counters(cspmu);
 582 }
 583
 584 static int arm_cspmu_get_event_idx(struct arm_cspmu_hw_events *hw_events,
 585                                 struct perf_event *event)
 586 {
 587         int idx, ret;
 588         struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
 589
 590         if (supports_cycle_counter(cspmu)) {
 591                 if (cspmu->impl.ops.is_cycle_counter_event(event)) {
 592                         /* Search for available cycle counter. */
 593                         if (test_and_set_bit(cspmu->cycle_counter_logical_idx,
 594                                              hw_events->used_ctrs))
 595                                 return -EAGAIN;
 596
 597                         return cspmu->cycle_counter_logical_idx;
 598                 }
 599
 600                 /*
 601                  * Search a regular counter from the used counter bitmap.
 602                  * The cycle counter divides the bitmap into two parts. Search
 603                  * the first then second half to exclude the cycle counter bit.
 604                  */
 605                 idx = find_first_zero_bit(hw_events->used_ctrs,
 606                                           cspmu->cycle_counter_logical_idx);
 607                 if (idx >= cspmu->cycle_counter_logical_idx) {
 608                         idx = find_next_zero_bit(
 609                                 hw_events->used_ctrs,
 610                                 cspmu->num_logical_ctrs,
 611                                 cspmu->cycle_counter_logical_idx + 1);
 612                 }
 613         } else {
 614                 idx = find_first_zero_bit(hw_events->used_ctrs,
 615                                           cspmu->num_logical_ctrs);
 616         }
 617
 618         if (idx >= cspmu->num_logical_ctrs)
 619                 return -EAGAIN;
 620
 621         if (cspmu->impl.ops.validate_event) {
 622                 ret = cspmu->impl.ops.validate_event(cspmu, event);
 623                 if (ret)
 624                         return ret;
 625         }
 626
 627         set_bit(idx, hw_events->used_ctrs);
 628
 629         return idx;
 630 }
 631
 632 static bool arm_cspmu_validate_event(struct pmu *pmu,
 633                                  struct arm_cspmu_hw_events *hw_events,
 634                                  struct perf_event *event)
 635 {
 636         if (is_software_event(event))
 637                 return true;
 638
 639         /* Reject groups spanning multiple HW PMUs. */
 640         if (event->pmu != pmu)
 641                 return false;
 642
 643         return (arm_cspmu_get_event_idx(hw_events, event) >= 0);
 644 }
 645
 646 /*
 647  * Make sure the group of events can be scheduled at once
 648  * on the PMU.
 649  */
 650 static bool arm_cspmu_validate_group(struct perf_event *event)
 651 {
 652         struct perf_event *sibling, *leader = event->group_leader;
 653         struct arm_cspmu_hw_events fake_hw_events;
 654
 655         if (event->group_leader == event)
 656                 return true;
 657
 658         memset(&fake_hw_events, 0, sizeof(fake_hw_events));
 659
 660         if (!arm_cspmu_validate_event(event->pmu, &fake_hw_events, leader))
 661                 return false;
 662
 663         for_each_sibling_event(sibling, leader) {
 664                 if (!arm_cspmu_validate_event(event->pmu, &fake_hw_events,
 665                                                   sibling))
 666                         return false;
 667         }
 668
 669         return arm_cspmu_validate_event(event->pmu, &fake_hw_events, event);
 670 }
 671
 672 static int arm_cspmu_event_init(struct perf_event *event)
 673 {
 674         struct arm_cspmu *cspmu;
 675         struct hw_perf_event *hwc = &event->hw;
 676
 677         cspmu = to_arm_cspmu(event->pmu);
 678
 679         if (event->attr.type != event->pmu->type)
 680                 return -ENOENT;
 681
 682         /*
 683          * Following other "uncore" PMUs, we do not support sampling mode or
 684          * attach to a task (per-process mode).
 685          */
 686         if (is_sampling_event(event)) {
 687                 dev_dbg(cspmu->pmu.dev,
 688                         "Can't support sampling events\n");
 689                 return -EOPNOTSUPP;
 690         }
 691
 692         if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) {
 693                 dev_dbg(cspmu->pmu.dev,
 694                         "Can't support per-task counters\n");
 695                 return -EINVAL;
 696         }
 697
 698         /*
 699          * Make sure the CPU assignment is on one of the CPUs associated with
 700          * this PMU.
 701          */
 702         if (!cpumask_test_cpu(event->cpu, &cspmu->associated_cpus)) {
 703                 dev_dbg(cspmu->pmu.dev,
 704                         "Requested cpu is not associated with the PMU\n");
 705                 return -EINVAL;
 706         }
 707
 708         /* Enforce the current active CPU to handle the events in this PMU. */
 709         event->cpu = cpumask_first(&cspmu->active_cpu);
 710         if (event->cpu >= nr_cpu_ids)
 711                 return -EINVAL;
 712
 713         if (!arm_cspmu_validate_group(event))
 714                 return -EINVAL;
 715
 716         /*
 717          * The logical counter id is tracked with hw_perf_event.extra_reg.idx.
 718          * The physical counter id is tracked with hw_perf_event.idx.
 719          * We don't assign an index until we actually place the event onto
 720          * hardware. Use -1 to signify that we haven't decided where to put it
 721          * yet.
 722          */
 723         hwc->idx = -1;
 724         hwc->extra_reg.idx = -1;
 725         hwc->config = cspmu->impl.ops.event_type(event);
 726
 727         return 0;
 728 }
 729
 730 static inline u32 counter_offset(u32 reg_sz, u32 ctr_idx)
 731 {
 732         return (PMEVCNTR_LO + (reg_sz * ctr_idx));
 733 }
 734
 735 static void arm_cspmu_write_counter(struct perf_event *event, u64 val)
 736 {
 737         u32 offset;
 738         struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
 739
 740         if (use_64b_counter_reg(cspmu)) {
 741                 offset = counter_offset(sizeof(u64), event->hw.idx);
 742
 743                 if (cspmu->has_atomic_dword)
 744                         writeq(val, cspmu->base1 + offset);
 745                 else
 746                         lo_hi_writeq(val, cspmu->base1 + offset);
 747         } else {
 748                 offset = counter_offset(sizeof(u32), event->hw.idx);
 749
 750                 writel(lower_32_bits(val), cspmu->base1 + offset);
 751         }
 752 }
 753
 754 static u64 arm_cspmu_read_counter(struct perf_event *event)
 755 {
 756         u32 offset;
 757         const void __iomem *counter_addr;
 758         struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
 759
 760         if (use_64b_counter_reg(cspmu)) {
 761                 offset = counter_offset(sizeof(u64), event->hw.idx);
 762                 counter_addr = cspmu->base1 + offset;
 763
 764                 return cspmu->has_atomic_dword ?
 765                                readq(counter_addr) :
 766                                read_reg64_hilohi(counter_addr, HILOHI_MAX_POLL);
 767         }
 768
 769         offset = counter_offset(sizeof(u32), event->hw.idx);
 770         return readl(cspmu->base1 + offset);
 771 }
 772
 773 /*
 774  * arm_cspmu_set_event_period: Set the period for the counter.
 775  *
 776  * To handle cases of extreme interrupt latency, we program
 777  * the counter with half of the max count for the counters.
 778  */
 779 static void arm_cspmu_set_event_period(struct perf_event *event)
 780 {
 781         struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
 782         u64 val = counter_mask(cspmu) >> 1ULL;
 783
 784         local64_set(&event->hw.prev_count, val);
 785         arm_cspmu_write_counter(event, val);
 786 }
 787
 788 static void arm_cspmu_enable_counter(struct arm_cspmu *cspmu, int idx)
 789 {
 790         u32 reg_id, reg_bit, inten_off, cnten_off;
 791
 792         reg_id = COUNTER_TO_SET_CLR_ID(idx);
 793         reg_bit = COUNTER_TO_SET_CLR_BIT(idx);
 794
 795         inten_off = PMINTENSET + (4 * reg_id);
 796         cnten_off = PMCNTENSET + (4 * reg_id);
 797
 798         writel(BIT(reg_bit), cspmu->base0 + inten_off);
 799         writel(BIT(reg_bit), cspmu->base0 + cnten_off);
 800 }
 801
 802 static void arm_cspmu_disable_counter(struct arm_cspmu *cspmu, int idx)
 803 {
 804         u32 reg_id, reg_bit, inten_off, cnten_off;
 805
 806         reg_id = COUNTER_TO_SET_CLR_ID(idx);
 807         reg_bit = COUNTER_TO_SET_CLR_BIT(idx);
 808
 809         inten_off = PMINTENCLR + (4 * reg_id);
 810         cnten_off = PMCNTENCLR + (4 * reg_id);
 811
 812         writel(BIT(reg_bit), cspmu->base0 + cnten_off);
 813         writel(BIT(reg_bit), cspmu->base0 + inten_off);
 814 }
 815
 816 static void arm_cspmu_event_update(struct perf_event *event)
 817 {
 818         struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
 819         struct hw_perf_event *hwc = &event->hw;
 820         u64 delta, prev, now;
 821
 822         do {
 823                 prev = local64_read(&hwc->prev_count);
 824                 now = arm_cspmu_read_counter(event);
 825         } while (local64_cmpxchg(&hwc->prev_count, prev, now) != prev);
 826
 827         delta = (now - prev) & counter_mask(cspmu);
 828         local64_add(delta, &event->count);
 829 }
 830
 831 static inline void arm_cspmu_set_event(struct arm_cspmu *cspmu,
 832                                         struct hw_perf_event *hwc)
 833 {
 834         u32 offset = PMEVTYPER + (4 * hwc->idx);
 835
 836         writel(hwc->config, cspmu->base0 + offset);
 837 }
 838
 839 static void arm_cspmu_set_ev_filter(struct arm_cspmu *cspmu,
 840                                         struct hw_perf_event *hwc,
 841                                         u32 filter)
 842 {
 843         u32 offset = PMEVFILTR + (4 * hwc->idx);
 844
 845         writel(filter, cspmu->base0 + offset);
 846 }
 847
 848 static inline void arm_cspmu_set_cc_filter(struct arm_cspmu *cspmu, u32 filter)
 849 {
 850         u32 offset = PMCCFILTR;
 851
 852         writel(filter, cspmu->base0 + offset);
 853 }
 854
 855 static void arm_cspmu_start(struct perf_event *event, int pmu_flags)
 856 {
 857         struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
 858         struct hw_perf_event *hwc = &event->hw;
 859         u32 filter;
 860
 861         /* We always reprogram the counter */
 862         if (pmu_flags & PERF_EF_RELOAD)
 863                 WARN_ON(!(hwc->state & PERF_HES_UPTODATE));
 864
 865         arm_cspmu_set_event_period(event);
 866
 867         filter = cspmu->impl.ops.event_filter(event);
 868
 869         if (event->hw.extra_reg.idx == cspmu->cycle_counter_logical_idx) {
 870                 arm_cspmu_set_cc_filter(cspmu, filter);
 871         } else {
 872                 arm_cspmu_set_event(cspmu, hwc);
 873                 cspmu->impl.ops.set_ev_filter(cspmu, hwc, filter);
 874         }
 875
 876         hwc->state = 0;
 877
 878         arm_cspmu_enable_counter(cspmu, hwc->idx);
 879 }
 880
 881 static void arm_cspmu_stop(struct perf_event *event, int pmu_flags)
 882 {
 883         struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
 884         struct hw_perf_event *hwc = &event->hw;
 885
 886         if (hwc->state & PERF_HES_STOPPED)
 887                 return;
 888
 889         arm_cspmu_disable_counter(cspmu, hwc->idx);
 890         arm_cspmu_event_update(event);
 891
 892         hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
 893 }
 894
 895 static inline u32 to_phys_idx(struct arm_cspmu *cspmu, u32 idx)
 896 {
 897         return (idx == cspmu->cycle_counter_logical_idx) ?
 898                 ARM_CSPMU_CYCLE_CNTR_IDX : idx;
 899 }
 900
 901 static int arm_cspmu_add(struct perf_event *event, int flags)
 902 {
 903         struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
 904         struct arm_cspmu_hw_events *hw_events = &cspmu->hw_events;
 905         struct hw_perf_event *hwc = &event->hw;
 906         int idx;
 907
 908         if (WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(),
 909                                            &cspmu->associated_cpus)))
 910                 return -ENOENT;
 911
 912         idx = arm_cspmu_get_event_idx(hw_events, event);
 913         if (idx < 0)
 914                 return idx;
 915
 916         hw_events->events[idx] = event;
 917         hwc->idx = to_phys_idx(cspmu, idx);
 918         hwc->extra_reg.idx = idx;
 919         hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
 920
 921         if (flags & PERF_EF_START)
 922                 arm_cspmu_start(event, PERF_EF_RELOAD);
 923
 924         /* Propagate changes to the userspace mapping. */
 925         perf_event_update_userpage(event);
 926
 927         return 0;
 928 }
 929
 930 static void arm_cspmu_del(struct perf_event *event, int flags)
 931 {
 932         struct arm_cspmu *cspmu = to_arm_cspmu(event->pmu);
 933         struct arm_cspmu_hw_events *hw_events = &cspmu->hw_events;
 934         struct hw_perf_event *hwc = &event->hw;
 935         int idx = hwc->extra_reg.idx;
 936
 937         arm_cspmu_stop(event, PERF_EF_UPDATE);
 938
 939         hw_events->events[idx] = NULL;
 940
 941         clear_bit(idx, hw_events->used_ctrs);
 942
 943         perf_event_update_userpage(event);
 944 }
 945
 946 static void arm_cspmu_read(struct perf_event *event)
 947 {
 948         arm_cspmu_event_update(event);
 949 }
 950
 951 static struct arm_cspmu *arm_cspmu_alloc(struct platform_device *pdev)
 952 {
 953         struct acpi_apmt_node *apmt_node;
 954         struct arm_cspmu *cspmu;
 955         struct device *dev = &pdev->dev;
 956
 957         cspmu = devm_kzalloc(dev, sizeof(*cspmu), GFP_KERNEL);
 958         if (!cspmu)
 959                 return NULL;
 960
 961         cspmu->dev = dev;
 962         platform_set_drvdata(pdev, cspmu);
 963
 964         apmt_node = arm_cspmu_apmt_node(dev);
 965         cspmu->has_atomic_dword = apmt_node->flags & ACPI_APMT_FLAGS_ATOMIC;
 966
 967         return cspmu;
 968 }
 969
 970 static int arm_cspmu_init_mmio(struct arm_cspmu *cspmu)
 971 {
 972         struct device *dev;
 973         struct platform_device *pdev;
 974
 975         dev = cspmu->dev;
 976         pdev = to_platform_device(dev);
 977
 978         /* Base address for page 0. */
 979         cspmu->base0 = devm_platform_ioremap_resource(pdev, 0);
 980         if (IS_ERR(cspmu->base0)) {
 981                 dev_err(dev, "ioremap failed for page-0 resource\n");
 982                 return PTR_ERR(cspmu->base0);
 983         }
 984
 985         /* Base address for page 1 if supported. Otherwise point to page 0. */
 986         cspmu->base1 = cspmu->base0;
 987         if (platform_get_resource(pdev, IORESOURCE_MEM, 1)) {
 988                 cspmu->base1 = devm_platform_ioremap_resource(pdev, 1);
 989                 if (IS_ERR(cspmu->base1)) {
 990                         dev_err(dev, "ioremap failed for page-1 resource\n");
 991                         return PTR_ERR(cspmu->base1);
 992                 }
 993         }
 994
 995         cspmu->pmcfgr = readl(cspmu->base0 + PMCFGR);
 996
 997         cspmu->num_logical_ctrs = FIELD_GET(PMCFGR_N, cspmu->pmcfgr) + 1;
 998
 999         cspmu->cycle_counter_logical_idx = ARM_CSPMU_MAX_HW_CNTRS;
1000
1001         if (supports_cycle_counter(cspmu)) {
1002                 /*
1003                  * The last logical counter is mapped to cycle counter if
1004                  * there is a gap between regular and cycle counter. Otherwise,
1005                  * logical and physical have 1-to-1 mapping.
1006                  */
1007                 cspmu->cycle_counter_logical_idx =
1008                         (cspmu->num_logical_ctrs <= ARM_CSPMU_CYCLE_CNTR_IDX) ?
1009                                 cspmu->num_logical_ctrs - 1 :
1010                                 ARM_CSPMU_CYCLE_CNTR_IDX;
1011         }
1012
1013         cspmu->num_set_clr_reg =
1014                 DIV_ROUND_UP(cspmu->num_logical_ctrs,
1015                                 ARM_CSPMU_SET_CLR_COUNTER_NUM);
1016
1017         cspmu->hw_events.events =
1018                 devm_kcalloc(dev, cspmu->num_logical_ctrs,
1019                              sizeof(*cspmu->hw_events.events), GFP_KERNEL);
1020
1021         if (!cspmu->hw_events.events)
1022                 return -ENOMEM;
1023
1024         return 0;
1025 }
1026
1027 static inline int arm_cspmu_get_reset_overflow(struct arm_cspmu *cspmu,
1028                                                u32 *pmovs)
1029 {
1030         int i;
1031         u32 pmovclr_offset = PMOVSCLR;
1032         u32 has_overflowed = 0;
1033
1034         for (i = 0; i < cspmu->num_set_clr_reg; ++i) {
1035                 pmovs[i] = readl(cspmu->base1 + pmovclr_offset);
1036                 has_overflowed |= pmovs[i];
1037                 writel(pmovs[i], cspmu->base1 + pmovclr_offset);
1038                 pmovclr_offset += sizeof(u32);
1039         }
1040
1041         return has_overflowed != 0;
1042 }
1043
1044 static irqreturn_t arm_cspmu_handle_irq(int irq_num, void *dev)
1045 {
1046         int idx, has_overflowed;
1047         struct perf_event *event;
1048         struct arm_cspmu *cspmu = dev;
1049         DECLARE_BITMAP(pmovs, ARM_CSPMU_MAX_HW_CNTRS);
1050         bool handled = false;
1051
1052         arm_cspmu_stop_counters(cspmu);
1053
1054         has_overflowed = arm_cspmu_get_reset_overflow(cspmu, (u32 *)pmovs);
1055         if (!has_overflowed)
1056                 goto done;
1057
1058         for_each_set_bit(idx, cspmu->hw_events.used_ctrs,
1059                         cspmu->num_logical_ctrs) {
1060                 event = cspmu->hw_events.events[idx];
1061
1062                 if (!event)
1063                         continue;
1064
1065                 if (!test_bit(event->hw.idx, pmovs))
1066                         continue;
1067
1068                 arm_cspmu_event_update(event);
1069                 arm_cspmu_set_event_period(event);
1070
1071                 handled = true;
1072         }
1073
1074 done:
1075         arm_cspmu_start_counters(cspmu);
1076         return IRQ_RETVAL(handled);
1077 }
1078
1079 static int arm_cspmu_request_irq(struct arm_cspmu *cspmu)
1080 {
1081         int irq, ret;
1082         struct device *dev;
1083         struct platform_device *pdev;
1084
1085         dev = cspmu->dev;
1086         pdev = to_platform_device(dev);
1087
1088         /* Skip IRQ request if the PMU does not support overflow interrupt. */
1089         irq = platform_get_irq_optional(pdev, 0);
1090         if (irq < 0)
1091                 return irq == -ENXIO ? 0 : irq;
1092
1093         ret = devm_request_irq(dev, irq, arm_cspmu_handle_irq,
1094                                IRQF_NOBALANCING | IRQF_NO_THREAD, dev_name(dev),
1095                                cspmu);
1096         if (ret) {
1097                 dev_err(dev, "Could not request IRQ %d\n", irq);
1098                 return ret;
1099         }
1100
1101         cspmu->irq = irq;
1102
1103         return 0;
1104 }
1105
1106 #if defined(CONFIG_ACPI) && defined(CONFIG_ARM64)
1107 #include <acpi/processor.h>
1108
1109 static inline int arm_cspmu_find_cpu_container(int cpu, u32 container_uid)
1110 {
1111         struct device *cpu_dev;
1112         struct acpi_device *acpi_dev;
1113
1114         cpu_dev = get_cpu_device(cpu);
1115         if (!cpu_dev)
1116                 return -ENODEV;
1117
1118         acpi_dev = ACPI_COMPANION(cpu_dev);
1119         while (acpi_dev) {
1120                 if (acpi_dev_hid_uid_match(acpi_dev, ACPI_PROCESSOR_CONTAINER_HID, container_uid))
1121                         return 0;
1122
1123                 acpi_dev = acpi_dev_parent(acpi_dev);
1124         }
1125
1126         return -ENODEV;
1127 }
1128
1129 static int arm_cspmu_acpi_get_cpus(struct arm_cspmu *cspmu)
1130 {
1131         struct acpi_apmt_node *apmt_node;
1132         int affinity_flag;
1133         int cpu;
1134
1135         apmt_node = arm_cspmu_apmt_node(cspmu->dev);
1136         affinity_flag = apmt_node->flags & ACPI_APMT_FLAGS_AFFINITY;
1137
1138         if (affinity_flag == ACPI_APMT_FLAGS_AFFINITY_PROC) {
1139                 for_each_possible_cpu(cpu) {
1140                         if (apmt_node->proc_affinity ==
1141                             get_acpi_id_for_cpu(cpu)) {
1142                                 cpumask_set_cpu(cpu, &cspmu->associated_cpus);
1143                                 break;
1144                         }
1145                 }
1146         } else {
1147                 for_each_possible_cpu(cpu) {
1148                         if (arm_cspmu_find_cpu_container(
1149                                     cpu, apmt_node->proc_affinity))
1150                                 continue;
1151
1152                         cpumask_set_cpu(cpu, &cspmu->associated_cpus);
1153                 }
1154         }
1155
1156         if (cpumask_empty(&cspmu->associated_cpus)) {
1157                 dev_dbg(cspmu->dev, "No cpu associated with the PMU\n");
1158                 return -ENODEV;
1159         }
1160
1161         return 0;
1162 }
1163 #else
1164 static int arm_cspmu_acpi_get_cpus(struct arm_cspmu *cspmu)
1165 {
1166         return -ENODEV;
1167 }
1168 #endif
1169
1170 static int arm_cspmu_get_cpus(struct arm_cspmu *cspmu)
1171 {
1172         return arm_cspmu_acpi_get_cpus(cspmu);
1173 }
1174
1175 static int arm_cspmu_register_pmu(struct arm_cspmu *cspmu)
1176 {
1177         int ret, capabilities;
1178         struct attribute_group **attr_groups;
1179
1180         attr_groups = arm_cspmu_alloc_attr_group(cspmu);
1181         if (!attr_groups)
1182                 return -ENOMEM;
1183
1184         ret = cpuhp_state_add_instance(arm_cspmu_cpuhp_state,
1185                                        &cspmu->cpuhp_node);
1186         if (ret)
1187                 return ret;
1188
1189         capabilities = PERF_PMU_CAP_NO_EXCLUDE;
1190         if (cspmu->irq == 0)
1191                 capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
1192
1193         cspmu->pmu = (struct pmu){
1194                 .task_ctx_nr    = perf_invalid_context,
1195                 .module         = cspmu->impl.module,
1196                 .pmu_enable     = arm_cspmu_enable,
1197                 .pmu_disable    = arm_cspmu_disable,
1198                 .event_init     = arm_cspmu_event_init,
1199                 .add            = arm_cspmu_add,
1200                 .del            = arm_cspmu_del,
1201                 .start          = arm_cspmu_start,
1202                 .stop           = arm_cspmu_stop,
1203                 .read           = arm_cspmu_read,
1204                 .attr_groups    = (const struct attribute_group **)attr_groups,
1205                 .capabilities   = capabilities,
1206         };
1207
1208         /* Hardware counter init */
1209         arm_cspmu_stop_counters(cspmu);
1210         arm_cspmu_reset_counters(cspmu);
1211
1212         ret = perf_pmu_register(&cspmu->pmu, cspmu->name, -1);
1213         if (ret) {
1214                 cpuhp_state_remove_instance(arm_cspmu_cpuhp_state,
1215                                             &cspmu->cpuhp_node);
1216         }
1217
1218         return ret;
1219 }
1220
1221 static int arm_cspmu_device_probe(struct platform_device *pdev)
1222 {
1223         int ret;
1224         struct arm_cspmu *cspmu;
1225
1226         cspmu = arm_cspmu_alloc(pdev);
1227         if (!cspmu)
1228                 return -ENOMEM;
1229
1230         ret = arm_cspmu_init_mmio(cspmu);
1231         if (ret)
1232                 return ret;
1233
1234         ret = arm_cspmu_request_irq(cspmu);
1235         if (ret)
1236                 return ret;
1237
1238         ret = arm_cspmu_get_cpus(cspmu);
1239         if (ret)
1240                 return ret;
1241
1242         ret = arm_cspmu_init_impl_ops(cspmu);
1243         if (ret)
1244                 return ret;
1245
1246         ret = arm_cspmu_register_pmu(cspmu);
1247
1248         /* Matches arm_cspmu_init_impl_ops() above. */
1249         if (cspmu->impl.module != THIS_MODULE)
1250                 module_put(cspmu->impl.module);
1251
1252         return ret;
1253 }
1254
1255 static int arm_cspmu_device_remove(struct platform_device *pdev)
1256 {
1257         struct arm_cspmu *cspmu = platform_get_drvdata(pdev);
1258
1259         perf_pmu_unregister(&cspmu->pmu);
1260         cpuhp_state_remove_instance(arm_cspmu_cpuhp_state, &cspmu->cpuhp_node);
1261
1262         return 0;
1263 }
1264
1265 static const struct platform_device_id arm_cspmu_id[] = {
1266         {DRVNAME, 0},
1267         { },
1268 };
1269 MODULE_DEVICE_TABLE(platform, arm_cspmu_id);
1270
1271 static struct platform_driver arm_cspmu_driver = {
1272         .driver = {
1273                         .name = DRVNAME,
1274                         .suppress_bind_attrs = true,
1275                 },
1276         .probe = arm_cspmu_device_probe,
1277         .remove = arm_cspmu_device_remove,
1278         .id_table = arm_cspmu_id,
1279 };
1280
1281 static void arm_cspmu_set_active_cpu(int cpu, struct arm_cspmu *cspmu)
1282 {
1283         cpumask_set_cpu(cpu, &cspmu->active_cpu);
1284         if (cspmu->irq)
1285                 WARN_ON(irq_set_affinity(cspmu->irq, &cspmu->active_cpu));
1286 }
1287
1288 static int arm_cspmu_cpu_online(unsigned int cpu, struct hlist_node *node)
1289 {
1290         struct arm_cspmu *cspmu =
1291                 hlist_entry_safe(node, struct arm_cspmu, cpuhp_node);
1292
1293         if (!cpumask_test_cpu(cpu, &cspmu->associated_cpus))
1294                 return 0;
1295
1296         /* If the PMU is already managed, there is nothing to do */
1297         if (!cpumask_empty(&cspmu->active_cpu))
1298                 return 0;
1299
1300         /* Use this CPU for event counting */
1301         arm_cspmu_set_active_cpu(cpu, cspmu);
1302
1303         return 0;
1304 }
1305
1306 static int arm_cspmu_cpu_teardown(unsigned int cpu, struct hlist_node *node)
1307 {
1308         int dst;
1309         struct cpumask online_supported;
1310
1311         struct arm_cspmu *cspmu =
1312                 hlist_entry_safe(node, struct arm_cspmu, cpuhp_node);
1313
1314         /* Nothing to do if this CPU doesn't own the PMU */
1315         if (!cpumask_test_and_clear_cpu(cpu, &cspmu->active_cpu))
1316                 return 0;
1317
1318         /* Choose a new CPU to migrate ownership of the PMU to */
1319         cpumask_and(&online_supported, &cspmu->associated_cpus,
1320                     cpu_online_mask);
1321         dst = cpumask_any_but(&online_supported, cpu);
1322         if (dst >= nr_cpu_ids)
1323                 return 0;
1324
1325         /* Use this CPU for event counting */
1326         perf_pmu_migrate_context(&cspmu->pmu, cpu, dst);
1327         arm_cspmu_set_active_cpu(dst, cspmu);
1328
1329         return 0;
1330 }
1331
1332 static int __init arm_cspmu_init(void)
1333 {
1334         int ret;
1335
1336         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
1337                                         "perf/arm/cspmu:online",
1338                                         arm_cspmu_cpu_online,
1339                                         arm_cspmu_cpu_teardown);
1340         if (ret < 0)
1341                 return ret;
1342         arm_cspmu_cpuhp_state = ret;
1343         return platform_driver_register(&arm_cspmu_driver);
1344 }
1345
1346 static void __exit arm_cspmu_exit(void)
1347 {
1348         platform_driver_unregister(&arm_cspmu_driver);
1349         cpuhp_remove_multi_state(arm_cspmu_cpuhp_state);
1350 }
1351
1352 int arm_cspmu_impl_register(const struct arm_cspmu_impl_match *impl_match)
1353 {
1354         struct arm_cspmu_impl_match *match;
1355         int ret = 0;
1356
1357         match = arm_cspmu_impl_match_get(impl_match->pmiidr_val);
1358
1359         if (match) {
1360                 mutex_lock(&arm_cspmu_lock);
1361
1362                 if (!match->impl_init_ops) {
1363                         match->module = impl_match->module;
1364                         match->impl_init_ops = impl_match->impl_init_ops;
1365                 } else {
1366                         /* Broken match table may contain non-unique entries */
1367                         WARN(1, "arm_cspmu backend already registered for module: %s, pmiidr: 0x%x, mask: 0x%x\n",
1368                                 match->module_name,
1369                                 match->pmiidr_val,
1370                                 match->pmiidr_mask);
1371
1372                         ret = -EINVAL;
1373                 }
1374
1375                 mutex_unlock(&arm_cspmu_lock);
1376
1377                 if (!ret)
1378                         ret = driver_attach(&arm_cspmu_driver.driver);
1379         } else {
1380                 pr_err("arm_cspmu reg failed, unable to find a match for pmiidr: 0x%x\n",
1381                         impl_match->pmiidr_val);
1382
1383                 ret = -EINVAL;
1384         }
1385
1386         return ret;
1387 }
1388 EXPORT_SYMBOL_GPL(arm_cspmu_impl_register);
1389
1390 static int arm_cspmu_match_device(struct device *dev, const void *match)
1391 {
1392         struct arm_cspmu *cspmu = platform_get_drvdata(to_platform_device(dev));
1393
1394         return (cspmu && cspmu->impl.match == match) ? 1 : 0;
1395 }
1396
1397 void arm_cspmu_impl_unregister(const struct arm_cspmu_impl_match *impl_match)
1398 {
1399         struct device *dev;
1400         struct arm_cspmu_impl_match *match;
1401
1402         match = arm_cspmu_impl_match_get(impl_match->pmiidr_val);
1403
1404         if (WARN_ON(!match))
1405                 return;
1406
1407         /* Unbind the driver from all matching backend devices. */
1408         while ((dev = driver_find_device(&arm_cspmu_driver.driver, NULL,
1409                         match, arm_cspmu_match_device)))
1410                 device_release_driver(dev);
1411
1412         mutex_lock(&arm_cspmu_lock);
1413
1414         match->module = NULL;
1415         match->impl_init_ops = NULL;
1416
1417         mutex_unlock(&arm_cspmu_lock);
1418 }
1419 EXPORT_SYMBOL_GPL(arm_cspmu_impl_unregister);
1420
1421 module_init(arm_cspmu_init);
1422 module_exit(arm_cspmu_exit);
1423
1424 MODULE_LICENSE("GPL v2");