arch/x86/kernel/cpu/perf_event_intel_cstate.c

   1 /*
   2  * perf_event_intel_cstate.c: support cstate residency counters
   3  *
   4  * Copyright (C) 2015, Intel Corp.
   5  * Author: Kan Liang (kan.liang@intel.com)
   6  *
   7  * This library is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU Library General Public
   9  * License as published by the Free Software Foundation; either
  10  * version 2 of the License, or (at your option) any later version.
  11  *
  12  * This library is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * Library General Public License for more details.
  16  *
  17  */
  18
  19 /*
  20  * This file export cstate related free running (read-only) counters
  21  * for perf. These counters may be use simultaneously by other tools,
  22  * such as turbostat. However, it still make sense to implement them
  23  * in perf. Because we can conveniently collect them together with
  24  * other events, and allow to use them from tools without special MSR
  25  * access code.
  26  *
  27  * The events only support system-wide mode counting. There is no
  28  * sampling support because it is not supported by the hardware.
  29  *
  30  * According to counters' scope and category, two PMUs are registered
  31  * with the perf_event core subsystem.
  32  *  - 'cstate_core': The counter is available for each physical core.
  33  *    The counters include CORE_C*_RESIDENCY.
  34  *  - 'cstate_pkg': The counter is available for each physical package.
  35  *    The counters include PKG_C*_RESIDENCY.
  36  *
  37  * All of these counters are specified in the Intel® 64 and IA-32
  38  * Architectures Software Developer.s Manual Vol3b.
  39  *
  40  * Model specific counters:
  41  *      MSR_CORE_C1_RES: CORE C1 Residency Counter
  42  *                       perf code: 0x00
  43  *                       Available model: SLM,AMT
  44  *                       Scope: Core (each processor core has a MSR)
  45  *      MSR_CORE_C3_RESIDENCY: CORE C3 Residency Counter
  46  *                             perf code: 0x01
  47  *                             Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL
  48  *                             Scope: Core
  49  *      MSR_CORE_C6_RESIDENCY: CORE C6 Residency Counter
  50  *                             perf code: 0x02
  51  *                             Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,SKL
  52  *                             Scope: Core
  53  *      MSR_CORE_C7_RESIDENCY: CORE C7 Residency Counter
  54  *                             perf code: 0x03
  55  *                             Available model: SNB,IVB,HSW,BDW,SKL
  56  *                             Scope: Core
  57  *      MSR_PKG_C2_RESIDENCY:  Package C2 Residency Counter.
  58  *                             perf code: 0x00
  59  *                             Available model: SNB,IVB,HSW,BDW,SKL
  60  *                             Scope: Package (physical package)
  61  *      MSR_PKG_C3_RESIDENCY:  Package C3 Residency Counter.
  62  *                             perf code: 0x01
  63  *                             Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL
  64  *                             Scope: Package (physical package)
  65  *      MSR_PKG_C6_RESIDENCY:  Package C6 Residency Counter.
  66  *                             perf code: 0x02
  67  *                             Available model: SLM,AMT,NHM,WSM,SNB,IVB,HSW,BDW,SKL
  68  *                             Scope: Package (physical package)
  69  *      MSR_PKG_C7_RESIDENCY:  Package C7 Residency Counter.
  70  *                             perf code: 0x03
  71  *                             Available model: NHM,WSM,SNB,IVB,HSW,BDW,SKL
  72  *                             Scope: Package (physical package)
  73  *      MSR_PKG_C8_RESIDENCY:  Package C8 Residency Counter.
  74  *                             perf code: 0x04
  75  *                             Available model: HSW ULT only
  76  *                             Scope: Package (physical package)
  77  *      MSR_PKG_C9_RESIDENCY:  Package C9 Residency Counter.
  78  *                             perf code: 0x05
  79  *                             Available model: HSW ULT only
  80  *                             Scope: Package (physical package)
  81  *      MSR_PKG_C10_RESIDENCY: Package C10 Residency Counter.
  82  *                             perf code: 0x06
  83  *                             Available model: HSW ULT only
  84  *                             Scope: Package (physical package)
  85  *
  86  */
  87
  88 #include <linux/module.h>
  89 #include <linux/slab.h>
  90 #include <linux/perf_event.h>
  91 #include <linux/nospec.h>
  92 #include <asm/cpu_device_id.h>
  93 #include "perf_event.h"
  94
  95 #define DEFINE_CSTATE_FORMAT_ATTR(_var, _name, _format)         \
  96 static ssize_t __cstate_##_var##_show(struct kobject *kobj,     \
  97                                 struct kobj_attribute *attr,    \
  98                                 char *page)                     \
  99 {                                                               \
 100         BUILD_BUG_ON(sizeof(_format) >= PAGE_SIZE);             \
 101         return sprintf(page, _format "\n");                     \
 102 }                                                               \
 103 static struct kobj_attribute format_attr_##_var =               \
 104         __ATTR(_name, 0444, __cstate_##_var##_show, NULL)
 105
 106 static ssize_t cstate_get_attr_cpumask(struct device *dev,
 107                                        struct device_attribute *attr,
 108                                        char *buf);
 109
 110 struct perf_cstate_msr {
 111         u64     msr;
 112         struct  perf_pmu_events_attr *attr;
 113         bool    (*test)(int idx);
 114 };
 115
 116
 117 /* cstate_core PMU */
 118
 119 static struct pmu cstate_core_pmu;
 120 static bool has_cstate_core;
 121
 122 enum perf_cstate_core_id {
 123         /*
 124          * cstate_core events
 125          */
 126         PERF_CSTATE_CORE_C1_RES = 0,
 127         PERF_CSTATE_CORE_C3_RES,
 128         PERF_CSTATE_CORE_C6_RES,
 129         PERF_CSTATE_CORE_C7_RES,
 130
 131         PERF_CSTATE_CORE_EVENT_MAX,
 132 };
 133
 134 bool test_core(int idx)
 135 {
 136         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL ||
 137             boot_cpu_data.x86 != 6)
 138                 return false;
 139
 140         switch (boot_cpu_data.x86_model) {
 141         case 30: /* 45nm Nehalem    */
 142         case 26: /* 45nm Nehalem-EP */
 143         case 46: /* 45nm Nehalem-EX */
 144
 145         case 37: /* 32nm Westmere    */
 146         case 44: /* 32nm Westmere-EP */
 147         case 47: /* 32nm Westmere-EX */
 148                 if (idx == PERF_CSTATE_CORE_C3_RES ||
 149                     idx == PERF_CSTATE_CORE_C6_RES)
 150                         return true;
 151                 break;
 152         case 42: /* 32nm SandyBridge         */
 153         case 45: /* 32nm SandyBridge-E/EN/EP */
 154
 155         case 58: /* 22nm IvyBridge       */
 156         case 62: /* 22nm IvyBridge-EP/EX */
 157
 158         case 60: /* 22nm Haswell Core */
 159         case 63: /* 22nm Haswell Server */
 160         case 69: /* 22nm Haswell ULT */
 161         case 70: /* 22nm Haswell + GT3e (Intel Iris Pro graphics) */
 162
 163         case 61: /* 14nm Broadwell Core-M */
 164         case 86: /* 14nm Broadwell Xeon D */
 165         case 71: /* 14nm Broadwell + GT3e (Intel Iris Pro graphics) */
 166         case 79: /* 14nm Broadwell Server */
 167
 168         case 78: /* 14nm Skylake Mobile */
 169         case 94: /* 14nm Skylake Desktop */
 170                 if (idx == PERF_CSTATE_CORE_C3_RES ||
 171                     idx == PERF_CSTATE_CORE_C6_RES ||
 172                     idx == PERF_CSTATE_CORE_C7_RES)
 173                         return true;
 174                 break;
 175         case 55: /* 22nm Atom "Silvermont"                */
 176         case 77: /* 22nm Atom "Silvermont Avoton/Rangely" */
 177         case 76: /* 14nm Atom "Airmont"                   */
 178                 if (idx == PERF_CSTATE_CORE_C1_RES ||
 179                     idx == PERF_CSTATE_CORE_C6_RES)
 180                         return true;
 181                 break;
 182         }
 183
 184         return false;
 185 }
 186
 187 PMU_EVENT_ATTR_STRING(c1-residency, evattr_cstate_core_c1, "event=0x00");
 188 PMU_EVENT_ATTR_STRING(c3-residency, evattr_cstate_core_c3, "event=0x01");
 189 PMU_EVENT_ATTR_STRING(c6-residency, evattr_cstate_core_c6, "event=0x02");
 190 PMU_EVENT_ATTR_STRING(c7-residency, evattr_cstate_core_c7, "event=0x03");
 191
 192 static struct perf_cstate_msr core_msr[] = {
 193         [PERF_CSTATE_CORE_C1_RES] = { MSR_CORE_C1_RES,          &evattr_cstate_core_c1, test_core, },
 194         [PERF_CSTATE_CORE_C3_RES] = { MSR_CORE_C3_RESIDENCY,    &evattr_cstate_core_c3, test_core, },
 195         [PERF_CSTATE_CORE_C6_RES] = { MSR_CORE_C6_RESIDENCY,    &evattr_cstate_core_c6, test_core, },
 196         [PERF_CSTATE_CORE_C7_RES] = { MSR_CORE_C7_RESIDENCY,    &evattr_cstate_core_c7, test_core, },
 197 };
 198
 199 static struct attribute *core_events_attrs[PERF_CSTATE_CORE_EVENT_MAX + 1] = {
 200         NULL,
 201 };
 202
 203 static struct attribute_group core_events_attr_group = {
 204         .name = "events",
 205         .attrs = core_events_attrs,
 206 };
 207
 208 DEFINE_CSTATE_FORMAT_ATTR(core_event, event, "config:0-63");
 209 static struct attribute *core_format_attrs[] = {
 210         &format_attr_core_event.attr,
 211         NULL,
 212 };
 213
 214 static struct attribute_group core_format_attr_group = {
 215         .name = "format",
 216         .attrs = core_format_attrs,
 217 };
 218
 219 static cpumask_t cstate_core_cpu_mask;
 220 static DEVICE_ATTR(cpumask, S_IRUGO, cstate_get_attr_cpumask, NULL);
 221
 222 static struct attribute *cstate_cpumask_attrs[] = {
 223         &dev_attr_cpumask.attr,
 224         NULL,
 225 };
 226
 227 static struct attribute_group cpumask_attr_group = {
 228         .attrs = cstate_cpumask_attrs,
 229 };
 230
 231 static const struct attribute_group *core_attr_groups[] = {
 232         &core_events_attr_group,
 233         &core_format_attr_group,
 234         &cpumask_attr_group,
 235         NULL,
 236 };
 237
 238 /* cstate_core PMU end */
 239
 240
 241 /* cstate_pkg PMU */
 242
 243 static struct pmu cstate_pkg_pmu;
 244 static bool has_cstate_pkg;
 245
 246 enum perf_cstate_pkg_id {
 247         /*
 248          * cstate_pkg events
 249          */
 250         PERF_CSTATE_PKG_C2_RES = 0,
 251         PERF_CSTATE_PKG_C3_RES,
 252         PERF_CSTATE_PKG_C6_RES,
 253         PERF_CSTATE_PKG_C7_RES,
 254         PERF_CSTATE_PKG_C8_RES,
 255         PERF_CSTATE_PKG_C9_RES,
 256         PERF_CSTATE_PKG_C10_RES,
 257
 258         PERF_CSTATE_PKG_EVENT_MAX,
 259 };
 260
 261 bool test_pkg(int idx)
 262 {
 263         if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL ||
 264             boot_cpu_data.x86 != 6)
 265                 return false;
 266
 267         switch (boot_cpu_data.x86_model) {
 268         case 30: /* 45nm Nehalem    */
 269         case 26: /* 45nm Nehalem-EP */
 270         case 46: /* 45nm Nehalem-EX */
 271
 272         case 37: /* 32nm Westmere    */
 273         case 44: /* 32nm Westmere-EP */
 274         case 47: /* 32nm Westmere-EX */
 275                 if (idx == PERF_CSTATE_CORE_C3_RES ||
 276                     idx == PERF_CSTATE_CORE_C6_RES ||
 277                     idx == PERF_CSTATE_CORE_C7_RES)
 278                         return true;
 279                 break;
 280         case 42: /* 32nm SandyBridge         */
 281         case 45: /* 32nm SandyBridge-E/EN/EP */
 282
 283         case 58: /* 22nm IvyBridge       */
 284         case 62: /* 22nm IvyBridge-EP/EX */
 285
 286         case 60: /* 22nm Haswell Core */
 287         case 63: /* 22nm Haswell Server */
 288         case 70: /* 22nm Haswell + GT3e (Intel Iris Pro graphics) */
 289
 290         case 61: /* 14nm Broadwell Core-M */
 291         case 86: /* 14nm Broadwell Xeon D */
 292         case 71: /* 14nm Broadwell + GT3e (Intel Iris Pro graphics) */
 293         case 79: /* 14nm Broadwell Server */
 294
 295         case 78: /* 14nm Skylake Mobile */
 296         case 94: /* 14nm Skylake Desktop */
 297                 if (idx == PERF_CSTATE_PKG_C2_RES ||
 298                     idx == PERF_CSTATE_PKG_C3_RES ||
 299                     idx == PERF_CSTATE_PKG_C6_RES ||
 300                     idx == PERF_CSTATE_PKG_C7_RES)
 301                         return true;
 302                 break;
 303         case 55: /* 22nm Atom "Silvermont"                */
 304         case 77: /* 22nm Atom "Silvermont Avoton/Rangely" */
 305         case 76: /* 14nm Atom "Airmont"                   */
 306                 if (idx == PERF_CSTATE_CORE_C6_RES)
 307                         return true;
 308                 break;
 309         case 69: /* 22nm Haswell ULT */
 310                 if (idx == PERF_CSTATE_PKG_C2_RES ||
 311                     idx == PERF_CSTATE_PKG_C3_RES ||
 312                     idx == PERF_CSTATE_PKG_C6_RES ||
 313                     idx == PERF_CSTATE_PKG_C7_RES ||
 314                     idx == PERF_CSTATE_PKG_C8_RES ||
 315                     idx == PERF_CSTATE_PKG_C9_RES ||
 316                     idx == PERF_CSTATE_PKG_C10_RES)
 317                         return true;
 318                 break;
 319         }
 320
 321         return false;
 322 }
 323
 324 PMU_EVENT_ATTR_STRING(c2-residency, evattr_cstate_pkg_c2, "event=0x00");
 325 PMU_EVENT_ATTR_STRING(c3-residency, evattr_cstate_pkg_c3, "event=0x01");
 326 PMU_EVENT_ATTR_STRING(c6-residency, evattr_cstate_pkg_c6, "event=0x02");
 327 PMU_EVENT_ATTR_STRING(c7-residency, evattr_cstate_pkg_c7, "event=0x03");
 328 PMU_EVENT_ATTR_STRING(c8-residency, evattr_cstate_pkg_c8, "event=0x04");
 329 PMU_EVENT_ATTR_STRING(c9-residency, evattr_cstate_pkg_c9, "event=0x05");
 330 PMU_EVENT_ATTR_STRING(c10-residency, evattr_cstate_pkg_c10, "event=0x06");
 331
 332 static struct perf_cstate_msr pkg_msr[] = {
 333         [PERF_CSTATE_PKG_C2_RES] = { MSR_PKG_C2_RESIDENCY,      &evattr_cstate_pkg_c2,  test_pkg, },
 334         [PERF_CSTATE_PKG_C3_RES] = { MSR_PKG_C3_RESIDENCY,      &evattr_cstate_pkg_c3,  test_pkg, },
 335         [PERF_CSTATE_PKG_C6_RES] = { MSR_PKG_C6_RESIDENCY,      &evattr_cstate_pkg_c6,  test_pkg, },
 336         [PERF_CSTATE_PKG_C7_RES] = { MSR_PKG_C7_RESIDENCY,      &evattr_cstate_pkg_c7,  test_pkg, },
 337         [PERF_CSTATE_PKG_C8_RES] = { MSR_PKG_C8_RESIDENCY,      &evattr_cstate_pkg_c8,  test_pkg, },
 338         [PERF_CSTATE_PKG_C9_RES] = { MSR_PKG_C9_RESIDENCY,      &evattr_cstate_pkg_c9,  test_pkg, },
 339         [PERF_CSTATE_PKG_C10_RES] = { MSR_PKG_C10_RESIDENCY,    &evattr_cstate_pkg_c10, test_pkg, },
 340 };
 341
 342 static struct attribute *pkg_events_attrs[PERF_CSTATE_PKG_EVENT_MAX + 1] = {
 343         NULL,
 344 };
 345
 346 static struct attribute_group pkg_events_attr_group = {
 347         .name = "events",
 348         .attrs = pkg_events_attrs,
 349 };
 350
 351 DEFINE_CSTATE_FORMAT_ATTR(pkg_event, event, "config:0-63");
 352 static struct attribute *pkg_format_attrs[] = {
 353         &format_attr_pkg_event.attr,
 354         NULL,
 355 };
 356 static struct attribute_group pkg_format_attr_group = {
 357         .name = "format",
 358         .attrs = pkg_format_attrs,
 359 };
 360
 361 static cpumask_t cstate_pkg_cpu_mask;
 362
 363 static const struct attribute_group *pkg_attr_groups[] = {
 364         &pkg_events_attr_group,
 365         &pkg_format_attr_group,
 366         &cpumask_attr_group,
 367         NULL,
 368 };
 369
 370 /* cstate_pkg PMU end*/
 371
 372 static ssize_t cstate_get_attr_cpumask(struct device *dev,
 373                                        struct device_attribute *attr,
 374                                        char *buf)
 375 {
 376         struct pmu *pmu = dev_get_drvdata(dev);
 377
 378         if (pmu == &cstate_core_pmu)
 379                 return cpumap_print_to_pagebuf(true, buf, &cstate_core_cpu_mask);
 380         else if (pmu == &cstate_pkg_pmu)
 381                 return cpumap_print_to_pagebuf(true, buf, &cstate_pkg_cpu_mask);
 382         else
 383                 return 0;
 384 }
 385
 386 static int cstate_pmu_event_init(struct perf_event *event)
 387 {
 388         u64 cfg = event->attr.config;
 389         int ret = 0;
 390
 391         if (event->attr.type != event->pmu->type)
 392                 return -ENOENT;
 393
 394         /* unsupported modes and filters */
 395         if (event->attr.exclude_user   ||
 396             event->attr.exclude_kernel ||
 397             event->attr.exclude_hv     ||
 398             event->attr.exclude_idle   ||
 399             event->attr.exclude_host   ||
 400             event->attr.exclude_guest  ||
 401             event->attr.sample_period) /* no sampling */
 402                 return -EINVAL;
 403
 404         if (event->pmu == &cstate_core_pmu) {
 405                 if (cfg >= PERF_CSTATE_CORE_EVENT_MAX)
 406                         return -EINVAL;
 407                 if (!core_msr[cfg].attr)
 408                         return -EINVAL;
 409                 event->hw.event_base = core_msr[cfg].msr;
 410         } else if (event->pmu == &cstate_pkg_pmu) {
 411                 if (cfg >= PERF_CSTATE_PKG_EVENT_MAX)
 412                         return -EINVAL;
 413                 cfg = array_index_nospec((unsigned long)cfg, PERF_CSTATE_PKG_EVENT_MAX);
 414                 if (!pkg_msr[cfg].attr)
 415                         return -EINVAL;
 416                 event->hw.event_base = pkg_msr[cfg].msr;
 417         } else
 418                 return -ENOENT;
 419
 420         /* must be done before validate_group */
 421         event->hw.config = cfg;
 422         event->hw.idx = -1;
 423
 424         return ret;
 425 }
 426
 427 static inline u64 cstate_pmu_read_counter(struct perf_event *event)
 428 {
 429         u64 val;
 430
 431         rdmsrl(event->hw.event_base, val);
 432         return val;
 433 }
 434
 435 static void cstate_pmu_event_update(struct perf_event *event)
 436 {
 437         struct hw_perf_event *hwc = &event->hw;
 438         u64 prev_raw_count, new_raw_count;
 439
 440 again:
 441         prev_raw_count = local64_read(&hwc->prev_count);
 442         new_raw_count = cstate_pmu_read_counter(event);
 443
 444         if (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
 445                             new_raw_count) != prev_raw_count)
 446                 goto again;
 447
 448         local64_add(new_raw_count - prev_raw_count, &event->count);
 449 }
 450
 451 static void cstate_pmu_event_start(struct perf_event *event, int mode)
 452 {
 453         local64_set(&event->hw.prev_count, cstate_pmu_read_counter(event));
 454 }
 455
 456 static void cstate_pmu_event_stop(struct perf_event *event, int mode)
 457 {
 458         cstate_pmu_event_update(event);
 459 }
 460
 461 static void cstate_pmu_event_del(struct perf_event *event, int mode)
 462 {
 463         cstate_pmu_event_stop(event, PERF_EF_UPDATE);
 464 }
 465
 466 static int cstate_pmu_event_add(struct perf_event *event, int mode)
 467 {
 468         if (mode & PERF_EF_START)
 469                 cstate_pmu_event_start(event, mode);
 470
 471         return 0;
 472 }
 473
 474 static void cstate_cpu_exit(int cpu)
 475 {
 476         int i, id, target;
 477
 478         /* cpu exit for cstate core */
 479         if (has_cstate_core) {
 480                 id = topology_core_id(cpu);
 481                 target = -1;
 482
 483                 for_each_online_cpu(i) {
 484                         if (i == cpu)
 485                                 continue;
 486                         if (id == topology_core_id(i)) {
 487                                 target = i;
 488                                 break;
 489                         }
 490                 }
 491                 if (cpumask_test_and_clear_cpu(cpu, &cstate_core_cpu_mask) && target >= 0)
 492                         cpumask_set_cpu(target, &cstate_core_cpu_mask);
 493                 WARN_ON(cpumask_empty(&cstate_core_cpu_mask));
 494                 if (target >= 0)
 495                         perf_pmu_migrate_context(&cstate_core_pmu, cpu, target);
 496         }
 497
 498         /* cpu exit for cstate pkg */
 499         if (has_cstate_pkg) {
 500                 id = topology_physical_package_id(cpu);
 501                 target = -1;
 502
 503                 for_each_online_cpu(i) {
 504                         if (i == cpu)
 505                                 continue;
 506                         if (id == topology_physical_package_id(i)) {
 507                                 target = i;
 508                                 break;
 509                         }
 510                 }
 511                 if (cpumask_test_and_clear_cpu(cpu, &cstate_pkg_cpu_mask) && target >= 0)
 512                         cpumask_set_cpu(target, &cstate_pkg_cpu_mask);
 513                 WARN_ON(cpumask_empty(&cstate_pkg_cpu_mask));
 514                 if (target >= 0)
 515                         perf_pmu_migrate_context(&cstate_pkg_pmu, cpu, target);
 516         }
 517 }
 518
 519 static void cstate_cpu_init(int cpu)
 520 {
 521         int i, id;
 522
 523         /* cpu init for cstate core */
 524         if (has_cstate_core) {
 525                 id = topology_core_id(cpu);
 526                 for_each_cpu(i, &cstate_core_cpu_mask) {
 527                         if (id == topology_core_id(i))
 528                                 break;
 529                 }
 530                 if (i >= nr_cpu_ids)
 531                         cpumask_set_cpu(cpu, &cstate_core_cpu_mask);
 532         }
 533
 534         /* cpu init for cstate pkg */
 535         if (has_cstate_pkg) {
 536                 id = topology_physical_package_id(cpu);
 537                 for_each_cpu(i, &cstate_pkg_cpu_mask) {
 538                         if (id == topology_physical_package_id(i))
 539                                 break;
 540                 }
 541                 if (i >= nr_cpu_ids)
 542                         cpumask_set_cpu(cpu, &cstate_pkg_cpu_mask);
 543         }
 544 }
 545
 546 static int cstate_cpu_notifier(struct notifier_block *self,
 547                                   unsigned long action, void *hcpu)
 548 {
 549         unsigned int cpu = (long)hcpu;
 550
 551         switch (action & ~CPU_TASKS_FROZEN) {
 552         case CPU_UP_PREPARE:
 553                 break;
 554         case CPU_STARTING:
 555                 cstate_cpu_init(cpu);
 556                 break;
 557         case CPU_UP_CANCELED:
 558         case CPU_DYING:
 559                 break;
 560         case CPU_ONLINE:
 561         case CPU_DEAD:
 562                 break;
 563         case CPU_DOWN_PREPARE:
 564                 cstate_cpu_exit(cpu);
 565                 break;
 566         default:
 567                 break;
 568         }
 569
 570         return NOTIFY_OK;
 571 }
 572
 573 /*
 574  * Probe the cstate events and insert the available one into sysfs attrs
 575  * Return false if there is no available events.
 576  */
 577 static bool cstate_probe_msr(struct perf_cstate_msr *msr,
 578                              struct attribute   **events_attrs,
 579                              int max_event_nr)
 580 {
 581         int i, j = 0;
 582         u64 val;
 583
 584         /* Probe the cstate events. */
 585         for (i = 0; i < max_event_nr; i++) {
 586                 if (!msr[i].test(i) || rdmsrl_safe(msr[i].msr, &val))
 587                         msr[i].attr = NULL;
 588         }
 589
 590         /* List remaining events in the sysfs attrs. */
 591         for (i = 0; i < max_event_nr; i++) {
 592                 if (msr[i].attr)
 593                         events_attrs[j++] = &msr[i].attr->attr.attr;
 594         }
 595         events_attrs[j] = NULL;
 596
 597         return (j > 0) ? true : false;
 598 }
 599
 600 static int __init cstate_init(void)
 601 {
 602         /* SLM has different MSR for PKG C6 */
 603         switch (boot_cpu_data.x86_model) {
 604         case 55:
 605         case 76:
 606         case 77:
 607                 pkg_msr[PERF_CSTATE_PKG_C6_RES].msr = MSR_PKG_C7_RESIDENCY;
 608         }
 609
 610         if (cstate_probe_msr(core_msr, core_events_attrs, PERF_CSTATE_CORE_EVENT_MAX))
 611                 has_cstate_core = true;
 612
 613         if (cstate_probe_msr(pkg_msr, pkg_events_attrs, PERF_CSTATE_PKG_EVENT_MAX))
 614                 has_cstate_pkg = true;
 615
 616         return (has_cstate_core || has_cstate_pkg) ? 0 : -ENODEV;
 617 }
 618
 619 static void __init cstate_cpumask_init(void)
 620 {
 621         int cpu;
 622
 623         cpu_notifier_register_begin();
 624
 625         for_each_online_cpu(cpu)
 626                 cstate_cpu_init(cpu);
 627
 628         __perf_cpu_notifier(cstate_cpu_notifier);
 629
 630         cpu_notifier_register_done();
 631 }
 632
 633 static struct pmu cstate_core_pmu = {
 634         .attr_groups    = core_attr_groups,
 635         .name           = "cstate_core",
 636         .task_ctx_nr    = perf_invalid_context,
 637         .event_init     = cstate_pmu_event_init,
 638         .add            = cstate_pmu_event_add, /* must have */
 639         .del            = cstate_pmu_event_del, /* must have */
 640         .start          = cstate_pmu_event_start,
 641         .stop           = cstate_pmu_event_stop,
 642         .read           = cstate_pmu_event_update,
 643         .capabilities   = PERF_PMU_CAP_NO_INTERRUPT,
 644 };
 645
 646 static struct pmu cstate_pkg_pmu = {
 647         .attr_groups    = pkg_attr_groups,
 648         .name           = "cstate_pkg",
 649         .task_ctx_nr    = perf_invalid_context,
 650         .event_init     = cstate_pmu_event_init,
 651         .add            = cstate_pmu_event_add, /* must have */
 652         .del            = cstate_pmu_event_del, /* must have */
 653         .start          = cstate_pmu_event_start,
 654         .stop           = cstate_pmu_event_stop,
 655         .read           = cstate_pmu_event_update,
 656         .capabilities   = PERF_PMU_CAP_NO_INTERRUPT,
 657 };
 658
 659 static void __init cstate_pmus_register(void)
 660 {
 661         int err;
 662
 663         if (has_cstate_core) {
 664                 err = perf_pmu_register(&cstate_core_pmu, cstate_core_pmu.name, -1);
 665                 if (WARN_ON(err))
 666                         pr_info("Failed to register PMU %s error %d\n",
 667                                 cstate_core_pmu.name, err);
 668         }
 669
 670         if (has_cstate_pkg) {
 671                 err = perf_pmu_register(&cstate_pkg_pmu, cstate_pkg_pmu.name, -1);
 672                 if (WARN_ON(err))
 673                         pr_info("Failed to register PMU %s error %d\n",
 674                                 cstate_pkg_pmu.name, err);
 675         }
 676 }
 677
 678 static int __init cstate_pmu_init(void)
 679 {
 680         int err;
 681
 682         if (cpu_has_hypervisor)
 683                 return -ENODEV;
 684
 685         err = cstate_init();
 686         if (err)
 687                 return err;
 688
 689         cstate_cpumask_init();
 690
 691         cstate_pmus_register();
 692
 693         return 0;
 694 }
 695
 696 device_initcall(cstate_pmu_init);