x86/events/amd/core.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 #include <linux/perf_event.h>
   3 #include <linux/jump_label.h>
   4 #include <linux/export.h>
   5 #include <linux/types.h>
   6 #include <linux/init.h>
   7 #include <linux/slab.h>
   8 #include <linux/delay.h>
   9 #include <linux/jiffies.h>
  10 #include <asm/apicdef.h>
  11 #include <asm/apic.h>
  12 #include <asm/nmi.h>
  13
  14 #include "../perf_event.h"
  15
  16 static DEFINE_PER_CPU(unsigned long, perf_nmi_tstamp);
  17 static unsigned long perf_nmi_window;
  18
  19 /* AMD Event 0xFFF: Merge.  Used with Large Increment per Cycle events */
  20 #define AMD_MERGE_EVENT ((0xFULL << 32) | 0xFFULL)
  21 #define AMD_MERGE_EVENT_ENABLE (AMD_MERGE_EVENT | ARCH_PERFMON_EVENTSEL_ENABLE)
  22
  23 /* PMC Enable and Overflow bits for PerfCntrGlobal* registers */
  24 static u64 amd_pmu_global_cntr_mask __read_mostly;
  25
  26 static __initconst const u64 amd_hw_cache_event_ids
  27                                 [PERF_COUNT_HW_CACHE_MAX]
  28                                 [PERF_COUNT_HW_CACHE_OP_MAX]
  29                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] =
  30 {
  31  [ C(L1D) ] = {
  32         [ C(OP_READ) ] = {
  33                 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
  34                 [ C(RESULT_MISS)   ] = 0x0141, /* Data Cache Misses          */
  35         },
  36         [ C(OP_WRITE) ] = {
  37                 [ C(RESULT_ACCESS) ] = 0,
  38                 [ C(RESULT_MISS)   ] = 0,
  39         },
  40         [ C(OP_PREFETCH) ] = {
  41                 [ C(RESULT_ACCESS) ] = 0x0267, /* Data Prefetcher :attempts  */
  42                 [ C(RESULT_MISS)   ] = 0x0167, /* Data Prefetcher :cancelled */
  43         },
  44  },
  45  [ C(L1I ) ] = {
  46         [ C(OP_READ) ] = {
  47                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction cache fetches  */
  48                 [ C(RESULT_MISS)   ] = 0x0081, /* Instruction cache misses   */
  49         },
  50         [ C(OP_WRITE) ] = {
  51                 [ C(RESULT_ACCESS) ] = -1,
  52                 [ C(RESULT_MISS)   ] = -1,
  53         },
  54         [ C(OP_PREFETCH) ] = {
  55                 [ C(RESULT_ACCESS) ] = 0x014B, /* Prefetch Instructions :Load */
  56                 [ C(RESULT_MISS)   ] = 0,
  57         },
  58  },
  59  [ C(LL  ) ] = {
  60         [ C(OP_READ) ] = {
  61                 [ C(RESULT_ACCESS) ] = 0x037D, /* Requests to L2 Cache :IC+DC */
  62                 [ C(RESULT_MISS)   ] = 0x037E, /* L2 Cache Misses : IC+DC     */
  63         },
  64         [ C(OP_WRITE) ] = {
  65                 [ C(RESULT_ACCESS) ] = 0x017F, /* L2 Fill/Writeback           */
  66                 [ C(RESULT_MISS)   ] = 0,
  67         },
  68         [ C(OP_PREFETCH) ] = {
  69                 [ C(RESULT_ACCESS) ] = 0,
  70                 [ C(RESULT_MISS)   ] = 0,
  71         },
  72  },
  73  [ C(DTLB) ] = {
  74         [ C(OP_READ) ] = {
  75                 [ C(RESULT_ACCESS) ] = 0x0040, /* Data Cache Accesses        */
  76                 [ C(RESULT_MISS)   ] = 0x0746, /* L1_DTLB_AND_L2_DLTB_MISS.ALL */
  77         },
  78         [ C(OP_WRITE) ] = {
  79                 [ C(RESULT_ACCESS) ] = 0,
  80                 [ C(RESULT_MISS)   ] = 0,
  81         },
  82         [ C(OP_PREFETCH) ] = {
  83                 [ C(RESULT_ACCESS) ] = 0,
  84                 [ C(RESULT_MISS)   ] = 0,
  85         },
  86  },
  87  [ C(ITLB) ] = {
  88         [ C(OP_READ) ] = {
  89                 [ C(RESULT_ACCESS) ] = 0x0080, /* Instruction fecthes        */
  90                 [ C(RESULT_MISS)   ] = 0x0385, /* L1_ITLB_AND_L2_ITLB_MISS.ALL */
  91         },
  92         [ C(OP_WRITE) ] = {
  93                 [ C(RESULT_ACCESS) ] = -1,
  94                 [ C(RESULT_MISS)   ] = -1,
  95         },
  96         [ C(OP_PREFETCH) ] = {
  97                 [ C(RESULT_ACCESS) ] = -1,
  98                 [ C(RESULT_MISS)   ] = -1,
  99         },
 100  },
 101  [ C(BPU ) ] = {
 102         [ C(OP_READ) ] = {
 103                 [ C(RESULT_ACCESS) ] = 0x00c2, /* Retired Branch Instr.      */
 104                 [ C(RESULT_MISS)   ] = 0x00c3, /* Retired Mispredicted BI    */
 105         },
 106         [ C(OP_WRITE) ] = {
 107                 [ C(RESULT_ACCESS) ] = -1,
 108                 [ C(RESULT_MISS)   ] = -1,
 109         },
 110         [ C(OP_PREFETCH) ] = {
 111                 [ C(RESULT_ACCESS) ] = -1,
 112                 [ C(RESULT_MISS)   ] = -1,
 113         },
 114  },
 115  [ C(NODE) ] = {
 116         [ C(OP_READ) ] = {
 117                 [ C(RESULT_ACCESS) ] = 0xb8e9, /* CPU Request to Memory, l+r */
 118                 [ C(RESULT_MISS)   ] = 0x98e9, /* CPU Request to Memory, r   */
 119         },
 120         [ C(OP_WRITE) ] = {
 121                 [ C(RESULT_ACCESS) ] = -1,
 122                 [ C(RESULT_MISS)   ] = -1,
 123         },
 124         [ C(OP_PREFETCH) ] = {
 125                 [ C(RESULT_ACCESS) ] = -1,
 126                 [ C(RESULT_MISS)   ] = -1,
 127         },
 128  },
 129 };
 130
 131 static __initconst const u64 amd_hw_cache_event_ids_f17h
 132                                 [PERF_COUNT_HW_CACHE_MAX]
 133                                 [PERF_COUNT_HW_CACHE_OP_MAX]
 134                                 [PERF_COUNT_HW_CACHE_RESULT_MAX] = {
 135 [C(L1D)] = {
 136         [C(OP_READ)] = {
 137                 [C(RESULT_ACCESS)] = 0x0040, /* Data Cache Accesses */
 138                 [C(RESULT_MISS)]   = 0xc860, /* L2$ access from DC Miss */
 139         },
 140         [C(OP_WRITE)] = {
 141                 [C(RESULT_ACCESS)] = 0,
 142                 [C(RESULT_MISS)]   = 0,
 143         },
 144         [C(OP_PREFETCH)] = {
 145                 [C(RESULT_ACCESS)] = 0xff5a, /* h/w prefetch DC Fills */
 146                 [C(RESULT_MISS)]   = 0,
 147         },
 148 },
 149 [C(L1I)] = {
 150         [C(OP_READ)] = {
 151                 [C(RESULT_ACCESS)] = 0x0080, /* Instruction cache fetches  */
 152                 [C(RESULT_MISS)]   = 0x0081, /* Instruction cache misses   */
 153         },
 154         [C(OP_WRITE)] = {
 155                 [C(RESULT_ACCESS)] = -1,
 156                 [C(RESULT_MISS)]   = -1,
 157         },
 158         [C(OP_PREFETCH)] = {
 159                 [C(RESULT_ACCESS)] = 0,
 160                 [C(RESULT_MISS)]   = 0,
 161         },
 162 },
 163 [C(LL)] = {
 164         [C(OP_READ)] = {
 165                 [C(RESULT_ACCESS)] = 0,
 166                 [C(RESULT_MISS)]   = 0,
 167         },
 168         [C(OP_WRITE)] = {
 169                 [C(RESULT_ACCESS)] = 0,
 170                 [C(RESULT_MISS)]   = 0,
 171         },
 172         [C(OP_PREFETCH)] = {
 173                 [C(RESULT_ACCESS)] = 0,
 174                 [C(RESULT_MISS)]   = 0,
 175         },
 176 },
 177 [C(DTLB)] = {
 178         [C(OP_READ)] = {
 179                 [C(RESULT_ACCESS)] = 0xff45, /* All L2 DTLB accesses */
 180                 [C(RESULT_MISS)]   = 0xf045, /* L2 DTLB misses (PT walks) */
 181         },
 182         [C(OP_WRITE)] = {
 183                 [C(RESULT_ACCESS)] = 0,
 184                 [C(RESULT_MISS)]   = 0,
 185         },
 186         [C(OP_PREFETCH)] = {
 187                 [C(RESULT_ACCESS)] = 0,
 188                 [C(RESULT_MISS)]   = 0,
 189         },
 190 },
 191 [C(ITLB)] = {
 192         [C(OP_READ)] = {
 193                 [C(RESULT_ACCESS)] = 0x0084, /* L1 ITLB misses, L2 ITLB hits */
 194                 [C(RESULT_MISS)]   = 0xff85, /* L1 ITLB misses, L2 misses */
 195         },
 196         [C(OP_WRITE)] = {
 197                 [C(RESULT_ACCESS)] = -1,
 198                 [C(RESULT_MISS)]   = -1,
 199         },
 200         [C(OP_PREFETCH)] = {
 201                 [C(RESULT_ACCESS)] = -1,
 202                 [C(RESULT_MISS)]   = -1,
 203         },
 204 },
 205 [C(BPU)] = {
 206         [C(OP_READ)] = {
 207                 [C(RESULT_ACCESS)] = 0x00c2, /* Retired Branch Instr.      */
 208                 [C(RESULT_MISS)]   = 0x00c3, /* Retired Mispredicted BI    */
 209         },
 210         [C(OP_WRITE)] = {
 211                 [C(RESULT_ACCESS)] = -1,
 212                 [C(RESULT_MISS)]   = -1,
 213         },
 214         [C(OP_PREFETCH)] = {
 215                 [C(RESULT_ACCESS)] = -1,
 216                 [C(RESULT_MISS)]   = -1,
 217         },
 218 },
 219 [C(NODE)] = {
 220         [C(OP_READ)] = {
 221                 [C(RESULT_ACCESS)] = 0,
 222                 [C(RESULT_MISS)]   = 0,
 223         },
 224         [C(OP_WRITE)] = {
 225                 [C(RESULT_ACCESS)] = -1,
 226                 [C(RESULT_MISS)]   = -1,
 227         },
 228         [C(OP_PREFETCH)] = {
 229                 [C(RESULT_ACCESS)] = -1,
 230                 [C(RESULT_MISS)]   = -1,
 231         },
 232 },
 233 };
 234
 235 /*
 236  * AMD Performance Monitor K7 and later, up to and including Family 16h:
 237  */
 238 static const u64 amd_perfmon_event_map[PERF_COUNT_HW_MAX] =
 239 {
 240         [PERF_COUNT_HW_CPU_CYCLES]              = 0x0076,
 241         [PERF_COUNT_HW_INSTRUCTIONS]            = 0x00c0,
 242         [PERF_COUNT_HW_CACHE_REFERENCES]        = 0x077d,
 243         [PERF_COUNT_HW_CACHE_MISSES]            = 0x077e,
 244         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = 0x00c2,
 245         [PERF_COUNT_HW_BRANCH_MISSES]           = 0x00c3,
 246         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x00d0, /* "Decoder empty" event */
 247         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = 0x00d1, /* "Dispatch stalls" event */
 248 };
 249
 250 /*
 251  * AMD Performance Monitor Family 17h and later:
 252  */
 253 static const u64 amd_f17h_perfmon_event_map[PERF_COUNT_HW_MAX] =
 254 {
 255         [PERF_COUNT_HW_CPU_CYCLES]              = 0x0076,
 256         [PERF_COUNT_HW_INSTRUCTIONS]            = 0x00c0,
 257         [PERF_COUNT_HW_CACHE_REFERENCES]        = 0xff60,
 258         [PERF_COUNT_HW_CACHE_MISSES]            = 0x0964,
 259         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = 0x00c2,
 260         [PERF_COUNT_HW_BRANCH_MISSES]           = 0x00c3,
 261         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x0287,
 262         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = 0x0187,
 263 };
 264
 265 static u64 amd_pmu_event_map(int hw_event)
 266 {
 267         if (boot_cpu_data.x86 >= 0x17)
 268                 return amd_f17h_perfmon_event_map[hw_event];
 269
 270         return amd_perfmon_event_map[hw_event];
 271 }
 272
 273 /*
 274  * Previously calculated offsets
 275  */
 276 static unsigned int event_offsets[X86_PMC_IDX_MAX] __read_mostly;
 277 static unsigned int count_offsets[X86_PMC_IDX_MAX] __read_mostly;
 278
 279 /*
 280  * Legacy CPUs:
 281  *   4 counters starting at 0xc0010000 each offset by 1
 282  *
 283  * CPUs with core performance counter extensions:
 284  *   6 counters starting at 0xc0010200 each offset by 2
 285  */
 286 static inline int amd_pmu_addr_offset(int index, bool eventsel)
 287 {
 288         int offset;
 289
 290         if (!index)
 291                 return index;
 292
 293         if (eventsel)
 294                 offset = event_offsets[index];
 295         else
 296                 offset = count_offsets[index];
 297
 298         if (offset)
 299                 return offset;
 300
 301         if (!boot_cpu_has(X86_FEATURE_PERFCTR_CORE))
 302                 offset = index;
 303         else
 304                 offset = index << 1;
 305
 306         if (eventsel)
 307                 event_offsets[index] = offset;
 308         else
 309                 count_offsets[index] = offset;
 310
 311         return offset;
 312 }
 313
 314 /*
 315  * AMD64 events are detected based on their event codes.
 316  */
 317 static inline unsigned int amd_get_event_code(struct hw_perf_event *hwc)
 318 {
 319         return ((hwc->config >> 24) & 0x0f00) | (hwc->config & 0x00ff);
 320 }
 321
 322 static inline bool amd_is_pair_event_code(struct hw_perf_event *hwc)
 323 {
 324         if (!(x86_pmu.flags & PMU_FL_PAIR))
 325                 return false;
 326
 327         switch (amd_get_event_code(hwc)) {
 328         case 0x003:     return true;    /* Retired SSE/AVX FLOPs */
 329         default:        return false;
 330         }
 331 }
 332
 333 DEFINE_STATIC_CALL_RET0(amd_pmu_branch_hw_config, *x86_pmu.hw_config);
 334
 335 static int amd_core_hw_config(struct perf_event *event)
 336 {
 337         if (event->attr.exclude_host && event->attr.exclude_guest)
 338                 /*
 339                  * When HO == GO == 1 the hardware treats that as GO == HO == 0
 340                  * and will count in both modes. We don't want to count in that
 341                  * case so we emulate no-counting by setting US = OS = 0.
 342                  */
 343                 event->hw.config &= ~(ARCH_PERFMON_EVENTSEL_USR |
 344                                       ARCH_PERFMON_EVENTSEL_OS);
 345         else if (event->attr.exclude_host)
 346                 event->hw.config |= AMD64_EVENTSEL_GUESTONLY;
 347         else if (event->attr.exclude_guest)
 348                 event->hw.config |= AMD64_EVENTSEL_HOSTONLY;
 349
 350         if ((x86_pmu.flags & PMU_FL_PAIR) && amd_is_pair_event_code(&event->hw))
 351                 event->hw.flags |= PERF_X86_EVENT_PAIR;
 352
 353         if (has_branch_stack(event))
 354                 return static_call(amd_pmu_branch_hw_config)(event);
 355
 356         return 0;
 357 }
 358
 359 static inline int amd_is_nb_event(struct hw_perf_event *hwc)
 360 {
 361         return (hwc->config & 0xe0) == 0xe0;
 362 }
 363
 364 static inline int amd_has_nb(struct cpu_hw_events *cpuc)
 365 {
 366         struct amd_nb *nb = cpuc->amd_nb;
 367
 368         return nb && nb->nb_id != -1;
 369 }
 370
 371 static int amd_pmu_hw_config(struct perf_event *event)
 372 {
 373         int ret;
 374
 375         /* pass precise event sampling to ibs: */
 376         if (event->attr.precise_ip && get_ibs_caps())
 377                 return forward_event_to_ibs(event);
 378
 379         if (has_branch_stack(event) && !x86_pmu.lbr_nr)
 380                 return -EOPNOTSUPP;
 381
 382         ret = x86_pmu_hw_config(event);
 383         if (ret)
 384                 return ret;
 385
 386         if (event->attr.type == PERF_TYPE_RAW)
 387                 event->hw.config |= event->attr.config & AMD64_RAW_EVENT_MASK;
 388
 389         return amd_core_hw_config(event);
 390 }
 391
 392 static void __amd_put_nb_event_constraints(struct cpu_hw_events *cpuc,
 393                                            struct perf_event *event)
 394 {
 395         struct amd_nb *nb = cpuc->amd_nb;
 396         int i;
 397
 398         /*
 399          * need to scan whole list because event may not have
 400          * been assigned during scheduling
 401          *
 402          * no race condition possible because event can only
 403          * be removed on one CPU at a time AND PMU is disabled
 404          * when we come here
 405          */
 406         for (i = 0; i < x86_pmu.num_counters; i++) {
 407                 if (cmpxchg(nb->owners + i, event, NULL) == event)
 408                         break;
 409         }
 410 }
 411
 412  /*
 413   * AMD64 NorthBridge events need special treatment because
 414   * counter access needs to be synchronized across all cores
 415   * of a package. Refer to BKDG section 3.12
 416   *
 417   * NB events are events measuring L3 cache, Hypertransport
 418   * traffic. They are identified by an event code >= 0xe00.
 419   * They measure events on the NorthBride which is shared
 420   * by all cores on a package. NB events are counted on a
 421   * shared set of counters. When a NB event is programmed
 422   * in a counter, the data actually comes from a shared
 423   * counter. Thus, access to those counters needs to be
 424   * synchronized.
 425   *
 426   * We implement the synchronization such that no two cores
 427   * can be measuring NB events using the same counters. Thus,
 428   * we maintain a per-NB allocation table. The available slot
 429   * is propagated using the event_constraint structure.
 430   *
 431   * We provide only one choice for each NB event based on
 432   * the fact that only NB events have restrictions. Consequently,
 433   * if a counter is available, there is a guarantee the NB event
 434   * will be assigned to it. If no slot is available, an empty
 435   * constraint is returned and scheduling will eventually fail
 436   * for this event.
 437   *
 438   * Note that all cores attached the same NB compete for the same
 439   * counters to host NB events, this is why we use atomic ops. Some
 440   * multi-chip CPUs may have more than one NB.
 441   *
 442   * Given that resources are allocated (cmpxchg), they must be
 443   * eventually freed for others to use. This is accomplished by
 444   * calling __amd_put_nb_event_constraints()
 445   *
 446   * Non NB events are not impacted by this restriction.
 447   */
 448 static struct event_constraint *
 449 __amd_get_nb_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event,
 450                                struct event_constraint *c)
 451 {
 452         struct hw_perf_event *hwc = &event->hw;
 453         struct amd_nb *nb = cpuc->amd_nb;
 454         struct perf_event *old;
 455         int idx, new = -1;
 456
 457         if (!c)
 458                 c = &unconstrained;
 459
 460         if (cpuc->is_fake)
 461                 return c;
 462
 463         /*
 464          * detect if already present, if so reuse
 465          *
 466          * cannot merge with actual allocation
 467          * because of possible holes
 468          *
 469          * event can already be present yet not assigned (in hwc->idx)
 470          * because of successive calls to x86_schedule_events() from
 471          * hw_perf_group_sched_in() without hw_perf_enable()
 472          */
 473         for_each_set_bit(idx, c->idxmsk, x86_pmu.num_counters) {
 474                 if (new == -1 || hwc->idx == idx)
 475                         /* assign free slot, prefer hwc->idx */
 476                         old = cmpxchg(nb->owners + idx, NULL, event);
 477                 else if (nb->owners[idx] == event)
 478                         /* event already present */
 479                         old = event;
 480                 else
 481                         continue;
 482
 483                 if (old && old != event)
 484                         continue;
 485
 486                 /* reassign to this slot */
 487                 if (new != -1)
 488                         cmpxchg(nb->owners + new, event, NULL);
 489                 new = idx;
 490
 491                 /* already present, reuse */
 492                 if (old == event)
 493                         break;
 494         }
 495
 496         if (new == -1)
 497                 return &emptyconstraint;
 498
 499         return &nb->event_constraints[new];
 500 }
 501
 502 static struct amd_nb *amd_alloc_nb(int cpu)
 503 {
 504         struct amd_nb *nb;
 505         int i;
 506
 507         nb = kzalloc_node(sizeof(struct amd_nb), GFP_KERNEL, cpu_to_node(cpu));
 508         if (!nb)
 509                 return NULL;
 510
 511         nb->nb_id = -1;
 512
 513         /*
 514          * initialize all possible NB constraints
 515          */
 516         for (i = 0; i < x86_pmu.num_counters; i++) {
 517                 __set_bit(i, nb->event_constraints[i].idxmsk);
 518                 nb->event_constraints[i].weight = 1;
 519         }
 520         return nb;
 521 }
 522
 523 typedef void (amd_pmu_branch_reset_t)(void);
 524 DEFINE_STATIC_CALL_NULL(amd_pmu_branch_reset, amd_pmu_branch_reset_t);
 525
 526 static void amd_pmu_cpu_reset(int cpu)
 527 {
 528         if (x86_pmu.lbr_nr)
 529                 static_call(amd_pmu_branch_reset)();
 530
 531         if (x86_pmu.version < 2)
 532                 return;
 533
 534         /* Clear enable bits i.e. PerfCntrGlobalCtl.PerfCntrEn */
 535         wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, 0);
 536
 537         /*
 538          * Clear freeze and overflow bits i.e. PerfCntrGLobalStatus.LbrFreeze
 539          * and PerfCntrGLobalStatus.PerfCntrOvfl
 540          */
 541         wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR,
 542                GLOBAL_STATUS_LBRS_FROZEN | amd_pmu_global_cntr_mask);
 543 }
 544
 545 static int amd_pmu_cpu_prepare(int cpu)
 546 {
 547         struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
 548
 549         cpuc->lbr_sel = kzalloc_node(sizeof(struct er_account), GFP_KERNEL,
 550                                      cpu_to_node(cpu));
 551         if (!cpuc->lbr_sel)
 552                 return -ENOMEM;
 553
 554         WARN_ON_ONCE(cpuc->amd_nb);
 555
 556         if (!x86_pmu.amd_nb_constraints)
 557                 return 0;
 558
 559         cpuc->amd_nb = amd_alloc_nb(cpu);
 560         if (cpuc->amd_nb)
 561                 return 0;
 562
 563         kfree(cpuc->lbr_sel);
 564         cpuc->lbr_sel = NULL;
 565
 566         return -ENOMEM;
 567 }
 568
 569 static void amd_pmu_cpu_starting(int cpu)
 570 {
 571         struct cpu_hw_events *cpuc = &per_cpu(cpu_hw_events, cpu);
 572         void **onln = &cpuc->kfree_on_online[X86_PERF_KFREE_SHARED];
 573         struct amd_nb *nb;
 574         int i, nb_id;
 575
 576         cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY;
 577         amd_pmu_cpu_reset(cpu);
 578
 579         if (!x86_pmu.amd_nb_constraints)
 580                 return;
 581
 582         nb_id = topology_die_id(cpu);
 583         WARN_ON_ONCE(nb_id == BAD_APICID);
 584
 585         for_each_online_cpu(i) {
 586                 nb = per_cpu(cpu_hw_events, i).amd_nb;
 587                 if (WARN_ON_ONCE(!nb))
 588                         continue;
 589
 590                 if (nb->nb_id == nb_id) {
 591                         *onln = cpuc->amd_nb;
 592                         cpuc->amd_nb = nb;
 593                         break;
 594                 }
 595         }
 596
 597         cpuc->amd_nb->nb_id = nb_id;
 598         cpuc->amd_nb->refcnt++;
 599 }
 600
 601 static void amd_pmu_cpu_dead(int cpu)
 602 {
 603         struct cpu_hw_events *cpuhw = &per_cpu(cpu_hw_events, cpu);
 604
 605         kfree(cpuhw->lbr_sel);
 606         cpuhw->lbr_sel = NULL;
 607
 608         if (!x86_pmu.amd_nb_constraints)
 609                 return;
 610
 611         if (cpuhw->amd_nb) {
 612                 struct amd_nb *nb = cpuhw->amd_nb;
 613
 614                 if (nb->nb_id == -1 || --nb->refcnt == 0)
 615                         kfree(nb);
 616
 617                 cpuhw->amd_nb = NULL;
 618         }
 619 }
 620
 621 static inline void amd_pmu_set_global_ctl(u64 ctl)
 622 {
 623         wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_CTL, ctl);
 624 }
 625
 626 static inline u64 amd_pmu_get_global_status(void)
 627 {
 628         u64 status;
 629
 630         /* PerfCntrGlobalStatus is read-only */
 631         rdmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS, status);
 632
 633         return status;
 634 }
 635
 636 static inline void amd_pmu_ack_global_status(u64 status)
 637 {
 638         /*
 639          * PerfCntrGlobalStatus is read-only but an overflow acknowledgment
 640          * mechanism exists; writing 1 to a bit in PerfCntrGlobalStatusClr
 641          * clears the same bit in PerfCntrGlobalStatus
 642          */
 643
 644         wrmsrl(MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR, status);
 645 }
 646
 647 static bool amd_pmu_test_overflow_topbit(int idx)
 648 {
 649         u64 counter;
 650
 651         rdmsrl(x86_pmu_event_addr(idx), counter);
 652
 653         return !(counter & BIT_ULL(x86_pmu.cntval_bits - 1));
 654 }
 655
 656 static bool amd_pmu_test_overflow_status(int idx)
 657 {
 658         return amd_pmu_get_global_status() & BIT_ULL(idx);
 659 }
 660
 661 DEFINE_STATIC_CALL(amd_pmu_test_overflow, amd_pmu_test_overflow_topbit);
 662
 663 /*
 664  * When a PMC counter overflows, an NMI is used to process the event and
 665  * reset the counter. NMI latency can result in the counter being updated
 666  * before the NMI can run, which can result in what appear to be spurious
 667  * NMIs. This function is intended to wait for the NMI to run and reset
 668  * the counter to avoid possible unhandled NMI messages.
 669  */
 670 #define OVERFLOW_WAIT_COUNT     50
 671
 672 static void amd_pmu_wait_on_overflow(int idx)
 673 {
 674         unsigned int i;
 675
 676         /*
 677          * Wait for the counter to be reset if it has overflowed. This loop
 678          * should exit very, very quickly, but just in case, don't wait
 679          * forever...
 680          */
 681         for (i = 0; i < OVERFLOW_WAIT_COUNT; i++) {
 682                 if (!static_call(amd_pmu_test_overflow)(idx))
 683                         break;
 684
 685                 /* Might be in IRQ context, so can't sleep */
 686                 udelay(1);
 687         }
 688 }
 689
 690 static void amd_pmu_check_overflow(void)
 691 {
 692         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 693         int idx;
 694
 695         /*
 696          * This shouldn't be called from NMI context, but add a safeguard here
 697          * to return, since if we're in NMI context we can't wait for an NMI
 698          * to reset an overflowed counter value.
 699          */
 700         if (in_nmi())
 701                 return;
 702
 703         /*
 704          * Check each counter for overflow and wait for it to be reset by the
 705          * NMI if it has overflowed. This relies on the fact that all active
 706          * counters are always enabled when this function is called and
 707          * ARCH_PERFMON_EVENTSEL_INT is always set.
 708          */
 709         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 710                 if (!test_bit(idx, cpuc->active_mask))
 711                         continue;
 712
 713                 amd_pmu_wait_on_overflow(idx);
 714         }
 715 }
 716
 717 static void amd_pmu_enable_event(struct perf_event *event)
 718 {
 719         x86_pmu_enable_event(event);
 720 }
 721
 722 static void amd_pmu_enable_all(int added)
 723 {
 724         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 725         int idx;
 726
 727         amd_brs_enable_all();
 728
 729         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 730                 /* only activate events which are marked as active */
 731                 if (!test_bit(idx, cpuc->active_mask))
 732                         continue;
 733
 734                 amd_pmu_enable_event(cpuc->events[idx]);
 735         }
 736 }
 737
 738 static void amd_pmu_v2_enable_event(struct perf_event *event)
 739 {
 740         struct hw_perf_event *hwc = &event->hw;
 741
 742         /*
 743          * Testing cpu_hw_events.enabled should be skipped in this case unlike
 744          * in x86_pmu_enable_event().
 745          *
 746          * Since cpu_hw_events.enabled is set only after returning from
 747          * x86_pmu_start(), the PMCs must be programmed and kept ready.
 748          * Counting starts only after x86_pmu_enable_all() is called.
 749          */
 750         __x86_pmu_enable_event(hwc, ARCH_PERFMON_EVENTSEL_ENABLE);
 751 }
 752
 753 static __always_inline void amd_pmu_core_enable_all(void)
 754 {
 755         amd_pmu_set_global_ctl(amd_pmu_global_cntr_mask);
 756 }
 757
 758 static void amd_pmu_v2_enable_all(int added)
 759 {
 760         amd_pmu_lbr_enable_all();
 761         amd_pmu_core_enable_all();
 762 }
 763
 764 static void amd_pmu_disable_event(struct perf_event *event)
 765 {
 766         x86_pmu_disable_event(event);
 767
 768         /*
 769          * This can be called from NMI context (via x86_pmu_stop). The counter
 770          * may have overflowed, but either way, we'll never see it get reset
 771          * by the NMI if we're already in the NMI. And the NMI latency support
 772          * below will take care of any pending NMI that might have been
 773          * generated by the overflow.
 774          */
 775         if (in_nmi())
 776                 return;
 777
 778         amd_pmu_wait_on_overflow(event->hw.idx);
 779 }
 780
 781 static void amd_pmu_disable_all(void)
 782 {
 783         amd_brs_disable_all();
 784         x86_pmu_disable_all();
 785         amd_pmu_check_overflow();
 786 }
 787
 788 static __always_inline void amd_pmu_core_disable_all(void)
 789 {
 790         amd_pmu_set_global_ctl(0);
 791 }
 792
 793 static void amd_pmu_v2_disable_all(void)
 794 {
 795         amd_pmu_core_disable_all();
 796         amd_pmu_lbr_disable_all();
 797         amd_pmu_check_overflow();
 798 }
 799
 800 DEFINE_STATIC_CALL_NULL(amd_pmu_branch_add, *x86_pmu.add);
 801
 802 static void amd_pmu_add_event(struct perf_event *event)
 803 {
 804         if (needs_branch_stack(event))
 805                 static_call(amd_pmu_branch_add)(event);
 806 }
 807
 808 DEFINE_STATIC_CALL_NULL(amd_pmu_branch_del, *x86_pmu.del);
 809
 810 static void amd_pmu_del_event(struct perf_event *event)
 811 {
 812         if (needs_branch_stack(event))
 813                 static_call(amd_pmu_branch_del)(event);
 814 }
 815
 816 /*
 817  * Because of NMI latency, if multiple PMC counters are active or other sources
 818  * of NMIs are received, the perf NMI handler can handle one or more overflowed
 819  * PMC counters outside of the NMI associated with the PMC overflow. If the NMI
 820  * doesn't arrive at the LAPIC in time to become a pending NMI, then the kernel
 821  * back-to-back NMI support won't be active. This PMC handler needs to take into
 822  * account that this can occur, otherwise this could result in unknown NMI
 823  * messages being issued. Examples of this is PMC overflow while in the NMI
 824  * handler when multiple PMCs are active or PMC overflow while handling some
 825  * other source of an NMI.
 826  *
 827  * Attempt to mitigate this by creating an NMI window in which un-handled NMIs
 828  * received during this window will be claimed. This prevents extending the
 829  * window past when it is possible that latent NMIs should be received. The
 830  * per-CPU perf_nmi_tstamp will be set to the window end time whenever perf has
 831  * handled a counter. When an un-handled NMI is received, it will be claimed
 832  * only if arriving within that window.
 833  */
 834 static inline int amd_pmu_adjust_nmi_window(int handled)
 835 {
 836         /*
 837          * If a counter was handled, record a timestamp such that un-handled
 838          * NMIs will be claimed if arriving within that window.
 839          */
 840         if (handled) {
 841                 this_cpu_write(perf_nmi_tstamp, jiffies + perf_nmi_window);
 842
 843                 return handled;
 844         }
 845
 846         if (time_after(jiffies, this_cpu_read(perf_nmi_tstamp)))
 847                 return NMI_DONE;
 848
 849         return NMI_HANDLED;
 850 }
 851
 852 static int amd_pmu_handle_irq(struct pt_regs *regs)
 853 {
 854         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 855         int handled;
 856         int pmu_enabled;
 857
 858         /*
 859          * Save the PMU state.
 860          * It needs to be restored when leaving the handler.
 861          */
 862         pmu_enabled = cpuc->enabled;
 863         cpuc->enabled = 0;
 864
 865         amd_brs_disable_all();
 866
 867         /* Drain BRS is in use (could be inactive) */
 868         if (cpuc->lbr_users)
 869                 amd_brs_drain();
 870
 871         /* Process any counter overflows */
 872         handled = x86_pmu_handle_irq(regs);
 873
 874         cpuc->enabled = pmu_enabled;
 875         if (pmu_enabled)
 876                 amd_brs_enable_all();
 877
 878         return amd_pmu_adjust_nmi_window(handled);
 879 }
 880
 881 static int amd_pmu_v2_handle_irq(struct pt_regs *regs)
 882 {
 883         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
 884         struct perf_sample_data data;
 885         struct hw_perf_event *hwc;
 886         struct perf_event *event;
 887         int handled = 0, idx;
 888         u64 reserved, status, mask;
 889         bool pmu_enabled;
 890
 891         /*
 892          * Save the PMU state as it needs to be restored when leaving the
 893          * handler
 894          */
 895         pmu_enabled = cpuc->enabled;
 896         cpuc->enabled = 0;
 897
 898         /* Stop counting but do not disable LBR */
 899         amd_pmu_core_disable_all();
 900
 901         status = amd_pmu_get_global_status();
 902
 903         /* Check if any overflows are pending */
 904         if (!status)
 905                 goto done;
 906
 907         /* Read branch records */
 908         if (x86_pmu.lbr_nr) {
 909                 amd_pmu_lbr_read();
 910                 status &= ~GLOBAL_STATUS_LBRS_FROZEN;
 911         }
 912
 913         reserved = status & ~amd_pmu_global_cntr_mask;
 914         if (reserved)
 915                 pr_warn_once("Reserved PerfCntrGlobalStatus bits are set (0x%llx), please consider updating microcode\n",
 916                              reserved);
 917
 918         /* Clear any reserved bits set by buggy microcode */
 919         status &= amd_pmu_global_cntr_mask;
 920
 921         for (idx = 0; idx < x86_pmu.num_counters; idx++) {
 922                 if (!test_bit(idx, cpuc->active_mask))
 923                         continue;
 924
 925                 event = cpuc->events[idx];
 926                 hwc = &event->hw;
 927                 x86_perf_event_update(event);
 928                 mask = BIT_ULL(idx);
 929
 930                 if (!(status & mask))
 931                         continue;
 932
 933                 /* Event overflow */
 934                 handled++;
 935                 status &= ~mask;
 936                 perf_sample_data_init(&data, 0, hwc->last_period);
 937
 938                 if (!x86_perf_event_set_period(event))
 939                         continue;
 940
 941                 if (has_branch_stack(event)) {
 942                         data.br_stack = &cpuc->lbr_stack;
 943                         data.sample_flags |= PERF_SAMPLE_BRANCH_STACK;
 944                 }
 945
 946                 if (perf_event_overflow(event, &data, regs))
 947                         x86_pmu_stop(event, 0);
 948         }
 949
 950         /*
 951          * It should never be the case that some overflows are not handled as
 952          * the corresponding PMCs are expected to be inactive according to the
 953          * active_mask
 954          */
 955         WARN_ON(status > 0);
 956
 957         /* Clear overflow and freeze bits */
 958         amd_pmu_ack_global_status(~status);
 959
 960         /*
 961          * Unmasking the LVTPC is not required as the Mask (M) bit of the LVT
 962          * PMI entry is not set by the local APIC when a PMC overflow occurs
 963          */
 964         inc_irq_stat(apic_perf_irqs);
 965
 966 done:
 967         cpuc->enabled = pmu_enabled;
 968
 969         /* Resume counting only if PMU is active */
 970         if (pmu_enabled)
 971                 amd_pmu_core_enable_all();
 972
 973         return amd_pmu_adjust_nmi_window(handled);
 974 }
 975
 976 static struct event_constraint *
 977 amd_get_event_constraints(struct cpu_hw_events *cpuc, int idx,
 978                           struct perf_event *event)
 979 {
 980         /*
 981          * if not NB event or no NB, then no constraints
 982          */
 983         if (!(amd_has_nb(cpuc) && amd_is_nb_event(&event->hw)))
 984                 return &unconstrained;
 985
 986         return __amd_get_nb_event_constraints(cpuc, event, NULL);
 987 }
 988
 989 static void amd_put_event_constraints(struct cpu_hw_events *cpuc,
 990                                       struct perf_event *event)
 991 {
 992         if (amd_has_nb(cpuc) && amd_is_nb_event(&event->hw))
 993                 __amd_put_nb_event_constraints(cpuc, event);
 994 }
 995
 996 PMU_FORMAT_ATTR(event,  "config:0-7,32-35");
 997 PMU_FORMAT_ATTR(umask,  "config:8-15"   );
 998 PMU_FORMAT_ATTR(edge,   "config:18"     );
 999 PMU_FORMAT_ATTR(inv,    "config:23"     );
1000 PMU_FORMAT_ATTR(cmask,  "config:24-31"  );
1001
1002 static struct attribute *amd_format_attr[] = {
1003         &format_attr_event.attr,
1004         &format_attr_umask.attr,
1005         &format_attr_edge.attr,
1006         &format_attr_inv.attr,
1007         &format_attr_cmask.attr,
1008         NULL,
1009 };
1010
1011 /* AMD Family 15h */
1012
1013 #define AMD_EVENT_TYPE_MASK     0x000000F0ULL
1014
1015 #define AMD_EVENT_FP            0x00000000ULL ... 0x00000010ULL
1016 #define AMD_EVENT_LS            0x00000020ULL ... 0x00000030ULL
1017 #define AMD_EVENT_DC            0x00000040ULL ... 0x00000050ULL
1018 #define AMD_EVENT_CU            0x00000060ULL ... 0x00000070ULL
1019 #define AMD_EVENT_IC_DE         0x00000080ULL ... 0x00000090ULL
1020 #define AMD_EVENT_EX_LS         0x000000C0ULL
1021 #define AMD_EVENT_DE            0x000000D0ULL
1022 #define AMD_EVENT_NB            0x000000E0ULL ... 0x000000F0ULL
1023
1024 /*
1025  * AMD family 15h event code/PMC mappings:
1026  *
1027  * type = event_code & 0x0F0:
1028  *
1029  * 0x000        FP      PERF_CTL[5:3]
1030  * 0x010        FP      PERF_CTL[5:3]
1031  * 0x020        LS      PERF_CTL[5:0]
1032  * 0x030        LS      PERF_CTL[5:0]
1033  * 0x040        DC      PERF_CTL[5:0]
1034  * 0x050        DC      PERF_CTL[5:0]
1035  * 0x060        CU      PERF_CTL[2:0]
1036  * 0x070        CU      PERF_CTL[2:0]
1037  * 0x080        IC/DE   PERF_CTL[2:0]
1038  * 0x090        IC/DE   PERF_CTL[2:0]
1039  * 0x0A0        ---
1040  * 0x0B0        ---
1041  * 0x0C0        EX/LS   PERF_CTL[5:0]
1042  * 0x0D0        DE      PERF_CTL[2:0]
1043  * 0x0E0        NB      NB_PERF_CTL[3:0]
1044  * 0x0F0        NB      NB_PERF_CTL[3:0]
1045  *
1046  * Exceptions:
1047  *
1048  * 0x000        FP      PERF_CTL[3], PERF_CTL[5:3] (*)
1049  * 0x003        FP      PERF_CTL[3]
1050  * 0x004        FP      PERF_CTL[3], PERF_CTL[5:3] (*)
1051  * 0x00B        FP      PERF_CTL[3]
1052  * 0x00D        FP      PERF_CTL[3]
1053  * 0x023        DE      PERF_CTL[2:0]
1054  * 0x02D        LS      PERF_CTL[3]
1055  * 0x02E        LS      PERF_CTL[3,0]
1056  * 0x031        LS      PERF_CTL[2:0] (**)
1057  * 0x043        CU      PERF_CTL[2:0]
1058  * 0x045        CU      PERF_CTL[2:0]
1059  * 0x046        CU      PERF_CTL[2:0]
1060  * 0x054        CU      PERF_CTL[2:0]
1061  * 0x055        CU      PERF_CTL[2:0]
1062  * 0x08F        IC      PERF_CTL[0]
1063  * 0x187        DE      PERF_CTL[0]
1064  * 0x188        DE      PERF_CTL[0]
1065  * 0x0DB        EX      PERF_CTL[5:0]
1066  * 0x0DC        LS      PERF_CTL[5:0]
1067  * 0x0DD        LS      PERF_CTL[5:0]
1068  * 0x0DE        LS      PERF_CTL[5:0]
1069  * 0x0DF        LS      PERF_CTL[5:0]
1070  * 0x1C0        EX      PERF_CTL[5:3]
1071  * 0x1D6        EX      PERF_CTL[5:0]
1072  * 0x1D8        EX      PERF_CTL[5:0]
1073  *
1074  * (*)  depending on the umask all FPU counters may be used
1075  * (**) only one unitmask enabled at a time
1076  */
1077
1078 static struct event_constraint amd_f15_PMC0  = EVENT_CONSTRAINT(0, 0x01, 0);
1079 static struct event_constraint amd_f15_PMC20 = EVENT_CONSTRAINT(0, 0x07, 0);
1080 static struct event_constraint amd_f15_PMC3  = EVENT_CONSTRAINT(0, 0x08, 0);
1081 static struct event_constraint amd_f15_PMC30 = EVENT_CONSTRAINT_OVERLAP(0, 0x09, 0);
1082 static struct event_constraint amd_f15_PMC50 = EVENT_CONSTRAINT(0, 0x3F, 0);
1083 static struct event_constraint amd_f15_PMC53 = EVENT_CONSTRAINT(0, 0x38, 0);
1084
1085 static struct event_constraint *
1086 amd_get_event_constraints_f15h(struct cpu_hw_events *cpuc, int idx,
1087                                struct perf_event *event)
1088 {
1089         struct hw_perf_event *hwc = &event->hw;
1090         unsigned int event_code = amd_get_event_code(hwc);
1091
1092         switch (event_code & AMD_EVENT_TYPE_MASK) {
1093         case AMD_EVENT_FP:
1094                 switch (event_code) {
1095                 case 0x000:
1096                         if (!(hwc->config & 0x0000F000ULL))
1097                                 break;
1098                         if (!(hwc->config & 0x00000F00ULL))
1099                                 break;
1100                         return &amd_f15_PMC3;
1101                 case 0x004:
1102                         if (hweight_long(hwc->config & ARCH_PERFMON_EVENTSEL_UMASK) <= 1)
1103                                 break;
1104                         return &amd_f15_PMC3;
1105                 case 0x003:
1106                 case 0x00B:
1107                 case 0x00D:
1108                         return &amd_f15_PMC3;
1109                 }
1110                 return &amd_f15_PMC53;
1111         case AMD_EVENT_LS:
1112         case AMD_EVENT_DC:
1113         case AMD_EVENT_EX_LS:
1114                 switch (event_code) {
1115                 case 0x023:
1116                 case 0x043:
1117                 case 0x045:
1118                 case 0x046:
1119                 case 0x054:
1120                 case 0x055:
1121                         return &amd_f15_PMC20;
1122                 case 0x02D:
1123                         return &amd_f15_PMC3;
1124                 case 0x02E:
1125                         return &amd_f15_PMC30;
1126                 case 0x031:
1127                         if (hweight_long(hwc->config & ARCH_PERFMON_EVENTSEL_UMASK) <= 1)
1128                                 return &amd_f15_PMC20;
1129                         return &emptyconstraint;
1130                 case 0x1C0:
1131                         return &amd_f15_PMC53;
1132                 default:
1133                         return &amd_f15_PMC50;
1134                 }
1135         case AMD_EVENT_CU:
1136         case AMD_EVENT_IC_DE:
1137         case AMD_EVENT_DE:
1138                 switch (event_code) {
1139                 case 0x08F:
1140                 case 0x187:
1141                 case 0x188:
1142                         return &amd_f15_PMC0;
1143                 case 0x0DB ... 0x0DF:
1144                 case 0x1D6:
1145                 case 0x1D8:
1146                         return &amd_f15_PMC50;
1147                 default:
1148                         return &amd_f15_PMC20;
1149                 }
1150         case AMD_EVENT_NB:
1151                 /* moved to uncore.c */
1152                 return &emptyconstraint;
1153         default:
1154                 return &emptyconstraint;
1155         }
1156 }
1157
1158 static struct event_constraint pair_constraint;
1159
1160 static struct event_constraint *
1161 amd_get_event_constraints_f17h(struct cpu_hw_events *cpuc, int idx,
1162                                struct perf_event *event)
1163 {
1164         struct hw_perf_event *hwc = &event->hw;
1165
1166         if (amd_is_pair_event_code(hwc))
1167                 return &pair_constraint;
1168
1169         return &unconstrained;
1170 }
1171
1172 static void amd_put_event_constraints_f17h(struct cpu_hw_events *cpuc,
1173                                            struct perf_event *event)
1174 {
1175         struct hw_perf_event *hwc = &event->hw;
1176
1177         if (is_counter_pair(hwc))
1178                 --cpuc->n_pair;
1179 }
1180
1181 /*
1182  * Because of the way BRS operates with an inactive and active phases, and
1183  * the link to one counter, it is not possible to have two events using BRS
1184  * scheduled at the same time. There would be an issue with enforcing the
1185  * period of each one and given that the BRS saturates, it would not be possible
1186  * to guarantee correlated content for all events. Therefore, in situations
1187  * where multiple events want to use BRS, the kernel enforces mutual exclusion.
1188  * Exclusion is enforced by chosing only one counter for events using BRS.
1189  * The event scheduling logic will then automatically multiplex the
1190  * events and ensure that at most one event is actively using BRS.
1191  *
1192  * The BRS counter could be any counter, but there is no constraint on Fam19h,
1193  * therefore all counters are equal and thus we pick the first one: PMC0
1194  */
1195 static struct event_constraint amd_fam19h_brs_cntr0_constraint =
1196         EVENT_CONSTRAINT(0, 0x1, AMD64_RAW_EVENT_MASK);
1197
1198 static struct event_constraint amd_fam19h_brs_pair_cntr0_constraint =
1199         __EVENT_CONSTRAINT(0, 0x1, AMD64_RAW_EVENT_MASK, 1, 0, PERF_X86_EVENT_PAIR);
1200
1201 static struct event_constraint *
1202 amd_get_event_constraints_f19h(struct cpu_hw_events *cpuc, int idx,
1203                           struct perf_event *event)
1204 {
1205         struct hw_perf_event *hwc = &event->hw;
1206         bool has_brs = has_amd_brs(hwc);
1207
1208         /*
1209          * In case BRS is used with an event requiring a counter pair,
1210          * the kernel allows it but only on counter 0 & 1 to enforce
1211          * multiplexing requiring to protect BRS in case of multiple
1212          * BRS users
1213          */
1214         if (amd_is_pair_event_code(hwc)) {
1215                 return has_brs ? &amd_fam19h_brs_pair_cntr0_constraint
1216                                : &pair_constraint;
1217         }
1218
1219         if (has_brs)
1220                 return &amd_fam19h_brs_cntr0_constraint;
1221
1222         return &unconstrained;
1223 }
1224
1225
1226 static ssize_t amd_event_sysfs_show(char *page, u64 config)
1227 {
1228         u64 event = (config & ARCH_PERFMON_EVENTSEL_EVENT) |
1229                     (config & AMD64_EVENTSEL_EVENT) >> 24;
1230
1231         return x86_event_sysfs_show(page, config, event);
1232 }
1233
1234 static void amd_pmu_limit_period(struct perf_event *event, s64 *left)
1235 {
1236         /*
1237          * Decrease period by the depth of the BRS feature to get the last N
1238          * taken branches and approximate the desired period
1239          */
1240         if (has_branch_stack(event) && *left > x86_pmu.lbr_nr)
1241                 *left -= x86_pmu.lbr_nr;
1242 }
1243
1244 static __initconst const struct x86_pmu amd_pmu = {
1245         .name                   = "AMD",
1246         .handle_irq             = amd_pmu_handle_irq,
1247         .disable_all            = amd_pmu_disable_all,
1248         .enable_all             = amd_pmu_enable_all,
1249         .enable                 = amd_pmu_enable_event,
1250         .disable                = amd_pmu_disable_event,
1251         .hw_config              = amd_pmu_hw_config,
1252         .schedule_events        = x86_schedule_events,
1253         .eventsel               = MSR_K7_EVNTSEL0,
1254         .perfctr                = MSR_K7_PERFCTR0,
1255         .addr_offset            = amd_pmu_addr_offset,
1256         .event_map              = amd_pmu_event_map,
1257         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
1258         .num_counters           = AMD64_NUM_COUNTERS,
1259         .add                    = amd_pmu_add_event,
1260         .del                    = amd_pmu_del_event,
1261         .cntval_bits            = 48,
1262         .cntval_mask            = (1ULL << 48) - 1,
1263         .apic                   = 1,
1264         /* use highest bit to detect overflow */
1265         .max_period             = (1ULL << 47) - 1,
1266         .get_event_constraints  = amd_get_event_constraints,
1267         .put_event_constraints  = amd_put_event_constraints,
1268
1269         .format_attrs           = amd_format_attr,
1270         .events_sysfs_show      = amd_event_sysfs_show,
1271
1272         .cpu_prepare            = amd_pmu_cpu_prepare,
1273         .cpu_starting           = amd_pmu_cpu_starting,
1274         .cpu_dead               = amd_pmu_cpu_dead,
1275
1276         .amd_nb_constraints     = 1,
1277 };
1278
1279 static ssize_t branches_show(struct device *cdev,
1280                               struct device_attribute *attr,
1281                               char *buf)
1282 {
1283         return snprintf(buf, PAGE_SIZE, "%d\n", x86_pmu.lbr_nr);
1284 }
1285
1286 static DEVICE_ATTR_RO(branches);
1287
1288 static struct attribute *amd_pmu_branches_attrs[] = {
1289         &dev_attr_branches.attr,
1290         NULL,
1291 };
1292
1293 static umode_t
1294 amd_branches_is_visible(struct kobject *kobj, struct attribute *attr, int i)
1295 {
1296         return x86_pmu.lbr_nr ? attr->mode : 0;
1297 }
1298
1299 static struct attribute_group group_caps_amd_branches = {
1300         .name  = "caps",
1301         .attrs = amd_pmu_branches_attrs,
1302         .is_visible = amd_branches_is_visible,
1303 };
1304
1305 #ifdef CONFIG_PERF_EVENTS_AMD_BRS
1306
1307 EVENT_ATTR_STR(branch-brs, amd_branch_brs,
1308                "event=" __stringify(AMD_FAM19H_BRS_EVENT)"\n");
1309
1310 static struct attribute *amd_brs_events_attrs[] = {
1311         EVENT_PTR(amd_branch_brs),
1312         NULL,
1313 };
1314
1315 static umode_t
1316 amd_brs_is_visible(struct kobject *kobj, struct attribute *attr, int i)
1317 {
1318         return static_cpu_has(X86_FEATURE_BRS) && x86_pmu.lbr_nr ?
1319                attr->mode : 0;
1320 }
1321
1322 static struct attribute_group group_events_amd_brs = {
1323         .name       = "events",
1324         .attrs      = amd_brs_events_attrs,
1325         .is_visible = amd_brs_is_visible,
1326 };
1327
1328 #endif  /* CONFIG_PERF_EVENTS_AMD_BRS */
1329
1330 static const struct attribute_group *amd_attr_update[] = {
1331         &group_caps_amd_branches,
1332 #ifdef CONFIG_PERF_EVENTS_AMD_BRS
1333         &group_events_amd_brs,
1334 #endif
1335         NULL,
1336 };
1337
1338 static int __init amd_core_pmu_init(void)
1339 {
1340         union cpuid_0x80000022_ebx ebx;
1341         u64 even_ctr_mask = 0ULL;
1342         int i;
1343
1344         if (!boot_cpu_has(X86_FEATURE_PERFCTR_CORE))
1345                 return 0;
1346
1347         /* Avoid calculating the value each time in the NMI handler */
1348         perf_nmi_window = msecs_to_jiffies(100);
1349
1350         /*
1351          * If core performance counter extensions exists, we must use
1352          * MSR_F15H_PERF_CTL/MSR_F15H_PERF_CTR msrs. See also
1353          * amd_pmu_addr_offset().
1354          */
1355         x86_pmu.eventsel        = MSR_F15H_PERF_CTL;
1356         x86_pmu.perfctr         = MSR_F15H_PERF_CTR;
1357         x86_pmu.num_counters    = AMD64_NUM_COUNTERS_CORE;
1358
1359         /* Check for Performance Monitoring v2 support */
1360         if (boot_cpu_has(X86_FEATURE_PERFMON_V2)) {
1361                 ebx.full = cpuid_ebx(EXT_PERFMON_DEBUG_FEATURES);
1362
1363                 /* Update PMU version for later usage */
1364                 x86_pmu.version = 2;
1365
1366                 /* Find the number of available Core PMCs */
1367                 x86_pmu.num_counters = ebx.split.num_core_pmc;
1368
1369                 amd_pmu_global_cntr_mask = (1ULL << x86_pmu.num_counters) - 1;
1370
1371                 /* Update PMC handling functions */
1372                 x86_pmu.enable_all = amd_pmu_v2_enable_all;
1373                 x86_pmu.disable_all = amd_pmu_v2_disable_all;
1374                 x86_pmu.enable = amd_pmu_v2_enable_event;
1375                 x86_pmu.handle_irq = amd_pmu_v2_handle_irq;
1376                 static_call_update(amd_pmu_test_overflow, amd_pmu_test_overflow_status);
1377         }
1378
1379         /*
1380          * AMD Core perfctr has separate MSRs for the NB events, see
1381          * the amd/uncore.c driver.
1382          */
1383         x86_pmu.amd_nb_constraints = 0;
1384
1385         if (boot_cpu_data.x86 == 0x15) {
1386                 pr_cont("Fam15h ");
1387                 x86_pmu.get_event_constraints = amd_get_event_constraints_f15h;
1388         }
1389         if (boot_cpu_data.x86 >= 0x17) {
1390                 pr_cont("Fam17h+ ");
1391                 /*
1392                  * Family 17h and compatibles have constraints for Large
1393                  * Increment per Cycle events: they may only be assigned an
1394                  * even numbered counter that has a consecutive adjacent odd
1395                  * numbered counter following it.
1396                  */
1397                 for (i = 0; i < x86_pmu.num_counters - 1; i += 2)
1398                         even_ctr_mask |= BIT_ULL(i);
1399
1400                 pair_constraint = (struct event_constraint)
1401                                     __EVENT_CONSTRAINT(0, even_ctr_mask, 0,
1402                                     x86_pmu.num_counters / 2, 0,
1403                                     PERF_X86_EVENT_PAIR);
1404
1405                 x86_pmu.get_event_constraints = amd_get_event_constraints_f17h;
1406                 x86_pmu.put_event_constraints = amd_put_event_constraints_f17h;
1407                 x86_pmu.perf_ctr_pair_en = AMD_MERGE_EVENT_ENABLE;
1408                 x86_pmu.flags |= PMU_FL_PAIR;
1409         }
1410
1411         /* LBR and BRS are mutually exclusive features */
1412         if (!amd_pmu_lbr_init()) {
1413                 /* LBR requires flushing on context switch */
1414                 x86_pmu.sched_task = amd_pmu_lbr_sched_task;
1415                 static_call_update(amd_pmu_branch_hw_config, amd_pmu_lbr_hw_config);
1416                 static_call_update(amd_pmu_branch_reset, amd_pmu_lbr_reset);
1417                 static_call_update(amd_pmu_branch_add, amd_pmu_lbr_add);
1418                 static_call_update(amd_pmu_branch_del, amd_pmu_lbr_del);
1419         } else if (!amd_brs_init()) {
1420                 /*
1421                  * BRS requires special event constraints and flushing on ctxsw.
1422                  */
1423                 x86_pmu.get_event_constraints = amd_get_event_constraints_f19h;
1424                 x86_pmu.sched_task = amd_pmu_brs_sched_task;
1425                 x86_pmu.limit_period = amd_pmu_limit_period;
1426
1427                 static_call_update(amd_pmu_branch_hw_config, amd_brs_hw_config);
1428                 static_call_update(amd_pmu_branch_reset, amd_brs_reset);
1429                 static_call_update(amd_pmu_branch_add, amd_pmu_brs_add);
1430                 static_call_update(amd_pmu_branch_del, amd_pmu_brs_del);
1431
1432                 /*
1433                  * put_event_constraints callback same as Fam17h, set above
1434                  */
1435
1436                 /* branch sampling must be stopped when entering low power */
1437                 amd_brs_lopwr_init();
1438         }
1439
1440         x86_pmu.attr_update = amd_attr_update;
1441
1442         pr_cont("core perfctr, ");
1443         return 0;
1444 }
1445
1446 __init int amd_pmu_init(void)
1447 {
1448         int ret;
1449
1450         /* Performance-monitoring supported from K7 and later: */
1451         if (boot_cpu_data.x86 < 6)
1452                 return -ENODEV;
1453
1454         x86_pmu = amd_pmu;
1455
1456         ret = amd_core_pmu_init();
1457         if (ret)
1458                 return ret;
1459
1460         if (num_possible_cpus() == 1) {
1461                 /*
1462                  * No point in allocating data structures to serialize
1463                  * against other CPUs, when there is only the one CPU.
1464                  */
1465                 x86_pmu.amd_nb_constraints = 0;
1466         }
1467
1468         if (boot_cpu_data.x86 >= 0x17)
1469                 memcpy(hw_cache_event_ids, amd_hw_cache_event_ids_f17h, sizeof(hw_cache_event_ids));
1470         else
1471                 memcpy(hw_cache_event_ids, amd_hw_cache_event_ids, sizeof(hw_cache_event_ids));
1472
1473         return 0;
1474 }
1475
1476 static inline void amd_pmu_reload_virt(void)
1477 {
1478         if (x86_pmu.version >= 2) {
1479                 /*
1480                  * Clear global enable bits, reprogram the PERF_CTL
1481                  * registers with updated perf_ctr_virt_mask and then
1482                  * set global enable bits once again
1483                  */
1484                 amd_pmu_v2_disable_all();
1485                 amd_pmu_enable_all(0);
1486                 amd_pmu_v2_enable_all(0);
1487                 return;
1488         }
1489
1490         amd_pmu_disable_all();
1491         amd_pmu_enable_all(0);
1492 }
1493
1494 void amd_pmu_enable_virt(void)
1495 {
1496         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1497
1498         cpuc->perf_ctr_virt_mask = 0;
1499
1500         /* Reload all events */
1501         amd_pmu_reload_virt();
1502 }
1503 EXPORT_SYMBOL_GPL(amd_pmu_enable_virt);
1504
1505 void amd_pmu_disable_virt(void)
1506 {
1507         struct cpu_hw_events *cpuc = this_cpu_ptr(&cpu_hw_events);
1508
1509         /*
1510          * We only mask out the Host-only bit so that host-only counting works
1511          * when SVM is disabled. If someone sets up a guest-only counter when
1512          * SVM is disabled the Guest-only bits still gets set and the counter
1513          * will not count anything.
1514          */
1515         cpuc->perf_ctr_virt_mask = AMD64_EVENTSEL_HOSTONLY;
1516
1517         /* Reload all events */
1518         amd_pmu_reload_virt();
1519 }
1520 EXPORT_SYMBOL_GPL(amd_pmu_disable_virt);