arch/xtensa/kernel/perf_event.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Xtensa Performance Monitor Module driver
   4  * See Tensilica Debug User's Guide for PMU registers documentation.
   5  *
   6  * Copyright (C) 2015 Cadence Design Systems Inc.
   7  */
   8
   9 #include <linux/interrupt.h>
  10 #include <linux/irqdomain.h>
  11 #include <linux/module.h>
  12 #include <linux/of.h>
  13 #include <linux/perf_event.h>
  14 #include <linux/platform_device.h>
  15
  16 #include <asm/core.h>
  17 #include <asm/processor.h>
  18 #include <asm/stacktrace.h>
  19
  20 #define XTENSA_HWVERSION_RG_2015_0      260000
  21
  22 #if XCHAL_HW_MIN_VERSION >= XTENSA_HWVERSION_RG_2015_0
  23 #define XTENSA_PMU_ERI_BASE             0x00101000
  24 #else
  25 #define XTENSA_PMU_ERI_BASE             0x00001000
  26 #endif
  27
  28 /* Global control/status for all perf counters */
  29 #define XTENSA_PMU_PMG                  XTENSA_PMU_ERI_BASE
  30 /* Perf counter values */
  31 #define XTENSA_PMU_PM(i)                (XTENSA_PMU_ERI_BASE + 0x80 + (i) * 4)
  32 /* Perf counter control registers */
  33 #define XTENSA_PMU_PMCTRL(i)            (XTENSA_PMU_ERI_BASE + 0x100 + (i) * 4)
  34 /* Perf counter status registers */
  35 #define XTENSA_PMU_PMSTAT(i)            (XTENSA_PMU_ERI_BASE + 0x180 + (i) * 4)
  36
  37 #define XTENSA_PMU_PMG_PMEN             0x1
  38
  39 #define XTENSA_PMU_COUNTER_MASK         0xffffffffULL
  40 #define XTENSA_PMU_COUNTER_MAX          0x7fffffff
  41
  42 #define XTENSA_PMU_PMCTRL_INTEN         0x00000001
  43 #define XTENSA_PMU_PMCTRL_KRNLCNT       0x00000008
  44 #define XTENSA_PMU_PMCTRL_TRACELEVEL    0x000000f0
  45 #define XTENSA_PMU_PMCTRL_SELECT_SHIFT  8
  46 #define XTENSA_PMU_PMCTRL_SELECT        0x00001f00
  47 #define XTENSA_PMU_PMCTRL_MASK_SHIFT    16
  48 #define XTENSA_PMU_PMCTRL_MASK          0xffff0000
  49
  50 #define XTENSA_PMU_MASK(select, mask) \
  51         (((select) << XTENSA_PMU_PMCTRL_SELECT_SHIFT) | \
  52          ((mask) << XTENSA_PMU_PMCTRL_MASK_SHIFT) | \
  53          XTENSA_PMU_PMCTRL_TRACELEVEL | \
  54          XTENSA_PMU_PMCTRL_INTEN)
  55
  56 #define XTENSA_PMU_PMSTAT_OVFL          0x00000001
  57 #define XTENSA_PMU_PMSTAT_INTASRT       0x00000010
  58
  59 struct xtensa_pmu_events {
  60         /* Array of events currently on this core */
  61         struct perf_event *event[XCHAL_NUM_PERF_COUNTERS];
  62         /* Bitmap of used hardware counters */
  63         unsigned long used_mask[BITS_TO_LONGS(XCHAL_NUM_PERF_COUNTERS)];
  64 };
  65 static DEFINE_PER_CPU(struct xtensa_pmu_events, xtensa_pmu_events);
  66
  67 static const u32 xtensa_hw_ctl[] = {
  68         [PERF_COUNT_HW_CPU_CYCLES]              = XTENSA_PMU_MASK(0, 0x1),
  69         [PERF_COUNT_HW_INSTRUCTIONS]            = XTENSA_PMU_MASK(2, 0xffff),
  70         [PERF_COUNT_HW_CACHE_REFERENCES]        = XTENSA_PMU_MASK(10, 0x1),
  71         [PERF_COUNT_HW_CACHE_MISSES]            = XTENSA_PMU_MASK(12, 0x1),
  72         /* Taken and non-taken branches + taken loop ends */
  73         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS]     = XTENSA_PMU_MASK(2, 0x490),
  74         /* Instruction-related + other global stall cycles */
  75         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = XTENSA_PMU_MASK(4, 0x1ff),
  76         /* Data-related global stall cycles */
  77         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND]  = XTENSA_PMU_MASK(3, 0x1ff),
  78 };
  79
  80 #define C(_x) PERF_COUNT_HW_CACHE_##_x
  81
  82 static const u32 xtensa_cache_ctl[][C(OP_MAX)][C(RESULT_MAX)] = {
  83         [C(L1D)] = {
  84                 [C(OP_READ)] = {
  85                         [C(RESULT_ACCESS)]      = XTENSA_PMU_MASK(10, 0x1),
  86                         [C(RESULT_MISS)]        = XTENSA_PMU_MASK(10, 0x2),
  87                 },
  88                 [C(OP_WRITE)] = {
  89                         [C(RESULT_ACCESS)]      = XTENSA_PMU_MASK(11, 0x1),
  90                         [C(RESULT_MISS)]        = XTENSA_PMU_MASK(11, 0x2),
  91                 },
  92         },
  93         [C(L1I)] = {
  94                 [C(OP_READ)] = {
  95                         [C(RESULT_ACCESS)]      = XTENSA_PMU_MASK(8, 0x1),
  96                         [C(RESULT_MISS)]        = XTENSA_PMU_MASK(8, 0x2),
  97                 },
  98         },
  99         [C(DTLB)] = {
 100                 [C(OP_READ)] = {
 101                         [C(RESULT_ACCESS)]      = XTENSA_PMU_MASK(9, 0x1),
 102                         [C(RESULT_MISS)]        = XTENSA_PMU_MASK(9, 0x8),
 103                 },
 104         },
 105         [C(ITLB)] = {
 106                 [C(OP_READ)] = {
 107                         [C(RESULT_ACCESS)]      = XTENSA_PMU_MASK(7, 0x1),
 108                         [C(RESULT_MISS)]        = XTENSA_PMU_MASK(7, 0x8),
 109                 },
 110         },
 111 };
 112
 113 static int xtensa_pmu_cache_event(u64 config)
 114 {
 115         unsigned int cache_type, cache_op, cache_result;
 116         int ret;
 117
 118         cache_type = (config >>  0) & 0xff;
 119         cache_op = (config >>  8) & 0xff;
 120         cache_result = (config >> 16) & 0xff;
 121
 122         if (cache_type >= ARRAY_SIZE(xtensa_cache_ctl) ||
 123             cache_op >= C(OP_MAX) ||
 124             cache_result >= C(RESULT_MAX))
 125                 return -EINVAL;
 126
 127         ret = xtensa_cache_ctl[cache_type][cache_op][cache_result];
 128
 129         if (ret == 0)
 130                 return -EINVAL;
 131
 132         return ret;
 133 }
 134
 135 static inline uint32_t xtensa_pmu_read_counter(int idx)
 136 {
 137         return get_er(XTENSA_PMU_PM(idx));
 138 }
 139
 140 static inline void xtensa_pmu_write_counter(int idx, uint32_t v)
 141 {
 142         set_er(v, XTENSA_PMU_PM(idx));
 143 }
 144
 145 static void xtensa_perf_event_update(struct perf_event *event,
 146                                      struct hw_perf_event *hwc, int idx)
 147 {
 148         uint64_t prev_raw_count, new_raw_count;
 149         int64_t delta;
 150
 151         do {
 152                 prev_raw_count = local64_read(&hwc->prev_count);
 153                 new_raw_count = xtensa_pmu_read_counter(event->hw.idx);
 154         } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
 155                                  new_raw_count) != prev_raw_count);
 156
 157         delta = (new_raw_count - prev_raw_count) & XTENSA_PMU_COUNTER_MASK;
 158
 159         local64_add(delta, &event->count);
 160         local64_sub(delta, &hwc->period_left);
 161 }
 162
 163 static bool xtensa_perf_event_set_period(struct perf_event *event,
 164                                          struct hw_perf_event *hwc, int idx)
 165 {
 166         bool rc = false;
 167         s64 left;
 168
 169         if (!is_sampling_event(event)) {
 170                 left = XTENSA_PMU_COUNTER_MAX;
 171         } else {
 172                 s64 period = hwc->sample_period;
 173
 174                 left = local64_read(&hwc->period_left);
 175                 if (left <= -period) {
 176                         left = period;
 177                         local64_set(&hwc->period_left, left);
 178                         hwc->last_period = period;
 179                         rc = true;
 180                 } else if (left <= 0) {
 181                         left += period;
 182                         local64_set(&hwc->period_left, left);
 183                         hwc->last_period = period;
 184                         rc = true;
 185                 }
 186                 if (left > XTENSA_PMU_COUNTER_MAX)
 187                         left = XTENSA_PMU_COUNTER_MAX;
 188         }
 189
 190         local64_set(&hwc->prev_count, -left);
 191         xtensa_pmu_write_counter(idx, -left);
 192         perf_event_update_userpage(event);
 193
 194         return rc;
 195 }
 196
 197 static void xtensa_pmu_enable(struct pmu *pmu)
 198 {
 199         set_er(get_er(XTENSA_PMU_PMG) | XTENSA_PMU_PMG_PMEN, XTENSA_PMU_PMG);
 200 }
 201
 202 static void xtensa_pmu_disable(struct pmu *pmu)
 203 {
 204         set_er(get_er(XTENSA_PMU_PMG) & ~XTENSA_PMU_PMG_PMEN, XTENSA_PMU_PMG);
 205 }
 206
 207 static int xtensa_pmu_event_init(struct perf_event *event)
 208 {
 209         int ret;
 210
 211         switch (event->attr.type) {
 212         case PERF_TYPE_HARDWARE:
 213                 if (event->attr.config >= ARRAY_SIZE(xtensa_hw_ctl) ||
 214                     xtensa_hw_ctl[event->attr.config] == 0)
 215                         return -EINVAL;
 216                 event->hw.config = xtensa_hw_ctl[event->attr.config];
 217                 return 0;
 218
 219         case PERF_TYPE_HW_CACHE:
 220                 ret = xtensa_pmu_cache_event(event->attr.config);
 221                 if (ret < 0)
 222                         return ret;
 223                 event->hw.config = ret;
 224                 return 0;
 225
 226         case PERF_TYPE_RAW:
 227                 /* Not 'previous counter' select */
 228                 if ((event->attr.config & XTENSA_PMU_PMCTRL_SELECT) ==
 229                     (1 << XTENSA_PMU_PMCTRL_SELECT_SHIFT))
 230                         return -EINVAL;
 231                 event->hw.config = (event->attr.config &
 232                                     (XTENSA_PMU_PMCTRL_KRNLCNT |
 233                                      XTENSA_PMU_PMCTRL_TRACELEVEL |
 234                                      XTENSA_PMU_PMCTRL_SELECT |
 235                                      XTENSA_PMU_PMCTRL_MASK)) |
 236                         XTENSA_PMU_PMCTRL_INTEN;
 237                 return 0;
 238
 239         default:
 240                 return -ENOENT;
 241         }
 242 }
 243
 244 /*
 245  * Starts/Stops a counter present on the PMU. The PMI handler
 246  * should stop the counter when perf_event_overflow() returns
 247  * !0. ->start() will be used to continue.
 248  */
 249 static void xtensa_pmu_start(struct perf_event *event, int flags)
 250 {
 251         struct hw_perf_event *hwc = &event->hw;
 252         int idx = hwc->idx;
 253
 254         if (WARN_ON_ONCE(idx == -1))
 255                 return;
 256
 257         if (flags & PERF_EF_RELOAD) {
 258                 WARN_ON_ONCE(!(event->hw.state & PERF_HES_UPTODATE));
 259                 xtensa_perf_event_set_period(event, hwc, idx);
 260         }
 261
 262         hwc->state = 0;
 263
 264         set_er(hwc->config, XTENSA_PMU_PMCTRL(idx));
 265 }
 266
 267 static void xtensa_pmu_stop(struct perf_event *event, int flags)
 268 {
 269         struct hw_perf_event *hwc = &event->hw;
 270         int idx = hwc->idx;
 271
 272         if (!(hwc->state & PERF_HES_STOPPED)) {
 273                 set_er(0, XTENSA_PMU_PMCTRL(idx));
 274                 set_er(get_er(XTENSA_PMU_PMSTAT(idx)),
 275                        XTENSA_PMU_PMSTAT(idx));
 276                 hwc->state |= PERF_HES_STOPPED;
 277         }
 278
 279         if ((flags & PERF_EF_UPDATE) &&
 280             !(event->hw.state & PERF_HES_UPTODATE)) {
 281                 xtensa_perf_event_update(event, &event->hw, idx);
 282                 event->hw.state |= PERF_HES_UPTODATE;
 283         }
 284 }
 285
 286 /*
 287  * Adds/Removes a counter to/from the PMU, can be done inside
 288  * a transaction, see the ->*_txn() methods.
 289  */
 290 static int xtensa_pmu_add(struct perf_event *event, int flags)
 291 {
 292         struct xtensa_pmu_events *ev = this_cpu_ptr(&xtensa_pmu_events);
 293         struct hw_perf_event *hwc = &event->hw;
 294         int idx = hwc->idx;
 295
 296         if (__test_and_set_bit(idx, ev->used_mask)) {
 297                 idx = find_first_zero_bit(ev->used_mask,
 298                                           XCHAL_NUM_PERF_COUNTERS);
 299                 if (idx == XCHAL_NUM_PERF_COUNTERS)
 300                         return -EAGAIN;
 301
 302                 __set_bit(idx, ev->used_mask);
 303                 hwc->idx = idx;
 304         }
 305         ev->event[idx] = event;
 306
 307         hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
 308
 309         if (flags & PERF_EF_START)
 310                 xtensa_pmu_start(event, PERF_EF_RELOAD);
 311
 312         perf_event_update_userpage(event);
 313         return 0;
 314 }
 315
 316 static void xtensa_pmu_del(struct perf_event *event, int flags)
 317 {
 318         struct xtensa_pmu_events *ev = this_cpu_ptr(&xtensa_pmu_events);
 319
 320         xtensa_pmu_stop(event, PERF_EF_UPDATE);
 321         __clear_bit(event->hw.idx, ev->used_mask);
 322         perf_event_update_userpage(event);
 323 }
 324
 325 static void xtensa_pmu_read(struct perf_event *event)
 326 {
 327         xtensa_perf_event_update(event, &event->hw, event->hw.idx);
 328 }
 329
 330 static int callchain_trace(struct stackframe *frame, void *data)
 331 {
 332         struct perf_callchain_entry_ctx *entry = data;
 333
 334         perf_callchain_store(entry, frame->pc);
 335         return 0;
 336 }
 337
 338 void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
 339                            struct pt_regs *regs)
 340 {
 341         xtensa_backtrace_kernel(regs, entry->max_stack,
 342                                 callchain_trace, NULL, entry);
 343 }
 344
 345 void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
 346                          struct pt_regs *regs)
 347 {
 348         xtensa_backtrace_user(regs, entry->max_stack,
 349                               callchain_trace, entry);
 350 }
 351
 352 void perf_event_print_debug(void)
 353 {
 354         unsigned long flags;
 355         unsigned i;
 356
 357         local_irq_save(flags);
 358         pr_info("CPU#%d: PMG: 0x%08lx\n", smp_processor_id(),
 359                 get_er(XTENSA_PMU_PMG));
 360         for (i = 0; i < XCHAL_NUM_PERF_COUNTERS; ++i)
 361                 pr_info("PM%d: 0x%08lx, PMCTRL%d: 0x%08lx, PMSTAT%d: 0x%08lx\n",
 362                         i, get_er(XTENSA_PMU_PM(i)),
 363                         i, get_er(XTENSA_PMU_PMCTRL(i)),
 364                         i, get_er(XTENSA_PMU_PMSTAT(i)));
 365         local_irq_restore(flags);
 366 }
 367
 368 irqreturn_t xtensa_pmu_irq_handler(int irq, void *dev_id)
 369 {
 370         irqreturn_t rc = IRQ_NONE;
 371         struct xtensa_pmu_events *ev = this_cpu_ptr(&xtensa_pmu_events);
 372         unsigned i;
 373
 374         for (i = find_first_bit(ev->used_mask, XCHAL_NUM_PERF_COUNTERS);
 375              i < XCHAL_NUM_PERF_COUNTERS;
 376              i = find_next_bit(ev->used_mask, XCHAL_NUM_PERF_COUNTERS, i + 1)) {
 377                 uint32_t v = get_er(XTENSA_PMU_PMSTAT(i));
 378                 struct perf_event *event = ev->event[i];
 379                 struct hw_perf_event *hwc = &event->hw;
 380                 u64 last_period;
 381
 382                 if (!(v & XTENSA_PMU_PMSTAT_OVFL))
 383                         continue;
 384
 385                 set_er(v, XTENSA_PMU_PMSTAT(i));
 386                 xtensa_perf_event_update(event, hwc, i);
 387                 last_period = hwc->last_period;
 388                 if (xtensa_perf_event_set_period(event, hwc, i)) {
 389                         struct perf_sample_data data;
 390                         struct pt_regs *regs = get_irq_regs();
 391
 392                         perf_sample_data_init(&data, 0, last_period);
 393                         if (perf_event_overflow(event, &data, regs))
 394                                 xtensa_pmu_stop(event, 0);
 395                 }
 396
 397                 rc = IRQ_HANDLED;
 398         }
 399         return rc;
 400 }
 401
 402 static struct pmu xtensa_pmu = {
 403         .pmu_enable = xtensa_pmu_enable,
 404         .pmu_disable = xtensa_pmu_disable,
 405         .event_init = xtensa_pmu_event_init,
 406         .add = xtensa_pmu_add,
 407         .del = xtensa_pmu_del,
 408         .start = xtensa_pmu_start,
 409         .stop = xtensa_pmu_stop,
 410         .read = xtensa_pmu_read,
 411 };
 412
 413 static int xtensa_pmu_setup(unsigned int cpu)
 414 {
 415         unsigned i;
 416
 417         set_er(0, XTENSA_PMU_PMG);
 418         for (i = 0; i < XCHAL_NUM_PERF_COUNTERS; ++i) {
 419                 set_er(0, XTENSA_PMU_PMCTRL(i));
 420                 set_er(get_er(XTENSA_PMU_PMSTAT(i)), XTENSA_PMU_PMSTAT(i));
 421         }
 422         return 0;
 423 }
 424
 425 static int __init xtensa_pmu_init(void)
 426 {
 427         int ret;
 428         int irq = irq_create_mapping(NULL, XCHAL_PROFILING_INTERRUPT);
 429
 430         ret = cpuhp_setup_state(CPUHP_AP_PERF_XTENSA_STARTING,
 431                                 "perf/xtensa:starting", xtensa_pmu_setup,
 432                                 NULL);
 433         if (ret) {
 434                 pr_err("xtensa_pmu: failed to register CPU-hotplug.\n");
 435                 return ret;
 436         }
 437 #if XTENSA_FAKE_NMI
 438         enable_irq(irq);
 439 #else
 440         ret = request_irq(irq, xtensa_pmu_irq_handler, IRQF_PERCPU,
 441                           "pmu", NULL);
 442         if (ret < 0)
 443                 return ret;
 444 #endif
 445
 446         ret = perf_pmu_register(&xtensa_pmu, "cpu", PERF_TYPE_RAW);
 447         if (ret)
 448                 free_irq(irq, NULL);
 449
 450         return ret;
 451 }
 452 early_initcall(xtensa_pmu_init);