arch/arc/kernel/perf_event.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 //
   3 // Linux performance counter support for ARC CPUs.
   4 // This code is inspired by the perf support of various other architectures.
   5 //
   6 // Copyright (C) 2013-2018 Synopsys, Inc. (www.synopsys.com)
   7
   8 #include <linux/errno.h>
   9 #include <linux/interrupt.h>
  10 #include <linux/module.h>
  11 #include <linux/of.h>
  12 #include <linux/perf_event.h>
  13 #include <linux/platform_device.h>
  14 #include <asm/arcregs.h>
  15 #include <asm/stacktrace.h>
  16
  17 /* HW holds 8 symbols + one for null terminator */
  18 #define ARCPMU_EVENT_NAME_LEN   9
  19
  20 /*
  21  * Some ARC pct quirks:
  22  *
  23  * PERF_COUNT_HW_STALLED_CYCLES_BACKEND
  24  * PERF_COUNT_HW_STALLED_CYCLES_FRONTEND
  25  *      The ARC 700 can either measure stalls per pipeline stage, or all stalls
  26  *      combined; for now we assign all stalls to STALLED_CYCLES_BACKEND
  27  *      and all pipeline flushes (e.g. caused by mispredicts, etc.) to
  28  *      STALLED_CYCLES_FRONTEND.
  29  *
  30  *      We could start multiple performance counters and combine everything
  31  *      afterwards, but that makes it complicated.
  32  *
  33  *      Note that I$ cache misses aren't counted by either of the two!
  34  */
  35
  36 /*
  37  * ARC PCT has hardware conditions with fixed "names" but variable "indexes"
  38  * (based on a specific RTL build)
  39  * Below is the static map between perf generic/arc specific event_id and
  40  * h/w condition names.
  41  * At the time of probe, we loop thru each index and find it's name to
  42  * complete the mapping of perf event_id to h/w index as latter is needed
  43  * to program the counter really
  44  */
  45 static const char * const arc_pmu_ev_hw_map[] = {
  46         /* count cycles */
  47         [PERF_COUNT_HW_CPU_CYCLES] = "crun",
  48         [PERF_COUNT_HW_REF_CPU_CYCLES] = "crun",
  49         [PERF_COUNT_HW_BUS_CYCLES] = "crun",
  50
  51         [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = "bflush",
  52         [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = "bstall",
  53
  54         /* counts condition */
  55         [PERF_COUNT_HW_INSTRUCTIONS] = "iall",
  56         /* All jump instructions that are taken */
  57         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = "ijmptak",
  58 #ifdef CONFIG_ISA_ARCV2
  59         [PERF_COUNT_HW_BRANCH_MISSES] = "bpmp",
  60 #else
  61         [PERF_COUNT_ARC_BPOK]         = "bpok",   /* NP-NT, PT-T, PNT-NT */
  62         [PERF_COUNT_HW_BRANCH_MISSES] = "bpfail", /* NP-T, PT-NT, PNT-T */
  63 #endif
  64         [PERF_COUNT_ARC_LDC] = "imemrdc",       /* Instr: mem read cached */
  65         [PERF_COUNT_ARC_STC] = "imemwrc",       /* Instr: mem write cached */
  66
  67         [PERF_COUNT_ARC_DCLM] = "dclm",         /* D-cache Load Miss */
  68         [PERF_COUNT_ARC_DCSM] = "dcsm",         /* D-cache Store Miss */
  69         [PERF_COUNT_ARC_ICM] = "icm",           /* I-cache Miss */
  70         [PERF_COUNT_ARC_EDTLB] = "edtlb",       /* D-TLB Miss */
  71         [PERF_COUNT_ARC_EITLB] = "eitlb",       /* I-TLB Miss */
  72
  73         [PERF_COUNT_HW_CACHE_REFERENCES] = "imemrdc",   /* Instr: mem read cached */
  74         [PERF_COUNT_HW_CACHE_MISSES] = "dclm",          /* D-cache Load Miss */
  75 };
  76
  77 #define C(_x)                   PERF_COUNT_HW_CACHE_##_x
  78 #define CACHE_OP_UNSUPPORTED    0xffff
  79
  80 static const unsigned int arc_pmu_cache_map[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = {
  81         [C(L1D)] = {
  82                 [C(OP_READ)] = {
  83                         [C(RESULT_ACCESS)]      = PERF_COUNT_ARC_LDC,
  84                         [C(RESULT_MISS)]        = PERF_COUNT_ARC_DCLM,
  85                 },
  86                 [C(OP_WRITE)] = {
  87                         [C(RESULT_ACCESS)]      = PERF_COUNT_ARC_STC,
  88                         [C(RESULT_MISS)]        = PERF_COUNT_ARC_DCSM,
  89                 },
  90                 [C(OP_PREFETCH)] = {
  91                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
  92                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
  93                 },
  94         },
  95         [C(L1I)] = {
  96                 [C(OP_READ)] = {
  97                         [C(RESULT_ACCESS)]      = PERF_COUNT_HW_INSTRUCTIONS,
  98                         [C(RESULT_MISS)]        = PERF_COUNT_ARC_ICM,
  99                 },
 100                 [C(OP_WRITE)] = {
 101                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 102                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 103                 },
 104                 [C(OP_PREFETCH)] = {
 105                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 106                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 107                 },
 108         },
 109         [C(LL)] = {
 110                 [C(OP_READ)] = {
 111                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 112                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 113                 },
 114                 [C(OP_WRITE)] = {
 115                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 116                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 117                 },
 118                 [C(OP_PREFETCH)] = {
 119                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 120                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 121                 },
 122         },
 123         [C(DTLB)] = {
 124                 [C(OP_READ)] = {
 125                         [C(RESULT_ACCESS)]      = PERF_COUNT_ARC_LDC,
 126                         [C(RESULT_MISS)]        = PERF_COUNT_ARC_EDTLB,
 127                 },
 128                         /* DTLB LD/ST Miss not segregated by h/w*/
 129                 [C(OP_WRITE)] = {
 130                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 131                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 132                 },
 133                 [C(OP_PREFETCH)] = {
 134                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 135                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 136                 },
 137         },
 138         [C(ITLB)] = {
 139                 [C(OP_READ)] = {
 140                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 141                         [C(RESULT_MISS)]        = PERF_COUNT_ARC_EITLB,
 142                 },
 143                 [C(OP_WRITE)] = {
 144                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 145                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 146                 },
 147                 [C(OP_PREFETCH)] = {
 148                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 149                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 150                 },
 151         },
 152         [C(BPU)] = {
 153                 [C(OP_READ)] = {
 154                         [C(RESULT_ACCESS)] = PERF_COUNT_HW_BRANCH_INSTRUCTIONS,
 155                         [C(RESULT_MISS)]        = PERF_COUNT_HW_BRANCH_MISSES,
 156                 },
 157                 [C(OP_WRITE)] = {
 158                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 159                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 160                 },
 161                 [C(OP_PREFETCH)] = {
 162                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 163                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 164                 },
 165         },
 166         [C(NODE)] = {
 167                 [C(OP_READ)] = {
 168                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 169                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 170                 },
 171                 [C(OP_WRITE)] = {
 172                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 173                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 174                 },
 175                 [C(OP_PREFETCH)] = {
 176                         [C(RESULT_ACCESS)]      = CACHE_OP_UNSUPPORTED,
 177                         [C(RESULT_MISS)]        = CACHE_OP_UNSUPPORTED,
 178                 },
 179         },
 180 };
 181
 182 enum arc_pmu_attr_groups {
 183         ARCPMU_ATTR_GR_EVENTS,
 184         ARCPMU_ATTR_GR_FORMATS,
 185         ARCPMU_NR_ATTR_GR
 186 };
 187
 188 struct arc_pmu_raw_event_entry {
 189         char name[ARCPMU_EVENT_NAME_LEN];
 190 };
 191
 192 struct arc_pmu {
 193         struct pmu      pmu;
 194         unsigned int    irq;
 195         int             n_counters;
 196         int             n_events;
 197         u64             max_period;
 198         int             ev_hw_idx[PERF_COUNT_ARC_HW_MAX];
 199
 200         struct arc_pmu_raw_event_entry  *raw_entry;
 201         struct attribute                **attrs;
 202         struct perf_pmu_events_attr     *attr;
 203         const struct attribute_group    *attr_groups[ARCPMU_NR_ATTR_GR + 1];
 204 };
 205
 206 struct arc_pmu_cpu {
 207         /*
 208          * A 1 bit for an index indicates that the counter is being used for
 209          * an event. A 0 means that the counter can be used.
 210          */
 211         unsigned long   used_mask[BITS_TO_LONGS(ARC_PERF_MAX_COUNTERS)];
 212
 213         /*
 214          * The events that are active on the PMU for the given index.
 215          */
 216         struct perf_event *act_counter[ARC_PERF_MAX_COUNTERS];
 217 };
 218
 219 struct arc_callchain_trace {
 220         int depth;
 221         void *perf_stuff;
 222 };
 223
 224 static int callchain_trace(unsigned int addr, void *data)
 225 {
 226         struct arc_callchain_trace *ctrl = data;
 227         struct perf_callchain_entry_ctx *entry = ctrl->perf_stuff;
 228
 229         perf_callchain_store(entry, addr);
 230
 231         if (ctrl->depth++ < 3)
 232                 return 0;
 233
 234         return -1;
 235 }
 236
 237 void perf_callchain_kernel(struct perf_callchain_entry_ctx *entry,
 238                            struct pt_regs *regs)
 239 {
 240         struct arc_callchain_trace ctrl = {
 241                 .depth = 0,
 242                 .perf_stuff = entry,
 243         };
 244
 245         arc_unwind_core(NULL, regs, callchain_trace, &ctrl);
 246 }
 247
 248 void perf_callchain_user(struct perf_callchain_entry_ctx *entry,
 249                          struct pt_regs *regs)
 250 {
 251         /*
 252          * User stack can't be unwound trivially with kernel dwarf unwinder
 253          * So for now just record the user PC
 254          */
 255         perf_callchain_store(entry, instruction_pointer(regs));
 256 }
 257
 258 static struct arc_pmu *arc_pmu;
 259 static DEFINE_PER_CPU(struct arc_pmu_cpu, arc_pmu_cpu);
 260
 261 /* read counter #idx; note that counter# != event# on ARC! */
 262 static u64 arc_pmu_read_counter(int idx)
 263 {
 264         u32 tmp;
 265         u64 result;
 266
 267         /*
 268          * ARC supports making 'snapshots' of the counters, so we don't
 269          * need to care about counters wrapping to 0 underneath our feet
 270          */
 271         write_aux_reg(ARC_REG_PCT_INDEX, idx);
 272         tmp = read_aux_reg(ARC_REG_PCT_CONTROL);
 273         write_aux_reg(ARC_REG_PCT_CONTROL, tmp | ARC_REG_PCT_CONTROL_SN);
 274         result = (u64) (read_aux_reg(ARC_REG_PCT_SNAPH)) << 32;
 275         result |= read_aux_reg(ARC_REG_PCT_SNAPL);
 276
 277         return result;
 278 }
 279
 280 static void arc_perf_event_update(struct perf_event *event,
 281                                   struct hw_perf_event *hwc, int idx)
 282 {
 283         u64 prev_raw_count = local64_read(&hwc->prev_count);
 284         u64 new_raw_count = arc_pmu_read_counter(idx);
 285         s64 delta = new_raw_count - prev_raw_count;
 286
 287         /*
 288          * We aren't afraid of hwc->prev_count changing beneath our feet
 289          * because there's no way for us to re-enter this function anytime.
 290          */
 291         local64_set(&hwc->prev_count, new_raw_count);
 292         local64_add(delta, &event->count);
 293         local64_sub(delta, &hwc->period_left);
 294 }
 295
 296 static void arc_pmu_read(struct perf_event *event)
 297 {
 298         arc_perf_event_update(event, &event->hw, event->hw.idx);
 299 }
 300
 301 static int arc_pmu_cache_event(u64 config)
 302 {
 303         unsigned int cache_type, cache_op, cache_result;
 304         int ret;
 305
 306         cache_type      = (config >>  0) & 0xff;
 307         cache_op        = (config >>  8) & 0xff;
 308         cache_result    = (config >> 16) & 0xff;
 309         if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
 310                 return -EINVAL;
 311         if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
 312                 return -EINVAL;
 313         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 314                 return -EINVAL;
 315
 316         ret = arc_pmu_cache_map[cache_type][cache_op][cache_result];
 317
 318         if (ret == CACHE_OP_UNSUPPORTED)
 319                 return -ENOENT;
 320
 321         pr_debug("init cache event: type/op/result %d/%d/%d with h/w %d \'%s\'\n",
 322                  cache_type, cache_op, cache_result, ret,
 323                  arc_pmu_ev_hw_map[ret]);
 324
 325         return ret;
 326 }
 327
 328 /* initializes hw_perf_event structure if event is supported */
 329 static int arc_pmu_event_init(struct perf_event *event)
 330 {
 331         struct hw_perf_event *hwc = &event->hw;
 332         int ret;
 333
 334         if (!is_sampling_event(event)) {
 335                 hwc->sample_period = arc_pmu->max_period;
 336                 hwc->last_period = hwc->sample_period;
 337                 local64_set(&hwc->period_left, hwc->sample_period);
 338         }
 339
 340         hwc->config = 0;
 341
 342         if (is_isa_arcv2()) {
 343                 /* "exclude user" means "count only kernel" */
 344                 if (event->attr.exclude_user)
 345                         hwc->config |= ARC_REG_PCT_CONFIG_KERN;
 346
 347                 /* "exclude kernel" means "count only user" */
 348                 if (event->attr.exclude_kernel)
 349                         hwc->config |= ARC_REG_PCT_CONFIG_USER;
 350         }
 351
 352         switch (event->attr.type) {
 353         case PERF_TYPE_HARDWARE:
 354                 if (event->attr.config >= PERF_COUNT_HW_MAX)
 355                         return -ENOENT;
 356                 if (arc_pmu->ev_hw_idx[event->attr.config] < 0)
 357                         return -ENOENT;
 358                 hwc->config |= arc_pmu->ev_hw_idx[event->attr.config];
 359                 pr_debug("init event %d with h/w %08x \'%s\'\n",
 360                          (int)event->attr.config, (int)hwc->config,
 361                          arc_pmu_ev_hw_map[event->attr.config]);
 362                 return 0;
 363
 364         case PERF_TYPE_HW_CACHE:
 365                 ret = arc_pmu_cache_event(event->attr.config);
 366                 if (ret < 0)
 367                         return ret;
 368                 hwc->config |= arc_pmu->ev_hw_idx[ret];
 369                 pr_debug("init cache event with h/w %08x \'%s\'\n",
 370                          (int)hwc->config, arc_pmu_ev_hw_map[ret]);
 371                 return 0;
 372
 373         case PERF_TYPE_RAW:
 374                 if (event->attr.config >= arc_pmu->n_events)
 375                         return -ENOENT;
 376
 377                 hwc->config |= event->attr.config;
 378                 pr_debug("init raw event with idx %lld \'%s\'\n",
 379                          event->attr.config,
 380                          arc_pmu->raw_entry[event->attr.config].name);
 381
 382                 return 0;
 383
 384         default:
 385                 return -ENOENT;
 386         }
 387 }
 388
 389 /* starts all counters */
 390 static void arc_pmu_enable(struct pmu *pmu)
 391 {
 392         u32 tmp;
 393         tmp = read_aux_reg(ARC_REG_PCT_CONTROL);
 394         write_aux_reg(ARC_REG_PCT_CONTROL, (tmp & 0xffff0000) | 0x1);
 395 }
 396
 397 /* stops all counters */
 398 static void arc_pmu_disable(struct pmu *pmu)
 399 {
 400         u32 tmp;
 401         tmp = read_aux_reg(ARC_REG_PCT_CONTROL);
 402         write_aux_reg(ARC_REG_PCT_CONTROL, (tmp & 0xffff0000) | 0x0);
 403 }
 404
 405 static int arc_pmu_event_set_period(struct perf_event *event)
 406 {
 407         struct hw_perf_event *hwc = &event->hw;
 408         s64 left = local64_read(&hwc->period_left);
 409         s64 period = hwc->sample_period;
 410         int idx = hwc->idx;
 411         int overflow = 0;
 412         u64 value;
 413
 414         if (unlikely(left <= -period)) {
 415                 /* left underflowed by more than period. */
 416                 left = period;
 417                 local64_set(&hwc->period_left, left);
 418                 hwc->last_period = period;
 419                 overflow = 1;
 420         } else if (unlikely(left <= 0)) {
 421                 /* left underflowed by less than period. */
 422                 left += period;
 423                 local64_set(&hwc->period_left, left);
 424                 hwc->last_period = period;
 425                 overflow = 1;
 426         }
 427
 428         if (left > arc_pmu->max_period)
 429                 left = arc_pmu->max_period;
 430
 431         value = arc_pmu->max_period - left;
 432         local64_set(&hwc->prev_count, value);
 433
 434         /* Select counter */
 435         write_aux_reg(ARC_REG_PCT_INDEX, idx);
 436
 437         /* Write value */
 438         write_aux_reg(ARC_REG_PCT_COUNTL, lower_32_bits(value));
 439         write_aux_reg(ARC_REG_PCT_COUNTH, upper_32_bits(value));
 440
 441         perf_event_update_userpage(event);
 442
 443         return overflow;
 444 }
 445
 446 /*
 447  * Assigns hardware counter to hardware condition.
 448  * Note that there is no separate start/stop mechanism;
 449  * stopping is achieved by assigning the 'never' condition
 450  */
 451 static void arc_pmu_start(struct perf_event *event, int flags)
 452 {
 453         struct hw_perf_event *hwc = &event->hw;
 454         int idx = hwc->idx;
 455
 456         if (WARN_ON_ONCE(idx == -1))
 457                 return;
 458
 459         if (flags & PERF_EF_RELOAD)
 460                 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
 461
 462         hwc->state = 0;
 463
 464         arc_pmu_event_set_period(event);
 465
 466         /* Enable interrupt for this counter */
 467         if (is_sampling_event(event))
 468                 write_aux_reg(ARC_REG_PCT_INT_CTRL,
 469                               read_aux_reg(ARC_REG_PCT_INT_CTRL) | BIT(idx));
 470
 471         /* enable ARC pmu here */
 472         write_aux_reg(ARC_REG_PCT_INDEX, idx);          /* counter # */
 473         write_aux_reg(ARC_REG_PCT_CONFIG, hwc->config); /* condition */
 474 }
 475
 476 static void arc_pmu_stop(struct perf_event *event, int flags)
 477 {
 478         struct hw_perf_event *hwc = &event->hw;
 479         int idx = hwc->idx;
 480
 481         /* Disable interrupt for this counter */
 482         if (is_sampling_event(event)) {
 483                 /*
 484                  * Reset interrupt flag by writing of 1. This is required
 485                  * to make sure pending interrupt was not left.
 486                  */
 487                 write_aux_reg(ARC_REG_PCT_INT_ACT, BIT(idx));
 488                 write_aux_reg(ARC_REG_PCT_INT_CTRL,
 489                               read_aux_reg(ARC_REG_PCT_INT_CTRL) & ~BIT(idx));
 490         }
 491
 492         if (!(event->hw.state & PERF_HES_STOPPED)) {
 493                 /* stop hw counter here */
 494                 write_aux_reg(ARC_REG_PCT_INDEX, idx);
 495
 496                 /* condition code #0 is always "never" */
 497                 write_aux_reg(ARC_REG_PCT_CONFIG, 0);
 498
 499                 event->hw.state |= PERF_HES_STOPPED;
 500         }
 501
 502         if ((flags & PERF_EF_UPDATE) &&
 503             !(event->hw.state & PERF_HES_UPTODATE)) {
 504                 arc_perf_event_update(event, &event->hw, idx);
 505                 event->hw.state |= PERF_HES_UPTODATE;
 506         }
 507 }
 508
 509 static void arc_pmu_del(struct perf_event *event, int flags)
 510 {
 511         struct arc_pmu_cpu *pmu_cpu = this_cpu_ptr(&arc_pmu_cpu);
 512
 513         arc_pmu_stop(event, PERF_EF_UPDATE);
 514         __clear_bit(event->hw.idx, pmu_cpu->used_mask);
 515
 516         pmu_cpu->act_counter[event->hw.idx] = 0;
 517
 518         perf_event_update_userpage(event);
 519 }
 520
 521 /* allocate hardware counter and optionally start counting */
 522 static int arc_pmu_add(struct perf_event *event, int flags)
 523 {
 524         struct arc_pmu_cpu *pmu_cpu = this_cpu_ptr(&arc_pmu_cpu);
 525         struct hw_perf_event *hwc = &event->hw;
 526         int idx;
 527
 528         idx = ffz(pmu_cpu->used_mask[0]);
 529         if (idx == arc_pmu->n_counters)
 530                 return -EAGAIN;
 531
 532         __set_bit(idx, pmu_cpu->used_mask);
 533         hwc->idx = idx;
 534
 535         write_aux_reg(ARC_REG_PCT_INDEX, idx);
 536
 537         pmu_cpu->act_counter[idx] = event;
 538
 539         if (is_sampling_event(event)) {
 540                 /* Mimic full counter overflow as other arches do */
 541                 write_aux_reg(ARC_REG_PCT_INT_CNTL,
 542                               lower_32_bits(arc_pmu->max_period));
 543                 write_aux_reg(ARC_REG_PCT_INT_CNTH,
 544                               upper_32_bits(arc_pmu->max_period));
 545         }
 546
 547         write_aux_reg(ARC_REG_PCT_CONFIG, 0);
 548         write_aux_reg(ARC_REG_PCT_COUNTL, 0);
 549         write_aux_reg(ARC_REG_PCT_COUNTH, 0);
 550         local64_set(&hwc->prev_count, 0);
 551
 552         hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
 553         if (flags & PERF_EF_START)
 554                 arc_pmu_start(event, PERF_EF_RELOAD);
 555
 556         perf_event_update_userpage(event);
 557
 558         return 0;
 559 }
 560
 561 #ifdef CONFIG_ISA_ARCV2
 562 static irqreturn_t arc_pmu_intr(int irq, void *dev)
 563 {
 564         struct perf_sample_data data;
 565         struct arc_pmu_cpu *pmu_cpu = this_cpu_ptr(&arc_pmu_cpu);
 566         struct pt_regs *regs;
 567         unsigned int active_ints;
 568         int idx;
 569
 570         arc_pmu_disable(&arc_pmu->pmu);
 571
 572         active_ints = read_aux_reg(ARC_REG_PCT_INT_ACT);
 573         if (!active_ints)
 574                 goto done;
 575
 576         regs = get_irq_regs();
 577
 578         do {
 579                 struct perf_event *event;
 580                 struct hw_perf_event *hwc;
 581
 582                 idx = __ffs(active_ints);
 583
 584                 /* Reset interrupt flag by writing of 1 */
 585                 write_aux_reg(ARC_REG_PCT_INT_ACT, BIT(idx));
 586
 587                 /*
 588                  * On reset of "interrupt active" bit corresponding
 589                  * "interrupt enable" bit gets automatically reset as well.
 590                  * Now we need to re-enable interrupt for the counter.
 591                  */
 592                 write_aux_reg(ARC_REG_PCT_INT_CTRL,
 593                         read_aux_reg(ARC_REG_PCT_INT_CTRL) | BIT(idx));
 594
 595                 event = pmu_cpu->act_counter[idx];
 596                 hwc = &event->hw;
 597
 598                 WARN_ON_ONCE(hwc->idx != idx);
 599
 600                 arc_perf_event_update(event, &event->hw, event->hw.idx);
 601                 perf_sample_data_init(&data, 0, hwc->last_period);
 602                 if (arc_pmu_event_set_period(event)) {
 603                         if (perf_event_overflow(event, &data, regs))
 604                                 arc_pmu_stop(event, 0);
 605                 }
 606
 607                 active_ints &= ~BIT(idx);
 608         } while (active_ints);
 609
 610 done:
 611         arc_pmu_enable(&arc_pmu->pmu);
 612
 613         return IRQ_HANDLED;
 614 }
 615 #else
 616
 617 static irqreturn_t arc_pmu_intr(int irq, void *dev)
 618 {
 619         return IRQ_NONE;
 620 }
 621
 622 #endif /* CONFIG_ISA_ARCV2 */
 623
 624 static void arc_cpu_pmu_irq_init(void *data)
 625 {
 626         int irq = *(int *)data;
 627
 628         enable_percpu_irq(irq, IRQ_TYPE_NONE);
 629
 630         /* Clear all pending interrupt flags */
 631         write_aux_reg(ARC_REG_PCT_INT_ACT, 0xffffffff);
 632 }
 633
 634 /* Event field occupies the bottom 15 bits of our config field */
 635 PMU_FORMAT_ATTR(event, "config:0-14");
 636 static struct attribute *arc_pmu_format_attrs[] = {
 637         &format_attr_event.attr,
 638         NULL,
 639 };
 640
 641 static struct attribute_group arc_pmu_format_attr_gr = {
 642         .name = "format",
 643         .attrs = arc_pmu_format_attrs,
 644 };
 645
 646 static ssize_t arc_pmu_events_sysfs_show(struct device *dev,
 647                                          struct device_attribute *attr,
 648                                          char *page)
 649 {
 650         struct perf_pmu_events_attr *pmu_attr;
 651
 652         pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
 653         return sprintf(page, "event=0x%04llx\n", pmu_attr->id);
 654 }
 655
 656 /*
 657  * We don't add attrs here as we don't have pre-defined list of perf events.
 658  * We will generate and add attrs dynamically in probe() after we read HW
 659  * configuration.
 660  */
 661 static struct attribute_group arc_pmu_events_attr_gr = {
 662         .name = "events",
 663 };
 664
 665 static void arc_pmu_add_raw_event_attr(int j, char *str)
 666 {
 667         memmove(arc_pmu->raw_entry[j].name, str, ARCPMU_EVENT_NAME_LEN - 1);
 668         arc_pmu->attr[j].attr.attr.name = arc_pmu->raw_entry[j].name;
 669         arc_pmu->attr[j].attr.attr.mode = VERIFY_OCTAL_PERMISSIONS(0444);
 670         arc_pmu->attr[j].attr.show = arc_pmu_events_sysfs_show;
 671         arc_pmu->attr[j].id = j;
 672         arc_pmu->attrs[j] = &(arc_pmu->attr[j].attr.attr);
 673 }
 674
 675 static int arc_pmu_raw_alloc(struct device *dev)
 676 {
 677         arc_pmu->attr = devm_kmalloc_array(dev, arc_pmu->n_events + 1,
 678                 sizeof(*arc_pmu->attr), GFP_KERNEL | __GFP_ZERO);
 679         if (!arc_pmu->attr)
 680                 return -ENOMEM;
 681
 682         arc_pmu->attrs = devm_kmalloc_array(dev, arc_pmu->n_events + 1,
 683                 sizeof(*arc_pmu->attrs), GFP_KERNEL | __GFP_ZERO);
 684         if (!arc_pmu->attrs)
 685                 return -ENOMEM;
 686
 687         arc_pmu->raw_entry = devm_kmalloc_array(dev, arc_pmu->n_events,
 688                 sizeof(*arc_pmu->raw_entry), GFP_KERNEL | __GFP_ZERO);
 689         if (!arc_pmu->raw_entry)
 690                 return -ENOMEM;
 691
 692         return 0;
 693 }
 694
 695 static inline bool event_in_hw_event_map(int i, char *name)
 696 {
 697         if (!arc_pmu_ev_hw_map[i])
 698                 return false;
 699
 700         if (!strlen(arc_pmu_ev_hw_map[i]))
 701                 return false;
 702
 703         if (strcmp(arc_pmu_ev_hw_map[i], name))
 704                 return false;
 705
 706         return true;
 707 }
 708
 709 static void arc_pmu_map_hw_event(int j, char *str)
 710 {
 711         int i;
 712
 713         /* See if HW condition has been mapped to a perf event_id */
 714         for (i = 0; i < ARRAY_SIZE(arc_pmu_ev_hw_map); i++) {
 715                 if (event_in_hw_event_map(i, str)) {
 716                         pr_debug("mapping perf event %2d to h/w event \'%8s\' (idx %d)\n",
 717                                  i, str, j);
 718                         arc_pmu->ev_hw_idx[i] = j;
 719                 }
 720         }
 721 }
 722
 723 static int arc_pmu_device_probe(struct platform_device *pdev)
 724 {
 725         struct arc_reg_pct_build pct_bcr;
 726         struct arc_reg_cc_build cc_bcr;
 727         int i, has_interrupts, irq = -1;
 728         int counter_size;       /* in bits */
 729
 730         union cc_name {
 731                 struct {
 732                         u32 word0, word1;
 733                         char sentinel;
 734                 } indiv;
 735                 char str[ARCPMU_EVENT_NAME_LEN];
 736         } cc_name;
 737
 738
 739         READ_BCR(ARC_REG_PCT_BUILD, pct_bcr);
 740         if (!pct_bcr.v) {
 741                 pr_err("This core does not have performance counters!\n");
 742                 return -ENODEV;
 743         }
 744         BUILD_BUG_ON(ARC_PERF_MAX_COUNTERS > 32);
 745         if (WARN_ON(pct_bcr.c > ARC_PERF_MAX_COUNTERS))
 746                 return -EINVAL;
 747
 748         READ_BCR(ARC_REG_CC_BUILD, cc_bcr);
 749         if (WARN(!cc_bcr.v, "Counters exist but No countable conditions?"))
 750                 return -EINVAL;
 751
 752         arc_pmu = devm_kzalloc(&pdev->dev, sizeof(struct arc_pmu), GFP_KERNEL);
 753         if (!arc_pmu)
 754                 return -ENOMEM;
 755
 756         arc_pmu->n_events = cc_bcr.c;
 757
 758         if (arc_pmu_raw_alloc(&pdev->dev))
 759                 return -ENOMEM;
 760
 761         has_interrupts = is_isa_arcv2() ? pct_bcr.i : 0;
 762
 763         arc_pmu->n_counters = pct_bcr.c;
 764         counter_size = 32 + (pct_bcr.s << 4);
 765
 766         arc_pmu->max_period = (1ULL << counter_size) / 2 - 1ULL;
 767
 768         pr_info("ARC perf\t: %d counters (%d bits), %d conditions%s\n",
 769                 arc_pmu->n_counters, counter_size, cc_bcr.c,
 770                 has_interrupts ? ", [overflow IRQ support]" : "");
 771
 772         cc_name.str[ARCPMU_EVENT_NAME_LEN - 1] = 0;
 773         for (i = 0; i < PERF_COUNT_ARC_HW_MAX; i++)
 774                 arc_pmu->ev_hw_idx[i] = -1;
 775
 776         /* loop thru all available h/w condition indexes */
 777         for (i = 0; i < cc_bcr.c; i++) {
 778                 write_aux_reg(ARC_REG_CC_INDEX, i);
 779                 cc_name.indiv.word0 = le32_to_cpu(read_aux_reg(ARC_REG_CC_NAME0));
 780                 cc_name.indiv.word1 = le32_to_cpu(read_aux_reg(ARC_REG_CC_NAME1));
 781
 782                 arc_pmu_map_hw_event(i, cc_name.str);
 783                 arc_pmu_add_raw_event_attr(i, cc_name.str);
 784         }
 785
 786         arc_pmu_events_attr_gr.attrs = arc_pmu->attrs;
 787         arc_pmu->attr_groups[ARCPMU_ATTR_GR_EVENTS] = &arc_pmu_events_attr_gr;
 788         arc_pmu->attr_groups[ARCPMU_ATTR_GR_FORMATS] = &arc_pmu_format_attr_gr;
 789
 790         arc_pmu->pmu = (struct pmu) {
 791                 .pmu_enable     = arc_pmu_enable,
 792                 .pmu_disable    = arc_pmu_disable,
 793                 .event_init     = arc_pmu_event_init,
 794                 .add            = arc_pmu_add,
 795                 .del            = arc_pmu_del,
 796                 .start          = arc_pmu_start,
 797                 .stop           = arc_pmu_stop,
 798                 .read           = arc_pmu_read,
 799                 .attr_groups    = arc_pmu->attr_groups,
 800         };
 801
 802         if (has_interrupts) {
 803                 irq = platform_get_irq(pdev, 0);
 804                 if (irq >= 0) {
 805                         int ret;
 806
 807                         arc_pmu->irq = irq;
 808
 809                         /* intc map function ensures irq_set_percpu_devid() called */
 810                         ret = request_percpu_irq(irq, arc_pmu_intr, "ARC perf counters",
 811                                                  this_cpu_ptr(&arc_pmu_cpu));
 812
 813                         if (!ret)
 814                                 on_each_cpu(arc_cpu_pmu_irq_init, &irq, 1);
 815                         else
 816                                 irq = -1;
 817                 }
 818
 819         }
 820
 821         if (irq == -1)
 822                 arc_pmu->pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
 823
 824         /*
 825          * perf parser doesn't really like '-' symbol in events name, so let's
 826          * use '_' in arc pct name as it goes to kernel PMU event prefix.
 827          */
 828         return perf_pmu_register(&arc_pmu->pmu, "arc_pct", PERF_TYPE_RAW);
 829 }
 830
 831 static const struct of_device_id arc_pmu_match[] = {
 832         { .compatible = "snps,arc700-pct" },
 833         { .compatible = "snps,archs-pct" },
 834         {},
 835 };
 836 MODULE_DEVICE_TABLE(of, arc_pmu_match);
 837
 838 static struct platform_driver arc_pmu_driver = {
 839         .driver = {
 840                 .name           = "arc-pct",
 841                 .of_match_table = of_match_ptr(arc_pmu_match),
 842         },
 843         .probe          = arc_pmu_device_probe,
 844 };
 845
 846 module_platform_driver(arc_pmu_driver);
 847
 848 MODULE_LICENSE("GPL");
 849 MODULE_AUTHOR("Mischa Jonker <mjonker@synopsys.com>");
 850 MODULE_DESCRIPTION("ARC PMU driver");