arch/powerpc/platforms/cell/pmu.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Cell Broadband Engine Performance Monitor
   4  *
   5  * (C) Copyright IBM Corporation 2001,2006
   6  *
   7  * Author:
   8  *    David Erb (djerb@us.ibm.com)
   9  *    Kevin Corry (kevcorry@us.ibm.com)
  10  */
  11
  12 #include <linux/interrupt.h>
  13 #include <linux/irqdomain.h>
  14 #include <linux/types.h>
  15 #include <linux/export.h>
  16 #include <asm/io.h>
  17 #include <asm/irq_regs.h>
  18 #include <asm/machdep.h>
  19 #include <asm/pmc.h>
  20 #include <asm/reg.h>
  21 #include <asm/spu.h>
  22 #include <asm/cell-regs.h>
  23
  24 #include "interrupt.h"
  25
  26 /*
  27  * When writing to write-only mmio addresses, save a shadow copy. All of the
  28  * registers are 32-bit, but stored in the upper-half of a 64-bit field in
  29  * pmd_regs.
  30  */
  31
  32 #define WRITE_WO_MMIO(reg, x)                                   \
  33         do {                                                    \
  34                 u32 _x = (x);                                   \
  35                 struct cbe_pmd_regs __iomem *pmd_regs;          \
  36                 struct cbe_pmd_shadow_regs *shadow_regs;        \
  37                 pmd_regs = cbe_get_cpu_pmd_regs(cpu);           \
  38                 shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); \
  39                 out_be64(&(pmd_regs->reg), (((u64)_x) << 32));  \
  40                 shadow_regs->reg = _x;                          \
  41         } while (0)
  42
  43 #define READ_SHADOW_REG(val, reg)                               \
  44         do {                                                    \
  45                 struct cbe_pmd_shadow_regs *shadow_regs;        \
  46                 shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu); \
  47                 (val) = shadow_regs->reg;                       \
  48         } while (0)
  49
  50 #define READ_MMIO_UPPER32(val, reg)                             \
  51         do {                                                    \
  52                 struct cbe_pmd_regs __iomem *pmd_regs;          \
  53                 pmd_regs = cbe_get_cpu_pmd_regs(cpu);           \
  54                 (val) = (u32)(in_be64(&pmd_regs->reg) >> 32);   \
  55         } while (0)
  56
  57 /*
  58  * Physical counter registers.
  59  * Each physical counter can act as one 32-bit counter or two 16-bit counters.
  60  */
  61
  62 u32 cbe_read_phys_ctr(u32 cpu, u32 phys_ctr)
  63 {
  64         u32 val_in_latch, val = 0;
  65
  66         if (phys_ctr < NR_PHYS_CTRS) {
  67                 READ_SHADOW_REG(val_in_latch, counter_value_in_latch);
  68
  69                 /* Read the latch or the actual counter, whichever is newer. */
  70                 if (val_in_latch & (1 << phys_ctr)) {
  71                         READ_SHADOW_REG(val, pm_ctr[phys_ctr]);
  72                 } else {
  73                         READ_MMIO_UPPER32(val, pm_ctr[phys_ctr]);
  74                 }
  75         }
  76
  77         return val;
  78 }
  79 EXPORT_SYMBOL_GPL(cbe_read_phys_ctr);
  80
  81 void cbe_write_phys_ctr(u32 cpu, u32 phys_ctr, u32 val)
  82 {
  83         struct cbe_pmd_shadow_regs *shadow_regs;
  84         u32 pm_ctrl;
  85
  86         if (phys_ctr < NR_PHYS_CTRS) {
  87                 /* Writing to a counter only writes to a hardware latch.
  88                  * The new value is not propagated to the actual counter
  89                  * until the performance monitor is enabled.
  90                  */
  91                 WRITE_WO_MMIO(pm_ctr[phys_ctr], val);
  92
  93                 pm_ctrl = cbe_read_pm(cpu, pm_control);
  94                 if (pm_ctrl & CBE_PM_ENABLE_PERF_MON) {
  95                         /* The counters are already active, so we need to
  96                          * rewrite the pm_control register to "re-enable"
  97                          * the PMU.
  98                          */
  99                         cbe_write_pm(cpu, pm_control, pm_ctrl);
 100                 } else {
 101                         shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu);
 102                         shadow_regs->counter_value_in_latch |= (1 << phys_ctr);
 103                 }
 104         }
 105 }
 106 EXPORT_SYMBOL_GPL(cbe_write_phys_ctr);
 107
 108 /*
 109  * "Logical" counter registers.
 110  * These will read/write 16-bits or 32-bits depending on the
 111  * current size of the counter. Counters 4 - 7 are always 16-bit.
 112  */
 113
 114 u32 cbe_read_ctr(u32 cpu, u32 ctr)
 115 {
 116         u32 val;
 117         u32 phys_ctr = ctr & (NR_PHYS_CTRS - 1);
 118
 119         val = cbe_read_phys_ctr(cpu, phys_ctr);
 120
 121         if (cbe_get_ctr_size(cpu, phys_ctr) == 16)
 122                 val = (ctr < NR_PHYS_CTRS) ? (val >> 16) : (val & 0xffff);
 123
 124         return val;
 125 }
 126 EXPORT_SYMBOL_GPL(cbe_read_ctr);
 127
 128 void cbe_write_ctr(u32 cpu, u32 ctr, u32 val)
 129 {
 130         u32 phys_ctr;
 131         u32 phys_val;
 132
 133         phys_ctr = ctr & (NR_PHYS_CTRS - 1);
 134
 135         if (cbe_get_ctr_size(cpu, phys_ctr) == 16) {
 136                 phys_val = cbe_read_phys_ctr(cpu, phys_ctr);
 137
 138                 if (ctr < NR_PHYS_CTRS)
 139                         val = (val << 16) | (phys_val & 0xffff);
 140                 else
 141                         val = (val & 0xffff) | (phys_val & 0xffff0000);
 142         }
 143
 144         cbe_write_phys_ctr(cpu, phys_ctr, val);
 145 }
 146 EXPORT_SYMBOL_GPL(cbe_write_ctr);
 147
 148 /*
 149  * Counter-control registers.
 150  * Each "logical" counter has a corresponding control register.
 151  */
 152
 153 u32 cbe_read_pm07_control(u32 cpu, u32 ctr)
 154 {
 155         u32 pm07_control = 0;
 156
 157         if (ctr < NR_CTRS)
 158                 READ_SHADOW_REG(pm07_control, pm07_control[ctr]);
 159
 160         return pm07_control;
 161 }
 162 EXPORT_SYMBOL_GPL(cbe_read_pm07_control);
 163
 164 void cbe_write_pm07_control(u32 cpu, u32 ctr, u32 val)
 165 {
 166         if (ctr < NR_CTRS)
 167                 WRITE_WO_MMIO(pm07_control[ctr], val);
 168 }
 169 EXPORT_SYMBOL_GPL(cbe_write_pm07_control);
 170
 171 /*
 172  * Other PMU control registers. Most of these are write-only.
 173  */
 174
 175 u32 cbe_read_pm(u32 cpu, enum pm_reg_name reg)
 176 {
 177         u32 val = 0;
 178
 179         switch (reg) {
 180         case group_control:
 181                 READ_SHADOW_REG(val, group_control);
 182                 break;
 183
 184         case debug_bus_control:
 185                 READ_SHADOW_REG(val, debug_bus_control);
 186                 break;
 187
 188         case trace_address:
 189                 READ_MMIO_UPPER32(val, trace_address);
 190                 break;
 191
 192         case ext_tr_timer:
 193                 READ_SHADOW_REG(val, ext_tr_timer);
 194                 break;
 195
 196         case pm_status:
 197                 READ_MMIO_UPPER32(val, pm_status);
 198                 break;
 199
 200         case pm_control:
 201                 READ_SHADOW_REG(val, pm_control);
 202                 break;
 203
 204         case pm_interval:
 205                 READ_MMIO_UPPER32(val, pm_interval);
 206                 break;
 207
 208         case pm_start_stop:
 209                 READ_SHADOW_REG(val, pm_start_stop);
 210                 break;
 211         }
 212
 213         return val;
 214 }
 215 EXPORT_SYMBOL_GPL(cbe_read_pm);
 216
 217 void cbe_write_pm(u32 cpu, enum pm_reg_name reg, u32 val)
 218 {
 219         switch (reg) {
 220         case group_control:
 221                 WRITE_WO_MMIO(group_control, val);
 222                 break;
 223
 224         case debug_bus_control:
 225                 WRITE_WO_MMIO(debug_bus_control, val);
 226                 break;
 227
 228         case trace_address:
 229                 WRITE_WO_MMIO(trace_address, val);
 230                 break;
 231
 232         case ext_tr_timer:
 233                 WRITE_WO_MMIO(ext_tr_timer, val);
 234                 break;
 235
 236         case pm_status:
 237                 WRITE_WO_MMIO(pm_status, val);
 238                 break;
 239
 240         case pm_control:
 241                 WRITE_WO_MMIO(pm_control, val);
 242                 break;
 243
 244         case pm_interval:
 245                 WRITE_WO_MMIO(pm_interval, val);
 246                 break;
 247
 248         case pm_start_stop:
 249                 WRITE_WO_MMIO(pm_start_stop, val);
 250                 break;
 251         }
 252 }
 253 EXPORT_SYMBOL_GPL(cbe_write_pm);
 254
 255 /*
 256  * Get/set the size of a physical counter to either 16 or 32 bits.
 257  */
 258
 259 u32 cbe_get_ctr_size(u32 cpu, u32 phys_ctr)
 260 {
 261         u32 pm_ctrl, size = 0;
 262
 263         if (phys_ctr < NR_PHYS_CTRS) {
 264                 pm_ctrl = cbe_read_pm(cpu, pm_control);
 265                 size = (pm_ctrl & CBE_PM_16BIT_CTR(phys_ctr)) ? 16 : 32;
 266         }
 267
 268         return size;
 269 }
 270 EXPORT_SYMBOL_GPL(cbe_get_ctr_size);
 271
 272 void cbe_set_ctr_size(u32 cpu, u32 phys_ctr, u32 ctr_size)
 273 {
 274         u32 pm_ctrl;
 275
 276         if (phys_ctr < NR_PHYS_CTRS) {
 277                 pm_ctrl = cbe_read_pm(cpu, pm_control);
 278                 switch (ctr_size) {
 279                 case 16:
 280                         pm_ctrl |= CBE_PM_16BIT_CTR(phys_ctr);
 281                         break;
 282
 283                 case 32:
 284                         pm_ctrl &= ~CBE_PM_16BIT_CTR(phys_ctr);
 285                         break;
 286                 }
 287                 cbe_write_pm(cpu, pm_control, pm_ctrl);
 288         }
 289 }
 290 EXPORT_SYMBOL_GPL(cbe_set_ctr_size);
 291
 292 /*
 293  * Enable/disable the entire performance monitoring unit.
 294  * When we enable the PMU, all pending writes to counters get committed.
 295  */
 296
 297 void cbe_enable_pm(u32 cpu)
 298 {
 299         struct cbe_pmd_shadow_regs *shadow_regs;
 300         u32 pm_ctrl;
 301
 302         shadow_regs = cbe_get_cpu_pmd_shadow_regs(cpu);
 303         shadow_regs->counter_value_in_latch = 0;
 304
 305         pm_ctrl = cbe_read_pm(cpu, pm_control) | CBE_PM_ENABLE_PERF_MON;
 306         cbe_write_pm(cpu, pm_control, pm_ctrl);
 307 }
 308 EXPORT_SYMBOL_GPL(cbe_enable_pm);
 309
 310 void cbe_disable_pm(u32 cpu)
 311 {
 312         u32 pm_ctrl;
 313         pm_ctrl = cbe_read_pm(cpu, pm_control) & ~CBE_PM_ENABLE_PERF_MON;
 314         cbe_write_pm(cpu, pm_control, pm_ctrl);
 315 }
 316 EXPORT_SYMBOL_GPL(cbe_disable_pm);
 317
 318 /*
 319  * Reading from the trace_buffer.
 320  * The trace buffer is two 64-bit registers. Reading from
 321  * the second half automatically increments the trace_address.
 322  */
 323
 324 void cbe_read_trace_buffer(u32 cpu, u64 *buf)
 325 {
 326         struct cbe_pmd_regs __iomem *pmd_regs = cbe_get_cpu_pmd_regs(cpu);
 327
 328         *buf++ = in_be64(&pmd_regs->trace_buffer_0_63);
 329         *buf++ = in_be64(&pmd_regs->trace_buffer_64_127);
 330 }
 331 EXPORT_SYMBOL_GPL(cbe_read_trace_buffer);
 332
 333 /*
 334  * Enabling/disabling interrupts for the entire performance monitoring unit.
 335  */
 336
 337 u32 cbe_get_and_clear_pm_interrupts(u32 cpu)
 338 {
 339         /* Reading pm_status clears the interrupt bits. */
 340         return cbe_read_pm(cpu, pm_status);
 341 }
 342 EXPORT_SYMBOL_GPL(cbe_get_and_clear_pm_interrupts);
 343
 344 void cbe_enable_pm_interrupts(u32 cpu, u32 thread, u32 mask)
 345 {
 346         /* Set which node and thread will handle the next interrupt. */
 347         iic_set_interrupt_routing(cpu, thread, 0);
 348
 349         /* Enable the interrupt bits in the pm_status register. */
 350         if (mask)
 351                 cbe_write_pm(cpu, pm_status, mask);
 352 }
 353 EXPORT_SYMBOL_GPL(cbe_enable_pm_interrupts);
 354
 355 void cbe_disable_pm_interrupts(u32 cpu)
 356 {
 357         cbe_get_and_clear_pm_interrupts(cpu);
 358         cbe_write_pm(cpu, pm_status, 0);
 359 }
 360 EXPORT_SYMBOL_GPL(cbe_disable_pm_interrupts);
 361
 362 static irqreturn_t cbe_pm_irq(int irq, void *dev_id)
 363 {
 364         perf_irq(get_irq_regs());
 365         return IRQ_HANDLED;
 366 }
 367
 368 static int __init cbe_init_pm_irq(void)
 369 {
 370         unsigned int irq;
 371         int rc, node;
 372
 373         for_each_online_node(node) {
 374                 irq = irq_create_mapping(NULL, IIC_IRQ_IOEX_PMI |
 375                                                (node << IIC_IRQ_NODE_SHIFT));
 376                 if (!irq) {
 377                         printk("ERROR: Unable to allocate irq for node %d\n",
 378                                node);
 379                         return -EINVAL;
 380                 }
 381
 382                 rc = request_irq(irq, cbe_pm_irq,
 383                                  0, "cbe-pmu-0", NULL);
 384                 if (rc) {
 385                         printk("ERROR: Request for irq on node %d failed\n",
 386                                node);
 387                         return rc;
 388                 }
 389         }
 390
 391         return 0;
 392 }
 393 machine_arch_initcall(cell, cbe_init_pm_irq);
 394
 395 void cbe_sync_irq(int node)
 396 {
 397         unsigned int irq;
 398
 399         irq = irq_find_mapping(NULL,
 400                                IIC_IRQ_IOEX_PMI
 401                                | (node << IIC_IRQ_NODE_SHIFT));
 402
 403         if (!irq) {
 404                 printk(KERN_WARNING "ERROR, unable to get existing irq %d " \
 405                 "for node %d\n", irq, node);
 406                 return;
 407         }
 408
 409         synchronize_irq(irq);
 410 }
 411 EXPORT_SYMBOL_GPL(cbe_sync_irq);
 412