drivers/hwtracing/coresight/coresight-etm4x-core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2014, The Linux Foundation. All rights reserved.
   4  */
   5
   6 #include <linux/kernel.h>
   7 #include <linux/moduleparam.h>
   8 #include <linux/init.h>
   9 #include <linux/types.h>
  10 #include <linux/device.h>
  11 #include <linux/io.h>
  12 #include <linux/err.h>
  13 #include <linux/fs.h>
  14 #include <linux/slab.h>
  15 #include <linux/delay.h>
  16 #include <linux/smp.h>
  17 #include <linux/sysfs.h>
  18 #include <linux/stat.h>
  19 #include <linux/clk.h>
  20 #include <linux/cpu.h>
  21 #include <linux/cpu_pm.h>
  22 #include <linux/coresight.h>
  23 #include <linux/coresight-pmu.h>
  24 #include <linux/pm_wakeup.h>
  25 #include <linux/amba/bus.h>
  26 #include <linux/seq_file.h>
  27 #include <linux/uaccess.h>
  28 #include <linux/perf_event.h>
  29 #include <linux/pm_runtime.h>
  30 #include <linux/property.h>
  31 #include <asm/sections.h>
  32 #include <asm/local.h>
  33 #include <asm/virt.h>
  34
  35 #include "coresight-etm4x.h"
  36 #include "coresight-etm-perf.h"
  37
  38 static int boot_enable;
  39 module_param(boot_enable, int, 0444);
  40 MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
  41
  42 #define PARAM_PM_SAVE_FIRMWARE    0 /* save self-hosted state as per firmware */
  43 #define PARAM_PM_SAVE_NEVER       1 /* never save any state */
  44 #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */
  45
  46 static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
  47 module_param(pm_save_enable, int, 0444);
  48 MODULE_PARM_DESC(pm_save_enable,
  49         "Save/restore state on power down: 1 = never, 2 = self-hosted");
  50
  51 static struct etmv4_drvdata *etmdrvdata[NR_CPUS];
  52 static void etm4_set_default_config(struct etmv4_config *config);
  53 static int etm4_set_event_filters(struct etmv4_drvdata *drvdata,
  54                                   struct perf_event *event);
  55 static u64 etm4_get_access_type(struct etmv4_config *config);
  56
  57 static enum cpuhp_state hp_online;
  58
  59 static void etm4_os_unlock(struct etmv4_drvdata *drvdata)
  60 {
  61         /* Writing 0 to TRCOSLAR unlocks the trace registers */
  62         writel_relaxed(0x0, drvdata->base + TRCOSLAR);
  63         drvdata->os_unlock = true;
  64         isb();
  65 }
  66
  67 static void etm4_os_lock(struct etmv4_drvdata *drvdata)
  68 {
  69         /* Writing 0x1 to TRCOSLAR locks the trace registers */
  70         writel_relaxed(0x1, drvdata->base + TRCOSLAR);
  71         drvdata->os_unlock = false;
  72         isb();
  73 }
  74
  75 static bool etm4_arch_supported(u8 arch)
  76 {
  77         /* Mask out the minor version number */
  78         switch (arch & 0xf0) {
  79         case ETM_ARCH_V4:
  80                 break;
  81         default:
  82                 return false;
  83         }
  84         return true;
  85 }
  86
  87 static int etm4_cpu_id(struct coresight_device *csdev)
  88 {
  89         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
  90
  91         return drvdata->cpu;
  92 }
  93
  94 static int etm4_trace_id(struct coresight_device *csdev)
  95 {
  96         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
  97
  98         return drvdata->trcid;
  99 }
 100
 101 struct etm4_enable_arg {
 102         struct etmv4_drvdata *drvdata;
 103         int rc;
 104 };
 105
 106 static int etm4_enable_hw(struct etmv4_drvdata *drvdata)
 107 {
 108         int i, rc;
 109         struct etmv4_config *config = &drvdata->config;
 110         struct device *etm_dev = &drvdata->csdev->dev;
 111
 112         CS_UNLOCK(drvdata->base);
 113
 114         etm4_os_unlock(drvdata);
 115
 116         rc = coresight_claim_device_unlocked(drvdata->base);
 117         if (rc)
 118                 goto done;
 119
 120         /* Disable the trace unit before programming trace registers */
 121         writel_relaxed(0, drvdata->base + TRCPRGCTLR);
 122
 123         /* wait for TRCSTATR.IDLE to go up */
 124         if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 1))
 125                 dev_err(etm_dev,
 126                         "timeout while waiting for Idle Trace Status\n");
 127         if (drvdata->nr_pe)
 128                 writel_relaxed(config->pe_sel, drvdata->base + TRCPROCSELR);
 129         writel_relaxed(config->cfg, drvdata->base + TRCCONFIGR);
 130         /* nothing specific implemented */
 131         writel_relaxed(0x0, drvdata->base + TRCAUXCTLR);
 132         writel_relaxed(config->eventctrl0, drvdata->base + TRCEVENTCTL0R);
 133         writel_relaxed(config->eventctrl1, drvdata->base + TRCEVENTCTL1R);
 134         if (drvdata->stallctl)
 135                 writel_relaxed(config->stall_ctrl, drvdata->base + TRCSTALLCTLR);
 136         writel_relaxed(config->ts_ctrl, drvdata->base + TRCTSCTLR);
 137         writel_relaxed(config->syncfreq, drvdata->base + TRCSYNCPR);
 138         writel_relaxed(config->ccctlr, drvdata->base + TRCCCCTLR);
 139         writel_relaxed(config->bb_ctrl, drvdata->base + TRCBBCTLR);
 140         writel_relaxed(drvdata->trcid, drvdata->base + TRCTRACEIDR);
 141         writel_relaxed(config->vinst_ctrl, drvdata->base + TRCVICTLR);
 142         writel_relaxed(config->viiectlr, drvdata->base + TRCVIIECTLR);
 143         writel_relaxed(config->vissctlr,
 144                        drvdata->base + TRCVISSCTLR);
 145         if (drvdata->nr_pe_cmp)
 146                 writel_relaxed(config->vipcssctlr,
 147                                drvdata->base + TRCVIPCSSCTLR);
 148         for (i = 0; i < drvdata->nrseqstate - 1; i++)
 149                 writel_relaxed(config->seq_ctrl[i],
 150                                drvdata->base + TRCSEQEVRn(i));
 151         writel_relaxed(config->seq_rst, drvdata->base + TRCSEQRSTEVR);
 152         writel_relaxed(config->seq_state, drvdata->base + TRCSEQSTR);
 153         writel_relaxed(config->ext_inp, drvdata->base + TRCEXTINSELR);
 154         for (i = 0; i < drvdata->nr_cntr; i++) {
 155                 writel_relaxed(config->cntrldvr[i],
 156                                drvdata->base + TRCCNTRLDVRn(i));
 157                 writel_relaxed(config->cntr_ctrl[i],
 158                                drvdata->base + TRCCNTCTLRn(i));
 159                 writel_relaxed(config->cntr_val[i],
 160                                drvdata->base + TRCCNTVRn(i));
 161         }
 162
 163         /*
 164          * Resource selector pair 0 is always implemented and reserved.  As
 165          * such start at 2.
 166          */
 167         for (i = 2; i < drvdata->nr_resource * 2; i++)
 168                 writel_relaxed(config->res_ctrl[i],
 169                                drvdata->base + TRCRSCTLRn(i));
 170
 171         for (i = 0; i < drvdata->nr_ss_cmp; i++) {
 172                 /* always clear status bit on restart if using single-shot */
 173                 if (config->ss_ctrl[i] || config->ss_pe_cmp[i])
 174                         config->ss_status[i] &= ~BIT(31);
 175                 writel_relaxed(config->ss_ctrl[i],
 176                                drvdata->base + TRCSSCCRn(i));
 177                 writel_relaxed(config->ss_status[i],
 178                                drvdata->base + TRCSSCSRn(i));
 179                 writel_relaxed(config->ss_pe_cmp[i],
 180                                drvdata->base + TRCSSPCICRn(i));
 181         }
 182         for (i = 0; i < drvdata->nr_addr_cmp; i++) {
 183                 writeq_relaxed(config->addr_val[i],
 184                                drvdata->base + TRCACVRn(i));
 185                 writeq_relaxed(config->addr_acc[i],
 186                                drvdata->base + TRCACATRn(i));
 187         }
 188         for (i = 0; i < drvdata->numcidc; i++)
 189                 writeq_relaxed(config->ctxid_pid[i],
 190                                drvdata->base + TRCCIDCVRn(i));
 191         writel_relaxed(config->ctxid_mask0, drvdata->base + TRCCIDCCTLR0);
 192         if (drvdata->numcidc > 4)
 193                 writel_relaxed(config->ctxid_mask1, drvdata->base + TRCCIDCCTLR1);
 194
 195         for (i = 0; i < drvdata->numvmidc; i++)
 196                 writeq_relaxed(config->vmid_val[i],
 197                                drvdata->base + TRCVMIDCVRn(i));
 198         writel_relaxed(config->vmid_mask0, drvdata->base + TRCVMIDCCTLR0);
 199         if (drvdata->numvmidc > 4)
 200                 writel_relaxed(config->vmid_mask1, drvdata->base + TRCVMIDCCTLR1);
 201
 202         if (!drvdata->skip_power_up) {
 203                 /*
 204                  * Request to keep the trace unit powered and also
 205                  * emulation of powerdown
 206                  */
 207                 writel_relaxed(readl_relaxed(drvdata->base + TRCPDCR) |
 208                                TRCPDCR_PU, drvdata->base + TRCPDCR);
 209         }
 210
 211         /* Enable the trace unit */
 212         writel_relaxed(1, drvdata->base + TRCPRGCTLR);
 213
 214         /* wait for TRCSTATR.IDLE to go back down to '0' */
 215         if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 0))
 216                 dev_err(etm_dev,
 217                         "timeout while waiting for Idle Trace Status\n");
 218
 219         /*
 220          * As recommended by section 4.3.7 ("Synchronization when using the
 221          * memory-mapped interface") of ARM IHI 0064D
 222          */
 223         dsb(sy);
 224         isb();
 225
 226 done:
 227         CS_LOCK(drvdata->base);
 228
 229         dev_dbg(etm_dev, "cpu: %d enable smp call done: %d\n",
 230                 drvdata->cpu, rc);
 231         return rc;
 232 }
 233
 234 static void etm4_enable_hw_smp_call(void *info)
 235 {
 236         struct etm4_enable_arg *arg = info;
 237
 238         if (WARN_ON(!arg))
 239                 return;
 240         arg->rc = etm4_enable_hw(arg->drvdata);
 241 }
 242
 243 /*
 244  * The goal of function etm4_config_timestamp_event() is to configure a
 245  * counter that will tell the tracer to emit a timestamp packet when it
 246  * reaches zero.  This is done in order to get a more fine grained idea
 247  * of when instructions are executed so that they can be correlated
 248  * with execution on other CPUs.
 249  *
 250  * To do this the counter itself is configured to self reload and
 251  * TRCRSCTLR1 (always true) used to get the counter to decrement.  From
 252  * there a resource selector is configured with the counter and the
 253  * timestamp control register to use the resource selector to trigger the
 254  * event that will insert a timestamp packet in the stream.
 255  */
 256 static int etm4_config_timestamp_event(struct etmv4_drvdata *drvdata)
 257 {
 258         int ctridx, ret = -EINVAL;
 259         int counter, rselector;
 260         u32 val = 0;
 261         struct etmv4_config *config = &drvdata->config;
 262
 263         /* No point in trying if we don't have at least one counter */
 264         if (!drvdata->nr_cntr)
 265                 goto out;
 266
 267         /* Find a counter that hasn't been initialised */
 268         for (ctridx = 0; ctridx < drvdata->nr_cntr; ctridx++)
 269                 if (config->cntr_val[ctridx] == 0)
 270                         break;
 271
 272         /* All the counters have been configured already, bail out */
 273         if (ctridx == drvdata->nr_cntr) {
 274                 pr_debug("%s: no available counter found\n", __func__);
 275                 ret = -ENOSPC;
 276                 goto out;
 277         }
 278
 279         /*
 280          * Searching for an available resource selector to use, starting at
 281          * '2' since every implementation has at least 2 resource selector.
 282          * ETMIDR4 gives the number of resource selector _pairs_,
 283          * hence multiply by 2.
 284          */
 285         for (rselector = 2; rselector < drvdata->nr_resource * 2; rselector++)
 286                 if (!config->res_ctrl[rselector])
 287                         break;
 288
 289         if (rselector == drvdata->nr_resource * 2) {
 290                 pr_debug("%s: no available resource selector found\n",
 291                          __func__);
 292                 ret = -ENOSPC;
 293                 goto out;
 294         }
 295
 296         /* Remember what counter we used */
 297         counter = 1 << ctridx;
 298
 299         /*
 300          * Initialise original and reload counter value to the smallest
 301          * possible value in order to get as much precision as we can.
 302          */
 303         config->cntr_val[ctridx] = 1;
 304         config->cntrldvr[ctridx] = 1;
 305
 306         /* Set the trace counter control register */
 307         val =  0x1 << 16        |  /* Bit 16, reload counter automatically */
 308                0x0 << 7         |  /* Select single resource selector */
 309                0x1;                /* Resource selector 1, i.e always true */
 310
 311         config->cntr_ctrl[ctridx] = val;
 312
 313         val = 0x2 << 16         | /* Group 0b0010 - Counter and sequencers */
 314               counter << 0;       /* Counter to use */
 315
 316         config->res_ctrl[rselector] = val;
 317
 318         val = 0x0 << 7          | /* Select single resource selector */
 319               rselector;          /* Resource selector */
 320
 321         config->ts_ctrl = val;
 322
 323         ret = 0;
 324 out:
 325         return ret;
 326 }
 327
 328 static int etm4_parse_event_config(struct etmv4_drvdata *drvdata,
 329                                    struct perf_event *event)
 330 {
 331         int ret = 0;
 332         struct etmv4_config *config = &drvdata->config;
 333         struct perf_event_attr *attr = &event->attr;
 334
 335         if (!attr) {
 336                 ret = -EINVAL;
 337                 goto out;
 338         }
 339
 340         /* Clear configuration from previous run */
 341         memset(config, 0, sizeof(struct etmv4_config));
 342
 343         if (attr->exclude_kernel)
 344                 config->mode = ETM_MODE_EXCL_KERN;
 345
 346         if (attr->exclude_user)
 347                 config->mode = ETM_MODE_EXCL_USER;
 348
 349         /* Always start from the default config */
 350         etm4_set_default_config(config);
 351
 352         /* Configure filters specified on the perf cmd line, if any. */
 353         ret = etm4_set_event_filters(drvdata, event);
 354         if (ret)
 355                 goto out;
 356
 357         /* Go from generic option to ETMv4 specifics */
 358         if (attr->config & BIT(ETM_OPT_CYCACC)) {
 359                 config->cfg |= BIT(4);
 360                 /* TRM: Must program this for cycacc to work */
 361                 config->ccctlr = ETM_CYC_THRESHOLD_DEFAULT;
 362         }
 363         if (attr->config & BIT(ETM_OPT_TS)) {
 364                 /*
 365                  * Configure timestamps to be emitted at regular intervals in
 366                  * order to correlate instructions executed on different CPUs
 367                  * (CPU-wide trace scenarios).
 368                  */
 369                 ret = etm4_config_timestamp_event(drvdata);
 370
 371                 /*
 372                  * No need to go further if timestamp intervals can't
 373                  * be configured.
 374                  */
 375                 if (ret)
 376                         goto out;
 377
 378                 /* bit[11], Global timestamp tracing bit */
 379                 config->cfg |= BIT(11);
 380         }
 381
 382         if (attr->config & BIT(ETM_OPT_CTXTID))
 383                 /* bit[6], Context ID tracing bit */
 384                 config->cfg |= BIT(ETM4_CFG_BIT_CTXTID);
 385
 386         /* return stack - enable if selected and supported */
 387         if ((attr->config & BIT(ETM_OPT_RETSTK)) && drvdata->retstack)
 388                 /* bit[12], Return stack enable bit */
 389                 config->cfg |= BIT(12);
 390
 391 out:
 392         return ret;
 393 }
 394
 395 static int etm4_enable_perf(struct coresight_device *csdev,
 396                             struct perf_event *event)
 397 {
 398         int ret = 0;
 399         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 400
 401         if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id())) {
 402                 ret = -EINVAL;
 403                 goto out;
 404         }
 405
 406         /* Configure the tracer based on the session's specifics */
 407         ret = etm4_parse_event_config(drvdata, event);
 408         if (ret)
 409                 goto out;
 410         /* And enable it */
 411         ret = etm4_enable_hw(drvdata);
 412
 413 out:
 414         return ret;
 415 }
 416
 417 static int etm4_enable_sysfs(struct coresight_device *csdev)
 418 {
 419         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 420         struct etm4_enable_arg arg = { };
 421         int ret;
 422
 423         spin_lock(&drvdata->spinlock);
 424
 425         /*
 426          * Executing etm4_enable_hw on the cpu whose ETM is being enabled
 427          * ensures that register writes occur when cpu is powered.
 428          */
 429         arg.drvdata = drvdata;
 430         ret = smp_call_function_single(drvdata->cpu,
 431                                        etm4_enable_hw_smp_call, &arg, 1);
 432         if (!ret)
 433                 ret = arg.rc;
 434         if (!ret)
 435                 drvdata->sticky_enable = true;
 436         spin_unlock(&drvdata->spinlock);
 437
 438         if (!ret)
 439                 dev_dbg(&csdev->dev, "ETM tracing enabled\n");
 440         return ret;
 441 }
 442
 443 static int etm4_enable(struct coresight_device *csdev,
 444                        struct perf_event *event, u32 mode)
 445 {
 446         int ret;
 447         u32 val;
 448         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 449
 450         val = local_cmpxchg(&drvdata->mode, CS_MODE_DISABLED, mode);
 451
 452         /* Someone is already using the tracer */
 453         if (val)
 454                 return -EBUSY;
 455
 456         switch (mode) {
 457         case CS_MODE_SYSFS:
 458                 ret = etm4_enable_sysfs(csdev);
 459                 break;
 460         case CS_MODE_PERF:
 461                 ret = etm4_enable_perf(csdev, event);
 462                 break;
 463         default:
 464                 ret = -EINVAL;
 465         }
 466
 467         /* The tracer didn't start */
 468         if (ret)
 469                 local_set(&drvdata->mode, CS_MODE_DISABLED);
 470
 471         return ret;
 472 }
 473
 474 static void etm4_disable_hw(void *info)
 475 {
 476         u32 control;
 477         struct etmv4_drvdata *drvdata = info;
 478         struct etmv4_config *config = &drvdata->config;
 479         struct device *etm_dev = &drvdata->csdev->dev;
 480         int i;
 481
 482         CS_UNLOCK(drvdata->base);
 483
 484         if (!drvdata->skip_power_up) {
 485                 /* power can be removed from the trace unit now */
 486                 control = readl_relaxed(drvdata->base + TRCPDCR);
 487                 control &= ~TRCPDCR_PU;
 488                 writel_relaxed(control, drvdata->base + TRCPDCR);
 489         }
 490
 491         control = readl_relaxed(drvdata->base + TRCPRGCTLR);
 492
 493         /* EN, bit[0] Trace unit enable bit */
 494         control &= ~0x1;
 495
 496         /*
 497          * Make sure everything completes before disabling, as recommended
 498          * by section 7.3.77 ("TRCVICTLR, ViewInst Main Control Register,
 499          * SSTATUS") of ARM IHI 0064D
 500          */
 501         dsb(sy);
 502         isb();
 503         writel_relaxed(control, drvdata->base + TRCPRGCTLR);
 504
 505         /* wait for TRCSTATR.PMSTABLE to go to '1' */
 506         if (coresight_timeout(drvdata->base, TRCSTATR,
 507                               TRCSTATR_PMSTABLE_BIT, 1))
 508                 dev_err(etm_dev,
 509                         "timeout while waiting for PM stable Trace Status\n");
 510
 511         /* read the status of the single shot comparators */
 512         for (i = 0; i < drvdata->nr_ss_cmp; i++) {
 513                 config->ss_status[i] =
 514                         readl_relaxed(drvdata->base + TRCSSCSRn(i));
 515         }
 516
 517         /* read back the current counter values */
 518         for (i = 0; i < drvdata->nr_cntr; i++) {
 519                 config->cntr_val[i] =
 520                         readl_relaxed(drvdata->base + TRCCNTVRn(i));
 521         }
 522
 523         coresight_disclaim_device_unlocked(drvdata->base);
 524
 525         CS_LOCK(drvdata->base);
 526
 527         dev_dbg(&drvdata->csdev->dev,
 528                 "cpu: %d disable smp call done\n", drvdata->cpu);
 529 }
 530
 531 static int etm4_disable_perf(struct coresight_device *csdev,
 532                              struct perf_event *event)
 533 {
 534         u32 control;
 535         struct etm_filters *filters = event->hw.addr_filters;
 536         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 537
 538         if (WARN_ON_ONCE(drvdata->cpu != smp_processor_id()))
 539                 return -EINVAL;
 540
 541         etm4_disable_hw(drvdata);
 542
 543         /*
 544          * Check if the start/stop logic was active when the unit was stopped.
 545          * That way we can re-enable the start/stop logic when the process is
 546          * scheduled again.  Configuration of the start/stop logic happens in
 547          * function etm4_set_event_filters().
 548          */
 549         control = readl_relaxed(drvdata->base + TRCVICTLR);
 550         /* TRCVICTLR::SSSTATUS, bit[9] */
 551         filters->ssstatus = (control & BIT(9));
 552
 553         return 0;
 554 }
 555
 556 static void etm4_disable_sysfs(struct coresight_device *csdev)
 557 {
 558         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 559
 560         /*
 561          * Taking hotplug lock here protects from clocks getting disabled
 562          * with tracing being left on (crash scenario) if user disable occurs
 563          * after cpu online mask indicates the cpu is offline but before the
 564          * DYING hotplug callback is serviced by the ETM driver.
 565          */
 566         cpus_read_lock();
 567         spin_lock(&drvdata->spinlock);
 568
 569         /*
 570          * Executing etm4_disable_hw on the cpu whose ETM is being disabled
 571          * ensures that register writes occur when cpu is powered.
 572          */
 573         smp_call_function_single(drvdata->cpu, etm4_disable_hw, drvdata, 1);
 574
 575         spin_unlock(&drvdata->spinlock);
 576         cpus_read_unlock();
 577
 578         dev_dbg(&csdev->dev, "ETM tracing disabled\n");
 579 }
 580
 581 static void etm4_disable(struct coresight_device *csdev,
 582                          struct perf_event *event)
 583 {
 584         u32 mode;
 585         struct etmv4_drvdata *drvdata = dev_get_drvdata(csdev->dev.parent);
 586
 587         /*
 588          * For as long as the tracer isn't disabled another entity can't
 589          * change its status.  As such we can read the status here without
 590          * fearing it will change under us.
 591          */
 592         mode = local_read(&drvdata->mode);
 593
 594         switch (mode) {
 595         case CS_MODE_DISABLED:
 596                 break;
 597         case CS_MODE_SYSFS:
 598                 etm4_disable_sysfs(csdev);
 599                 break;
 600         case CS_MODE_PERF:
 601                 etm4_disable_perf(csdev, event);
 602                 break;
 603         }
 604
 605         if (mode)
 606                 local_set(&drvdata->mode, CS_MODE_DISABLED);
 607 }
 608
 609 static const struct coresight_ops_source etm4_source_ops = {
 610         .cpu_id         = etm4_cpu_id,
 611         .trace_id       = etm4_trace_id,
 612         .enable         = etm4_enable,
 613         .disable        = etm4_disable,
 614 };
 615
 616 static const struct coresight_ops etm4_cs_ops = {
 617         .source_ops     = &etm4_source_ops,
 618 };
 619
 620 static void etm4_init_arch_data(void *info)
 621 {
 622         u32 etmidr0;
 623         u32 etmidr1;
 624         u32 etmidr2;
 625         u32 etmidr3;
 626         u32 etmidr4;
 627         u32 etmidr5;
 628         struct etmv4_drvdata *drvdata = info;
 629         int i;
 630
 631         /* Make sure all registers are accessible */
 632         etm4_os_unlock(drvdata);
 633
 634         CS_UNLOCK(drvdata->base);
 635
 636         /* find all capabilities of the tracing unit */
 637         etmidr0 = readl_relaxed(drvdata->base + TRCIDR0);
 638
 639         /* INSTP0, bits[2:1] P0 tracing support field */
 640         if (BMVAL(etmidr0, 1, 1) && BMVAL(etmidr0, 2, 2))
 641                 drvdata->instrp0 = true;
 642         else
 643                 drvdata->instrp0 = false;
 644
 645         /* TRCBB, bit[5] Branch broadcast tracing support bit */
 646         if (BMVAL(etmidr0, 5, 5))
 647                 drvdata->trcbb = true;
 648         else
 649                 drvdata->trcbb = false;
 650
 651         /* TRCCOND, bit[6] Conditional instruction tracing support bit */
 652         if (BMVAL(etmidr0, 6, 6))
 653                 drvdata->trccond = true;
 654         else
 655                 drvdata->trccond = false;
 656
 657         /* TRCCCI, bit[7] Cycle counting instruction bit */
 658         if (BMVAL(etmidr0, 7, 7))
 659                 drvdata->trccci = true;
 660         else
 661                 drvdata->trccci = false;
 662
 663         /* RETSTACK, bit[9] Return stack bit */
 664         if (BMVAL(etmidr0, 9, 9))
 665                 drvdata->retstack = true;
 666         else
 667                 drvdata->retstack = false;
 668
 669         /* NUMEVENT, bits[11:10] Number of events field */
 670         drvdata->nr_event = BMVAL(etmidr0, 10, 11);
 671         /* QSUPP, bits[16:15] Q element support field */
 672         drvdata->q_support = BMVAL(etmidr0, 15, 16);
 673         /* TSSIZE, bits[28:24] Global timestamp size field */
 674         drvdata->ts_size = BMVAL(etmidr0, 24, 28);
 675
 676         /* base architecture of trace unit */
 677         etmidr1 = readl_relaxed(drvdata->base + TRCIDR1);
 678         /*
 679          * TRCARCHMIN, bits[7:4] architecture the minor version number
 680          * TRCARCHMAJ, bits[11:8] architecture major versin number
 681          */
 682         drvdata->arch = BMVAL(etmidr1, 4, 11);
 683         drvdata->config.arch = drvdata->arch;
 684
 685         /* maximum size of resources */
 686         etmidr2 = readl_relaxed(drvdata->base + TRCIDR2);
 687         /* CIDSIZE, bits[9:5] Indicates the Context ID size */
 688         drvdata->ctxid_size = BMVAL(etmidr2, 5, 9);
 689         /* VMIDSIZE, bits[14:10] Indicates the VMID size */
 690         drvdata->vmid_size = BMVAL(etmidr2, 10, 14);
 691         /* CCSIZE, bits[28:25] size of the cycle counter in bits minus 12 */
 692         drvdata->ccsize = BMVAL(etmidr2, 25, 28);
 693
 694         etmidr3 = readl_relaxed(drvdata->base + TRCIDR3);
 695         /* CCITMIN, bits[11:0] minimum threshold value that can be programmed */
 696         drvdata->ccitmin = BMVAL(etmidr3, 0, 11);
 697         /* EXLEVEL_S, bits[19:16] Secure state instruction tracing */
 698         drvdata->s_ex_level = BMVAL(etmidr3, 16, 19);
 699         /* EXLEVEL_NS, bits[23:20] Non-secure state instruction tracing */
 700         drvdata->ns_ex_level = BMVAL(etmidr3, 20, 23);
 701
 702         /*
 703          * TRCERR, bit[24] whether a trace unit can trace a
 704          * system error exception.
 705          */
 706         if (BMVAL(etmidr3, 24, 24))
 707                 drvdata->trc_error = true;
 708         else
 709                 drvdata->trc_error = false;
 710
 711         /* SYNCPR, bit[25] implementation has a fixed synchronization period? */
 712         if (BMVAL(etmidr3, 25, 25))
 713                 drvdata->syncpr = true;
 714         else
 715                 drvdata->syncpr = false;
 716
 717         /* STALLCTL, bit[26] is stall control implemented? */
 718         if (BMVAL(etmidr3, 26, 26))
 719                 drvdata->stallctl = true;
 720         else
 721                 drvdata->stallctl = false;
 722
 723         /* SYSSTALL, bit[27] implementation can support stall control? */
 724         if (BMVAL(etmidr3, 27, 27))
 725                 drvdata->sysstall = true;
 726         else
 727                 drvdata->sysstall = false;
 728
 729         /* NUMPROC, bits[30:28] the number of PEs available for tracing */
 730         drvdata->nr_pe = BMVAL(etmidr3, 28, 30);
 731
 732         /* NOOVERFLOW, bit[31] is trace overflow prevention supported */
 733         if (BMVAL(etmidr3, 31, 31))
 734                 drvdata->nooverflow = true;
 735         else
 736                 drvdata->nooverflow = false;
 737
 738         /* number of resources trace unit supports */
 739         etmidr4 = readl_relaxed(drvdata->base + TRCIDR4);
 740         /* NUMACPAIRS, bits[0:3] number of addr comparator pairs for tracing */
 741         drvdata->nr_addr_cmp = BMVAL(etmidr4, 0, 3);
 742         /* NUMPC, bits[15:12] number of PE comparator inputs for tracing */
 743         drvdata->nr_pe_cmp = BMVAL(etmidr4, 12, 15);
 744         /*
 745          * NUMRSPAIR, bits[19:16]
 746          * The number of resource pairs conveyed by the HW starts at 0, i.e a
 747          * value of 0x0 indicate 1 resource pair, 0x1 indicate two and so on.
 748          * As such add 1 to the value of NUMRSPAIR for a better representation.
 749          *
 750          * For ETM v4.3 and later, 0x0 means 0, and no pairs are available -
 751          * the default TRUE and FALSE resource selectors are omitted.
 752          * Otherwise for values 0x1 and above the number is N + 1 as per v4.2.
 753          */
 754         drvdata->nr_resource = BMVAL(etmidr4, 16, 19);
 755         if ((drvdata->arch < ETM4X_ARCH_4V3) || (drvdata->nr_resource > 0))
 756                 drvdata->nr_resource += 1;
 757         /*
 758          * NUMSSCC, bits[23:20] the number of single-shot
 759          * comparator control for tracing. Read any status regs as these
 760          * also contain RO capability data.
 761          */
 762         drvdata->nr_ss_cmp = BMVAL(etmidr4, 20, 23);
 763         for (i = 0; i < drvdata->nr_ss_cmp; i++) {
 764                 drvdata->config.ss_status[i] =
 765                         readl_relaxed(drvdata->base + TRCSSCSRn(i));
 766         }
 767         /* NUMCIDC, bits[27:24] number of Context ID comparators for tracing */
 768         drvdata->numcidc = BMVAL(etmidr4, 24, 27);
 769         /* NUMVMIDC, bits[31:28] number of VMID comparators for tracing */
 770         drvdata->numvmidc = BMVAL(etmidr4, 28, 31);
 771
 772         etmidr5 = readl_relaxed(drvdata->base + TRCIDR5);
 773         /* NUMEXTIN, bits[8:0] number of external inputs implemented */
 774         drvdata->nr_ext_inp = BMVAL(etmidr5, 0, 8);
 775         /* TRACEIDSIZE, bits[21:16] indicates the trace ID width */
 776         drvdata->trcid_size = BMVAL(etmidr5, 16, 21);
 777         /* ATBTRIG, bit[22] implementation can support ATB triggers? */
 778         if (BMVAL(etmidr5, 22, 22))
 779                 drvdata->atbtrig = true;
 780         else
 781                 drvdata->atbtrig = false;
 782         /*
 783          * LPOVERRIDE, bit[23] implementation supports
 784          * low-power state override
 785          */
 786         if (BMVAL(etmidr5, 23, 23) && (!drvdata->skip_power_up))
 787                 drvdata->lpoverride = true;
 788         else
 789                 drvdata->lpoverride = false;
 790         /* NUMSEQSTATE, bits[27:25] number of sequencer states implemented */
 791         drvdata->nrseqstate = BMVAL(etmidr5, 25, 27);
 792         /* NUMCNTR, bits[30:28] number of counters available for tracing */
 793         drvdata->nr_cntr = BMVAL(etmidr5, 28, 30);
 794         CS_LOCK(drvdata->base);
 795 }
 796
 797 /* Set ELx trace filter access in the TRCVICTLR register */
 798 static void etm4_set_victlr_access(struct etmv4_config *config)
 799 {
 800         u64 access_type;
 801
 802         config->vinst_ctrl &= ~(ETM_EXLEVEL_S_VICTLR_MASK | ETM_EXLEVEL_NS_VICTLR_MASK);
 803
 804         /*
 805          * TRCVICTLR::EXLEVEL_NS:EXLEVELS: Set kernel / user filtering
 806          * bits in vinst_ctrl, same bit pattern as TRCACATRn values returned by
 807          * etm4_get_access_type() but with a relative shift in this register.
 808          */
 809         access_type = etm4_get_access_type(config) << ETM_EXLEVEL_LSHIFT_TRCVICTLR;
 810         config->vinst_ctrl |= (u32)access_type;
 811 }
 812
 813 static void etm4_set_default_config(struct etmv4_config *config)
 814 {
 815         /* disable all events tracing */
 816         config->eventctrl0 = 0x0;
 817         config->eventctrl1 = 0x0;
 818
 819         /* disable stalling */
 820         config->stall_ctrl = 0x0;
 821
 822         /* enable trace synchronization every 4096 bytes, if available */
 823         config->syncfreq = 0xC;
 824
 825         /* disable timestamp event */
 826         config->ts_ctrl = 0x0;
 827
 828         /* TRCVICTLR::EVENT = 0x01, select the always on logic */
 829         config->vinst_ctrl = BIT(0);
 830
 831         /* TRCVICTLR::EXLEVEL_NS:EXLEVELS: Set kernel / user filtering */
 832         etm4_set_victlr_access(config);
 833 }
 834
 835 static u64 etm4_get_ns_access_type(struct etmv4_config *config)
 836 {
 837         u64 access_type = 0;
 838
 839         /*
 840          * EXLEVEL_NS, bits[15:12]
 841          * The Exception levels are:
 842          *   Bit[12] Exception level 0 - Application
 843          *   Bit[13] Exception level 1 - OS
 844          *   Bit[14] Exception level 2 - Hypervisor
 845          *   Bit[15] Never implemented
 846          */
 847         if (!is_kernel_in_hyp_mode()) {
 848                 /* Stay away from hypervisor mode for non-VHE */
 849                 access_type =  ETM_EXLEVEL_NS_HYP;
 850                 if (config->mode & ETM_MODE_EXCL_KERN)
 851                         access_type |= ETM_EXLEVEL_NS_OS;
 852         } else if (config->mode & ETM_MODE_EXCL_KERN) {
 853                 access_type = ETM_EXLEVEL_NS_HYP;
 854         }
 855
 856         if (config->mode & ETM_MODE_EXCL_USER)
 857                 access_type |= ETM_EXLEVEL_NS_APP;
 858
 859         return access_type;
 860 }
 861
 862 static u64 etm4_get_access_type(struct etmv4_config *config)
 863 {
 864         u64 access_type = etm4_get_ns_access_type(config);
 865         u64 s_hyp = (config->arch & 0x0f) >= 0x4 ? ETM_EXLEVEL_S_HYP : 0;
 866
 867         /*
 868          * EXLEVEL_S, bits[11:8], don't trace anything happening
 869          * in secure state.
 870          */
 871         access_type |= (ETM_EXLEVEL_S_APP       |
 872                         ETM_EXLEVEL_S_OS        |
 873                         s_hyp                   |
 874                         ETM_EXLEVEL_S_MON);
 875
 876         return access_type;
 877 }
 878
 879 static void etm4_set_comparator_filter(struct etmv4_config *config,
 880                                        u64 start, u64 stop, int comparator)
 881 {
 882         u64 access_type = etm4_get_access_type(config);
 883
 884         /* First half of default address comparator */
 885         config->addr_val[comparator] = start;
 886         config->addr_acc[comparator] = access_type;
 887         config->addr_type[comparator] = ETM_ADDR_TYPE_RANGE;
 888
 889         /* Second half of default address comparator */
 890         config->addr_val[comparator + 1] = stop;
 891         config->addr_acc[comparator + 1] = access_type;
 892         config->addr_type[comparator + 1] = ETM_ADDR_TYPE_RANGE;
 893
 894         /*
 895          * Configure the ViewInst function to include this address range
 896          * comparator.
 897          *
 898          * @comparator is divided by two since it is the index in the
 899          * etmv4_config::addr_val array but register TRCVIIECTLR deals with
 900          * address range comparator _pairs_.
 901          *
 902          * Therefore:
 903          *      index 0 -> compatator pair 0
 904          *      index 2 -> comparator pair 1
 905          *      index 4 -> comparator pair 2
 906          *      ...
 907          *      index 14 -> comparator pair 7
 908          */
 909         config->viiectlr |= BIT(comparator / 2);
 910 }
 911
 912 static void etm4_set_start_stop_filter(struct etmv4_config *config,
 913                                        u64 address, int comparator,
 914                                        enum etm_addr_type type)
 915 {
 916         int shift;
 917         u64 access_type = etm4_get_access_type(config);
 918
 919         /* Configure the comparator */
 920         config->addr_val[comparator] = address;
 921         config->addr_acc[comparator] = access_type;
 922         config->addr_type[comparator] = type;
 923
 924         /*
 925          * Configure ViewInst Start-Stop control register.
 926          * Addresses configured to start tracing go from bit 0 to n-1,
 927          * while those configured to stop tracing from 16 to 16 + n-1.
 928          */
 929         shift = (type == ETM_ADDR_TYPE_START ? 0 : 16);
 930         config->vissctlr |= BIT(shift + comparator);
 931 }
 932
 933 static void etm4_set_default_filter(struct etmv4_config *config)
 934 {
 935         /* Trace everything 'default' filter achieved by no filtering */
 936         config->viiectlr = 0x0;
 937
 938         /*
 939          * TRCVICTLR::SSSTATUS == 1, the start-stop logic is
 940          * in the started state
 941          */
 942         config->vinst_ctrl |= BIT(9);
 943         config->mode |= ETM_MODE_VIEWINST_STARTSTOP;
 944
 945         /* No start-stop filtering for ViewInst */
 946         config->vissctlr = 0x0;
 947 }
 948
 949 static void etm4_set_default(struct etmv4_config *config)
 950 {
 951         if (WARN_ON_ONCE(!config))
 952                 return;
 953
 954         /*
 955          * Make default initialisation trace everything
 956          *
 957          * This is done by a minimum default config sufficient to enable
 958          * full instruction trace - with a default filter for trace all
 959          * achieved by having no filtering.
 960          */
 961         etm4_set_default_config(config);
 962         etm4_set_default_filter(config);
 963 }
 964
 965 static int etm4_get_next_comparator(struct etmv4_drvdata *drvdata, u32 type)
 966 {
 967         int nr_comparator, index = 0;
 968         struct etmv4_config *config = &drvdata->config;
 969
 970         /*
 971          * nr_addr_cmp holds the number of comparator _pair_, so time 2
 972          * for the total number of comparators.
 973          */
 974         nr_comparator = drvdata->nr_addr_cmp * 2;
 975
 976         /* Go through the tally of comparators looking for a free one. */
 977         while (index < nr_comparator) {
 978                 switch (type) {
 979                 case ETM_ADDR_TYPE_RANGE:
 980                         if (config->addr_type[index] == ETM_ADDR_TYPE_NONE &&
 981                             config->addr_type[index + 1] == ETM_ADDR_TYPE_NONE)
 982                                 return index;
 983
 984                         /* Address range comparators go in pairs */
 985                         index += 2;
 986                         break;
 987                 case ETM_ADDR_TYPE_START:
 988                 case ETM_ADDR_TYPE_STOP:
 989                         if (config->addr_type[index] == ETM_ADDR_TYPE_NONE)
 990                                 return index;
 991
 992                         /* Start/stop address can have odd indexes */
 993                         index += 1;
 994                         break;
 995                 default:
 996                         return -EINVAL;
 997                 }
 998         }
 999
1000         /* If we are here all the comparators have been used. */
1001         return -ENOSPC;
1002 }
1003
1004 static int etm4_set_event_filters(struct etmv4_drvdata *drvdata,
1005                                   struct perf_event *event)
1006 {
1007         int i, comparator, ret = 0;
1008         u64 address;
1009         struct etmv4_config *config = &drvdata->config;
1010         struct etm_filters *filters = event->hw.addr_filters;
1011
1012         if (!filters)
1013                 goto default_filter;
1014
1015         /* Sync events with what Perf got */
1016         perf_event_addr_filters_sync(event);
1017
1018         /*
1019          * If there are no filters to deal with simply go ahead with
1020          * the default filter, i.e the entire address range.
1021          */
1022         if (!filters->nr_filters)
1023                 goto default_filter;
1024
1025         for (i = 0; i < filters->nr_filters; i++) {
1026                 struct etm_filter *filter = &filters->etm_filter[i];
1027                 enum etm_addr_type type = filter->type;
1028
1029                 /* See if a comparator is free. */
1030                 comparator = etm4_get_next_comparator(drvdata, type);
1031                 if (comparator < 0) {
1032                         ret = comparator;
1033                         goto out;
1034                 }
1035
1036                 switch (type) {
1037                 case ETM_ADDR_TYPE_RANGE:
1038                         etm4_set_comparator_filter(config,
1039                                                    filter->start_addr,
1040                                                    filter->stop_addr,
1041                                                    comparator);
1042                         /*
1043                          * TRCVICTLR::SSSTATUS == 1, the start-stop logic is
1044                          * in the started state
1045                          */
1046                         config->vinst_ctrl |= BIT(9);
1047
1048                         /* No start-stop filtering for ViewInst */
1049                         config->vissctlr = 0x0;
1050                         break;
1051                 case ETM_ADDR_TYPE_START:
1052                 case ETM_ADDR_TYPE_STOP:
1053                         /* Get the right start or stop address */
1054                         address = (type == ETM_ADDR_TYPE_START ?
1055                                    filter->start_addr :
1056                                    filter->stop_addr);
1057
1058                         /* Configure comparator */
1059                         etm4_set_start_stop_filter(config, address,
1060                                                    comparator, type);
1061
1062                         /*
1063                          * If filters::ssstatus == 1, trace acquisition was
1064                          * started but the process was yanked away before the
1065                          * the stop address was hit.  As such the start/stop
1066                          * logic needs to be re-started so that tracing can
1067                          * resume where it left.
1068                          *
1069                          * The start/stop logic status when a process is
1070                          * scheduled out is checked in function
1071                          * etm4_disable_perf().
1072                          */
1073                         if (filters->ssstatus)
1074                                 config->vinst_ctrl |= BIT(9);
1075
1076                         /* No include/exclude filtering for ViewInst */
1077                         config->viiectlr = 0x0;
1078                         break;
1079                 default:
1080                         ret = -EINVAL;
1081                         goto out;
1082                 }
1083         }
1084
1085         goto out;
1086
1087
1088 default_filter:
1089         etm4_set_default_filter(config);
1090
1091 out:
1092         return ret;
1093 }
1094
1095 void etm4_config_trace_mode(struct etmv4_config *config)
1096 {
1097         u32 mode;
1098
1099         mode = config->mode;
1100         mode &= (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER);
1101
1102         /* excluding kernel AND user space doesn't make sense */
1103         WARN_ON_ONCE(mode == (ETM_MODE_EXCL_KERN | ETM_MODE_EXCL_USER));
1104
1105         /* nothing to do if neither flags are set */
1106         if (!(mode & ETM_MODE_EXCL_KERN) && !(mode & ETM_MODE_EXCL_USER))
1107                 return;
1108
1109         etm4_set_victlr_access(config);
1110 }
1111
1112 static int etm4_online_cpu(unsigned int cpu)
1113 {
1114         if (!etmdrvdata[cpu])
1115                 return 0;
1116
1117         if (etmdrvdata[cpu]->boot_enable && !etmdrvdata[cpu]->sticky_enable)
1118                 coresight_enable(etmdrvdata[cpu]->csdev);
1119         return 0;
1120 }
1121
1122 static int etm4_starting_cpu(unsigned int cpu)
1123 {
1124         if (!etmdrvdata[cpu])
1125                 return 0;
1126
1127         spin_lock(&etmdrvdata[cpu]->spinlock);
1128         if (!etmdrvdata[cpu]->os_unlock)
1129                 etm4_os_unlock(etmdrvdata[cpu]);
1130
1131         if (local_read(&etmdrvdata[cpu]->mode))
1132                 etm4_enable_hw(etmdrvdata[cpu]);
1133         spin_unlock(&etmdrvdata[cpu]->spinlock);
1134         return 0;
1135 }
1136
1137 static int etm4_dying_cpu(unsigned int cpu)
1138 {
1139         if (!etmdrvdata[cpu])
1140                 return 0;
1141
1142         spin_lock(&etmdrvdata[cpu]->spinlock);
1143         if (local_read(&etmdrvdata[cpu]->mode))
1144                 etm4_disable_hw(etmdrvdata[cpu]);
1145         spin_unlock(&etmdrvdata[cpu]->spinlock);
1146         return 0;
1147 }
1148
1149 static void etm4_init_trace_id(struct etmv4_drvdata *drvdata)
1150 {
1151         drvdata->trcid = coresight_get_trace_id(drvdata->cpu);
1152 }
1153
1154 static int etm4_cpu_save(struct etmv4_drvdata *drvdata)
1155 {
1156         int i, ret = 0;
1157         struct etmv4_save_state *state;
1158         struct device *etm_dev = &drvdata->csdev->dev;
1159
1160         /*
1161          * As recommended by 3.4.1 ("The procedure when powering down the PE")
1162          * of ARM IHI 0064D
1163          */
1164         dsb(sy);
1165         isb();
1166
1167         CS_UNLOCK(drvdata->base);
1168
1169         /* Lock the OS lock to disable trace and external debugger access */
1170         etm4_os_lock(drvdata);
1171
1172         /* wait for TRCSTATR.PMSTABLE to go up */
1173         if (coresight_timeout(drvdata->base, TRCSTATR,
1174                               TRCSTATR_PMSTABLE_BIT, 1)) {
1175                 dev_err(etm_dev,
1176                         "timeout while waiting for PM Stable Status\n");
1177                 etm4_os_unlock(drvdata);
1178                 ret = -EBUSY;
1179                 goto out;
1180         }
1181
1182         state = drvdata->save_state;
1183
1184         state->trcprgctlr = readl(drvdata->base + TRCPRGCTLR);
1185         if (drvdata->nr_pe)
1186                 state->trcprocselr = readl(drvdata->base + TRCPROCSELR);
1187         state->trcconfigr = readl(drvdata->base + TRCCONFIGR);
1188         state->trcauxctlr = readl(drvdata->base + TRCAUXCTLR);
1189         state->trceventctl0r = readl(drvdata->base + TRCEVENTCTL0R);
1190         state->trceventctl1r = readl(drvdata->base + TRCEVENTCTL1R);
1191         if (drvdata->stallctl)
1192                 state->trcstallctlr = readl(drvdata->base + TRCSTALLCTLR);
1193         state->trctsctlr = readl(drvdata->base + TRCTSCTLR);
1194         state->trcsyncpr = readl(drvdata->base + TRCSYNCPR);
1195         state->trcccctlr = readl(drvdata->base + TRCCCCTLR);
1196         state->trcbbctlr = readl(drvdata->base + TRCBBCTLR);
1197         state->trctraceidr = readl(drvdata->base + TRCTRACEIDR);
1198         state->trcqctlr = readl(drvdata->base + TRCQCTLR);
1199
1200         state->trcvictlr = readl(drvdata->base + TRCVICTLR);
1201         state->trcviiectlr = readl(drvdata->base + TRCVIIECTLR);
1202         state->trcvissctlr = readl(drvdata->base + TRCVISSCTLR);
1203         if (drvdata->nr_pe_cmp)
1204                 state->trcvipcssctlr = readl(drvdata->base + TRCVIPCSSCTLR);
1205         state->trcvdctlr = readl(drvdata->base + TRCVDCTLR);
1206         state->trcvdsacctlr = readl(drvdata->base + TRCVDSACCTLR);
1207         state->trcvdarcctlr = readl(drvdata->base + TRCVDARCCTLR);
1208
1209         for (i = 0; i < drvdata->nrseqstate - 1; i++)
1210                 state->trcseqevr[i] = readl(drvdata->base + TRCSEQEVRn(i));
1211
1212         state->trcseqrstevr = readl(drvdata->base + TRCSEQRSTEVR);
1213         state->trcseqstr = readl(drvdata->base + TRCSEQSTR);
1214         state->trcextinselr = readl(drvdata->base + TRCEXTINSELR);
1215
1216         for (i = 0; i < drvdata->nr_cntr; i++) {
1217                 state->trccntrldvr[i] = readl(drvdata->base + TRCCNTRLDVRn(i));
1218                 state->trccntctlr[i] = readl(drvdata->base + TRCCNTCTLRn(i));
1219                 state->trccntvr[i] = readl(drvdata->base + TRCCNTVRn(i));
1220         }
1221
1222         for (i = 0; i < drvdata->nr_resource * 2; i++)
1223                 state->trcrsctlr[i] = readl(drvdata->base + TRCRSCTLRn(i));
1224
1225         for (i = 0; i < drvdata->nr_ss_cmp; i++) {
1226                 state->trcssccr[i] = readl(drvdata->base + TRCSSCCRn(i));
1227                 state->trcsscsr[i] = readl(drvdata->base + TRCSSCSRn(i));
1228                 state->trcsspcicr[i] = readl(drvdata->base + TRCSSPCICRn(i));
1229         }
1230
1231         for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) {
1232                 state->trcacvr[i] = readq(drvdata->base + TRCACVRn(i));
1233                 state->trcacatr[i] = readq(drvdata->base + TRCACATRn(i));
1234         }
1235
1236         /*
1237          * Data trace stream is architecturally prohibited for A profile cores
1238          * so we don't save (or later restore) trcdvcvr and trcdvcmr - As per
1239          * section 1.3.4 ("Possible functional configurations of an ETMv4 trace
1240          * unit") of ARM IHI 0064D.
1241          */
1242
1243         for (i = 0; i < drvdata->numcidc; i++)
1244                 state->trccidcvr[i] = readq(drvdata->base + TRCCIDCVRn(i));
1245
1246         for (i = 0; i < drvdata->numvmidc; i++)
1247                 state->trcvmidcvr[i] = readq(drvdata->base + TRCVMIDCVRn(i));
1248
1249         state->trccidcctlr0 = readl(drvdata->base + TRCCIDCCTLR0);
1250         if (drvdata->numcidc > 4)
1251                 state->trccidcctlr1 = readl(drvdata->base + TRCCIDCCTLR1);
1252
1253         state->trcvmidcctlr0 = readl(drvdata->base + TRCVMIDCCTLR0);
1254         if (drvdata->numvmidc > 4)
1255                 state->trcvmidcctlr1 = readl(drvdata->base + TRCVMIDCCTLR1);
1256
1257         state->trcclaimset = readl(drvdata->base + TRCCLAIMCLR);
1258
1259         if (!drvdata->skip_power_up)
1260                 state->trcpdcr = readl(drvdata->base + TRCPDCR);
1261
1262         /* wait for TRCSTATR.IDLE to go up */
1263         if (coresight_timeout(drvdata->base, TRCSTATR, TRCSTATR_IDLE_BIT, 1)) {
1264                 dev_err(etm_dev,
1265                         "timeout while waiting for Idle Trace Status\n");
1266                 etm4_os_unlock(drvdata);
1267                 ret = -EBUSY;
1268                 goto out;
1269         }
1270
1271         drvdata->state_needs_restore = true;
1272
1273         /*
1274          * Power can be removed from the trace unit now. We do this to
1275          * potentially save power on systems that respect the TRCPDCR_PU
1276          * despite requesting software to save/restore state.
1277          */
1278         if (!drvdata->skip_power_up)
1279                 writel_relaxed((state->trcpdcr & ~TRCPDCR_PU),
1280                                 drvdata->base + TRCPDCR);
1281 out:
1282         CS_LOCK(drvdata->base);
1283         return ret;
1284 }
1285
1286 static void etm4_cpu_restore(struct etmv4_drvdata *drvdata)
1287 {
1288         int i;
1289         struct etmv4_save_state *state = drvdata->save_state;
1290
1291         CS_UNLOCK(drvdata->base);
1292
1293         writel_relaxed(state->trcclaimset, drvdata->base + TRCCLAIMSET);
1294
1295         writel_relaxed(state->trcprgctlr, drvdata->base + TRCPRGCTLR);
1296         if (drvdata->nr_pe)
1297                 writel_relaxed(state->trcprocselr, drvdata->base + TRCPROCSELR);
1298         writel_relaxed(state->trcconfigr, drvdata->base + TRCCONFIGR);
1299         writel_relaxed(state->trcauxctlr, drvdata->base + TRCAUXCTLR);
1300         writel_relaxed(state->trceventctl0r, drvdata->base + TRCEVENTCTL0R);
1301         writel_relaxed(state->trceventctl1r, drvdata->base + TRCEVENTCTL1R);
1302         if (drvdata->stallctl)
1303                 writel_relaxed(state->trcstallctlr, drvdata->base + TRCSTALLCTLR);
1304         writel_relaxed(state->trctsctlr, drvdata->base + TRCTSCTLR);
1305         writel_relaxed(state->trcsyncpr, drvdata->base + TRCSYNCPR);
1306         writel_relaxed(state->trcccctlr, drvdata->base + TRCCCCTLR);
1307         writel_relaxed(state->trcbbctlr, drvdata->base + TRCBBCTLR);
1308         writel_relaxed(state->trctraceidr, drvdata->base + TRCTRACEIDR);
1309         writel_relaxed(state->trcqctlr, drvdata->base + TRCQCTLR);
1310
1311         writel_relaxed(state->trcvictlr, drvdata->base + TRCVICTLR);
1312         writel_relaxed(state->trcviiectlr, drvdata->base + TRCVIIECTLR);
1313         writel_relaxed(state->trcvissctlr, drvdata->base + TRCVISSCTLR);
1314         if (drvdata->nr_pe_cmp)
1315                 writel_relaxed(state->trcvipcssctlr, drvdata->base + TRCVIPCSSCTLR);
1316         writel_relaxed(state->trcvdctlr, drvdata->base + TRCVDCTLR);
1317         writel_relaxed(state->trcvdsacctlr, drvdata->base + TRCVDSACCTLR);
1318         writel_relaxed(state->trcvdarcctlr, drvdata->base + TRCVDARCCTLR);
1319
1320         for (i = 0; i < drvdata->nrseqstate - 1; i++)
1321                 writel_relaxed(state->trcseqevr[i],
1322                                drvdata->base + TRCSEQEVRn(i));
1323
1324         writel_relaxed(state->trcseqrstevr, drvdata->base + TRCSEQRSTEVR);
1325         writel_relaxed(state->trcseqstr, drvdata->base + TRCSEQSTR);
1326         writel_relaxed(state->trcextinselr, drvdata->base + TRCEXTINSELR);
1327
1328         for (i = 0; i < drvdata->nr_cntr; i++) {
1329                 writel_relaxed(state->trccntrldvr[i],
1330                                drvdata->base + TRCCNTRLDVRn(i));
1331                 writel_relaxed(state->trccntctlr[i],
1332                                drvdata->base + TRCCNTCTLRn(i));
1333                 writel_relaxed(state->trccntvr[i],
1334                                drvdata->base + TRCCNTVRn(i));
1335         }
1336
1337         for (i = 0; i < drvdata->nr_resource * 2; i++)
1338                 writel_relaxed(state->trcrsctlr[i],
1339                                drvdata->base + TRCRSCTLRn(i));
1340
1341         for (i = 0; i < drvdata->nr_ss_cmp; i++) {
1342                 writel_relaxed(state->trcssccr[i],
1343                                drvdata->base + TRCSSCCRn(i));
1344                 writel_relaxed(state->trcsscsr[i],
1345                                drvdata->base + TRCSSCSRn(i));
1346                 writel_relaxed(state->trcsspcicr[i],
1347                                drvdata->base + TRCSSPCICRn(i));
1348         }
1349
1350         for (i = 0; i < drvdata->nr_addr_cmp * 2; i++) {
1351                 writeq_relaxed(state->trcacvr[i],
1352                                drvdata->base + TRCACVRn(i));
1353                 writeq_relaxed(state->trcacatr[i],
1354                                drvdata->base + TRCACATRn(i));
1355         }
1356
1357         for (i = 0; i < drvdata->numcidc; i++)
1358                 writeq_relaxed(state->trccidcvr[i],
1359                                drvdata->base + TRCCIDCVRn(i));
1360
1361         for (i = 0; i < drvdata->numvmidc; i++)
1362                 writeq_relaxed(state->trcvmidcvr[i],
1363                                drvdata->base + TRCVMIDCVRn(i));
1364
1365         writel_relaxed(state->trccidcctlr0, drvdata->base + TRCCIDCCTLR0);
1366         if (drvdata->numcidc > 4)
1367                 writel_relaxed(state->trccidcctlr1, drvdata->base + TRCCIDCCTLR1);
1368
1369         writel_relaxed(state->trcvmidcctlr0, drvdata->base + TRCVMIDCCTLR0);
1370         if (drvdata->numvmidc > 4)
1371                 writel_relaxed(state->trcvmidcctlr1, drvdata->base + TRCVMIDCCTLR1);
1372
1373         writel_relaxed(state->trcclaimset, drvdata->base + TRCCLAIMSET);
1374
1375         if (!drvdata->skip_power_up)
1376                 writel_relaxed(state->trcpdcr, drvdata->base + TRCPDCR);
1377
1378         drvdata->state_needs_restore = false;
1379
1380         /*
1381          * As recommended by section 4.3.7 ("Synchronization when using the
1382          * memory-mapped interface") of ARM IHI 0064D
1383          */
1384         dsb(sy);
1385         isb();
1386
1387         /* Unlock the OS lock to re-enable trace and external debug access */
1388         etm4_os_unlock(drvdata);
1389         CS_LOCK(drvdata->base);
1390 }
1391
1392 static int etm4_cpu_pm_notify(struct notifier_block *nb, unsigned long cmd,
1393                               void *v)
1394 {
1395         struct etmv4_drvdata *drvdata;
1396         unsigned int cpu = smp_processor_id();
1397
1398         if (!etmdrvdata[cpu])
1399                 return NOTIFY_OK;
1400
1401         drvdata = etmdrvdata[cpu];
1402
1403         if (!drvdata->save_state)
1404                 return NOTIFY_OK;
1405
1406         if (WARN_ON_ONCE(drvdata->cpu != cpu))
1407                 return NOTIFY_BAD;
1408
1409         switch (cmd) {
1410         case CPU_PM_ENTER:
1411                 /* save the state if self-hosted coresight is in use */
1412                 if (local_read(&drvdata->mode))
1413                         if (etm4_cpu_save(drvdata))
1414                                 return NOTIFY_BAD;
1415                 break;
1416         case CPU_PM_EXIT:
1417         case CPU_PM_ENTER_FAILED:
1418                 if (drvdata->state_needs_restore)
1419                         etm4_cpu_restore(drvdata);
1420                 break;
1421         default:
1422                 return NOTIFY_DONE;
1423         }
1424
1425         return NOTIFY_OK;
1426 }
1427
1428 static struct notifier_block etm4_cpu_pm_nb = {
1429         .notifier_call = etm4_cpu_pm_notify,
1430 };
1431
1432 /* Setup PM. Deals with error conditions and counts */
1433 static int __init etm4_pm_setup(void)
1434 {
1435         int ret;
1436
1437         ret = cpu_pm_register_notifier(&etm4_cpu_pm_nb);
1438         if (ret)
1439                 return ret;
1440
1441         ret = cpuhp_setup_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING,
1442                                         "arm/coresight4:starting",
1443                                         etm4_starting_cpu, etm4_dying_cpu);
1444
1445         if (ret)
1446                 goto unregister_notifier;
1447
1448         ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
1449                                         "arm/coresight4:online",
1450                                         etm4_online_cpu, NULL);
1451
1452         /* HP dyn state ID returned in ret on success */
1453         if (ret > 0) {
1454                 hp_online = ret;
1455                 return 0;
1456         }
1457
1458         /* failed dyn state - remove others */
1459         cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
1460
1461 unregister_notifier:
1462         cpu_pm_unregister_notifier(&etm4_cpu_pm_nb);
1463         return ret;
1464 }
1465
1466 static void etm4_pm_clear(void)
1467 {
1468         cpu_pm_unregister_notifier(&etm4_cpu_pm_nb);
1469         cpuhp_remove_state_nocalls(CPUHP_AP_ARM_CORESIGHT_STARTING);
1470         if (hp_online) {
1471                 cpuhp_remove_state_nocalls(hp_online);
1472                 hp_online = 0;
1473         }
1474 }
1475
1476 static int etm4_probe(struct amba_device *adev, const struct amba_id *id)
1477 {
1478         int ret;
1479         void __iomem *base;
1480         struct device *dev = &adev->dev;
1481         struct coresight_platform_data *pdata = NULL;
1482         struct etmv4_drvdata *drvdata;
1483         struct resource *res = &adev->res;
1484         struct coresight_desc desc = { 0 };
1485
1486         drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL);
1487         if (!drvdata)
1488                 return -ENOMEM;
1489
1490         dev_set_drvdata(dev, drvdata);
1491
1492         if (pm_save_enable == PARAM_PM_SAVE_FIRMWARE)
1493                 pm_save_enable = coresight_loses_context_with_cpu(dev) ?
1494                                PARAM_PM_SAVE_SELF_HOSTED : PARAM_PM_SAVE_NEVER;
1495
1496         if (pm_save_enable != PARAM_PM_SAVE_NEVER) {
1497                 drvdata->save_state = devm_kmalloc(dev,
1498                                 sizeof(struct etmv4_save_state), GFP_KERNEL);
1499                 if (!drvdata->save_state)
1500                         return -ENOMEM;
1501         }
1502
1503         if (fwnode_property_present(dev_fwnode(dev), "qcom,skip-power-up"))
1504                 drvdata->skip_power_up = true;
1505
1506         /* Validity for the resource is already checked by the AMBA core */
1507         base = devm_ioremap_resource(dev, res);
1508         if (IS_ERR(base))
1509                 return PTR_ERR(base);
1510
1511         drvdata->base = base;
1512
1513         spin_lock_init(&drvdata->spinlock);
1514
1515         drvdata->cpu = coresight_get_cpu(dev);
1516         if (drvdata->cpu < 0)
1517                 return drvdata->cpu;
1518
1519         desc.name = devm_kasprintf(dev, GFP_KERNEL, "etm%d", drvdata->cpu);
1520         if (!desc.name)
1521                 return -ENOMEM;
1522
1523         if (smp_call_function_single(drvdata->cpu,
1524                                 etm4_init_arch_data,  drvdata, 1))
1525                 dev_err(dev, "ETM arch init failed\n");
1526
1527         if (etm4_arch_supported(drvdata->arch) == false)
1528                 return -EINVAL;
1529
1530         etm4_init_trace_id(drvdata);
1531         etm4_set_default(&drvdata->config);
1532
1533         pdata = coresight_get_platform_data(dev);
1534         if (IS_ERR(pdata))
1535                 return PTR_ERR(pdata);
1536
1537         adev->dev.platform_data = pdata;
1538
1539         desc.type = CORESIGHT_DEV_TYPE_SOURCE;
1540         desc.subtype.source_subtype = CORESIGHT_DEV_SUBTYPE_SOURCE_PROC;
1541         desc.ops = &etm4_cs_ops;
1542         desc.pdata = pdata;
1543         desc.dev = dev;
1544         desc.groups = coresight_etmv4_groups;
1545         drvdata->csdev = coresight_register(&desc);
1546         if (IS_ERR(drvdata->csdev))
1547                 return PTR_ERR(drvdata->csdev);
1548
1549         ret = etm_perf_symlink(drvdata->csdev, true);
1550         if (ret) {
1551                 coresight_unregister(drvdata->csdev);
1552                 return ret;
1553         }
1554
1555         etmdrvdata[drvdata->cpu] = drvdata;
1556
1557         pm_runtime_put(&adev->dev);
1558         dev_info(&drvdata->csdev->dev, "CPU%d: ETM v%d.%d initialized\n",
1559                  drvdata->cpu, drvdata->arch >> 4, drvdata->arch & 0xf);
1560
1561         if (boot_enable) {
1562                 coresight_enable(drvdata->csdev);
1563                 drvdata->boot_enable = true;
1564         }
1565
1566         return 0;
1567 }
1568
1569 static struct amba_cs_uci_id uci_id_etm4[] = {
1570         {
1571                 /*  ETMv4 UCI data */
1572                 .devarch        = 0x47704a13,
1573                 .devarch_mask   = 0xfff0ffff,
1574                 .devtype        = 0x00000013,
1575         }
1576 };
1577
1578 static void clear_etmdrvdata(void *info)
1579 {
1580         int cpu = *(int *)info;
1581
1582         etmdrvdata[cpu] = NULL;
1583 }
1584
1585 static void etm4_remove(struct amba_device *adev)
1586 {
1587         struct etmv4_drvdata *drvdata = dev_get_drvdata(&adev->dev);
1588
1589         etm_perf_symlink(drvdata->csdev, false);
1590
1591         /*
1592          * Taking hotplug lock here to avoid racing between etm4_remove and
1593          * CPU hotplug call backs.
1594          */
1595         cpus_read_lock();
1596         /*
1597          * The readers for etmdrvdata[] are CPU hotplug call backs
1598          * and PM notification call backs. Change etmdrvdata[i] on
1599          * CPU i ensures these call backs has consistent view
1600          * inside one call back function.
1601          */
1602         if (smp_call_function_single(drvdata->cpu, clear_etmdrvdata, &drvdata->cpu, 1))
1603                 etmdrvdata[drvdata->cpu] = NULL;
1604
1605         cpus_read_unlock();
1606
1607         coresight_unregister(drvdata->csdev);
1608 }
1609
1610 static const struct amba_id etm4_ids[] = {
1611         CS_AMBA_ID(0x000bb95d),                 /* Cortex-A53 */
1612         CS_AMBA_ID(0x000bb95e),                 /* Cortex-A57 */
1613         CS_AMBA_ID(0x000bb95a),                 /* Cortex-A72 */
1614         CS_AMBA_ID(0x000bb959),                 /* Cortex-A73 */
1615         CS_AMBA_UCI_ID(0x000bb9da, uci_id_etm4),/* Cortex-A35 */
1616         CS_AMBA_UCI_ID(0x000bbd0c, uci_id_etm4),/* Neoverse N1 */
1617         CS_AMBA_UCI_ID(0x000f0205, uci_id_etm4),/* Qualcomm Kryo */
1618         CS_AMBA_UCI_ID(0x000f0211, uci_id_etm4),/* Qualcomm Kryo */
1619         CS_AMBA_UCI_ID(0x000bb802, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A55 */
1620         CS_AMBA_UCI_ID(0x000bb803, uci_id_etm4),/* Qualcomm Kryo 385 Cortex-A75 */
1621         CS_AMBA_UCI_ID(0x000bb805, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A55 */
1622         CS_AMBA_UCI_ID(0x000bb804, uci_id_etm4),/* Qualcomm Kryo 4XX Cortex-A76 */
1623         CS_AMBA_UCI_ID(0x000cc0af, uci_id_etm4),/* Marvell ThunderX2 */
1624         CS_AMBA_UCI_ID(0x000b6d01, uci_id_etm4),/* HiSilicon-Hip08 */
1625         CS_AMBA_UCI_ID(0x000b6d02, uci_id_etm4),/* HiSilicon-Hip09 */
1626         {},
1627 };
1628
1629 MODULE_DEVICE_TABLE(amba, etm4_ids);
1630
1631 static struct amba_driver etm4x_driver = {
1632         .drv = {
1633                 .name   = "coresight-etm4x",
1634                 .owner  = THIS_MODULE,
1635                 .suppress_bind_attrs = true,
1636         },
1637         .probe          = etm4_probe,
1638         .remove         = etm4_remove,
1639         .id_table       = etm4_ids,
1640 };
1641
1642 static int __init etm4x_init(void)
1643 {
1644         int ret;
1645
1646         ret = etm4_pm_setup();
1647
1648         /* etm4_pm_setup() does its own cleanup - exit on error */
1649         if (ret)
1650                 return ret;
1651
1652         ret = amba_driver_register(&etm4x_driver);
1653         if (ret) {
1654                 pr_err("Error registering etm4x driver\n");
1655                 etm4_pm_clear();
1656         }
1657
1658         return ret;
1659 }
1660
1661 static void __exit etm4x_exit(void)
1662 {
1663         amba_driver_unregister(&etm4x_driver);
1664         etm4_pm_clear();
1665 }
1666
1667 module_init(etm4x_init);
1668 module_exit(etm4x_exit);
1669
1670 MODULE_AUTHOR("Pratik Patel <pratikp@codeaurora.org>");
1671 MODULE_AUTHOR("Mathieu Poirier <mathieu.poirier@linaro.org>");
1672 MODULE_DESCRIPTION("Arm CoreSight Program Flow Trace v4.x driver");
1673 MODULE_LICENSE("GPL v2");