drivers/clocksource/arm_arch_timer.c

   1 /*
   2  *  linux/drivers/clocksource/arm_arch_timer.c
   3  *
   4  *  Copyright (C) 2011 ARM Ltd.
   5  *  All Rights Reserved
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11
  12 #define pr_fmt(fmt)     "arm_arch_timer: " fmt
  13
  14 #include <linux/init.h>
  15 #include <linux/kernel.h>
  16 #include <linux/device.h>
  17 #include <linux/smp.h>
  18 #include <linux/cpu.h>
  19 #include <linux/cpu_pm.h>
  20 #include <linux/clockchips.h>
  21 #include <linux/clocksource.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/of_irq.h>
  24 #include <linux/of_address.h>
  25 #include <linux/io.h>
  26 #include <linux/slab.h>
  27 #include <linux/sched_clock.h>
  28 #include <linux/acpi.h>
  29
  30 #include <asm/arch_timer.h>
  31 #include <asm/virt.h>
  32
  33 #include <clocksource/arm_arch_timer.h>
  34
  35 #define CNTTIDR         0x08
  36 #define CNTTIDR_VIRT(n) (BIT(1) << ((n) * 4))
  37
  38 #define CNTACR(n)       (0x40 + ((n) * 4))
  39 #define CNTACR_RPCT     BIT(0)
  40 #define CNTACR_RVCT     BIT(1)
  41 #define CNTACR_RFRQ     BIT(2)
  42 #define CNTACR_RVOFF    BIT(3)
  43 #define CNTACR_RWVT     BIT(4)
  44 #define CNTACR_RWPT     BIT(5)
  45
  46 #define CNTVCT_LO       0x08
  47 #define CNTVCT_HI       0x0c
  48 #define CNTFRQ          0x10
  49 #define CNTP_TVAL       0x28
  50 #define CNTP_CTL        0x2c
  51 #define CNTV_TVAL       0x38
  52 #define CNTV_CTL        0x3c
  53
  54 #define ARCH_CP15_TIMER BIT(0)
  55 #define ARCH_MEM_TIMER  BIT(1)
  56 static unsigned arch_timers_present __initdata;
  57
  58 static void __iomem *arch_counter_base;
  59
  60 struct arch_timer {
  61         void __iomem *base;
  62         struct clock_event_device evt;
  63 };
  64
  65 #define to_arch_timer(e) container_of(e, struct arch_timer, evt)
  66
  67 static u32 arch_timer_rate;
  68
  69 enum ppi_nr {
  70         PHYS_SECURE_PPI,
  71         PHYS_NONSECURE_PPI,
  72         VIRT_PPI,
  73         HYP_PPI,
  74         MAX_TIMER_PPI
  75 };
  76
  77 static int arch_timer_ppi[MAX_TIMER_PPI];
  78
  79 static struct clock_event_device __percpu *arch_timer_evt;
  80
  81 static enum ppi_nr arch_timer_uses_ppi = VIRT_PPI;
  82 static bool arch_timer_c3stop;
  83 static bool arch_timer_mem_use_virtual;
  84 static bool arch_counter_suspend_stop;
  85
  86 static bool evtstrm_enable = IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM);
  87
  88 static int __init early_evtstrm_cfg(char *buf)
  89 {
  90         return strtobool(buf, &evtstrm_enable);
  91 }
  92 early_param("clocksource.arm_arch_timer.evtstrm", early_evtstrm_cfg);
  93
  94 /*
  95  * Architected system timer support.
  96  */
  97
  98 #ifdef CONFIG_FSL_ERRATUM_A008585
  99 DEFINE_STATIC_KEY_FALSE(arch_timer_read_ool_enabled);
 100 EXPORT_SYMBOL_GPL(arch_timer_read_ool_enabled);
 101
 102 static int fsl_a008585_enable = -1;
 103
 104 static int __init early_fsl_a008585_cfg(char *buf)
 105 {
 106         int ret;
 107         bool val;
 108
 109         ret = strtobool(buf, &val);
 110         if (ret)
 111                 return ret;
 112
 113         fsl_a008585_enable = val;
 114         return 0;
 115 }
 116 early_param("clocksource.arm_arch_timer.fsl-a008585", early_fsl_a008585_cfg);
 117
 118 u32 __fsl_a008585_read_cntp_tval_el0(void)
 119 {
 120         return __fsl_a008585_read_reg(cntp_tval_el0);
 121 }
 122
 123 u32 __fsl_a008585_read_cntv_tval_el0(void)
 124 {
 125         return __fsl_a008585_read_reg(cntv_tval_el0);
 126 }
 127
 128 u64 __fsl_a008585_read_cntvct_el0(void)
 129 {
 130         return __fsl_a008585_read_reg(cntvct_el0);
 131 }
 132 EXPORT_SYMBOL(__fsl_a008585_read_cntvct_el0);
 133 #endif /* CONFIG_FSL_ERRATUM_A008585 */
 134
 135 static __always_inline
 136 void arch_timer_reg_write(int access, enum arch_timer_reg reg, u32 val,
 137                           struct clock_event_device *clk)
 138 {
 139         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 140                 struct arch_timer *timer = to_arch_timer(clk);
 141                 switch (reg) {
 142                 case ARCH_TIMER_REG_CTRL:
 143                         writel_relaxed(val, timer->base + CNTP_CTL);
 144                         break;
 145                 case ARCH_TIMER_REG_TVAL:
 146                         writel_relaxed(val, timer->base + CNTP_TVAL);
 147                         break;
 148                 }
 149         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 150                 struct arch_timer *timer = to_arch_timer(clk);
 151                 switch (reg) {
 152                 case ARCH_TIMER_REG_CTRL:
 153                         writel_relaxed(val, timer->base + CNTV_CTL);
 154                         break;
 155                 case ARCH_TIMER_REG_TVAL:
 156                         writel_relaxed(val, timer->base + CNTV_TVAL);
 157                         break;
 158                 }
 159         } else {
 160                 arch_timer_reg_write_cp15(access, reg, val);
 161         }
 162 }
 163
 164 static __always_inline
 165 u32 arch_timer_reg_read(int access, enum arch_timer_reg reg,
 166                         struct clock_event_device *clk)
 167 {
 168         u32 val;
 169
 170         if (access == ARCH_TIMER_MEM_PHYS_ACCESS) {
 171                 struct arch_timer *timer = to_arch_timer(clk);
 172                 switch (reg) {
 173                 case ARCH_TIMER_REG_CTRL:
 174                         val = readl_relaxed(timer->base + CNTP_CTL);
 175                         break;
 176                 case ARCH_TIMER_REG_TVAL:
 177                         val = readl_relaxed(timer->base + CNTP_TVAL);
 178                         break;
 179                 }
 180         } else if (access == ARCH_TIMER_MEM_VIRT_ACCESS) {
 181                 struct arch_timer *timer = to_arch_timer(clk);
 182                 switch (reg) {
 183                 case ARCH_TIMER_REG_CTRL:
 184                         val = readl_relaxed(timer->base + CNTV_CTL);
 185                         break;
 186                 case ARCH_TIMER_REG_TVAL:
 187                         val = readl_relaxed(timer->base + CNTV_TVAL);
 188                         break;
 189                 }
 190         } else {
 191                 val = arch_timer_reg_read_cp15(access, reg);
 192         }
 193
 194         return val;
 195 }
 196
 197 static __always_inline irqreturn_t timer_handler(const int access,
 198                                         struct clock_event_device *evt)
 199 {
 200         unsigned long ctrl;
 201
 202         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, evt);
 203         if (ctrl & ARCH_TIMER_CTRL_IT_STAT) {
 204                 ctrl |= ARCH_TIMER_CTRL_IT_MASK;
 205                 arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, evt);
 206                 evt->event_handler(evt);
 207                 return IRQ_HANDLED;
 208         }
 209
 210         return IRQ_NONE;
 211 }
 212
 213 static irqreturn_t arch_timer_handler_virt(int irq, void *dev_id)
 214 {
 215         struct clock_event_device *evt = dev_id;
 216
 217         return timer_handler(ARCH_TIMER_VIRT_ACCESS, evt);
 218 }
 219
 220 static irqreturn_t arch_timer_handler_phys(int irq, void *dev_id)
 221 {
 222         struct clock_event_device *evt = dev_id;
 223
 224         return timer_handler(ARCH_TIMER_PHYS_ACCESS, evt);
 225 }
 226
 227 static irqreturn_t arch_timer_handler_phys_mem(int irq, void *dev_id)
 228 {
 229         struct clock_event_device *evt = dev_id;
 230
 231         return timer_handler(ARCH_TIMER_MEM_PHYS_ACCESS, evt);
 232 }
 233
 234 static irqreturn_t arch_timer_handler_virt_mem(int irq, void *dev_id)
 235 {
 236         struct clock_event_device *evt = dev_id;
 237
 238         return timer_handler(ARCH_TIMER_MEM_VIRT_ACCESS, evt);
 239 }
 240
 241 static __always_inline int timer_shutdown(const int access,
 242                                           struct clock_event_device *clk)
 243 {
 244         unsigned long ctrl;
 245
 246         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 247         ctrl &= ~ARCH_TIMER_CTRL_ENABLE;
 248         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 249
 250         return 0;
 251 }
 252
 253 static int arch_timer_shutdown_virt(struct clock_event_device *clk)
 254 {
 255         return timer_shutdown(ARCH_TIMER_VIRT_ACCESS, clk);
 256 }
 257
 258 static int arch_timer_shutdown_phys(struct clock_event_device *clk)
 259 {
 260         return timer_shutdown(ARCH_TIMER_PHYS_ACCESS, clk);
 261 }
 262
 263 static int arch_timer_shutdown_virt_mem(struct clock_event_device *clk)
 264 {
 265         return timer_shutdown(ARCH_TIMER_MEM_VIRT_ACCESS, clk);
 266 }
 267
 268 static int arch_timer_shutdown_phys_mem(struct clock_event_device *clk)
 269 {
 270         return timer_shutdown(ARCH_TIMER_MEM_PHYS_ACCESS, clk);
 271 }
 272
 273 static __always_inline void set_next_event(const int access, unsigned long evt,
 274                                            struct clock_event_device *clk)
 275 {
 276         unsigned long ctrl;
 277         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 278         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 279         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 280         arch_timer_reg_write(access, ARCH_TIMER_REG_TVAL, evt, clk);
 281         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 282 }
 283
 284 #ifdef CONFIG_FSL_ERRATUM_A008585
 285 static __always_inline void fsl_a008585_set_next_event(const int access,
 286                 unsigned long evt, struct clock_event_device *clk)
 287 {
 288         unsigned long ctrl;
 289         u64 cval = evt + arch_counter_get_cntvct();
 290
 291         ctrl = arch_timer_reg_read(access, ARCH_TIMER_REG_CTRL, clk);
 292         ctrl |= ARCH_TIMER_CTRL_ENABLE;
 293         ctrl &= ~ARCH_TIMER_CTRL_IT_MASK;
 294
 295         if (access == ARCH_TIMER_PHYS_ACCESS)
 296                 write_sysreg(cval, cntp_cval_el0);
 297         else if (access == ARCH_TIMER_VIRT_ACCESS)
 298                 write_sysreg(cval, cntv_cval_el0);
 299
 300         arch_timer_reg_write(access, ARCH_TIMER_REG_CTRL, ctrl, clk);
 301 }
 302
 303 static int fsl_a008585_set_next_event_virt(unsigned long evt,
 304                                            struct clock_event_device *clk)
 305 {
 306         fsl_a008585_set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 307         return 0;
 308 }
 309
 310 static int fsl_a008585_set_next_event_phys(unsigned long evt,
 311                                            struct clock_event_device *clk)
 312 {
 313         fsl_a008585_set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 314         return 0;
 315 }
 316 #endif /* CONFIG_FSL_ERRATUM_A008585 */
 317
 318 static int arch_timer_set_next_event_virt(unsigned long evt,
 319                                           struct clock_event_device *clk)
 320 {
 321         set_next_event(ARCH_TIMER_VIRT_ACCESS, evt, clk);
 322         return 0;
 323 }
 324
 325 static int arch_timer_set_next_event_phys(unsigned long evt,
 326                                           struct clock_event_device *clk)
 327 {
 328         set_next_event(ARCH_TIMER_PHYS_ACCESS, evt, clk);
 329         return 0;
 330 }
 331
 332 static int arch_timer_set_next_event_virt_mem(unsigned long evt,
 333                                               struct clock_event_device *clk)
 334 {
 335         set_next_event(ARCH_TIMER_MEM_VIRT_ACCESS, evt, clk);
 336         return 0;
 337 }
 338
 339 static int arch_timer_set_next_event_phys_mem(unsigned long evt,
 340                                               struct clock_event_device *clk)
 341 {
 342         set_next_event(ARCH_TIMER_MEM_PHYS_ACCESS, evt, clk);
 343         return 0;
 344 }
 345
 346 static void fsl_a008585_set_sne(struct clock_event_device *clk)
 347 {
 348 #ifdef CONFIG_FSL_ERRATUM_A008585
 349         if (!static_branch_unlikely(&arch_timer_read_ool_enabled))
 350                 return;
 351
 352         if (arch_timer_uses_ppi == VIRT_PPI)
 353                 clk->set_next_event = fsl_a008585_set_next_event_virt;
 354         else
 355                 clk->set_next_event = fsl_a008585_set_next_event_phys;
 356 #endif
 357 }
 358
 359 static void __arch_timer_setup(unsigned type,
 360                                struct clock_event_device *clk)
 361 {
 362         clk->features = CLOCK_EVT_FEAT_ONESHOT;
 363
 364         if (type == ARCH_CP15_TIMER) {
 365                 if (arch_timer_c3stop)
 366                         clk->features |= CLOCK_EVT_FEAT_C3STOP;
 367                 clk->name = "arch_sys_timer";
 368                 clk->rating = 450;
 369                 clk->cpumask = cpumask_of(smp_processor_id());
 370                 clk->irq = arch_timer_ppi[arch_timer_uses_ppi];
 371                 switch (arch_timer_uses_ppi) {
 372                 case VIRT_PPI:
 373                         clk->set_state_shutdown = arch_timer_shutdown_virt;
 374                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt;
 375                         clk->set_next_event = arch_timer_set_next_event_virt;
 376                         break;
 377                 case PHYS_SECURE_PPI:
 378                 case PHYS_NONSECURE_PPI:
 379                 case HYP_PPI:
 380                         clk->set_state_shutdown = arch_timer_shutdown_phys;
 381                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys;
 382                         clk->set_next_event = arch_timer_set_next_event_phys;
 383                         break;
 384                 default:
 385                         BUG();
 386                 }
 387
 388                 fsl_a008585_set_sne(clk);
 389         } else {
 390                 clk->features |= CLOCK_EVT_FEAT_DYNIRQ;
 391                 clk->name = "arch_mem_timer";
 392                 clk->rating = 400;
 393                 clk->cpumask = cpu_all_mask;
 394                 if (arch_timer_mem_use_virtual) {
 395                         clk->set_state_shutdown = arch_timer_shutdown_virt_mem;
 396                         clk->set_state_oneshot_stopped = arch_timer_shutdown_virt_mem;
 397                         clk->set_next_event =
 398                                 arch_timer_set_next_event_virt_mem;
 399                 } else {
 400                         clk->set_state_shutdown = arch_timer_shutdown_phys_mem;
 401                         clk->set_state_oneshot_stopped = arch_timer_shutdown_phys_mem;
 402                         clk->set_next_event =
 403                                 arch_timer_set_next_event_phys_mem;
 404                 }
 405         }
 406
 407         clk->set_state_shutdown(clk);
 408
 409         clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff);
 410 }
 411
 412 static void arch_timer_evtstrm_enable(int divider)
 413 {
 414         u32 cntkctl = arch_timer_get_cntkctl();
 415
 416         cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK;
 417         /* Set the divider and enable virtual event stream */
 418         cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT)
 419                         | ARCH_TIMER_VIRT_EVT_EN;
 420         arch_timer_set_cntkctl(cntkctl);
 421         elf_hwcap |= HWCAP_EVTSTRM;
 422 #ifdef CONFIG_COMPAT
 423         compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
 424 #endif
 425 }
 426
 427 static void arch_timer_configure_evtstream(void)
 428 {
 429         int evt_stream_div, lsb;
 430
 431         /*
 432          * As the event stream can at most be generated at half the frequency
 433          * of the counter, use half the frequency when computing the divider.
 434          */
 435         evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ / 2;
 436
 437         /*
 438          * Find the closest power of two to the divisor. If the adjacent bit
 439          * of lsb (last set bit, starts from 0) is set, then we use (lsb + 1).
 440          */
 441         lsb = fls(evt_stream_div) - 1;
 442         if (lsb > 0 && (evt_stream_div & BIT(lsb - 1)))
 443                 lsb++;
 444
 445         /* enable event stream */
 446         arch_timer_evtstrm_enable(max(0, min(lsb, 15)));
 447 }
 448
 449 static void arch_counter_set_user_access(void)
 450 {
 451         u32 cntkctl = arch_timer_get_cntkctl();
 452
 453         /* Disable user access to the timers and the physical counter */
 454         /* Also disable virtual event stream */
 455         cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN
 456                         | ARCH_TIMER_USR_VT_ACCESS_EN
 457                         | ARCH_TIMER_VIRT_EVT_EN
 458                         | ARCH_TIMER_USR_PCT_ACCESS_EN);
 459
 460         /* Enable user access to the virtual counter */
 461         cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN;
 462
 463         arch_timer_set_cntkctl(cntkctl);
 464 }
 465
 466 static bool arch_timer_has_nonsecure_ppi(void)
 467 {
 468         return (arch_timer_uses_ppi == PHYS_SECURE_PPI &&
 469                 arch_timer_ppi[PHYS_NONSECURE_PPI]);
 470 }
 471
 472 static u32 check_ppi_trigger(int irq)
 473 {
 474         u32 flags = irq_get_trigger_type(irq);
 475
 476         if (flags != IRQF_TRIGGER_HIGH && flags != IRQF_TRIGGER_LOW) {
 477                 pr_warn("WARNING: Invalid trigger for IRQ%d, assuming level low\n", irq);
 478                 pr_warn("WARNING: Please fix your firmware\n");
 479                 flags = IRQF_TRIGGER_LOW;
 480         }
 481
 482         return flags;
 483 }
 484
 485 static int arch_timer_starting_cpu(unsigned int cpu)
 486 {
 487         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
 488         u32 flags;
 489
 490         __arch_timer_setup(ARCH_CP15_TIMER, clk);
 491
 492         flags = check_ppi_trigger(arch_timer_ppi[arch_timer_uses_ppi]);
 493         enable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], flags);
 494
 495         if (arch_timer_has_nonsecure_ppi()) {
 496                 flags = check_ppi_trigger(arch_timer_ppi[PHYS_NONSECURE_PPI]);
 497                 enable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI], flags);
 498         }
 499
 500         arch_counter_set_user_access();
 501         if (evtstrm_enable)
 502                 arch_timer_configure_evtstream();
 503
 504         return 0;
 505 }
 506
 507 static void
 508 arch_timer_detect_rate(void __iomem *cntbase, struct device_node *np)
 509 {
 510         /* Who has more than one independent system counter? */
 511         if (arch_timer_rate)
 512                 return;
 513
 514         /*
 515          * Try to determine the frequency from the device tree or CNTFRQ,
 516          * if ACPI is enabled, get the frequency from CNTFRQ ONLY.
 517          */
 518         if (!acpi_disabled ||
 519             of_property_read_u32(np, "clock-frequency", &arch_timer_rate)) {
 520                 if (cntbase)
 521                         arch_timer_rate = readl_relaxed(cntbase + CNTFRQ);
 522                 else
 523                         arch_timer_rate = arch_timer_get_cntfrq();
 524         }
 525
 526         /* Check the timer frequency. */
 527         if (arch_timer_rate == 0)
 528                 pr_warn("Architected timer frequency not available\n");
 529 }
 530
 531 static void arch_timer_banner(unsigned type)
 532 {
 533         pr_info("Architected %s%s%s timer(s) running at %lu.%02luMHz (%s%s%s).\n",
 534                      type & ARCH_CP15_TIMER ? "cp15" : "",
 535                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  " and " : "",
 536                      type & ARCH_MEM_TIMER ? "mmio" : "",
 537                      (unsigned long)arch_timer_rate / 1000000,
 538                      (unsigned long)(arch_timer_rate / 10000) % 100,
 539                      type & ARCH_CP15_TIMER ?
 540                      (arch_timer_uses_ppi == VIRT_PPI) ? "virt" : "phys" :
 541                         "",
 542                      type == (ARCH_CP15_TIMER | ARCH_MEM_TIMER) ?  "/" : "",
 543                      type & ARCH_MEM_TIMER ?
 544                         arch_timer_mem_use_virtual ? "virt" : "phys" :
 545                         "");
 546 }
 547
 548 u32 arch_timer_get_rate(void)
 549 {
 550         return arch_timer_rate;
 551 }
 552
 553 static u64 arch_counter_get_cntvct_mem(void)
 554 {
 555         u32 vct_lo, vct_hi, tmp_hi;
 556
 557         do {
 558                 vct_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 559                 vct_lo = readl_relaxed(arch_counter_base + CNTVCT_LO);
 560                 tmp_hi = readl_relaxed(arch_counter_base + CNTVCT_HI);
 561         } while (vct_hi != tmp_hi);
 562
 563         return ((u64) vct_hi << 32) | vct_lo;
 564 }
 565
 566 /*
 567  * Default to cp15 based access because arm64 uses this function for
 568  * sched_clock() before DT is probed and the cp15 method is guaranteed
 569  * to exist on arm64. arm doesn't use this before DT is probed so even
 570  * if we don't have the cp15 accessors we won't have a problem.
 571  */
 572 u64 (*arch_timer_read_counter)(void) = arch_counter_get_cntvct;
 573
 574 static cycle_t arch_counter_read(struct clocksource *cs)
 575 {
 576         return arch_timer_read_counter();
 577 }
 578
 579 static cycle_t arch_counter_read_cc(const struct cyclecounter *cc)
 580 {
 581         return arch_timer_read_counter();
 582 }
 583
 584 static struct clocksource clocksource_counter = {
 585         .name   = "arch_sys_counter",
 586         .rating = 400,
 587         .read   = arch_counter_read,
 588         .mask   = CLOCKSOURCE_MASK(56),
 589         .flags  = CLOCK_SOURCE_IS_CONTINUOUS,
 590 };
 591
 592 static struct cyclecounter cyclecounter = {
 593         .read   = arch_counter_read_cc,
 594         .mask   = CLOCKSOURCE_MASK(56),
 595 };
 596
 597 static struct arch_timer_kvm_info arch_timer_kvm_info;
 598
 599 struct arch_timer_kvm_info *arch_timer_get_kvm_info(void)
 600 {
 601         return &arch_timer_kvm_info;
 602 }
 603
 604 static void __init arch_counter_register(unsigned type)
 605 {
 606         u64 start_count;
 607
 608         /* Register the CP15 based counter if we have one */
 609         if (type & ARCH_CP15_TIMER) {
 610                 if (IS_ENABLED(CONFIG_ARM64) || arch_timer_uses_ppi == VIRT_PPI)
 611                         arch_timer_read_counter = arch_counter_get_cntvct;
 612                 else
 613                         arch_timer_read_counter = arch_counter_get_cntpct;
 614
 615                 clocksource_counter.archdata.vdso_direct = true;
 616
 617 #ifdef CONFIG_FSL_ERRATUM_A008585
 618                 /*
 619                  * Don't use the vdso fastpath if errata require using
 620                  * the out-of-line counter accessor.
 621                  */
 622                 if (static_branch_unlikely(&arch_timer_read_ool_enabled))
 623                         clocksource_counter.archdata.vdso_direct = false;
 624 #endif
 625         } else {
 626                 arch_timer_read_counter = arch_counter_get_cntvct_mem;
 627         }
 628
 629         if (!arch_counter_suspend_stop)
 630                 clocksource_counter.flags |= CLOCK_SOURCE_SUSPEND_NONSTOP;
 631         start_count = arch_timer_read_counter();
 632         clocksource_register_hz(&clocksource_counter, arch_timer_rate);
 633         cyclecounter.mult = clocksource_counter.mult;
 634         cyclecounter.shift = clocksource_counter.shift;
 635         timecounter_init(&arch_timer_kvm_info.timecounter,
 636                          &cyclecounter, start_count);
 637
 638         /* 56 bits minimum, so we assume worst case rollover */
 639         sched_clock_register(arch_timer_read_counter, 56, arch_timer_rate);
 640 }
 641
 642 static void arch_timer_stop(struct clock_event_device *clk)
 643 {
 644         pr_debug("arch_timer_teardown disable IRQ%d cpu #%d\n",
 645                  clk->irq, smp_processor_id());
 646
 647         disable_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi]);
 648         if (arch_timer_has_nonsecure_ppi())
 649                 disable_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI]);
 650
 651         clk->set_state_shutdown(clk);
 652 }
 653
 654 static int arch_timer_dying_cpu(unsigned int cpu)
 655 {
 656         struct clock_event_device *clk = this_cpu_ptr(arch_timer_evt);
 657
 658         arch_timer_stop(clk);
 659         return 0;
 660 }
 661
 662 #ifdef CONFIG_CPU_PM
 663 static unsigned int saved_cntkctl;
 664 static int arch_timer_cpu_pm_notify(struct notifier_block *self,
 665                                     unsigned long action, void *hcpu)
 666 {
 667         if (action == CPU_PM_ENTER)
 668                 saved_cntkctl = arch_timer_get_cntkctl();
 669         else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT)
 670                 arch_timer_set_cntkctl(saved_cntkctl);
 671         return NOTIFY_OK;
 672 }
 673
 674 static struct notifier_block arch_timer_cpu_pm_notifier = {
 675         .notifier_call = arch_timer_cpu_pm_notify,
 676 };
 677
 678 static int __init arch_timer_cpu_pm_init(void)
 679 {
 680         return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier);
 681 }
 682
 683 static void __init arch_timer_cpu_pm_deinit(void)
 684 {
 685         WARN_ON(cpu_pm_unregister_notifier(&arch_timer_cpu_pm_notifier));
 686 }
 687
 688 #else
 689 static int __init arch_timer_cpu_pm_init(void)
 690 {
 691         return 0;
 692 }
 693
 694 static void __init arch_timer_cpu_pm_deinit(void)
 695 {
 696 }
 697 #endif
 698
 699 static int __init arch_timer_register(void)
 700 {
 701         int err;
 702         int ppi;
 703
 704         arch_timer_evt = alloc_percpu(struct clock_event_device);
 705         if (!arch_timer_evt) {
 706                 err = -ENOMEM;
 707                 goto out;
 708         }
 709
 710         ppi = arch_timer_ppi[arch_timer_uses_ppi];
 711         switch (arch_timer_uses_ppi) {
 712         case VIRT_PPI:
 713                 err = request_percpu_irq(ppi, arch_timer_handler_virt,
 714                                          "arch_timer", arch_timer_evt);
 715                 break;
 716         case PHYS_SECURE_PPI:
 717         case PHYS_NONSECURE_PPI:
 718                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
 719                                          "arch_timer", arch_timer_evt);
 720                 if (!err && arch_timer_ppi[PHYS_NONSECURE_PPI]) {
 721                         ppi = arch_timer_ppi[PHYS_NONSECURE_PPI];
 722                         err = request_percpu_irq(ppi, arch_timer_handler_phys,
 723                                                  "arch_timer", arch_timer_evt);
 724                         if (err)
 725                                 free_percpu_irq(arch_timer_ppi[PHYS_SECURE_PPI],
 726                                                 arch_timer_evt);
 727                 }
 728                 break;
 729         case HYP_PPI:
 730                 err = request_percpu_irq(ppi, arch_timer_handler_phys,
 731                                          "arch_timer", arch_timer_evt);
 732                 break;
 733         default:
 734                 BUG();
 735         }
 736
 737         if (err) {
 738                 pr_err("arch_timer: can't register interrupt %d (%d)\n",
 739                        ppi, err);
 740                 goto out_free;
 741         }
 742
 743         err = arch_timer_cpu_pm_init();
 744         if (err)
 745                 goto out_unreg_notify;
 746
 747
 748         /* Register and immediately configure the timer on the boot CPU */
 749         err = cpuhp_setup_state(CPUHP_AP_ARM_ARCH_TIMER_STARTING,
 750                                 "AP_ARM_ARCH_TIMER_STARTING",
 751                                 arch_timer_starting_cpu, arch_timer_dying_cpu);
 752         if (err)
 753                 goto out_unreg_cpupm;
 754         return 0;
 755
 756 out_unreg_cpupm:
 757         arch_timer_cpu_pm_deinit();
 758
 759 out_unreg_notify:
 760         free_percpu_irq(arch_timer_ppi[arch_timer_uses_ppi], arch_timer_evt);
 761         if (arch_timer_has_nonsecure_ppi())
 762                 free_percpu_irq(arch_timer_ppi[PHYS_NONSECURE_PPI],
 763                                 arch_timer_evt);
 764
 765 out_free:
 766         free_percpu(arch_timer_evt);
 767 out:
 768         return err;
 769 }
 770
 771 static int __init arch_timer_mem_register(void __iomem *base, unsigned int irq)
 772 {
 773         int ret;
 774         irq_handler_t func;
 775         struct arch_timer *t;
 776
 777         t = kzalloc(sizeof(*t), GFP_KERNEL);
 778         if (!t)
 779                 return -ENOMEM;
 780
 781         t->base = base;
 782         t->evt.irq = irq;
 783         __arch_timer_setup(ARCH_MEM_TIMER, &t->evt);
 784
 785         if (arch_timer_mem_use_virtual)
 786                 func = arch_timer_handler_virt_mem;
 787         else
 788                 func = arch_timer_handler_phys_mem;
 789
 790         ret = request_irq(irq, func, IRQF_TIMER, "arch_mem_timer", &t->evt);
 791         if (ret) {
 792                 pr_err("arch_timer: Failed to request mem timer irq\n");
 793                 kfree(t);
 794         }
 795
 796         return ret;
 797 }
 798
 799 static const struct of_device_id arch_timer_of_match[] __initconst = {
 800         { .compatible   = "arm,armv7-timer",    },
 801         { .compatible   = "arm,armv8-timer",    },
 802         {},
 803 };
 804
 805 static const struct of_device_id arch_timer_mem_of_match[] __initconst = {
 806         { .compatible   = "arm,armv7-timer-mem", },
 807         {},
 808 };
 809
 810 static bool __init
 811 arch_timer_needs_probing(int type, const struct of_device_id *matches)
 812 {
 813         struct device_node *dn;
 814         bool needs_probing = false;
 815
 816         dn = of_find_matching_node(NULL, matches);
 817         if (dn && of_device_is_available(dn) && !(arch_timers_present & type))
 818                 needs_probing = true;
 819         of_node_put(dn);
 820
 821         return needs_probing;
 822 }
 823
 824 static int __init arch_timer_common_init(void)
 825 {
 826         unsigned mask = ARCH_CP15_TIMER | ARCH_MEM_TIMER;
 827
 828         /* Wait until both nodes are probed if we have two timers */
 829         if ((arch_timers_present & mask) != mask) {
 830                 if (arch_timer_needs_probing(ARCH_MEM_TIMER, arch_timer_mem_of_match))
 831                         return 0;
 832                 if (arch_timer_needs_probing(ARCH_CP15_TIMER, arch_timer_of_match))
 833                         return 0;
 834         }
 835
 836         arch_timer_banner(arch_timers_present);
 837         arch_counter_register(arch_timers_present);
 838         return arch_timer_arch_init();
 839 }
 840
 841 static int __init arch_timer_init(void)
 842 {
 843         int ret;
 844         /*
 845          * If HYP mode is available, we know that the physical timer
 846          * has been configured to be accessible from PL1. Use it, so
 847          * that a guest can use the virtual timer instead.
 848          *
 849          * If no interrupt provided for virtual timer, we'll have to
 850          * stick to the physical timer. It'd better be accessible...
 851          *
 852          * On ARMv8.1 with VH extensions, the kernel runs in HYP. VHE
 853          * accesses to CNTP_*_EL1 registers are silently redirected to
 854          * their CNTHP_*_EL2 counterparts, and use a different PPI
 855          * number.
 856          */
 857         if (is_hyp_mode_available() || !arch_timer_ppi[VIRT_PPI]) {
 858                 bool has_ppi;
 859
 860                 if (is_kernel_in_hyp_mode()) {
 861                         arch_timer_uses_ppi = HYP_PPI;
 862                         has_ppi = !!arch_timer_ppi[HYP_PPI];
 863                 } else {
 864                         arch_timer_uses_ppi = PHYS_SECURE_PPI;
 865                         has_ppi = (!!arch_timer_ppi[PHYS_SECURE_PPI] ||
 866                                    !!arch_timer_ppi[PHYS_NONSECURE_PPI]);
 867                 }
 868
 869                 if (!has_ppi) {
 870                         pr_warn("arch_timer: No interrupt available, giving up\n");
 871                         return -EINVAL;
 872                 }
 873         }
 874
 875         ret = arch_timer_register();
 876         if (ret)
 877                 return ret;
 878
 879         ret = arch_timer_common_init();
 880         if (ret)
 881                 return ret;
 882
 883         arch_timer_kvm_info.virtual_irq = arch_timer_ppi[VIRT_PPI];
 884
 885         return 0;
 886 }
 887
 888 static int __init arch_timer_of_init(struct device_node *np)
 889 {
 890         int i;
 891
 892         if (arch_timers_present & ARCH_CP15_TIMER) {
 893                 pr_warn("arch_timer: multiple nodes in dt, skipping\n");
 894                 return 0;
 895         }
 896
 897         arch_timers_present |= ARCH_CP15_TIMER;
 898         for (i = PHYS_SECURE_PPI; i < MAX_TIMER_PPI; i++)
 899                 arch_timer_ppi[i] = irq_of_parse_and_map(np, i);
 900
 901         arch_timer_detect_rate(NULL, np);
 902
 903         arch_timer_c3stop = !of_property_read_bool(np, "always-on");
 904
 905 #ifdef CONFIG_FSL_ERRATUM_A008585
 906         if (fsl_a008585_enable < 0)
 907                 fsl_a008585_enable = of_property_read_bool(np, "fsl,erratum-a008585");
 908         if (fsl_a008585_enable) {
 909                 static_branch_enable(&arch_timer_read_ool_enabled);
 910                 pr_info("Enabling workaround for FSL erratum A-008585\n");
 911         }
 912 #endif
 913
 914         /*
 915          * If we cannot rely on firmware initializing the timer registers then
 916          * we should use the physical timers instead.
 917          */
 918         if (IS_ENABLED(CONFIG_ARM) &&
 919             of_property_read_bool(np, "arm,cpu-registers-not-fw-configured"))
 920                 arch_timer_uses_ppi = PHYS_SECURE_PPI;
 921
 922         /* On some systems, the counter stops ticking when in suspend. */
 923         arch_counter_suspend_stop = of_property_read_bool(np,
 924                                                          "arm,no-tick-in-suspend");
 925
 926         return arch_timer_init();
 927 }
 928 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer, "arm,armv7-timer", arch_timer_of_init);
 929 CLOCKSOURCE_OF_DECLARE(armv8_arch_timer, "arm,armv8-timer", arch_timer_of_init);
 930
 931 static int __init arch_timer_mem_init(struct device_node *np)
 932 {
 933         struct device_node *frame, *best_frame = NULL;
 934         void __iomem *cntctlbase, *base;
 935         unsigned int irq, ret = -EINVAL;
 936         u32 cnttidr;
 937
 938         arch_timers_present |= ARCH_MEM_TIMER;
 939         cntctlbase = of_iomap(np, 0);
 940         if (!cntctlbase) {
 941                 pr_err("arch_timer: Can't find CNTCTLBase\n");
 942                 return -ENXIO;
 943         }
 944
 945         cnttidr = readl_relaxed(cntctlbase + CNTTIDR);
 946
 947         /*
 948          * Try to find a virtual capable frame. Otherwise fall back to a
 949          * physical capable frame.
 950          */
 951         for_each_available_child_of_node(np, frame) {
 952                 int n;
 953                 u32 cntacr;
 954
 955                 if (of_property_read_u32(frame, "frame-number", &n)) {
 956                         pr_err("arch_timer: Missing frame-number\n");
 957                         of_node_put(frame);
 958                         goto out;
 959                 }
 960
 961                 /* Try enabling everything, and see what sticks */
 962                 cntacr = CNTACR_RFRQ | CNTACR_RWPT | CNTACR_RPCT |
 963                          CNTACR_RWVT | CNTACR_RVOFF | CNTACR_RVCT;
 964                 writel_relaxed(cntacr, cntctlbase + CNTACR(n));
 965                 cntacr = readl_relaxed(cntctlbase + CNTACR(n));
 966
 967                 if ((cnttidr & CNTTIDR_VIRT(n)) &&
 968                     !(~cntacr & (CNTACR_RWVT | CNTACR_RVCT))) {
 969                         of_node_put(best_frame);
 970                         best_frame = frame;
 971                         arch_timer_mem_use_virtual = true;
 972                         break;
 973                 }
 974
 975                 if (~cntacr & (CNTACR_RWPT | CNTACR_RPCT))
 976                         continue;
 977
 978                 of_node_put(best_frame);
 979                 best_frame = of_node_get(frame);
 980         }
 981
 982         ret= -ENXIO;
 983         base = arch_counter_base = of_iomap(best_frame, 0);
 984         if (!base) {
 985                 pr_err("arch_timer: Can't map frame's registers\n");
 986                 goto out;
 987         }
 988
 989         if (arch_timer_mem_use_virtual)
 990                 irq = irq_of_parse_and_map(best_frame, 1);
 991         else
 992                 irq = irq_of_parse_and_map(best_frame, 0);
 993
 994         ret = -EINVAL;
 995         if (!irq) {
 996                 pr_err("arch_timer: Frame missing %s irq",
 997                        arch_timer_mem_use_virtual ? "virt" : "phys");
 998                 goto out;
 999         }
1000
1001         arch_timer_detect_rate(base, np);
1002         ret = arch_timer_mem_register(base, irq);
1003         if (ret)
1004                 goto out;
1005
1006         return arch_timer_common_init();
1007 out:
1008         iounmap(cntctlbase);
1009         of_node_put(best_frame);
1010         return ret;
1011 }
1012 CLOCKSOURCE_OF_DECLARE(armv7_arch_timer_mem, "arm,armv7-timer-mem",
1013                        arch_timer_mem_init);
1014
1015 #ifdef CONFIG_ACPI
1016 static int __init map_generic_timer_interrupt(u32 interrupt, u32 flags)
1017 {
1018         int trigger, polarity;
1019
1020         if (!interrupt)
1021                 return 0;
1022
1023         trigger = (flags & ACPI_GTDT_INTERRUPT_MODE) ? ACPI_EDGE_SENSITIVE
1024                         : ACPI_LEVEL_SENSITIVE;
1025
1026         polarity = (flags & ACPI_GTDT_INTERRUPT_POLARITY) ? ACPI_ACTIVE_LOW
1027                         : ACPI_ACTIVE_HIGH;
1028
1029         return acpi_register_gsi(NULL, interrupt, trigger, polarity);
1030 }
1031
1032 /* Initialize per-processor generic timer */
1033 static int __init arch_timer_acpi_init(struct acpi_table_header *table)
1034 {
1035         struct acpi_table_gtdt *gtdt;
1036
1037         if (arch_timers_present & ARCH_CP15_TIMER) {
1038                 pr_warn("arch_timer: already initialized, skipping\n");
1039                 return -EINVAL;
1040         }
1041
1042         gtdt = container_of(table, struct acpi_table_gtdt, header);
1043
1044         arch_timers_present |= ARCH_CP15_TIMER;
1045
1046         arch_timer_ppi[PHYS_SECURE_PPI] =
1047                 map_generic_timer_interrupt(gtdt->secure_el1_interrupt,
1048                 gtdt->secure_el1_flags);
1049
1050         arch_timer_ppi[PHYS_NONSECURE_PPI] =
1051                 map_generic_timer_interrupt(gtdt->non_secure_el1_interrupt,
1052                 gtdt->non_secure_el1_flags);
1053
1054         arch_timer_ppi[VIRT_PPI] =
1055                 map_generic_timer_interrupt(gtdt->virtual_timer_interrupt,
1056                 gtdt->virtual_timer_flags);
1057
1058         arch_timer_ppi[HYP_PPI] =
1059                 map_generic_timer_interrupt(gtdt->non_secure_el2_interrupt,
1060                 gtdt->non_secure_el2_flags);
1061
1062         /* Get the frequency from CNTFRQ */
1063         arch_timer_detect_rate(NULL, NULL);
1064
1065         /* Always-on capability */
1066         arch_timer_c3stop = !(gtdt->non_secure_el1_flags & ACPI_GTDT_ALWAYS_ON);
1067
1068         arch_timer_init();
1069         return 0;
1070 }
1071 CLOCKSOURCE_ACPI_DECLARE(arch_timer, ACPI_SIG_GTDT, arch_timer_acpi_init);
1072 #endif