arch/arm64/include/asm/arch_timer.h

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  * arch/arm64/include/asm/arch_timer.h
   4  *
   5  * Copyright (C) 2012 ARM Ltd.
   6  * Author: Marc Zyngier <marc.zyngier@arm.com>
   7  */
   8 #ifndef __ASM_ARCH_TIMER_H
   9 #define __ASM_ARCH_TIMER_H
  10
  11 #include <asm/barrier.h>
  12 #include <asm/hwcap.h>
  13 #include <asm/sysreg.h>
  14
  15 #include <linux/bug.h>
  16 #include <linux/init.h>
  17 #include <linux/jump_label.h>
  18 #include <linux/smp.h>
  19 #include <linux/types.h>
  20
  21 #include <clocksource/arm_arch_timer.h>
  22
  23 #if IS_ENABLED(CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND)
  24 #define has_erratum_handler(h)                                          \
  25         ({                                                              \
  26                 const struct arch_timer_erratum_workaround *__wa;       \
  27                 __wa = __this_cpu_read(timer_unstable_counter_workaround); \
  28                 (__wa && __wa->h);                                      \
  29         })
  30
  31 #define erratum_handler(h)                                              \
  32         ({                                                              \
  33                 const struct arch_timer_erratum_workaround *__wa;       \
  34                 __wa = __this_cpu_read(timer_unstable_counter_workaround); \
  35                 (__wa && __wa->h) ? ({ isb(); __wa->h;}) : arch_timer_##h; \
  36         })
  37
  38 #else
  39 #define has_erratum_handler(h)                     false
  40 #define erratum_handler(h)                         (arch_timer_##h)
  41 #endif
  42
  43 enum arch_timer_erratum_match_type {
  44         ate_match_dt,
  45         ate_match_local_cap_id,
  46         ate_match_acpi_oem_info,
  47 };
  48
  49 struct clock_event_device;
  50
  51 struct arch_timer_erratum_workaround {
  52         enum arch_timer_erratum_match_type match_type;
  53         const void *id;
  54         const char *desc;
  55         u64 (*read_cntpct_el0)(void);
  56         u64 (*read_cntvct_el0)(void);
  57         int (*set_next_event_phys)(unsigned long, struct clock_event_device *);
  58         int (*set_next_event_virt)(unsigned long, struct clock_event_device *);
  59         bool disable_compat_vdso;
  60 };
  61
  62 DECLARE_PER_CPU(const struct arch_timer_erratum_workaround *,
  63                 timer_unstable_counter_workaround);
  64
  65 static inline notrace u64 arch_timer_read_cntpct_el0(void)
  66 {
  67         u64 cnt;
  68
  69         asm volatile(ALTERNATIVE("isb\n mrs %0, cntpct_el0",
  70                                  "nop\n" __mrs_s("%0", SYS_CNTPCTSS_EL0),
  71                                  ARM64_HAS_ECV)
  72                      : "=r" (cnt));
  73
  74         return cnt;
  75 }
  76
  77 static inline notrace u64 arch_timer_read_cntvct_el0(void)
  78 {
  79         u64 cnt;
  80
  81         asm volatile(ALTERNATIVE("isb\n mrs %0, cntvct_el0",
  82                                  "nop\n" __mrs_s("%0", SYS_CNTVCTSS_EL0),
  83                                  ARM64_HAS_ECV)
  84                      : "=r" (cnt));
  85
  86         return cnt;
  87 }
  88
  89 #define arch_timer_reg_read_stable(reg)                                 \
  90         ({                                                              \
  91                 erratum_handler(read_ ## reg)();                        \
  92         })
  93
  94 /*
  95  * These register accessors are marked inline so the compiler can
  96  * nicely work out which register we want, and chuck away the rest of
  97  * the code.
  98  */
  99 static __always_inline
 100 void arch_timer_reg_write_cp15(int access, enum arch_timer_reg reg, u64 val)
 101 {
 102         if (access == ARCH_TIMER_PHYS_ACCESS) {
 103                 switch (reg) {
 104                 case ARCH_TIMER_REG_CTRL:
 105                         write_sysreg(val, cntp_ctl_el0);
 106                         isb();
 107                         break;
 108                 case ARCH_TIMER_REG_CVAL:
 109                         write_sysreg(val, cntp_cval_el0);
 110                         break;
 111                 default:
 112                         BUILD_BUG();
 113                 }
 114         } else if (access == ARCH_TIMER_VIRT_ACCESS) {
 115                 switch (reg) {
 116                 case ARCH_TIMER_REG_CTRL:
 117                         write_sysreg(val, cntv_ctl_el0);
 118                         isb();
 119                         break;
 120                 case ARCH_TIMER_REG_CVAL:
 121                         write_sysreg(val, cntv_cval_el0);
 122                         break;
 123                 default:
 124                         BUILD_BUG();
 125                 }
 126         } else {
 127                 BUILD_BUG();
 128         }
 129 }
 130
 131 static __always_inline
 132 u64 arch_timer_reg_read_cp15(int access, enum arch_timer_reg reg)
 133 {
 134         if (access == ARCH_TIMER_PHYS_ACCESS) {
 135                 switch (reg) {
 136                 case ARCH_TIMER_REG_CTRL:
 137                         return read_sysreg(cntp_ctl_el0);
 138                 default:
 139                         BUILD_BUG();
 140                 }
 141         } else if (access == ARCH_TIMER_VIRT_ACCESS) {
 142                 switch (reg) {
 143                 case ARCH_TIMER_REG_CTRL:
 144                         return read_sysreg(cntv_ctl_el0);
 145                 default:
 146                         BUILD_BUG();
 147                 }
 148         }
 149
 150         BUILD_BUG();
 151         unreachable();
 152 }
 153
 154 static inline u32 arch_timer_get_cntfrq(void)
 155 {
 156         return read_sysreg(cntfrq_el0);
 157 }
 158
 159 static inline u32 arch_timer_get_cntkctl(void)
 160 {
 161         return read_sysreg(cntkctl_el1);
 162 }
 163
 164 static inline void arch_timer_set_cntkctl(u32 cntkctl)
 165 {
 166         write_sysreg(cntkctl, cntkctl_el1);
 167         isb();
 168 }
 169
 170 static __always_inline u64 __arch_counter_get_cntpct_stable(void)
 171 {
 172         u64 cnt;
 173
 174         cnt = arch_timer_reg_read_stable(cntpct_el0);
 175         arch_counter_enforce_ordering(cnt);
 176         return cnt;
 177 }
 178
 179 static __always_inline u64 __arch_counter_get_cntpct(void)
 180 {
 181         u64 cnt;
 182
 183         asm volatile(ALTERNATIVE("isb\n mrs %0, cntpct_el0",
 184                                  "nop\n" __mrs_s("%0", SYS_CNTPCTSS_EL0),
 185                                  ARM64_HAS_ECV)
 186                      : "=r" (cnt));
 187         arch_counter_enforce_ordering(cnt);
 188         return cnt;
 189 }
 190
 191 static __always_inline u64 __arch_counter_get_cntvct_stable(void)
 192 {
 193         u64 cnt;
 194
 195         cnt = arch_timer_reg_read_stable(cntvct_el0);
 196         arch_counter_enforce_ordering(cnt);
 197         return cnt;
 198 }
 199
 200 static __always_inline u64 __arch_counter_get_cntvct(void)
 201 {
 202         u64 cnt;
 203
 204         asm volatile(ALTERNATIVE("isb\n mrs %0, cntvct_el0",
 205                                  "nop\n" __mrs_s("%0", SYS_CNTVCTSS_EL0),
 206                                  ARM64_HAS_ECV)
 207                      : "=r" (cnt));
 208         arch_counter_enforce_ordering(cnt);
 209         return cnt;
 210 }
 211
 212 static inline int arch_timer_arch_init(void)
 213 {
 214         return 0;
 215 }
 216
 217 static inline void arch_timer_set_evtstrm_feature(void)
 218 {
 219         cpu_set_named_feature(EVTSTRM);
 220 #ifdef CONFIG_COMPAT
 221         compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
 222 #endif
 223 }
 224
 225 static inline bool arch_timer_have_evtstrm_feature(void)
 226 {
 227         return cpu_have_named_feature(EVTSTRM);
 228 }
 229 #endif