lib/vdso/gettimeofday.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Generic userspace implementations of gettimeofday() and similar.
   4  */
   5 #include <linux/compiler.h>
   6 #include <linux/math64.h>
   7 #include <linux/time.h>
   8 #include <linux/kernel.h>
   9 #include <linux/hrtimer_defs.h>
  10 #include <vdso/datapage.h>
  11 #include <vdso/helpers.h>
  12
  13 /*
  14  * The generic vDSO implementation requires that gettimeofday.h
  15  * provides:
  16  * - __arch_get_vdso_data(): to get the vdso datapage.
  17  * - __arch_get_hw_counter(): to get the hw counter based on the
  18  *   clock_mode.
  19  * - gettimeofday_fallback(): fallback for gettimeofday.
  20  * - clock_gettime_fallback(): fallback for clock_gettime.
  21  * - clock_getres_fallback(): fallback for clock_getres.
  22  */
  23 #ifdef ENABLE_COMPAT_VDSO
  24 #include <asm/vdso/compat_gettimeofday.h>
  25 #else
  26 #include <asm/vdso/gettimeofday.h>
  27 #endif /* ENABLE_COMPAT_VDSO */
  28
  29 #ifndef vdso_calc_delta
  30 /*
  31  * Default implementation which works for all sane clocksources. That
  32  * obviously excludes x86/TSC.
  33  */
  34 static __always_inline
  35 u64 vdso_calc_delta(u64 cycles, u64 last, u64 mask, u32 mult)
  36 {
  37         return ((cycles - last) & mask) * mult;
  38 }
  39 #endif
  40
  41 static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk,
  42                    struct __kernel_timespec *ts)
  43 {
  44         const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
  45         u64 cycles, last, sec, ns;
  46         u32 seq;
  47
  48         do {
  49                 seq = vdso_read_begin(vd);
  50                 cycles = __arch_get_hw_counter(vd->clock_mode);
  51                 ns = vdso_ts->nsec;
  52                 last = vd->cycle_last;
  53                 if (unlikely((s64)cycles < 0))
  54                         return -1;
  55
  56                 ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult);
  57                 ns >>= vd->shift;
  58                 sec = vdso_ts->sec;
  59         } while (unlikely(vdso_read_retry(vd, seq)));
  60
  61         /*
  62          * Do this outside the loop: a race inside the loop could result
  63          * in __iter_div_u64_rem() being extremely slow.
  64          */
  65         ts->tv_sec = sec + __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
  66         ts->tv_nsec = ns;
  67
  68         return 0;
  69 }
  70
  71 static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk,
  72                                      struct __kernel_timespec *ts)
  73 {
  74         const struct vdso_timestamp *vdso_ts = &vd->basetime[clk];
  75         u32 seq;
  76
  77         do {
  78                 seq = vdso_read_begin(vd);
  79                 ts->tv_sec = vdso_ts->sec;
  80                 ts->tv_nsec = vdso_ts->nsec;
  81         } while (unlikely(vdso_read_retry(vd, seq)));
  82
  83         return 0;
  84 }
  85
  86 static __maybe_unused int
  87 __cvdso_clock_gettime_common(clockid_t clock, struct __kernel_timespec *ts)
  88 {
  89         const struct vdso_data *vd = __arch_get_vdso_data();
  90         u32 msk;
  91
  92         /* Check for negative values or invalid clocks */
  93         if (unlikely((u32) clock >= MAX_CLOCKS))
  94                 return -1;
  95
  96         /*
  97          * Convert the clockid to a bitmask and use it to check which
  98          * clocks are handled in the VDSO directly.
  99          */
 100         msk = 1U << clock;
 101         if (likely(msk & VDSO_HRES))
 102                 vd = &vd[CS_HRES_COARSE];
 103         else if (msk & VDSO_COARSE)
 104                 return do_coarse(&vd[CS_HRES_COARSE], clock, ts);
 105         else if (msk & VDSO_RAW)
 106                 vd = &vd[CS_RAW];
 107         else
 108                 return -1;
 109
 110         return do_hres(vd, clock, ts);
 111 }
 112
 113 static __maybe_unused int
 114 __cvdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts)
 115 {
 116         int ret = __cvdso_clock_gettime_common(clock, ts);
 117
 118         if (unlikely(ret))
 119                 return clock_gettime_fallback(clock, ts);
 120         return 0;
 121 }
 122
 123 static __maybe_unused int
 124 __cvdso_clock_gettime32(clockid_t clock, struct old_timespec32 *res)
 125 {
 126         struct __kernel_timespec ts;
 127         int ret;
 128
 129         ret = __cvdso_clock_gettime_common(clock, &ts);
 130
 131 #ifdef VDSO_HAS_32BIT_FALLBACK
 132         if (unlikely(ret))
 133                 return clock_gettime32_fallback(clock, res);
 134 #else
 135         if (unlikely(ret))
 136                 ret = clock_gettime_fallback(clock, &ts);
 137 #endif
 138
 139         if (likely(!ret)) {
 140                 res->tv_sec = ts.tv_sec;
 141                 res->tv_nsec = ts.tv_nsec;
 142         }
 143         return ret;
 144 }
 145
 146 static __maybe_unused int
 147 __cvdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
 148 {
 149         const struct vdso_data *vd = __arch_get_vdso_data();
 150
 151         if (likely(tv != NULL)) {
 152                 struct __kernel_timespec ts;
 153
 154                 if (do_hres(&vd[CS_HRES_COARSE], CLOCK_REALTIME, &ts))
 155                         return gettimeofday_fallback(tv, tz);
 156
 157                 tv->tv_sec = ts.tv_sec;
 158                 tv->tv_usec = (u32)ts.tv_nsec / NSEC_PER_USEC;
 159         }
 160
 161         if (unlikely(tz != NULL)) {
 162                 tz->tz_minuteswest = vd[CS_HRES_COARSE].tz_minuteswest;
 163                 tz->tz_dsttime = vd[CS_HRES_COARSE].tz_dsttime;
 164         }
 165
 166         return 0;
 167 }
 168
 169 #ifdef VDSO_HAS_TIME
 170 static __maybe_unused time_t __cvdso_time(time_t *time)
 171 {
 172         const struct vdso_data *vd = __arch_get_vdso_data();
 173         time_t t = READ_ONCE(vd[CS_HRES_COARSE].basetime[CLOCK_REALTIME].sec);
 174
 175         if (time)
 176                 *time = t;
 177
 178         return t;
 179 }
 180 #endif /* VDSO_HAS_TIME */
 181
 182 #ifdef VDSO_HAS_CLOCK_GETRES
 183 static __maybe_unused
 184 int __cvdso_clock_getres_common(clockid_t clock, struct __kernel_timespec *res)
 185 {
 186         const struct vdso_data *vd = __arch_get_vdso_data();
 187         u64 hrtimer_res;
 188         u32 msk;
 189         u64 ns;
 190
 191         /* Check for negative values or invalid clocks */
 192         if (unlikely((u32) clock >= MAX_CLOCKS))
 193                 return -1;
 194
 195         hrtimer_res = READ_ONCE(vd[CS_HRES_COARSE].hrtimer_res);
 196         /*
 197          * Convert the clockid to a bitmask and use it to check which
 198          * clocks are handled in the VDSO directly.
 199          */
 200         msk = 1U << clock;
 201         if (msk & VDSO_HRES) {
 202                 /*
 203                  * Preserves the behaviour of posix_get_hrtimer_res().
 204                  */
 205                 ns = hrtimer_res;
 206         } else if (msk & VDSO_COARSE) {
 207                 /*
 208                  * Preserves the behaviour of posix_get_coarse_res().
 209                  */
 210                 ns = LOW_RES_NSEC;
 211         } else if (msk & VDSO_RAW) {
 212                 /*
 213                  * Preserves the behaviour of posix_get_hrtimer_res().
 214                  */
 215                 ns = hrtimer_res;
 216         } else {
 217                 return -1;
 218         }
 219
 220         if (likely(res)) {
 221                 res->tv_sec = 0;
 222                 res->tv_nsec = ns;
 223         }
 224         return 0;
 225 }
 226
 227 int __cvdso_clock_getres(clockid_t clock, struct __kernel_timespec *res)
 228 {
 229         int ret = __cvdso_clock_getres_common(clock, res);
 230
 231         if (unlikely(ret))
 232                 return clock_getres_fallback(clock, res);
 233         return 0;
 234 }
 235
 236 static __maybe_unused int
 237 __cvdso_clock_getres_time32(clockid_t clock, struct old_timespec32 *res)
 238 {
 239         struct __kernel_timespec ts;
 240         int ret;
 241
 242         ret = __cvdso_clock_getres_common(clock, &ts);
 243
 244 #ifdef VDSO_HAS_32BIT_FALLBACK
 245         if (unlikely(ret))
 246                 return clock_getres32_fallback(clock, res);
 247 #else
 248         if (unlikely(ret))
 249                 ret = clock_getres_fallback(clock, &ts);
 250 #endif
 251
 252         if (likely(!ret && res)) {
 253                 res->tv_sec = ts.tv_sec;
 254                 res->tv_nsec = ts.tv_nsec;
 255         }
 256         return ret;
 257 }
 258 #endif /* VDSO_HAS_CLOCK_GETRES */