1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright 2006 Andi Kleen, SUSE Labs.
5 * Fast user context implementation of clock_gettime, gettimeofday, and time.
7 * The code should have no internal unresolved relocations.
8 * Check with readelf after changing.
9 * Also alternative() doesn't work.
12 * Copyright (c) 2017 Oracle and/or its affiliates. All rights reserved.
15 #include <linux/kernel.h>
16 #include <linux/time.h>
17 #include <linux/string.h>
19 #include <asm/unistd.h>
20 #include <asm/timex.h>
21 #include <asm/clocksource.h>
25 #define SYSCALL_STRING \
28 " sub %%g0, %%o0, %%o0;" \
31 #define SYSCALL_STRING \
34 " sub %%g0, %%o0, %%o0;" \
38 #define SYSCALL_CLOBBERS \
39 "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7", \
40 "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", \
41 "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23", \
42 "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31", \
43 "f32", "f34", "f36", "f38", "f40", "f42", "f44", "f46", \
44 "f48", "f50", "f52", "f54", "f56", "f58", "f60", "f62", \
48 * Compute the vvar page's address in the process address space, and return it
49 * as a pointer to the vvar_data.
51 notrace static __always_inline struct vvar_data *get_vvar_data(void)
56 * vdso data page is the first vDSO page so grab the PC
57 * and move up a page to get to the data page.
59 __asm__("rd %%pc, %0" : "=r" (ret));
63 return (struct vvar_data *) ret;
66 notrace static long vdso_fallback_gettime(long clock, struct __kernel_old_timespec *ts)
68 register long num __asm__("g1") = __NR_clock_gettime;
69 register long o0 __asm__("o0") = clock;
70 register long o1 __asm__("o1") = (long) ts;
72 __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
73 "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
77 notrace static long vdso_fallback_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
79 register long num __asm__("g1") = __NR_gettimeofday;
80 register long o0 __asm__("o0") = (long) tv;
81 register long o1 __asm__("o1") = (long) tz;
83 __asm__ __volatile__(SYSCALL_STRING : "=r" (o0) : "r" (num),
84 "0" (o0), "r" (o1) : SYSCALL_CLOBBERS);
89 notrace static __always_inline u64 vread_tick(void)
93 __asm__ __volatile__("rd %%tick, %0" : "=r" (ret));
97 notrace static __always_inline u64 vread_tick_stick(void)
101 __asm__ __volatile__("rd %%asr24, %0" : "=r" (ret));
105 notrace static __always_inline u64 vread_tick(void)
107 register unsigned long long ret asm("o4");
109 __asm__ __volatile__("rd %%tick, %L0\n\t"
115 notrace static __always_inline u64 vread_tick_stick(void)
117 register unsigned long long ret asm("o4");
119 __asm__ __volatile__("rd %%asr24, %L0\n\t"
126 notrace static __always_inline u64 vgetsns(struct vvar_data *vvar)
131 cycles = vread_tick();
132 v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask;
133 return v * vvar->clock.mult;
136 notrace static __always_inline u64 vgetsns_stick(struct vvar_data *vvar)
141 cycles = vread_tick_stick();
142 v = (cycles - vvar->clock.cycle_last) & vvar->clock.mask;
143 return v * vvar->clock.mult;
146 notrace static __always_inline int do_realtime(struct vvar_data *vvar,
147 struct __kernel_old_timespec *ts)
153 seq = vvar_read_begin(vvar);
154 ts->tv_sec = vvar->wall_time_sec;
155 ns = vvar->wall_time_snsec;
157 ns >>= vvar->clock.shift;
158 } while (unlikely(vvar_read_retry(vvar, seq)));
160 ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
166 notrace static __always_inline int do_realtime_stick(struct vvar_data *vvar,
167 struct __kernel_old_timespec *ts)
173 seq = vvar_read_begin(vvar);
174 ts->tv_sec = vvar->wall_time_sec;
175 ns = vvar->wall_time_snsec;
176 ns += vgetsns_stick(vvar);
177 ns >>= vvar->clock.shift;
178 } while (unlikely(vvar_read_retry(vvar, seq)));
180 ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
186 notrace static __always_inline int do_monotonic(struct vvar_data *vvar,
187 struct __kernel_old_timespec *ts)
193 seq = vvar_read_begin(vvar);
194 ts->tv_sec = vvar->monotonic_time_sec;
195 ns = vvar->monotonic_time_snsec;
197 ns >>= vvar->clock.shift;
198 } while (unlikely(vvar_read_retry(vvar, seq)));
200 ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
206 notrace static __always_inline int do_monotonic_stick(struct vvar_data *vvar,
207 struct __kernel_old_timespec *ts)
213 seq = vvar_read_begin(vvar);
214 ts->tv_sec = vvar->monotonic_time_sec;
215 ns = vvar->monotonic_time_snsec;
216 ns += vgetsns_stick(vvar);
217 ns >>= vvar->clock.shift;
218 } while (unlikely(vvar_read_retry(vvar, seq)));
220 ts->tv_sec += __iter_div_u64_rem(ns, NSEC_PER_SEC, &ns);
226 notrace static int do_realtime_coarse(struct vvar_data *vvar,
227 struct __kernel_old_timespec *ts)
232 seq = vvar_read_begin(vvar);
233 ts->tv_sec = vvar->wall_time_coarse_sec;
234 ts->tv_nsec = vvar->wall_time_coarse_nsec;
235 } while (unlikely(vvar_read_retry(vvar, seq)));
239 notrace static int do_monotonic_coarse(struct vvar_data *vvar,
240 struct __kernel_old_timespec *ts)
245 seq = vvar_read_begin(vvar);
246 ts->tv_sec = vvar->monotonic_time_coarse_sec;
247 ts->tv_nsec = vvar->monotonic_time_coarse_nsec;
248 } while (unlikely(vvar_read_retry(vvar, seq)));
254 __vdso_clock_gettime(clockid_t clock, struct __kernel_old_timespec *ts)
256 struct vvar_data *vvd = get_vvar_data();
260 if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
262 return do_realtime(vvd, ts);
263 case CLOCK_MONOTONIC:
264 if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
266 return do_monotonic(vvd, ts);
267 case CLOCK_REALTIME_COARSE:
268 return do_realtime_coarse(vvd, ts);
269 case CLOCK_MONOTONIC_COARSE:
270 return do_monotonic_coarse(vvd, ts);
273 * Unknown clock ID ? Fall back to the syscall.
275 return vdso_fallback_gettime(clock, ts);
278 clock_gettime(clockid_t, struct __kernel_old_timespec *)
279 __attribute__((weak, alias("__vdso_clock_gettime")));
282 __vdso_clock_gettime_stick(clockid_t clock, struct __kernel_old_timespec *ts)
284 struct vvar_data *vvd = get_vvar_data();
288 if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
290 return do_realtime_stick(vvd, ts);
291 case CLOCK_MONOTONIC:
292 if (unlikely(vvd->vclock_mode == VCLOCK_NONE))
294 return do_monotonic_stick(vvd, ts);
295 case CLOCK_REALTIME_COARSE:
296 return do_realtime_coarse(vvd, ts);
297 case CLOCK_MONOTONIC_COARSE:
298 return do_monotonic_coarse(vvd, ts);
301 * Unknown clock ID ? Fall back to the syscall.
303 return vdso_fallback_gettime(clock, ts);
307 __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz)
309 struct vvar_data *vvd = get_vvar_data();
311 if (likely(vvd->vclock_mode != VCLOCK_NONE)) {
312 if (likely(tv != NULL)) {
314 struct __kernel_old_timespec ts;
315 struct __kernel_old_timeval tv;
316 } *tstv = (union tstv_t *) tv;
317 do_realtime(vvd, &tstv->ts);
319 * Assign before dividing to ensure that the division is
320 * done in the type of tv_usec, not tv_nsec.
322 * There cannot be > 1 billion usec in a second:
323 * do_realtime() has already distributed such overflow
324 * into tv_sec. So we can assign it to an int safely.
326 tstv->tv.tv_usec = tstv->ts.tv_nsec;
327 tstv->tv.tv_usec /= 1000;
329 if (unlikely(tz != NULL)) {
330 /* Avoid memcpy. Some old compilers fail to inline it */
331 tz->tz_minuteswest = vvd->tz_minuteswest;
332 tz->tz_dsttime = vvd->tz_dsttime;
336 return vdso_fallback_gettimeofday(tv, tz);
339 gettimeofday(struct __kernel_old_timeval *, struct timezone *)
340 __attribute__((weak, alias("__vdso_gettimeofday")));
343 __vdso_gettimeofday_stick(struct __kernel_old_timeval *tv, struct timezone *tz)
345 struct vvar_data *vvd = get_vvar_data();
347 if (likely(vvd->vclock_mode != VCLOCK_NONE)) {
348 if (likely(tv != NULL)) {
350 struct __kernel_old_timespec ts;
351 struct __kernel_old_timeval tv;
352 } *tstv = (union tstv_t *) tv;
353 do_realtime_stick(vvd, &tstv->ts);
355 * Assign before dividing to ensure that the division is
356 * done in the type of tv_usec, not tv_nsec.
358 * There cannot be > 1 billion usec in a second:
359 * do_realtime() has already distributed such overflow
360 * into tv_sec. So we can assign it to an int safely.
362 tstv->tv.tv_usec = tstv->ts.tv_nsec;
363 tstv->tv.tv_usec /= 1000;
365 if (unlikely(tz != NULL)) {
366 /* Avoid memcpy. Some old compilers fail to inline it */
367 tz->tz_minuteswest = vvd->tz_minuteswest;
368 tz->tz_dsttime = vvd->tz_dsttime;
372 return vdso_fallback_gettimeofday(tv, tz);