1 // SPDX-License-Identifier: GPL-2.0
3 * Xen time implementation.
5 * This is implemented in terms of a clocksource driver which uses
6 * the hypervisor clock as a nanosecond timebase, and a clockevent
7 * driver which uses the hypervisor's timer mechanism.
9 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
11 #include <linux/kernel.h>
12 #include <linux/interrupt.h>
13 #include <linux/clocksource.h>
14 #include <linux/clockchips.h>
15 #include <linux/gfp.h>
16 #include <linux/slab.h>
17 #include <linux/pvclock_gtod.h>
18 #include <linux/timekeeper_internal.h>
20 #include <asm/pvclock.h>
21 #include <asm/xen/hypervisor.h>
22 #include <asm/xen/hypercall.h>
24 #include <xen/events.h>
25 #include <xen/features.h>
26 #include <xen/interface/xen.h>
27 #include <xen/interface/vcpu.h>
31 /* Minimum amount of time until next clock event fires */
32 #define TIMER_SLOP 100000
34 static u64 xen_sched_clock_offset __read_mostly;
36 /* Get the TSC speed from Xen */
37 static unsigned long xen_tsc_khz(void)
39 struct pvclock_vcpu_time_info *info =
40 &HYPERVISOR_shared_info->vcpu_info[0].time;
42 setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ);
43 return pvclock_tsc_khz(info);
46 static u64 xen_clocksource_read(void)
48 struct pvclock_vcpu_time_info *src;
51 preempt_disable_notrace();
52 src = &__this_cpu_read(xen_vcpu)->time;
53 ret = pvclock_clocksource_read(src);
54 preempt_enable_notrace();
58 static u64 xen_clocksource_get_cycles(struct clocksource *cs)
60 return xen_clocksource_read();
63 static u64 xen_sched_clock(void)
65 return xen_clocksource_read() - xen_sched_clock_offset;
68 static void xen_read_wallclock(struct timespec64 *ts)
70 struct shared_info *s = HYPERVISOR_shared_info;
71 struct pvclock_wall_clock *wall_clock = &(s->wc);
72 struct pvclock_vcpu_time_info *vcpu_time;
74 vcpu_time = &get_cpu_var(xen_vcpu)->time;
75 pvclock_read_wallclock(wall_clock, vcpu_time, ts);
76 put_cpu_var(xen_vcpu);
79 static void xen_get_wallclock(struct timespec64 *now)
81 xen_read_wallclock(now);
84 static int xen_set_wallclock(const struct timespec64 *now)
89 static int xen_pvclock_gtod_notify(struct notifier_block *nb,
90 unsigned long was_set, void *priv)
92 /* Protected by the calling core code serialization */
93 static struct timespec64 next_sync;
95 struct xen_platform_op op;
96 struct timespec64 now;
97 struct timekeeper *tk = priv;
98 static bool settime64_supported = true;
101 now.tv_sec = tk->xtime_sec;
102 now.tv_nsec = (long)(tk->tkr_mono.xtime_nsec >> tk->tkr_mono.shift);
105 * We only take the expensive HV call when the clock was set
106 * or when the 11 minutes RTC synchronization time elapsed.
108 if (!was_set && timespec64_compare(&now, &next_sync) < 0)
112 if (settime64_supported) {
113 op.cmd = XENPF_settime64;
114 op.u.settime64.mbz = 0;
115 op.u.settime64.secs = now.tv_sec;
116 op.u.settime64.nsecs = now.tv_nsec;
117 op.u.settime64.system_time = xen_clocksource_read();
119 op.cmd = XENPF_settime32;
120 op.u.settime32.secs = now.tv_sec;
121 op.u.settime32.nsecs = now.tv_nsec;
122 op.u.settime32.system_time = xen_clocksource_read();
125 ret = HYPERVISOR_platform_op(&op);
127 if (ret == -ENOSYS && settime64_supported) {
128 settime64_supported = false;
135 * Move the next drift compensation time 11 minutes
136 * ahead. That's emulating the sync_cmos_clock() update for
140 next_sync.tv_sec += 11 * 60;
145 static struct notifier_block xen_pvclock_gtod_notifier = {
146 .notifier_call = xen_pvclock_gtod_notify,
149 static int xen_cs_enable(struct clocksource *cs)
151 vclocks_set_used(VDSO_CLOCKMODE_PVCLOCK);
155 static struct clocksource xen_clocksource __read_mostly = {
158 .read = xen_clocksource_get_cycles,
159 .mask = CLOCKSOURCE_MASK(64),
160 .flags = CLOCK_SOURCE_IS_CONTINUOUS,
161 .enable = xen_cs_enable,
165 Xen clockevent implementation
167 Xen has two clockevent implementations:
169 The old timer_op one works with all released versions of Xen prior
170 to version 3.0.4. This version of the hypervisor provides a
171 single-shot timer with nanosecond resolution. However, sharing the
172 same event channel is a 100Hz tick which is delivered while the
173 vcpu is running. We don't care about or use this tick, but it will
174 cause the core time code to think the timer fired too soon, and
175 will end up resetting it each time. It could be filtered, but
176 doing so has complications when the ktime clocksource is not yet
177 the xen clocksource (ie, at boot time).
179 The new vcpu_op-based timer interface allows the tick timer period
180 to be changed or turned off. The tick timer is not useful as a
181 periodic timer because events are only delivered to running vcpus.
182 The one-shot timer can report when a timeout is in the past, so
183 set_next_event is capable of returning -ETIME when appropriate.
184 This interface is used when available.
189 Get a hypervisor absolute time. In theory we could maintain an
190 offset between the kernel's time and the hypervisor's time, and
191 apply that to a kernel's absolute timeout. Unfortunately the
192 hypervisor and kernel times can drift even if the kernel is using
193 the Xen clocksource, because ntp can warp the kernel's clocksource.
195 static s64 get_abs_timeout(unsigned long delta)
197 return xen_clocksource_read() + delta;
200 static int xen_timerop_shutdown(struct clock_event_device *evt)
203 HYPERVISOR_set_timer_op(0);
208 static int xen_timerop_set_next_event(unsigned long delta,
209 struct clock_event_device *evt)
211 WARN_ON(!clockevent_state_oneshot(evt));
213 if (HYPERVISOR_set_timer_op(get_abs_timeout(delta)) < 0)
216 /* We may have missed the deadline, but there's no real way of
217 knowing for sure. If the event was in the past, then we'll
218 get an immediate interrupt. */
223 static struct clock_event_device xen_timerop_clockevent __ro_after_init = {
225 .features = CLOCK_EVT_FEAT_ONESHOT,
227 .max_delta_ns = 0xffffffff,
228 .max_delta_ticks = 0xffffffff,
229 .min_delta_ns = TIMER_SLOP,
230 .min_delta_ticks = TIMER_SLOP,
236 .set_state_shutdown = xen_timerop_shutdown,
237 .set_next_event = xen_timerop_set_next_event,
240 static int xen_vcpuop_shutdown(struct clock_event_device *evt)
242 int cpu = smp_processor_id();
244 if (HYPERVISOR_vcpu_op(VCPUOP_stop_singleshot_timer, xen_vcpu_nr(cpu),
246 HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu),
253 static int xen_vcpuop_set_oneshot(struct clock_event_device *evt)
255 int cpu = smp_processor_id();
257 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu),
264 static int xen_vcpuop_set_next_event(unsigned long delta,
265 struct clock_event_device *evt)
267 int cpu = smp_processor_id();
268 struct vcpu_set_singleshot_timer single;
271 WARN_ON(!clockevent_state_oneshot(evt));
273 single.timeout_abs_ns = get_abs_timeout(delta);
274 /* Get an event anyway, even if the timeout is already expired */
277 ret = HYPERVISOR_vcpu_op(VCPUOP_set_singleshot_timer, xen_vcpu_nr(cpu),
284 static struct clock_event_device xen_vcpuop_clockevent __ro_after_init = {
286 .features = CLOCK_EVT_FEAT_ONESHOT,
288 .max_delta_ns = 0xffffffff,
289 .max_delta_ticks = 0xffffffff,
290 .min_delta_ns = TIMER_SLOP,
291 .min_delta_ticks = TIMER_SLOP,
297 .set_state_shutdown = xen_vcpuop_shutdown,
298 .set_state_oneshot = xen_vcpuop_set_oneshot,
299 .set_next_event = xen_vcpuop_set_next_event,
302 static const struct clock_event_device *xen_clockevent =
303 &xen_timerop_clockevent;
305 struct xen_clock_event_device {
306 struct clock_event_device evt;
309 static DEFINE_PER_CPU(struct xen_clock_event_device, xen_clock_events) = { .evt.irq = -1 };
311 static irqreturn_t xen_timer_interrupt(int irq, void *dev_id)
313 struct clock_event_device *evt = this_cpu_ptr(&xen_clock_events.evt);
317 if (evt->event_handler) {
318 evt->event_handler(evt);
325 void xen_teardown_timer(int cpu)
327 struct clock_event_device *evt;
328 evt = &per_cpu(xen_clock_events, cpu).evt;
331 unbind_from_irqhandler(evt->irq, NULL);
336 void xen_setup_timer(int cpu)
338 struct xen_clock_event_device *xevt = &per_cpu(xen_clock_events, cpu);
339 struct clock_event_device *evt = &xevt->evt;
342 WARN(evt->irq >= 0, "IRQ%d for CPU%d is already allocated\n", evt->irq, cpu);
344 xen_teardown_timer(cpu);
346 printk(KERN_INFO "installing Xen timer for CPU %d\n", cpu);
348 snprintf(xevt->name, sizeof(xevt->name), "timer%d", cpu);
350 irq = bind_virq_to_irqhandler(VIRQ_TIMER, cpu, xen_timer_interrupt,
351 IRQF_PERCPU|IRQF_NOBALANCING|IRQF_TIMER|
352 IRQF_FORCE_RESUME|IRQF_EARLY_RESUME,
354 (void)xen_set_irq_priority(irq, XEN_IRQ_PRIORITY_MAX);
356 memcpy(evt, xen_clockevent, sizeof(*evt));
358 evt->cpumask = cpumask_of(cpu);
363 void xen_setup_cpu_clockevents(void)
365 clockevents_register_device(this_cpu_ptr(&xen_clock_events.evt));
368 void xen_timer_resume(void)
372 if (xen_clockevent != &xen_vcpuop_clockevent)
375 for_each_online_cpu(cpu) {
376 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer,
377 xen_vcpu_nr(cpu), NULL))
382 static const struct pv_time_ops xen_time_ops __initconst = {
383 .sched_clock = xen_sched_clock,
384 .steal_clock = xen_steal_clock,
387 static struct pvclock_vsyscall_time_info *xen_clock __read_mostly;
388 static u64 xen_clock_value_saved;
390 void xen_save_time_memory_area(void)
392 struct vcpu_register_time_memory_area t;
395 xen_clock_value_saved = xen_clocksource_read() - xen_sched_clock_offset;
402 ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t);
404 pr_notice("Cannot save secondary vcpu_time_info (err %d)",
407 clear_page(xen_clock);
410 void xen_restore_time_memory_area(void)
412 struct vcpu_register_time_memory_area t;
418 t.addr.v = &xen_clock->pvti;
420 ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t);
423 * We don't disable VDSO_CLOCKMODE_PVCLOCK entirely if it fails to
424 * register the secondary time info with Xen or if we migrated to a
425 * host without the necessary flags. On both of these cases what
426 * happens is either process seeing a zeroed out pvti or seeing no
427 * PVCLOCK_TSC_STABLE_BIT bit set. Userspace checks the latter and
428 * if 0, it discards the data in pvti and fallbacks to a system
429 * call for a reliable timestamp.
432 pr_notice("Cannot restore secondary vcpu_time_info (err %d)",
436 /* Need pvclock_resume() before using xen_clocksource_read(). */
438 xen_sched_clock_offset = xen_clocksource_read() - xen_clock_value_saved;
441 static void xen_setup_vsyscall_time_info(void)
443 struct vcpu_register_time_memory_area t;
444 struct pvclock_vsyscall_time_info *ti;
447 ti = (struct pvclock_vsyscall_time_info *)get_zeroed_page(GFP_KERNEL);
451 t.addr.v = &ti->pvti;
453 ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area, 0, &t);
455 pr_notice("xen: VDSO_CLOCKMODE_PVCLOCK not supported (err %d)\n", ret);
456 free_page((unsigned long)ti);
461 * If primary time info had this bit set, secondary should too since
462 * it's the same data on both just different memory regions. But we
463 * still check it in case hypervisor is buggy.
465 if (!(ti->pvti.flags & PVCLOCK_TSC_STABLE_BIT)) {
467 ret = HYPERVISOR_vcpu_op(VCPUOP_register_vcpu_time_memory_area,
470 free_page((unsigned long)ti);
472 pr_notice("xen: VDSO_CLOCKMODE_PVCLOCK not supported (tsc unstable)\n");
477 pvclock_set_pvti_cpu0_va(xen_clock);
479 xen_clocksource.vdso_clock_mode = VDSO_CLOCKMODE_PVCLOCK;
482 static void __init xen_time_init(void)
484 struct pvclock_vcpu_time_info *pvti;
485 int cpu = smp_processor_id();
486 struct timespec64 tp;
488 /* As Dom0 is never moved, no penalty on using TSC there */
489 if (xen_initial_domain())
490 xen_clocksource.rating = 275;
492 clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC);
494 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, xen_vcpu_nr(cpu),
496 /* Successfully turned off 100Hz tick, so we have the
497 vcpuop-based timer interface */
498 printk(KERN_DEBUG "Xen: using vcpuop timer interface\n");
499 xen_clockevent = &xen_vcpuop_clockevent;
502 /* Set initial system time with full resolution */
503 xen_read_wallclock(&tp);
504 do_settimeofday64(&tp);
506 setup_force_cpu_cap(X86_FEATURE_TSC);
509 * We check ahead on the primary time info if this
510 * bit is supported hence speeding up Xen clocksource.
512 pvti = &__this_cpu_read(xen_vcpu)->time;
513 if (pvti->flags & PVCLOCK_TSC_STABLE_BIT) {
514 pvclock_set_flags(PVCLOCK_TSC_STABLE_BIT);
515 xen_setup_vsyscall_time_info();
518 xen_setup_runstate_info(cpu);
519 xen_setup_timer(cpu);
520 xen_setup_cpu_clockevents();
522 xen_time_setup_guest();
524 if (xen_initial_domain())
525 pvclock_gtod_register_notifier(&xen_pvclock_gtod_notifier);
528 void __init xen_init_time_ops(void)
530 xen_sched_clock_offset = xen_clocksource_read();
531 pv_ops.time = xen_time_ops;
533 x86_init.timers.timer_init = xen_time_init;
534 x86_init.timers.setup_percpu_clockev = x86_init_noop;
535 x86_cpuinit.setup_percpu_clockev = x86_init_noop;
537 x86_platform.calibrate_tsc = xen_tsc_khz;
538 x86_platform.get_wallclock = xen_get_wallclock;
539 /* Dom0 uses the native method to set the hardware RTC. */
540 if (!xen_initial_domain())
541 x86_platform.set_wallclock = xen_set_wallclock;
544 #ifdef CONFIG_XEN_PVHVM
545 static void xen_hvm_setup_cpu_clockevents(void)
547 int cpu = smp_processor_id();
548 xen_setup_runstate_info(cpu);
550 * xen_setup_timer(cpu) - snprintf is bad in atomic context. Hence
551 * doing it xen_hvm_cpu_notify (which gets called by smp_init during
552 * early bootup and also during CPU hotplug events).
554 xen_setup_cpu_clockevents();
557 void __init xen_hvm_init_time_ops(void)
559 static bool hvm_time_initialized;
561 if (hvm_time_initialized)
565 * vector callback is needed otherwise we cannot receive interrupts
566 * on cpu > 0 and at this point we don't know how many cpus are
569 if (!xen_have_vector_callback)
572 if (!xen_feature(XENFEAT_hvm_safe_pvclock)) {
573 pr_info_once("Xen doesn't support pvclock on HVM, disable pv timer");
578 * Only MAX_VIRT_CPUS 'vcpu_info' are embedded inside 'shared_info'.
579 * The __this_cpu_read(xen_vcpu) is still NULL when Xen HVM guest
580 * boots on vcpu >= MAX_VIRT_CPUS (e.g., kexec), To access
581 * __this_cpu_read(xen_vcpu) via xen_clocksource_read() will panic.
583 * The xen_hvm_init_time_ops() should be called again later after
584 * __this_cpu_read(xen_vcpu) is available.
586 if (!__this_cpu_read(xen_vcpu)) {
587 pr_info("Delay xen_init_time_common() as kernel is running on vcpu=%d\n",
592 xen_sched_clock_offset = xen_clocksource_read();
593 pv_ops.time = xen_time_ops;
594 x86_init.timers.setup_percpu_clockev = xen_time_init;
595 x86_cpuinit.setup_percpu_clockev = xen_hvm_setup_cpu_clockevents;
597 x86_platform.calibrate_tsc = xen_tsc_khz;
598 x86_platform.get_wallclock = xen_get_wallclock;
599 x86_platform.set_wallclock = xen_set_wallclock;
601 hvm_time_initialized = true;
605 /* Kernel parameter to specify Xen timer slop */
606 static int __init parse_xen_timer_slop(char *ptr)
608 unsigned long slop = memparse(ptr, NULL);
610 xen_timerop_clockevent.min_delta_ns = slop;
611 xen_timerop_clockevent.min_delta_ticks = slop;
612 xen_vcpuop_clockevent.min_delta_ns = slop;
613 xen_vcpuop_clockevent.min_delta_ticks = slop;
617 early_param("xen_timer_slop", parse_xen_timer_slop);