virt/kvm/arm/arch_timer.c

   1 /*
   2  * Copyright (C) 2012 ARM Ltd.
   3  * Author: Marc Zyngier <marc.zyngier@arm.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License version 2 as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  17  */
  18
  19 #include <linux/cpu.h>
  20 #include <linux/kvm.h>
  21 #include <linux/kvm_host.h>
  22 #include <linux/interrupt.h>
  23 #include <linux/irq.h>
  24 #include <linux/uaccess.h>
  25
  26 #include <clocksource/arm_arch_timer.h>
  27 #include <asm/arch_timer.h>
  28 #include <asm/kvm_hyp.h>
  29
  30 #include <kvm/arm_vgic.h>
  31 #include <kvm/arm_arch_timer.h>
  32
  33 #include "trace.h"
  34
  35 static struct timecounter *timecounter;
  36 static unsigned int host_vtimer_irq;
  37 static u32 host_vtimer_irq_flags;
  38
  39 static DEFINE_STATIC_KEY_FALSE(has_gic_active_state);
  40
  41 static const struct kvm_irq_level default_ptimer_irq = {
  42         .irq    = 30,
  43         .level  = 1,
  44 };
  45
  46 static const struct kvm_irq_level default_vtimer_irq = {
  47         .irq    = 27,
  48         .level  = 1,
  49 };
  50
  51 static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx);
  52 static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
  53                                  struct arch_timer_context *timer_ctx);
  54 static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx);
  55
  56 u64 kvm_phys_timer_read(void)
  57 {
  58         return timecounter->cc->read(timecounter->cc);
  59 }
  60
  61 static inline bool userspace_irqchip(struct kvm *kvm)
  62 {
  63         return static_branch_unlikely(&userspace_irqchip_in_use) &&
  64                 unlikely(!irqchip_in_kernel(kvm));
  65 }
  66
  67 static void soft_timer_start(struct hrtimer *hrt, u64 ns)
  68 {
  69         hrtimer_start(hrt, ktime_add_ns(ktime_get(), ns),
  70                       HRTIMER_MODE_ABS);
  71 }
  72
  73 static void soft_timer_cancel(struct hrtimer *hrt, struct work_struct *work)
  74 {
  75         hrtimer_cancel(hrt);
  76         if (work)
  77                 cancel_work_sync(work);
  78 }
  79
  80 static irqreturn_t kvm_arch_timer_handler(int irq, void *dev_id)
  81 {
  82         struct kvm_vcpu *vcpu = *(struct kvm_vcpu **)dev_id;
  83         struct arch_timer_context *vtimer;
  84
  85         /*
  86          * We may see a timer interrupt after vcpu_put() has been called which
  87          * sets the CPU's vcpu pointer to NULL, because even though the timer
  88          * has been disabled in vtimer_save_state(), the hardware interrupt
  89          * signal may not have been retired from the interrupt controller yet.
  90          */
  91         if (!vcpu)
  92                 return IRQ_HANDLED;
  93
  94         vtimer = vcpu_vtimer(vcpu);
  95         if (kvm_timer_should_fire(vtimer))
  96                 kvm_timer_update_irq(vcpu, true, vtimer);
  97
  98         if (userspace_irqchip(vcpu->kvm) &&
  99             !static_branch_unlikely(&has_gic_active_state))
 100                 disable_percpu_irq(host_vtimer_irq);
 101
 102         return IRQ_HANDLED;
 103 }
 104
 105 /*
 106  * Work function for handling the backup timer that we schedule when a vcpu is
 107  * no longer running, but had a timer programmed to fire in the future.
 108  */
 109 static void kvm_timer_inject_irq_work(struct work_struct *work)
 110 {
 111         struct kvm_vcpu *vcpu;
 112
 113         vcpu = container_of(work, struct kvm_vcpu, arch.timer_cpu.expired);
 114
 115         /*
 116          * If the vcpu is blocked we want to wake it up so that it will see
 117          * the timer has expired when entering the guest.
 118          */
 119         kvm_vcpu_wake_up(vcpu);
 120 }
 121
 122 static u64 kvm_timer_compute_delta(struct arch_timer_context *timer_ctx)
 123 {
 124         u64 cval, now;
 125
 126         cval = timer_ctx->cnt_cval;
 127         now = kvm_phys_timer_read() - timer_ctx->cntvoff;
 128
 129         if (now < cval) {
 130                 u64 ns;
 131
 132                 ns = cyclecounter_cyc2ns(timecounter->cc,
 133                                          cval - now,
 134                                          timecounter->mask,
 135                                          &timecounter->frac);
 136                 return ns;
 137         }
 138
 139         return 0;
 140 }
 141
 142 static bool kvm_timer_irq_can_fire(struct arch_timer_context *timer_ctx)
 143 {
 144         return !(timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_IT_MASK) &&
 145                 (timer_ctx->cnt_ctl & ARCH_TIMER_CTRL_ENABLE);
 146 }
 147
 148 /*
 149  * Returns the earliest expiration time in ns among guest timers.
 150  * Note that it will return 0 if none of timers can fire.
 151  */
 152 static u64 kvm_timer_earliest_exp(struct kvm_vcpu *vcpu)
 153 {
 154         u64 min_virt = ULLONG_MAX, min_phys = ULLONG_MAX;
 155         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 156         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 157
 158         if (kvm_timer_irq_can_fire(vtimer))
 159                 min_virt = kvm_timer_compute_delta(vtimer);
 160
 161         if (kvm_timer_irq_can_fire(ptimer))
 162                 min_phys = kvm_timer_compute_delta(ptimer);
 163
 164         /* If none of timers can fire, then return 0 */
 165         if ((min_virt == ULLONG_MAX) && (min_phys == ULLONG_MAX))
 166                 return 0;
 167
 168         return min(min_virt, min_phys);
 169 }
 170
 171 static enum hrtimer_restart kvm_bg_timer_expire(struct hrtimer *hrt)
 172 {
 173         struct arch_timer_cpu *timer;
 174         struct kvm_vcpu *vcpu;
 175         u64 ns;
 176
 177         timer = container_of(hrt, struct arch_timer_cpu, bg_timer);
 178         vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu);
 179
 180         /*
 181          * Check that the timer has really expired from the guest's
 182          * PoV (NTP on the host may have forced it to expire
 183          * early). If we should have slept longer, restart it.
 184          */
 185         ns = kvm_timer_earliest_exp(vcpu);
 186         if (unlikely(ns)) {
 187                 hrtimer_forward_now(hrt, ns_to_ktime(ns));
 188                 return HRTIMER_RESTART;
 189         }
 190
 191         schedule_work(&timer->expired);
 192         return HRTIMER_NORESTART;
 193 }
 194
 195 static enum hrtimer_restart kvm_phys_timer_expire(struct hrtimer *hrt)
 196 {
 197         struct arch_timer_context *ptimer;
 198         struct arch_timer_cpu *timer;
 199         struct kvm_vcpu *vcpu;
 200         u64 ns;
 201
 202         timer = container_of(hrt, struct arch_timer_cpu, phys_timer);
 203         vcpu = container_of(timer, struct kvm_vcpu, arch.timer_cpu);
 204         ptimer = vcpu_ptimer(vcpu);
 205
 206         /*
 207          * Check that the timer has really expired from the guest's
 208          * PoV (NTP on the host may have forced it to expire
 209          * early). If not ready, schedule for a later time.
 210          */
 211         ns = kvm_timer_compute_delta(ptimer);
 212         if (unlikely(ns)) {
 213                 hrtimer_forward_now(hrt, ns_to_ktime(ns));
 214                 return HRTIMER_RESTART;
 215         }
 216
 217         kvm_timer_update_irq(vcpu, true, ptimer);
 218         return HRTIMER_NORESTART;
 219 }
 220
 221 static bool kvm_timer_should_fire(struct arch_timer_context *timer_ctx)
 222 {
 223         u64 cval, now;
 224
 225         if (timer_ctx->loaded) {
 226                 u32 cnt_ctl;
 227
 228                 /* Only the virtual timer can be loaded so far */
 229                 cnt_ctl = read_sysreg_el0(cntv_ctl);
 230                 return  (cnt_ctl & ARCH_TIMER_CTRL_ENABLE) &&
 231                         (cnt_ctl & ARCH_TIMER_CTRL_IT_STAT) &&
 232                        !(cnt_ctl & ARCH_TIMER_CTRL_IT_MASK);
 233         }
 234
 235         if (!kvm_timer_irq_can_fire(timer_ctx))
 236                 return false;
 237
 238         cval = timer_ctx->cnt_cval;
 239         now = kvm_phys_timer_read() - timer_ctx->cntvoff;
 240
 241         return cval <= now;
 242 }
 243
 244 bool kvm_timer_is_pending(struct kvm_vcpu *vcpu)
 245 {
 246         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 247         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 248
 249         if (kvm_timer_should_fire(vtimer))
 250                 return true;
 251
 252         return kvm_timer_should_fire(ptimer);
 253 }
 254
 255 /*
 256  * Reflect the timer output level into the kvm_run structure
 257  */
 258 void kvm_timer_update_run(struct kvm_vcpu *vcpu)
 259 {
 260         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 261         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 262         struct kvm_sync_regs *regs = &vcpu->run->s.regs;
 263
 264         /* Populate the device bitmap with the timer states */
 265         regs->device_irq_level &= ~(KVM_ARM_DEV_EL1_VTIMER |
 266                                     KVM_ARM_DEV_EL1_PTIMER);
 267         if (kvm_timer_should_fire(vtimer))
 268                 regs->device_irq_level |= KVM_ARM_DEV_EL1_VTIMER;
 269         if (kvm_timer_should_fire(ptimer))
 270                 regs->device_irq_level |= KVM_ARM_DEV_EL1_PTIMER;
 271 }
 272
 273 static void kvm_timer_update_irq(struct kvm_vcpu *vcpu, bool new_level,
 274                                  struct arch_timer_context *timer_ctx)
 275 {
 276         int ret;
 277
 278         timer_ctx->irq.level = new_level;
 279         trace_kvm_timer_update_irq(vcpu->vcpu_id, timer_ctx->irq.irq,
 280                                    timer_ctx->irq.level);
 281
 282         if (!userspace_irqchip(vcpu->kvm)) {
 283                 ret = kvm_vgic_inject_irq(vcpu->kvm, vcpu->vcpu_id,
 284                                           timer_ctx->irq.irq,
 285                                           timer_ctx->irq.level,
 286                                           timer_ctx);
 287                 WARN_ON(ret);
 288         }
 289 }
 290
 291 /* Schedule the background timer for the emulated timer. */
 292 static void phys_timer_emulate(struct kvm_vcpu *vcpu)
 293 {
 294         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 295         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 296
 297         /*
 298          * If the timer can fire now, we don't need to have a soft timer
 299          * scheduled for the future.  If the timer cannot fire at all,
 300          * then we also don't need a soft timer.
 301          */
 302         if (kvm_timer_should_fire(ptimer) || !kvm_timer_irq_can_fire(ptimer)) {
 303                 soft_timer_cancel(&timer->phys_timer, NULL);
 304                 return;
 305         }
 306
 307         soft_timer_start(&timer->phys_timer, kvm_timer_compute_delta(ptimer));
 308 }
 309
 310 /*
 311  * Check if there was a change in the timer state, so that we should either
 312  * raise or lower the line level to the GIC or schedule a background timer to
 313  * emulate the physical timer.
 314  */
 315 static void kvm_timer_update_state(struct kvm_vcpu *vcpu)
 316 {
 317         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 318         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 319         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 320         bool level;
 321
 322         if (unlikely(!timer->enabled))
 323                 return;
 324
 325         /*
 326          * The vtimer virtual interrupt is a 'mapped' interrupt, meaning part
 327          * of its lifecycle is offloaded to the hardware, and we therefore may
 328          * not have lowered the irq.level value before having to signal a new
 329          * interrupt, but have to signal an interrupt every time the level is
 330          * asserted.
 331          */
 332         level = kvm_timer_should_fire(vtimer);
 333         kvm_timer_update_irq(vcpu, level, vtimer);
 334
 335         phys_timer_emulate(vcpu);
 336
 337         if (kvm_timer_should_fire(ptimer) != ptimer->irq.level)
 338                 kvm_timer_update_irq(vcpu, !ptimer->irq.level, ptimer);
 339 }
 340
 341 static void vtimer_save_state(struct kvm_vcpu *vcpu)
 342 {
 343         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 344         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 345         unsigned long flags;
 346
 347         local_irq_save(flags);
 348
 349         if (!vtimer->loaded)
 350                 goto out;
 351
 352         if (timer->enabled) {
 353                 vtimer->cnt_ctl = read_sysreg_el0(cntv_ctl);
 354                 vtimer->cnt_cval = read_sysreg_el0(cntv_cval);
 355         }
 356
 357         /* Disable the virtual timer */
 358         write_sysreg_el0(0, cntv_ctl);
 359         isb();
 360
 361         vtimer->loaded = false;
 362 out:
 363         local_irq_restore(flags);
 364 }
 365
 366 /*
 367  * Schedule the background timer before calling kvm_vcpu_block, so that this
 368  * thread is removed from its waitqueue and made runnable when there's a timer
 369  * interrupt to handle.
 370  */
 371 void kvm_timer_schedule(struct kvm_vcpu *vcpu)
 372 {
 373         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 374         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 375         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 376
 377         vtimer_save_state(vcpu);
 378
 379         /*
 380          * No need to schedule a background timer if any guest timer has
 381          * already expired, because kvm_vcpu_block will return before putting
 382          * the thread to sleep.
 383          */
 384         if (kvm_timer_should_fire(vtimer) || kvm_timer_should_fire(ptimer))
 385                 return;
 386
 387         /*
 388          * If both timers are not capable of raising interrupts (disabled or
 389          * masked), then there's no more work for us to do.
 390          */
 391         if (!kvm_timer_irq_can_fire(vtimer) && !kvm_timer_irq_can_fire(ptimer))
 392                 return;
 393
 394         /*
 395          * The guest timers have not yet expired, schedule a background timer.
 396          * Set the earliest expiration time among the guest timers.
 397          */
 398         soft_timer_start(&timer->bg_timer, kvm_timer_earliest_exp(vcpu));
 399 }
 400
 401 static void vtimer_restore_state(struct kvm_vcpu *vcpu)
 402 {
 403         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 404         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 405         unsigned long flags;
 406
 407         local_irq_save(flags);
 408
 409         if (vtimer->loaded)
 410                 goto out;
 411
 412         if (timer->enabled) {
 413                 write_sysreg_el0(vtimer->cnt_cval, cntv_cval);
 414                 isb();
 415                 write_sysreg_el0(vtimer->cnt_ctl, cntv_ctl);
 416         }
 417
 418         vtimer->loaded = true;
 419 out:
 420         local_irq_restore(flags);
 421 }
 422
 423 void kvm_timer_unschedule(struct kvm_vcpu *vcpu)
 424 {
 425         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 426
 427         vtimer_restore_state(vcpu);
 428
 429         soft_timer_cancel(&timer->bg_timer, &timer->expired);
 430 }
 431
 432 static void set_cntvoff(u64 cntvoff)
 433 {
 434         u32 low = lower_32_bits(cntvoff);
 435         u32 high = upper_32_bits(cntvoff);
 436
 437         /*
 438          * Since kvm_call_hyp doesn't fully support the ARM PCS especially on
 439          * 32-bit systems, but rather passes register by register shifted one
 440          * place (we put the function address in r0/x0), we cannot simply pass
 441          * a 64-bit value as an argument, but have to split the value in two
 442          * 32-bit halves.
 443          */
 444         kvm_call_hyp(__kvm_timer_set_cntvoff, low, high);
 445 }
 446
 447 static inline void set_vtimer_irq_phys_active(struct kvm_vcpu *vcpu, bool active)
 448 {
 449         int r;
 450         r = irq_set_irqchip_state(host_vtimer_irq, IRQCHIP_STATE_ACTIVE, active);
 451         WARN_ON(r);
 452 }
 453
 454 static void kvm_timer_vcpu_load_gic(struct kvm_vcpu *vcpu)
 455 {
 456         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 457         bool phys_active;
 458
 459         if (irqchip_in_kernel(vcpu->kvm))
 460                 phys_active = kvm_vgic_map_is_active(vcpu, vtimer->irq.irq);
 461         else
 462                 phys_active = vtimer->irq.level;
 463         set_vtimer_irq_phys_active(vcpu, phys_active);
 464 }
 465
 466 static void kvm_timer_vcpu_load_nogic(struct kvm_vcpu *vcpu)
 467 {
 468         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 469
 470         /*
 471          * When using a userspace irqchip with the architected timers and a
 472          * host interrupt controller that doesn't support an active state, we
 473          * must still prevent continuously exiting from the guest, and
 474          * therefore mask the physical interrupt by disabling it on the host
 475          * interrupt controller when the virtual level is high, such that the
 476          * guest can make forward progress.  Once we detect the output level
 477          * being de-asserted, we unmask the interrupt again so that we exit
 478          * from the guest when the timer fires.
 479          */
 480         if (vtimer->irq.level)
 481                 disable_percpu_irq(host_vtimer_irq);
 482         else
 483                 enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
 484 }
 485
 486 void kvm_timer_vcpu_load(struct kvm_vcpu *vcpu)
 487 {
 488         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 489         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 490         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 491
 492         if (unlikely(!timer->enabled))
 493                 return;
 494
 495         if (static_branch_likely(&has_gic_active_state))
 496                 kvm_timer_vcpu_load_gic(vcpu);
 497         else
 498                 kvm_timer_vcpu_load_nogic(vcpu);
 499
 500         set_cntvoff(vtimer->cntvoff);
 501
 502         vtimer_restore_state(vcpu);
 503
 504         /* Set the background timer for the physical timer emulation. */
 505         phys_timer_emulate(vcpu);
 506
 507         /* If the timer fired while we weren't running, inject it now */
 508         if (kvm_timer_should_fire(ptimer) != ptimer->irq.level)
 509                 kvm_timer_update_irq(vcpu, !ptimer->irq.level, ptimer);
 510 }
 511
 512 bool kvm_timer_should_notify_user(struct kvm_vcpu *vcpu)
 513 {
 514         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 515         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 516         struct kvm_sync_regs *sregs = &vcpu->run->s.regs;
 517         bool vlevel, plevel;
 518
 519         if (likely(irqchip_in_kernel(vcpu->kvm)))
 520                 return false;
 521
 522         vlevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_VTIMER;
 523         plevel = sregs->device_irq_level & KVM_ARM_DEV_EL1_PTIMER;
 524
 525         return kvm_timer_should_fire(vtimer) != vlevel ||
 526                kvm_timer_should_fire(ptimer) != plevel;
 527 }
 528
 529 void kvm_timer_vcpu_put(struct kvm_vcpu *vcpu)
 530 {
 531         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 532
 533         if (unlikely(!timer->enabled))
 534                 return;
 535
 536         vtimer_save_state(vcpu);
 537
 538         /*
 539          * Cancel the physical timer emulation, because the only case where we
 540          * need it after a vcpu_put is in the context of a sleeping VCPU, and
 541          * in that case we already factor in the deadline for the physical
 542          * timer when scheduling the bg_timer.
 543          *
 544          * In any case, we re-schedule the hrtimer for the physical timer when
 545          * coming back to the VCPU thread in kvm_timer_vcpu_load().
 546          */
 547         soft_timer_cancel(&timer->phys_timer, NULL);
 548
 549         /*
 550          * The kernel may decide to run userspace after calling vcpu_put, so
 551          * we reset cntvoff to 0 to ensure a consistent read between user
 552          * accesses to the virtual counter and kernel access to the physical
 553          * counter of non-VHE case. For VHE, the virtual counter uses a fixed
 554          * virtual offset of zero, so no need to zero CNTVOFF_EL2 register.
 555          */
 556         if (!has_vhe())
 557                 set_cntvoff(0);
 558 }
 559
 560 /*
 561  * With a userspace irqchip we have to check if the guest de-asserted the
 562  * timer and if so, unmask the timer irq signal on the host interrupt
 563  * controller to ensure that we see future timer signals.
 564  */
 565 static void unmask_vtimer_irq_user(struct kvm_vcpu *vcpu)
 566 {
 567         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 568
 569         if (!kvm_timer_should_fire(vtimer)) {
 570                 kvm_timer_update_irq(vcpu, false, vtimer);
 571                 if (static_branch_likely(&has_gic_active_state))
 572                         set_vtimer_irq_phys_active(vcpu, false);
 573                 else
 574                         enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
 575         }
 576 }
 577
 578 void kvm_timer_sync_hwstate(struct kvm_vcpu *vcpu)
 579 {
 580         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 581
 582         if (unlikely(!timer->enabled))
 583                 return;
 584
 585         if (unlikely(!irqchip_in_kernel(vcpu->kvm)))
 586                 unmask_vtimer_irq_user(vcpu);
 587 }
 588
 589 int kvm_timer_vcpu_reset(struct kvm_vcpu *vcpu)
 590 {
 591         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 592         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 593         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 594
 595         /*
 596          * The bits in CNTV_CTL are architecturally reset to UNKNOWN for ARMv8
 597          * and to 0 for ARMv7.  We provide an implementation that always
 598          * resets the timer to be disabled and unmasked and is compliant with
 599          * the ARMv7 architecture.
 600          */
 601         vtimer->cnt_ctl = 0;
 602         ptimer->cnt_ctl = 0;
 603         kvm_timer_update_state(vcpu);
 604
 605         if (timer->enabled && irqchip_in_kernel(vcpu->kvm))
 606                 kvm_vgic_reset_mapped_irq(vcpu, vtimer->irq.irq);
 607
 608         return 0;
 609 }
 610
 611 /* Make the updates of cntvoff for all vtimer contexts atomic */
 612 static void update_vtimer_cntvoff(struct kvm_vcpu *vcpu, u64 cntvoff)
 613 {
 614         int i;
 615         struct kvm *kvm = vcpu->kvm;
 616         struct kvm_vcpu *tmp;
 617
 618         mutex_lock(&kvm->lock);
 619         kvm_for_each_vcpu(i, tmp, kvm)
 620                 vcpu_vtimer(tmp)->cntvoff = cntvoff;
 621
 622         /*
 623          * When called from the vcpu create path, the CPU being created is not
 624          * included in the loop above, so we just set it here as well.
 625          */
 626         vcpu_vtimer(vcpu)->cntvoff = cntvoff;
 627         mutex_unlock(&kvm->lock);
 628 }
 629
 630 void kvm_timer_vcpu_init(struct kvm_vcpu *vcpu)
 631 {
 632         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 633         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 634         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 635
 636         /* Synchronize cntvoff across all vtimers of a VM. */
 637         update_vtimer_cntvoff(vcpu, kvm_phys_timer_read());
 638         vcpu_ptimer(vcpu)->cntvoff = 0;
 639
 640         INIT_WORK(&timer->expired, kvm_timer_inject_irq_work);
 641         hrtimer_init(&timer->bg_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 642         timer->bg_timer.function = kvm_bg_timer_expire;
 643
 644         hrtimer_init(&timer->phys_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
 645         timer->phys_timer.function = kvm_phys_timer_expire;
 646
 647         vtimer->irq.irq = default_vtimer_irq.irq;
 648         ptimer->irq.irq = default_ptimer_irq.irq;
 649 }
 650
 651 static void kvm_timer_init_interrupt(void *info)
 652 {
 653         enable_percpu_irq(host_vtimer_irq, host_vtimer_irq_flags);
 654 }
 655
 656 int kvm_arm_timer_set_reg(struct kvm_vcpu *vcpu, u64 regid, u64 value)
 657 {
 658         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 659         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 660
 661         switch (regid) {
 662         case KVM_REG_ARM_TIMER_CTL:
 663                 vtimer->cnt_ctl = value & ~ARCH_TIMER_CTRL_IT_STAT;
 664                 break;
 665         case KVM_REG_ARM_TIMER_CNT:
 666                 update_vtimer_cntvoff(vcpu, kvm_phys_timer_read() - value);
 667                 break;
 668         case KVM_REG_ARM_TIMER_CVAL:
 669                 vtimer->cnt_cval = value;
 670                 break;
 671         case KVM_REG_ARM_PTIMER_CTL:
 672                 ptimer->cnt_ctl = value & ~ARCH_TIMER_CTRL_IT_STAT;
 673                 break;
 674         case KVM_REG_ARM_PTIMER_CVAL:
 675                 ptimer->cnt_cval = value;
 676                 break;
 677
 678         default:
 679                 return -1;
 680         }
 681
 682         kvm_timer_update_state(vcpu);
 683         return 0;
 684 }
 685
 686 static u64 read_timer_ctl(struct arch_timer_context *timer)
 687 {
 688         /*
 689          * Set ISTATUS bit if it's expired.
 690          * Note that according to ARMv8 ARM Issue A.k, ISTATUS bit is
 691          * UNKNOWN when ENABLE bit is 0, so we chose to set ISTATUS bit
 692          * regardless of ENABLE bit for our implementation convenience.
 693          */
 694         if (!kvm_timer_compute_delta(timer))
 695                 return timer->cnt_ctl | ARCH_TIMER_CTRL_IT_STAT;
 696         else
 697                 return timer->cnt_ctl;
 698 }
 699
 700 u64 kvm_arm_timer_get_reg(struct kvm_vcpu *vcpu, u64 regid)
 701 {
 702         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 703         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 704
 705         switch (regid) {
 706         case KVM_REG_ARM_TIMER_CTL:
 707                 return read_timer_ctl(vtimer);
 708         case KVM_REG_ARM_TIMER_CNT:
 709                 return kvm_phys_timer_read() - vtimer->cntvoff;
 710         case KVM_REG_ARM_TIMER_CVAL:
 711                 return vtimer->cnt_cval;
 712         case KVM_REG_ARM_PTIMER_CTL:
 713                 return read_timer_ctl(ptimer);
 714         case KVM_REG_ARM_PTIMER_CVAL:
 715                 return ptimer->cnt_cval;
 716         case KVM_REG_ARM_PTIMER_CNT:
 717                 return kvm_phys_timer_read();
 718         }
 719         return (u64)-1;
 720 }
 721
 722 static int kvm_timer_starting_cpu(unsigned int cpu)
 723 {
 724         kvm_timer_init_interrupt(NULL);
 725         return 0;
 726 }
 727
 728 static int kvm_timer_dying_cpu(unsigned int cpu)
 729 {
 730         disable_percpu_irq(host_vtimer_irq);
 731         return 0;
 732 }
 733
 734 int kvm_timer_hyp_init(bool has_gic)
 735 {
 736         struct arch_timer_kvm_info *info;
 737         int err;
 738
 739         info = arch_timer_get_kvm_info();
 740         timecounter = &info->timecounter;
 741
 742         if (!timecounter->cc) {
 743                 kvm_err("kvm_arch_timer: uninitialized timecounter\n");
 744                 return -ENODEV;
 745         }
 746
 747         if (info->virtual_irq <= 0) {
 748                 kvm_err("kvm_arch_timer: invalid virtual timer IRQ: %d\n",
 749                         info->virtual_irq);
 750                 return -ENODEV;
 751         }
 752         host_vtimer_irq = info->virtual_irq;
 753
 754         host_vtimer_irq_flags = irq_get_trigger_type(host_vtimer_irq);
 755         if (host_vtimer_irq_flags != IRQF_TRIGGER_HIGH &&
 756             host_vtimer_irq_flags != IRQF_TRIGGER_LOW) {
 757                 kvm_err("Invalid trigger for IRQ%d, assuming level low\n",
 758                         host_vtimer_irq);
 759                 host_vtimer_irq_flags = IRQF_TRIGGER_LOW;
 760         }
 761
 762         err = request_percpu_irq(host_vtimer_irq, kvm_arch_timer_handler,
 763                                  "kvm guest timer", kvm_get_running_vcpus());
 764         if (err) {
 765                 kvm_err("kvm_arch_timer: can't request interrupt %d (%d)\n",
 766                         host_vtimer_irq, err);
 767                 return err;
 768         }
 769
 770         if (has_gic) {
 771                 err = irq_set_vcpu_affinity(host_vtimer_irq,
 772                                             kvm_get_running_vcpus());
 773                 if (err) {
 774                         kvm_err("kvm_arch_timer: error setting vcpu affinity\n");
 775                         goto out_free_irq;
 776                 }
 777
 778                 static_branch_enable(&has_gic_active_state);
 779         }
 780
 781         kvm_debug("virtual timer IRQ%d\n", host_vtimer_irq);
 782
 783         cpuhp_setup_state(CPUHP_AP_KVM_ARM_TIMER_STARTING,
 784                           "kvm/arm/timer:starting", kvm_timer_starting_cpu,
 785                           kvm_timer_dying_cpu);
 786         return 0;
 787 out_free_irq:
 788         free_percpu_irq(host_vtimer_irq, kvm_get_running_vcpus());
 789         return err;
 790 }
 791
 792 void kvm_timer_vcpu_terminate(struct kvm_vcpu *vcpu)
 793 {
 794         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 795         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 796
 797         soft_timer_cancel(&timer->bg_timer, &timer->expired);
 798         soft_timer_cancel(&timer->phys_timer, NULL);
 799         kvm_vgic_unmap_phys_irq(vcpu, vtimer->irq.irq);
 800 }
 801
 802 static bool timer_irqs_are_valid(struct kvm_vcpu *vcpu)
 803 {
 804         int vtimer_irq, ptimer_irq;
 805         int i, ret;
 806
 807         vtimer_irq = vcpu_vtimer(vcpu)->irq.irq;
 808         ret = kvm_vgic_set_owner(vcpu, vtimer_irq, vcpu_vtimer(vcpu));
 809         if (ret)
 810                 return false;
 811
 812         ptimer_irq = vcpu_ptimer(vcpu)->irq.irq;
 813         ret = kvm_vgic_set_owner(vcpu, ptimer_irq, vcpu_ptimer(vcpu));
 814         if (ret)
 815                 return false;
 816
 817         kvm_for_each_vcpu(i, vcpu, vcpu->kvm) {
 818                 if (vcpu_vtimer(vcpu)->irq.irq != vtimer_irq ||
 819                     vcpu_ptimer(vcpu)->irq.irq != ptimer_irq)
 820                         return false;
 821         }
 822
 823         return true;
 824 }
 825
 826 bool kvm_arch_timer_get_input_level(int vintid)
 827 {
 828         struct kvm_vcpu *vcpu = kvm_arm_get_running_vcpu();
 829         struct arch_timer_context *timer;
 830
 831         if (vintid == vcpu_vtimer(vcpu)->irq.irq)
 832                 timer = vcpu_vtimer(vcpu);
 833         else
 834                 BUG(); /* We only map the vtimer so far */
 835
 836         return kvm_timer_should_fire(timer);
 837 }
 838
 839 int kvm_timer_enable(struct kvm_vcpu *vcpu)
 840 {
 841         struct arch_timer_cpu *timer = &vcpu->arch.timer_cpu;
 842         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 843         int ret;
 844
 845         if (timer->enabled)
 846                 return 0;
 847
 848         /* Without a VGIC we do not map virtual IRQs to physical IRQs */
 849         if (!irqchip_in_kernel(vcpu->kvm))
 850                 goto no_vgic;
 851
 852         if (!vgic_initialized(vcpu->kvm))
 853                 return -ENODEV;
 854
 855         if (!timer_irqs_are_valid(vcpu)) {
 856                 kvm_debug("incorrectly configured timer irqs\n");
 857                 return -EINVAL;
 858         }
 859
 860         ret = kvm_vgic_map_phys_irq(vcpu, host_vtimer_irq, vtimer->irq.irq,
 861                                     kvm_arch_timer_get_input_level);
 862         if (ret)
 863                 return ret;
 864
 865 no_vgic:
 866         timer->enabled = 1;
 867         return 0;
 868 }
 869
 870 /*
 871  * On VHE system, we only need to configure trap on physical timer and counter
 872  * accesses in EL0 and EL1 once, not for every world switch.
 873  * The host kernel runs at EL2 with HCR_EL2.TGE == 1,
 874  * and this makes those bits have no effect for the host kernel execution.
 875  */
 876 void kvm_timer_init_vhe(void)
 877 {
 878         /* When HCR_EL2.E2H ==1, EL1PCEN and EL1PCTEN are shifted by 10 */
 879         u32 cnthctl_shift = 10;
 880         u64 val;
 881
 882         /*
 883          * Disallow physical timer access for the guest.
 884          * Physical counter access is allowed.
 885          */
 886         val = read_sysreg(cnthctl_el2);
 887         val &= ~(CNTHCTL_EL1PCEN << cnthctl_shift);
 888         val |= (CNTHCTL_EL1PCTEN << cnthctl_shift);
 889         write_sysreg(val, cnthctl_el2);
 890 }
 891
 892 static void set_timer_irqs(struct kvm *kvm, int vtimer_irq, int ptimer_irq)
 893 {
 894         struct kvm_vcpu *vcpu;
 895         int i;
 896
 897         kvm_for_each_vcpu(i, vcpu, kvm) {
 898                 vcpu_vtimer(vcpu)->irq.irq = vtimer_irq;
 899                 vcpu_ptimer(vcpu)->irq.irq = ptimer_irq;
 900         }
 901 }
 902
 903 int kvm_arm_timer_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
 904 {
 905         int __user *uaddr = (int __user *)(long)attr->addr;
 906         struct arch_timer_context *vtimer = vcpu_vtimer(vcpu);
 907         struct arch_timer_context *ptimer = vcpu_ptimer(vcpu);
 908         int irq;
 909
 910         if (!irqchip_in_kernel(vcpu->kvm))
 911                 return -EINVAL;
 912
 913         if (get_user(irq, uaddr))
 914                 return -EFAULT;
 915
 916         if (!(irq_is_ppi(irq)))
 917                 return -EINVAL;
 918
 919         if (vcpu->arch.timer_cpu.enabled)
 920                 return -EBUSY;
 921
 922         switch (attr->attr) {
 923         case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
 924                 set_timer_irqs(vcpu->kvm, irq, ptimer->irq.irq);
 925                 break;
 926         case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
 927                 set_timer_irqs(vcpu->kvm, vtimer->irq.irq, irq);
 928                 break;
 929         default:
 930                 return -ENXIO;
 931         }
 932
 933         return 0;
 934 }
 935
 936 int kvm_arm_timer_get_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
 937 {
 938         int __user *uaddr = (int __user *)(long)attr->addr;
 939         struct arch_timer_context *timer;
 940         int irq;
 941
 942         switch (attr->attr) {
 943         case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
 944                 timer = vcpu_vtimer(vcpu);
 945                 break;
 946         case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
 947                 timer = vcpu_ptimer(vcpu);
 948                 break;
 949         default:
 950                 return -ENXIO;
 951         }
 952
 953         irq = timer->irq.irq;
 954         return put_user(irq, uaddr);
 955 }
 956
 957 int kvm_arm_timer_has_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
 958 {
 959         switch (attr->attr) {
 960         case KVM_ARM_VCPU_TIMER_IRQ_VTIMER:
 961         case KVM_ARM_VCPU_TIMER_IRQ_PTIMER:
 962                 return 0;
 963         }
 964
 965         return -ENXIO;
 966 }