arch/x86/kvm/lapic.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2
   3 /*
   4  * Local APIC virtualization
   5  *
   6  * Copyright (C) 2006 Qumranet, Inc.
   7  * Copyright (C) 2007 Novell
   8  * Copyright (C) 2007 Intel
   9  * Copyright 2009 Red Hat, Inc. and/or its affiliates.
  10  *
  11  * Authors:
  12  *   Dor Laor <dor.laor@qumranet.com>
  13  *   Gregory Haskins <ghaskins@novell.com>
  14  *   Yaozu (Eddie) Dong <eddie.dong@intel.com>
  15  *
  16  * Based on Xen 3.1 code, Copyright (c) 2004, Intel Corporation.
  17  */
  18
  19 #include <linux/kvm_host.h>
  20 #include <linux/kvm.h>
  21 #include <linux/mm.h>
  22 #include <linux/highmem.h>
  23 #include <linux/smp.h>
  24 #include <linux/hrtimer.h>
  25 #include <linux/io.h>
  26 #include <linux/export.h>
  27 #include <linux/math64.h>
  28 #include <linux/slab.h>
  29 #include <asm/processor.h>
  30 #include <asm/msr.h>
  31 #include <asm/page.h>
  32 #include <asm/current.h>
  33 #include <asm/apicdef.h>
  34 #include <asm/delay.h>
  35 #include <linux/atomic.h>
  36 #include <linux/jump_label.h>
  37 #include "kvm_cache_regs.h"
  38 #include "irq.h"
  39 #include "ioapic.h"
  40 #include "trace.h"
  41 #include "x86.h"
  42 #include "cpuid.h"
  43 #include "hyperv.h"
  44
  45 #ifndef CONFIG_X86_64
  46 #define mod_64(x, y) ((x) - (y) * div64_u64(x, y))
  47 #else
  48 #define mod_64(x, y) ((x) % (y))
  49 #endif
  50
  51 #define PRId64 "d"
  52 #define PRIx64 "llx"
  53 #define PRIu64 "u"
  54 #define PRIo64 "o"
  55
  56 /* 14 is the version for Xeon and Pentium 8.4.8*/
  57 #define APIC_VERSION                    (0x14UL | ((KVM_APIC_LVT_NUM - 1) << 16))
  58 #define LAPIC_MMIO_LENGTH               (1 << 12)
  59 /* followed define is not in apicdef.h */
  60 #define MAX_APIC_VECTOR                 256
  61 #define APIC_VECTORS_PER_REG            32
  62
  63 static bool lapic_timer_advance_dynamic __read_mostly;
  64 #define LAPIC_TIMER_ADVANCE_ADJUST_MIN  100     /* clock cycles */
  65 #define LAPIC_TIMER_ADVANCE_ADJUST_MAX  10000   /* clock cycles */
  66 #define LAPIC_TIMER_ADVANCE_NS_INIT     1000
  67 #define LAPIC_TIMER_ADVANCE_NS_MAX     5000
  68 /* step-by-step approximation to mitigate fluctuation */
  69 #define LAPIC_TIMER_ADVANCE_ADJUST_STEP 8
  70
  71 static inline int apic_test_vector(int vec, void *bitmap)
  72 {
  73         return test_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
  74 }
  75
  76 bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector)
  77 {
  78         struct kvm_lapic *apic = vcpu->arch.apic;
  79
  80         return apic_test_vector(vector, apic->regs + APIC_ISR) ||
  81                 apic_test_vector(vector, apic->regs + APIC_IRR);
  82 }
  83
  84 static inline int __apic_test_and_set_vector(int vec, void *bitmap)
  85 {
  86         return __test_and_set_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
  87 }
  88
  89 static inline int __apic_test_and_clear_vector(int vec, void *bitmap)
  90 {
  91         return __test_and_clear_bit(VEC_POS(vec), (bitmap) + REG_POS(vec));
  92 }
  93
  94 __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_hw_disabled, HZ);
  95 __read_mostly DEFINE_STATIC_KEY_DEFERRED_FALSE(apic_sw_disabled, HZ);
  96
  97 static inline int apic_enabled(struct kvm_lapic *apic)
  98 {
  99         return kvm_apic_sw_enabled(apic) &&     kvm_apic_hw_enabled(apic);
 100 }
 101
 102 #define LVT_MASK        \
 103         (APIC_LVT_MASKED | APIC_SEND_PENDING | APIC_VECTOR_MASK)
 104
 105 #define LINT_MASK       \
 106         (LVT_MASK | APIC_MODE_MASK | APIC_INPUT_POLARITY | \
 107          APIC_LVT_REMOTE_IRR | APIC_LVT_LEVEL_TRIGGER)
 108
 109 static inline u32 kvm_x2apic_id(struct kvm_lapic *apic)
 110 {
 111         return apic->vcpu->vcpu_id;
 112 }
 113
 114 static bool kvm_can_post_timer_interrupt(struct kvm_vcpu *vcpu)
 115 {
 116         return pi_inject_timer && kvm_vcpu_apicv_active(vcpu) &&
 117                 (kvm_mwait_in_guest(vcpu->kvm) || kvm_hlt_in_guest(vcpu->kvm));
 118 }
 119
 120 bool kvm_can_use_hv_timer(struct kvm_vcpu *vcpu)
 121 {
 122         return kvm_x86_ops.set_hv_timer
 123                && !(kvm_mwait_in_guest(vcpu->kvm) ||
 124                     kvm_can_post_timer_interrupt(vcpu));
 125 }
 126 EXPORT_SYMBOL_GPL(kvm_can_use_hv_timer);
 127
 128 static bool kvm_use_posted_timer_interrupt(struct kvm_vcpu *vcpu)
 129 {
 130         return kvm_can_post_timer_interrupt(vcpu) && vcpu->mode == IN_GUEST_MODE;
 131 }
 132
 133 static inline bool kvm_apic_map_get_logical_dest(struct kvm_apic_map *map,
 134                 u32 dest_id, struct kvm_lapic ***cluster, u16 *mask) {
 135         switch (map->mode) {
 136         case KVM_APIC_MODE_X2APIC: {
 137                 u32 offset = (dest_id >> 16) * 16;
 138                 u32 max_apic_id = map->max_apic_id;
 139
 140                 if (offset <= max_apic_id) {
 141                         u8 cluster_size = min(max_apic_id - offset + 1, 16U);
 142
 143                         offset = array_index_nospec(offset, map->max_apic_id + 1);
 144                         *cluster = &map->phys_map[offset];
 145                         *mask = dest_id & (0xffff >> (16 - cluster_size));
 146                 } else {
 147                         *mask = 0;
 148                 }
 149
 150                 return true;
 151                 }
 152         case KVM_APIC_MODE_XAPIC_FLAT:
 153                 *cluster = map->xapic_flat_map;
 154                 *mask = dest_id & 0xff;
 155                 return true;
 156         case KVM_APIC_MODE_XAPIC_CLUSTER:
 157                 *cluster = map->xapic_cluster_map[(dest_id >> 4) & 0xf];
 158                 *mask = dest_id & 0xf;
 159                 return true;
 160         default:
 161                 /* Not optimized. */
 162                 return false;
 163         }
 164 }
 165
 166 static void kvm_apic_map_free(struct rcu_head *rcu)
 167 {
 168         struct kvm_apic_map *map = container_of(rcu, struct kvm_apic_map, rcu);
 169
 170         kvfree(map);
 171 }
 172
 173 /*
 174  * CLEAN -> DIRTY and UPDATE_IN_PROGRESS -> DIRTY changes happen without a lock.
 175  *
 176  * DIRTY -> UPDATE_IN_PROGRESS and UPDATE_IN_PROGRESS -> CLEAN happen with
 177  * apic_map_lock_held.
 178  */
 179 enum {
 180         CLEAN,
 181         UPDATE_IN_PROGRESS,
 182         DIRTY
 183 };
 184
 185 void kvm_recalculate_apic_map(struct kvm *kvm)
 186 {
 187         struct kvm_apic_map *new, *old = NULL;
 188         struct kvm_vcpu *vcpu;
 189         int i;
 190         u32 max_id = 255; /* enough space for any xAPIC ID */
 191
 192         /* Read kvm->arch.apic_map_dirty before kvm->arch.apic_map.  */
 193         if (atomic_read_acquire(&kvm->arch.apic_map_dirty) == CLEAN)
 194                 return;
 195
 196         WARN_ONCE(!irqchip_in_kernel(kvm),
 197                   "Dirty APIC map without an in-kernel local APIC");
 198
 199         mutex_lock(&kvm->arch.apic_map_lock);
 200         /*
 201          * Read kvm->arch.apic_map_dirty before kvm->arch.apic_map
 202          * (if clean) or the APIC registers (if dirty).
 203          */
 204         if (atomic_cmpxchg_acquire(&kvm->arch.apic_map_dirty,
 205                                    DIRTY, UPDATE_IN_PROGRESS) == CLEAN) {
 206                 /* Someone else has updated the map. */
 207                 mutex_unlock(&kvm->arch.apic_map_lock);
 208                 return;
 209         }
 210
 211         kvm_for_each_vcpu(i, vcpu, kvm)
 212                 if (kvm_apic_present(vcpu))
 213                         max_id = max(max_id, kvm_x2apic_id(vcpu->arch.apic));
 214
 215         new = kvzalloc(sizeof(struct kvm_apic_map) +
 216                            sizeof(struct kvm_lapic *) * ((u64)max_id + 1),
 217                            GFP_KERNEL_ACCOUNT);
 218
 219         if (!new)
 220                 goto out;
 221
 222         new->max_apic_id = max_id;
 223
 224         kvm_for_each_vcpu(i, vcpu, kvm) {
 225                 struct kvm_lapic *apic = vcpu->arch.apic;
 226                 struct kvm_lapic **cluster;
 227                 u16 mask;
 228                 u32 ldr;
 229                 u8 xapic_id;
 230                 u32 x2apic_id;
 231
 232                 if (!kvm_apic_present(vcpu))
 233                         continue;
 234
 235                 xapic_id = kvm_xapic_id(apic);
 236                 x2apic_id = kvm_x2apic_id(apic);
 237
 238                 /* Hotplug hack: see kvm_apic_match_physical_addr(), ... */
 239                 if ((apic_x2apic_mode(apic) || x2apic_id > 0xff) &&
 240                                 x2apic_id <= new->max_apic_id)
 241                         new->phys_map[x2apic_id] = apic;
 242                 /*
 243                  * ... xAPIC ID of VCPUs with APIC ID > 0xff will wrap-around,
 244                  * prevent them from masking VCPUs with APIC ID <= 0xff.
 245                  */
 246                 if (!apic_x2apic_mode(apic) && !new->phys_map[xapic_id])
 247                         new->phys_map[xapic_id] = apic;
 248
 249                 if (!kvm_apic_sw_enabled(apic))
 250                         continue;
 251
 252                 ldr = kvm_lapic_get_reg(apic, APIC_LDR);
 253
 254                 if (apic_x2apic_mode(apic)) {
 255                         new->mode |= KVM_APIC_MODE_X2APIC;
 256                 } else if (ldr) {
 257                         ldr = GET_APIC_LOGICAL_ID(ldr);
 258                         if (kvm_lapic_get_reg(apic, APIC_DFR) == APIC_DFR_FLAT)
 259                                 new->mode |= KVM_APIC_MODE_XAPIC_FLAT;
 260                         else
 261                                 new->mode |= KVM_APIC_MODE_XAPIC_CLUSTER;
 262                 }
 263
 264                 if (!kvm_apic_map_get_logical_dest(new, ldr, &cluster, &mask))
 265                         continue;
 266
 267                 if (mask)
 268                         cluster[ffs(mask) - 1] = apic;
 269         }
 270 out:
 271         old = rcu_dereference_protected(kvm->arch.apic_map,
 272                         lockdep_is_held(&kvm->arch.apic_map_lock));
 273         rcu_assign_pointer(kvm->arch.apic_map, new);
 274         /*
 275          * Write kvm->arch.apic_map before clearing apic->apic_map_dirty.
 276          * If another update has come in, leave it DIRTY.
 277          */
 278         atomic_cmpxchg_release(&kvm->arch.apic_map_dirty,
 279                                UPDATE_IN_PROGRESS, CLEAN);
 280         mutex_unlock(&kvm->arch.apic_map_lock);
 281
 282         if (old)
 283                 call_rcu(&old->rcu, kvm_apic_map_free);
 284
 285         kvm_make_scan_ioapic_request(kvm);
 286 }
 287
 288 static inline void apic_set_spiv(struct kvm_lapic *apic, u32 val)
 289 {
 290         bool enabled = val & APIC_SPIV_APIC_ENABLED;
 291
 292         kvm_lapic_set_reg(apic, APIC_SPIV, val);
 293
 294         if (enabled != apic->sw_enabled) {
 295                 apic->sw_enabled = enabled;
 296                 if (enabled)
 297                         static_branch_slow_dec_deferred(&apic_sw_disabled);
 298                 else
 299                         static_branch_inc(&apic_sw_disabled.key);
 300
 301                 atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
 302         }
 303
 304         /* Check if there are APF page ready requests pending */
 305         if (enabled)
 306                 kvm_make_request(KVM_REQ_APF_READY, apic->vcpu);
 307 }
 308
 309 static inline void kvm_apic_set_xapic_id(struct kvm_lapic *apic, u8 id)
 310 {
 311         kvm_lapic_set_reg(apic, APIC_ID, id << 24);
 312         atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
 313 }
 314
 315 static inline void kvm_apic_set_ldr(struct kvm_lapic *apic, u32 id)
 316 {
 317         kvm_lapic_set_reg(apic, APIC_LDR, id);
 318         atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
 319 }
 320
 321 static inline void kvm_apic_set_dfr(struct kvm_lapic *apic, u32 val)
 322 {
 323         kvm_lapic_set_reg(apic, APIC_DFR, val);
 324         atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
 325 }
 326
 327 static inline u32 kvm_apic_calc_x2apic_ldr(u32 id)
 328 {
 329         return ((id >> 4) << 16) | (1 << (id & 0xf));
 330 }
 331
 332 static inline void kvm_apic_set_x2apic_id(struct kvm_lapic *apic, u32 id)
 333 {
 334         u32 ldr = kvm_apic_calc_x2apic_ldr(id);
 335
 336         WARN_ON_ONCE(id != apic->vcpu->vcpu_id);
 337
 338         kvm_lapic_set_reg(apic, APIC_ID, id);
 339         kvm_lapic_set_reg(apic, APIC_LDR, ldr);
 340         atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
 341 }
 342
 343 static inline int apic_lvt_enabled(struct kvm_lapic *apic, int lvt_type)
 344 {
 345         return !(kvm_lapic_get_reg(apic, lvt_type) & APIC_LVT_MASKED);
 346 }
 347
 348 static inline int apic_lvtt_oneshot(struct kvm_lapic *apic)
 349 {
 350         return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_ONESHOT;
 351 }
 352
 353 static inline int apic_lvtt_period(struct kvm_lapic *apic)
 354 {
 355         return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_PERIODIC;
 356 }
 357
 358 static inline int apic_lvtt_tscdeadline(struct kvm_lapic *apic)
 359 {
 360         return apic->lapic_timer.timer_mode == APIC_LVT_TIMER_TSCDEADLINE;
 361 }
 362
 363 static inline int apic_lvt_nmi_mode(u32 lvt_val)
 364 {
 365         return (lvt_val & (APIC_MODE_MASK | APIC_LVT_MASKED)) == APIC_DM_NMI;
 366 }
 367
 368 void kvm_apic_set_version(struct kvm_vcpu *vcpu)
 369 {
 370         struct kvm_lapic *apic = vcpu->arch.apic;
 371         u32 v = APIC_VERSION;
 372
 373         if (!lapic_in_kernel(vcpu))
 374                 return;
 375
 376         /*
 377          * KVM emulates 82093AA datasheet (with in-kernel IOAPIC implementation)
 378          * which doesn't have EOI register; Some buggy OSes (e.g. Windows with
 379          * Hyper-V role) disable EOI broadcast in lapic not checking for IOAPIC
 380          * version first and level-triggered interrupts never get EOIed in
 381          * IOAPIC.
 382          */
 383         if (guest_cpuid_has(vcpu, X86_FEATURE_X2APIC) &&
 384             !ioapic_in_kernel(vcpu->kvm))
 385                 v |= APIC_LVR_DIRECTED_EOI;
 386         kvm_lapic_set_reg(apic, APIC_LVR, v);
 387 }
 388
 389 static const unsigned int apic_lvt_mask[KVM_APIC_LVT_NUM] = {
 390         LVT_MASK ,      /* part LVTT mask, timer mode mask added at runtime */
 391         LVT_MASK | APIC_MODE_MASK,      /* LVTTHMR */
 392         LVT_MASK | APIC_MODE_MASK,      /* LVTPC */
 393         LINT_MASK, LINT_MASK,   /* LVT0-1 */
 394         LVT_MASK                /* LVTERR */
 395 };
 396
 397 static int find_highest_vector(void *bitmap)
 398 {
 399         int vec;
 400         u32 *reg;
 401
 402         for (vec = MAX_APIC_VECTOR - APIC_VECTORS_PER_REG;
 403              vec >= 0; vec -= APIC_VECTORS_PER_REG) {
 404                 reg = bitmap + REG_POS(vec);
 405                 if (*reg)
 406                         return __fls(*reg) + vec;
 407         }
 408
 409         return -1;
 410 }
 411
 412 static u8 count_vectors(void *bitmap)
 413 {
 414         int vec;
 415         u32 *reg;
 416         u8 count = 0;
 417
 418         for (vec = 0; vec < MAX_APIC_VECTOR; vec += APIC_VECTORS_PER_REG) {
 419                 reg = bitmap + REG_POS(vec);
 420                 count += hweight32(*reg);
 421         }
 422
 423         return count;
 424 }
 425
 426 bool __kvm_apic_update_irr(u32 *pir, void *regs, int *max_irr)
 427 {
 428         u32 i, vec;
 429         u32 pir_val, irr_val, prev_irr_val;
 430         int max_updated_irr;
 431
 432         max_updated_irr = -1;
 433         *max_irr = -1;
 434
 435         for (i = vec = 0; i <= 7; i++, vec += 32) {
 436                 pir_val = READ_ONCE(pir[i]);
 437                 irr_val = *((u32 *)(regs + APIC_IRR + i * 0x10));
 438                 if (pir_val) {
 439                         prev_irr_val = irr_val;
 440                         irr_val |= xchg(&pir[i], 0);
 441                         *((u32 *)(regs + APIC_IRR + i * 0x10)) = irr_val;
 442                         if (prev_irr_val != irr_val) {
 443                                 max_updated_irr =
 444                                         __fls(irr_val ^ prev_irr_val) + vec;
 445                         }
 446                 }
 447                 if (irr_val)
 448                         *max_irr = __fls(irr_val) + vec;
 449         }
 450
 451         return ((max_updated_irr != -1) &&
 452                 (max_updated_irr == *max_irr));
 453 }
 454 EXPORT_SYMBOL_GPL(__kvm_apic_update_irr);
 455
 456 bool kvm_apic_update_irr(struct kvm_vcpu *vcpu, u32 *pir, int *max_irr)
 457 {
 458         struct kvm_lapic *apic = vcpu->arch.apic;
 459
 460         return __kvm_apic_update_irr(pir, apic->regs, max_irr);
 461 }
 462 EXPORT_SYMBOL_GPL(kvm_apic_update_irr);
 463
 464 static inline int apic_search_irr(struct kvm_lapic *apic)
 465 {
 466         return find_highest_vector(apic->regs + APIC_IRR);
 467 }
 468
 469 static inline int apic_find_highest_irr(struct kvm_lapic *apic)
 470 {
 471         int result;
 472
 473         /*
 474          * Note that irr_pending is just a hint. It will be always
 475          * true with virtual interrupt delivery enabled.
 476          */
 477         if (!apic->irr_pending)
 478                 return -1;
 479
 480         result = apic_search_irr(apic);
 481         ASSERT(result == -1 || result >= 16);
 482
 483         return result;
 484 }
 485
 486 static inline void apic_clear_irr(int vec, struct kvm_lapic *apic)
 487 {
 488         struct kvm_vcpu *vcpu;
 489
 490         vcpu = apic->vcpu;
 491
 492         if (unlikely(vcpu->arch.apicv_active)) {
 493                 /* need to update RVI */
 494                 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
 495                 static_call(kvm_x86_hwapic_irr_update)(vcpu,
 496                                 apic_find_highest_irr(apic));
 497         } else {
 498                 apic->irr_pending = false;
 499                 kvm_lapic_clear_vector(vec, apic->regs + APIC_IRR);
 500                 if (apic_search_irr(apic) != -1)
 501                         apic->irr_pending = true;
 502         }
 503 }
 504
 505 void kvm_apic_clear_irr(struct kvm_vcpu *vcpu, int vec)
 506 {
 507         apic_clear_irr(vec, vcpu->arch.apic);
 508 }
 509 EXPORT_SYMBOL_GPL(kvm_apic_clear_irr);
 510
 511 static inline void apic_set_isr(int vec, struct kvm_lapic *apic)
 512 {
 513         struct kvm_vcpu *vcpu;
 514
 515         if (__apic_test_and_set_vector(vec, apic->regs + APIC_ISR))
 516                 return;
 517
 518         vcpu = apic->vcpu;
 519
 520         /*
 521          * With APIC virtualization enabled, all caching is disabled
 522          * because the processor can modify ISR under the hood.  Instead
 523          * just set SVI.
 524          */
 525         if (unlikely(vcpu->arch.apicv_active))
 526                 static_call(kvm_x86_hwapic_isr_update)(vcpu, vec);
 527         else {
 528                 ++apic->isr_count;
 529                 BUG_ON(apic->isr_count > MAX_APIC_VECTOR);
 530                 /*
 531                  * ISR (in service register) bit is set when injecting an interrupt.
 532                  * The highest vector is injected. Thus the latest bit set matches
 533                  * the highest bit in ISR.
 534                  */
 535                 apic->highest_isr_cache = vec;
 536         }
 537 }
 538
 539 static inline int apic_find_highest_isr(struct kvm_lapic *apic)
 540 {
 541         int result;
 542
 543         /*
 544          * Note that isr_count is always 1, and highest_isr_cache
 545          * is always -1, with APIC virtualization enabled.
 546          */
 547         if (!apic->isr_count)
 548                 return -1;
 549         if (likely(apic->highest_isr_cache != -1))
 550                 return apic->highest_isr_cache;
 551
 552         result = find_highest_vector(apic->regs + APIC_ISR);
 553         ASSERT(result == -1 || result >= 16);
 554
 555         return result;
 556 }
 557
 558 static inline void apic_clear_isr(int vec, struct kvm_lapic *apic)
 559 {
 560         struct kvm_vcpu *vcpu;
 561         if (!__apic_test_and_clear_vector(vec, apic->regs + APIC_ISR))
 562                 return;
 563
 564         vcpu = apic->vcpu;
 565
 566         /*
 567          * We do get here for APIC virtualization enabled if the guest
 568          * uses the Hyper-V APIC enlightenment.  In this case we may need
 569          * to trigger a new interrupt delivery by writing the SVI field;
 570          * on the other hand isr_count and highest_isr_cache are unused
 571          * and must be left alone.
 572          */
 573         if (unlikely(vcpu->arch.apicv_active))
 574                 static_call(kvm_x86_hwapic_isr_update)(vcpu,
 575                                                 apic_find_highest_isr(apic));
 576         else {
 577                 --apic->isr_count;
 578                 BUG_ON(apic->isr_count < 0);
 579                 apic->highest_isr_cache = -1;
 580         }
 581 }
 582
 583 int kvm_lapic_find_highest_irr(struct kvm_vcpu *vcpu)
 584 {
 585         /* This may race with setting of irr in __apic_accept_irq() and
 586          * value returned may be wrong, but kvm_vcpu_kick() in __apic_accept_irq
 587          * will cause vmexit immediately and the value will be recalculated
 588          * on the next vmentry.
 589          */
 590         return apic_find_highest_irr(vcpu->arch.apic);
 591 }
 592 EXPORT_SYMBOL_GPL(kvm_lapic_find_highest_irr);
 593
 594 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
 595                              int vector, int level, int trig_mode,
 596                              struct dest_map *dest_map);
 597
 598 int kvm_apic_set_irq(struct kvm_vcpu *vcpu, struct kvm_lapic_irq *irq,
 599                      struct dest_map *dest_map)
 600 {
 601         struct kvm_lapic *apic = vcpu->arch.apic;
 602
 603         return __apic_accept_irq(apic, irq->delivery_mode, irq->vector,
 604                         irq->level, irq->trig_mode, dest_map);
 605 }
 606
 607 static int __pv_send_ipi(unsigned long *ipi_bitmap, struct kvm_apic_map *map,
 608                          struct kvm_lapic_irq *irq, u32 min)
 609 {
 610         int i, count = 0;
 611         struct kvm_vcpu *vcpu;
 612
 613         if (min > map->max_apic_id)
 614                 return 0;
 615
 616         for_each_set_bit(i, ipi_bitmap,
 617                 min((u32)BITS_PER_LONG, (map->max_apic_id - min + 1))) {
 618                 if (map->phys_map[min + i]) {
 619                         vcpu = map->phys_map[min + i]->vcpu;
 620                         count += kvm_apic_set_irq(vcpu, irq, NULL);
 621                 }
 622         }
 623
 624         return count;
 625 }
 626
 627 int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
 628                     unsigned long ipi_bitmap_high, u32 min,
 629                     unsigned long icr, int op_64_bit)
 630 {
 631         struct kvm_apic_map *map;
 632         struct kvm_lapic_irq irq = {0};
 633         int cluster_size = op_64_bit ? 64 : 32;
 634         int count;
 635
 636         if (icr & (APIC_DEST_MASK | APIC_SHORT_MASK))
 637                 return -KVM_EINVAL;
 638
 639         irq.vector = icr & APIC_VECTOR_MASK;
 640         irq.delivery_mode = icr & APIC_MODE_MASK;
 641         irq.level = (icr & APIC_INT_ASSERT) != 0;
 642         irq.trig_mode = icr & APIC_INT_LEVELTRIG;
 643
 644         rcu_read_lock();
 645         map = rcu_dereference(kvm->arch.apic_map);
 646
 647         count = -EOPNOTSUPP;
 648         if (likely(map)) {
 649                 count = __pv_send_ipi(&ipi_bitmap_low, map, &irq, min);
 650                 min += cluster_size;
 651                 count += __pv_send_ipi(&ipi_bitmap_high, map, &irq, min);
 652         }
 653
 654         rcu_read_unlock();
 655         return count;
 656 }
 657
 658 static int pv_eoi_put_user(struct kvm_vcpu *vcpu, u8 val)
 659 {
 660
 661         return kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, &val,
 662                                       sizeof(val));
 663 }
 664
 665 static int pv_eoi_get_user(struct kvm_vcpu *vcpu, u8 *val)
 666 {
 667
 668         return kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.pv_eoi.data, val,
 669                                       sizeof(*val));
 670 }
 671
 672 static inline bool pv_eoi_enabled(struct kvm_vcpu *vcpu)
 673 {
 674         return vcpu->arch.pv_eoi.msr_val & KVM_MSR_ENABLED;
 675 }
 676
 677 static bool pv_eoi_get_pending(struct kvm_vcpu *vcpu)
 678 {
 679         u8 val;
 680         if (pv_eoi_get_user(vcpu, &val) < 0)
 681                 return false;
 682
 683         return val & KVM_PV_EOI_ENABLED;
 684 }
 685
 686 static void pv_eoi_set_pending(struct kvm_vcpu *vcpu)
 687 {
 688         if (pv_eoi_put_user(vcpu, KVM_PV_EOI_ENABLED) < 0)
 689                 return;
 690
 691         __set_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
 692 }
 693
 694 static void pv_eoi_clr_pending(struct kvm_vcpu *vcpu)
 695 {
 696         if (pv_eoi_put_user(vcpu, KVM_PV_EOI_DISABLED) < 0)
 697                 return;
 698
 699         __clear_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention);
 700 }
 701
 702 static int apic_has_interrupt_for_ppr(struct kvm_lapic *apic, u32 ppr)
 703 {
 704         int highest_irr;
 705         if (kvm_x86_ops.sync_pir_to_irr)
 706                 highest_irr = static_call(kvm_x86_sync_pir_to_irr)(apic->vcpu);
 707         else
 708                 highest_irr = apic_find_highest_irr(apic);
 709         if (highest_irr == -1 || (highest_irr & 0xF0) <= ppr)
 710                 return -1;
 711         return highest_irr;
 712 }
 713
 714 static bool __apic_update_ppr(struct kvm_lapic *apic, u32 *new_ppr)
 715 {
 716         u32 tpr, isrv, ppr, old_ppr;
 717         int isr;
 718
 719         old_ppr = kvm_lapic_get_reg(apic, APIC_PROCPRI);
 720         tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI);
 721         isr = apic_find_highest_isr(apic);
 722         isrv = (isr != -1) ? isr : 0;
 723
 724         if ((tpr & 0xf0) >= (isrv & 0xf0))
 725                 ppr = tpr & 0xff;
 726         else
 727                 ppr = isrv & 0xf0;
 728
 729         *new_ppr = ppr;
 730         if (old_ppr != ppr)
 731                 kvm_lapic_set_reg(apic, APIC_PROCPRI, ppr);
 732
 733         return ppr < old_ppr;
 734 }
 735
 736 static void apic_update_ppr(struct kvm_lapic *apic)
 737 {
 738         u32 ppr;
 739
 740         if (__apic_update_ppr(apic, &ppr) &&
 741             apic_has_interrupt_for_ppr(apic, ppr) != -1)
 742                 kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
 743 }
 744
 745 void kvm_apic_update_ppr(struct kvm_vcpu *vcpu)
 746 {
 747         apic_update_ppr(vcpu->arch.apic);
 748 }
 749 EXPORT_SYMBOL_GPL(kvm_apic_update_ppr);
 750
 751 static void apic_set_tpr(struct kvm_lapic *apic, u32 tpr)
 752 {
 753         kvm_lapic_set_reg(apic, APIC_TASKPRI, tpr);
 754         apic_update_ppr(apic);
 755 }
 756
 757 static bool kvm_apic_broadcast(struct kvm_lapic *apic, u32 mda)
 758 {
 759         return mda == (apic_x2apic_mode(apic) ?
 760                         X2APIC_BROADCAST : APIC_BROADCAST);
 761 }
 762
 763 static bool kvm_apic_match_physical_addr(struct kvm_lapic *apic, u32 mda)
 764 {
 765         if (kvm_apic_broadcast(apic, mda))
 766                 return true;
 767
 768         if (apic_x2apic_mode(apic))
 769                 return mda == kvm_x2apic_id(apic);
 770
 771         /*
 772          * Hotplug hack: Make LAPIC in xAPIC mode also accept interrupts as if
 773          * it were in x2APIC mode.  Hotplugged VCPUs start in xAPIC mode and
 774          * this allows unique addressing of VCPUs with APIC ID over 0xff.
 775          * The 0xff condition is needed because writeable xAPIC ID.
 776          */
 777         if (kvm_x2apic_id(apic) > 0xff && mda == kvm_x2apic_id(apic))
 778                 return true;
 779
 780         return mda == kvm_xapic_id(apic);
 781 }
 782
 783 static bool kvm_apic_match_logical_addr(struct kvm_lapic *apic, u32 mda)
 784 {
 785         u32 logical_id;
 786
 787         if (kvm_apic_broadcast(apic, mda))
 788                 return true;
 789
 790         logical_id = kvm_lapic_get_reg(apic, APIC_LDR);
 791
 792         if (apic_x2apic_mode(apic))
 793                 return ((logical_id >> 16) == (mda >> 16))
 794                        && (logical_id & mda & 0xffff) != 0;
 795
 796         logical_id = GET_APIC_LOGICAL_ID(logical_id);
 797
 798         switch (kvm_lapic_get_reg(apic, APIC_DFR)) {
 799         case APIC_DFR_FLAT:
 800                 return (logical_id & mda) != 0;
 801         case APIC_DFR_CLUSTER:
 802                 return ((logical_id >> 4) == (mda >> 4))
 803                        && (logical_id & mda & 0xf) != 0;
 804         default:
 805                 return false;
 806         }
 807 }
 808
 809 /* The KVM local APIC implementation has two quirks:
 810  *
 811  *  - Real hardware delivers interrupts destined to x2APIC ID > 0xff to LAPICs
 812  *    in xAPIC mode if the "destination & 0xff" matches its xAPIC ID.
 813  *    KVM doesn't do that aliasing.
 814  *
 815  *  - in-kernel IOAPIC messages have to be delivered directly to
 816  *    x2APIC, because the kernel does not support interrupt remapping.
 817  *    In order to support broadcast without interrupt remapping, x2APIC
 818  *    rewrites the destination of non-IPI messages from APIC_BROADCAST
 819  *    to X2APIC_BROADCAST.
 820  *
 821  * The broadcast quirk can be disabled with KVM_CAP_X2APIC_API.  This is
 822  * important when userspace wants to use x2APIC-format MSIs, because
 823  * APIC_BROADCAST (0xff) is a legal route for "cluster 0, CPUs 0-7".
 824  */
 825 static u32 kvm_apic_mda(struct kvm_vcpu *vcpu, unsigned int dest_id,
 826                 struct kvm_lapic *source, struct kvm_lapic *target)
 827 {
 828         bool ipi = source != NULL;
 829
 830         if (!vcpu->kvm->arch.x2apic_broadcast_quirk_disabled &&
 831             !ipi && dest_id == APIC_BROADCAST && apic_x2apic_mode(target))
 832                 return X2APIC_BROADCAST;
 833
 834         return dest_id;
 835 }
 836
 837 bool kvm_apic_match_dest(struct kvm_vcpu *vcpu, struct kvm_lapic *source,
 838                            int shorthand, unsigned int dest, int dest_mode)
 839 {
 840         struct kvm_lapic *target = vcpu->arch.apic;
 841         u32 mda = kvm_apic_mda(vcpu, dest, source, target);
 842
 843         ASSERT(target);
 844         switch (shorthand) {
 845         case APIC_DEST_NOSHORT:
 846                 if (dest_mode == APIC_DEST_PHYSICAL)
 847                         return kvm_apic_match_physical_addr(target, mda);
 848                 else
 849                         return kvm_apic_match_logical_addr(target, mda);
 850         case APIC_DEST_SELF:
 851                 return target == source;
 852         case APIC_DEST_ALLINC:
 853                 return true;
 854         case APIC_DEST_ALLBUT:
 855                 return target != source;
 856         default:
 857                 return false;
 858         }
 859 }
 860 EXPORT_SYMBOL_GPL(kvm_apic_match_dest);
 861
 862 int kvm_vector_to_index(u32 vector, u32 dest_vcpus,
 863                        const unsigned long *bitmap, u32 bitmap_size)
 864 {
 865         u32 mod;
 866         int i, idx = -1;
 867
 868         mod = vector % dest_vcpus;
 869
 870         for (i = 0; i <= mod; i++) {
 871                 idx = find_next_bit(bitmap, bitmap_size, idx + 1);
 872                 BUG_ON(idx == bitmap_size);
 873         }
 874
 875         return idx;
 876 }
 877
 878 static void kvm_apic_disabled_lapic_found(struct kvm *kvm)
 879 {
 880         if (!kvm->arch.disabled_lapic_found) {
 881                 kvm->arch.disabled_lapic_found = true;
 882                 printk(KERN_INFO
 883                        "Disabled LAPIC found during irq injection\n");
 884         }
 885 }
 886
 887 static bool kvm_apic_is_broadcast_dest(struct kvm *kvm, struct kvm_lapic **src,
 888                 struct kvm_lapic_irq *irq, struct kvm_apic_map *map)
 889 {
 890         if (kvm->arch.x2apic_broadcast_quirk_disabled) {
 891                 if ((irq->dest_id == APIC_BROADCAST &&
 892                                 map->mode != KVM_APIC_MODE_X2APIC))
 893                         return true;
 894                 if (irq->dest_id == X2APIC_BROADCAST)
 895                         return true;
 896         } else {
 897                 bool x2apic_ipi = src && *src && apic_x2apic_mode(*src);
 898                 if (irq->dest_id == (x2apic_ipi ?
 899                                      X2APIC_BROADCAST : APIC_BROADCAST))
 900                         return true;
 901         }
 902
 903         return false;
 904 }
 905
 906 /* Return true if the interrupt can be handled by using *bitmap as index mask
 907  * for valid destinations in *dst array.
 908  * Return false if kvm_apic_map_get_dest_lapic did nothing useful.
 909  * Note: we may have zero kvm_lapic destinations when we return true, which
 910  * means that the interrupt should be dropped.  In this case, *bitmap would be
 911  * zero and *dst undefined.
 912  */
 913 static inline bool kvm_apic_map_get_dest_lapic(struct kvm *kvm,
 914                 struct kvm_lapic **src, struct kvm_lapic_irq *irq,
 915                 struct kvm_apic_map *map, struct kvm_lapic ***dst,
 916                 unsigned long *bitmap)
 917 {
 918         int i, lowest;
 919
 920         if (irq->shorthand == APIC_DEST_SELF && src) {
 921                 *dst = src;
 922                 *bitmap = 1;
 923                 return true;
 924         } else if (irq->shorthand)
 925                 return false;
 926
 927         if (!map || kvm_apic_is_broadcast_dest(kvm, src, irq, map))
 928                 return false;
 929
 930         if (irq->dest_mode == APIC_DEST_PHYSICAL) {
 931                 if (irq->dest_id > map->max_apic_id) {
 932                         *bitmap = 0;
 933                 } else {
 934                         u32 dest_id = array_index_nospec(irq->dest_id, map->max_apic_id + 1);
 935                         *dst = &map->phys_map[dest_id];
 936                         *bitmap = 1;
 937                 }
 938                 return true;
 939         }
 940
 941         *bitmap = 0;
 942         if (!kvm_apic_map_get_logical_dest(map, irq->dest_id, dst,
 943                                 (u16 *)bitmap))
 944                 return false;
 945
 946         if (!kvm_lowest_prio_delivery(irq))
 947                 return true;
 948
 949         if (!kvm_vector_hashing_enabled()) {
 950                 lowest = -1;
 951                 for_each_set_bit(i, bitmap, 16) {
 952                         if (!(*dst)[i])
 953                                 continue;
 954                         if (lowest < 0)
 955                                 lowest = i;
 956                         else if (kvm_apic_compare_prio((*dst)[i]->vcpu,
 957                                                 (*dst)[lowest]->vcpu) < 0)
 958                                 lowest = i;
 959                 }
 960         } else {
 961                 if (!*bitmap)
 962                         return true;
 963
 964                 lowest = kvm_vector_to_index(irq->vector, hweight16(*bitmap),
 965                                 bitmap, 16);
 966
 967                 if (!(*dst)[lowest]) {
 968                         kvm_apic_disabled_lapic_found(kvm);
 969                         *bitmap = 0;
 970                         return true;
 971                 }
 972         }
 973
 974         *bitmap = (lowest >= 0) ? 1 << lowest : 0;
 975
 976         return true;
 977 }
 978
 979 bool kvm_irq_delivery_to_apic_fast(struct kvm *kvm, struct kvm_lapic *src,
 980                 struct kvm_lapic_irq *irq, int *r, struct dest_map *dest_map)
 981 {
 982         struct kvm_apic_map *map;
 983         unsigned long bitmap;
 984         struct kvm_lapic **dst = NULL;
 985         int i;
 986         bool ret;
 987
 988         *r = -1;
 989
 990         if (irq->shorthand == APIC_DEST_SELF) {
 991                 if (KVM_BUG_ON(!src, kvm)) {
 992                         *r = 0;
 993                         return true;
 994                 }
 995                 *r = kvm_apic_set_irq(src->vcpu, irq, dest_map);
 996                 return true;
 997         }
 998
 999         rcu_read_lock();
1000         map = rcu_dereference(kvm->arch.apic_map);
1001
1002         ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dst, &bitmap);
1003         if (ret) {
1004                 *r = 0;
1005                 for_each_set_bit(i, &bitmap, 16) {
1006                         if (!dst[i])
1007                                 continue;
1008                         *r += kvm_apic_set_irq(dst[i]->vcpu, irq, dest_map);
1009                 }
1010         }
1011
1012         rcu_read_unlock();
1013         return ret;
1014 }
1015
1016 /*
1017  * This routine tries to handle interrupts in posted mode, here is how
1018  * it deals with different cases:
1019  * - For single-destination interrupts, handle it in posted mode
1020  * - Else if vector hashing is enabled and it is a lowest-priority
1021  *   interrupt, handle it in posted mode and use the following mechanism
1022  *   to find the destination vCPU.
1023  *      1. For lowest-priority interrupts, store all the possible
1024  *         destination vCPUs in an array.
1025  *      2. Use "guest vector % max number of destination vCPUs" to find
1026  *         the right destination vCPU in the array for the lowest-priority
1027  *         interrupt.
1028  * - Otherwise, use remapped mode to inject the interrupt.
1029  */
1030 bool kvm_intr_is_single_vcpu_fast(struct kvm *kvm, struct kvm_lapic_irq *irq,
1031                         struct kvm_vcpu **dest_vcpu)
1032 {
1033         struct kvm_apic_map *map;
1034         unsigned long bitmap;
1035         struct kvm_lapic **dst = NULL;
1036         bool ret = false;
1037
1038         if (irq->shorthand)
1039                 return false;
1040
1041         rcu_read_lock();
1042         map = rcu_dereference(kvm->arch.apic_map);
1043
1044         if (kvm_apic_map_get_dest_lapic(kvm, NULL, irq, map, &dst, &bitmap) &&
1045                         hweight16(bitmap) == 1) {
1046                 unsigned long i = find_first_bit(&bitmap, 16);
1047
1048                 if (dst[i]) {
1049                         *dest_vcpu = dst[i]->vcpu;
1050                         ret = true;
1051                 }
1052         }
1053
1054         rcu_read_unlock();
1055         return ret;
1056 }
1057
1058 /*
1059  * Add a pending IRQ into lapic.
1060  * Return 1 if successfully added and 0 if discarded.
1061  */
1062 static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode,
1063                              int vector, int level, int trig_mode,
1064                              struct dest_map *dest_map)
1065 {
1066         int result = 0;
1067         struct kvm_vcpu *vcpu = apic->vcpu;
1068
1069         trace_kvm_apic_accept_irq(vcpu->vcpu_id, delivery_mode,
1070                                   trig_mode, vector);
1071         switch (delivery_mode) {
1072         case APIC_DM_LOWEST:
1073                 vcpu->arch.apic_arb_prio++;
1074                 fallthrough;
1075         case APIC_DM_FIXED:
1076                 if (unlikely(trig_mode && !level))
1077                         break;
1078
1079                 /* FIXME add logic for vcpu on reset */
1080                 if (unlikely(!apic_enabled(apic)))
1081                         break;
1082
1083                 result = 1;
1084
1085                 if (dest_map) {
1086                         __set_bit(vcpu->vcpu_id, dest_map->map);
1087                         dest_map->vectors[vcpu->vcpu_id] = vector;
1088                 }
1089
1090                 if (apic_test_vector(vector, apic->regs + APIC_TMR) != !!trig_mode) {
1091                         if (trig_mode)
1092                                 kvm_lapic_set_vector(vector,
1093                                                      apic->regs + APIC_TMR);
1094                         else
1095                                 kvm_lapic_clear_vector(vector,
1096                                                        apic->regs + APIC_TMR);
1097                 }
1098
1099                 if (static_call(kvm_x86_deliver_posted_interrupt)(vcpu, vector)) {
1100                         kvm_lapic_set_irr(vector, apic);
1101                         kvm_make_request(KVM_REQ_EVENT, vcpu);
1102                         kvm_vcpu_kick(vcpu);
1103                 }
1104                 break;
1105
1106         case APIC_DM_REMRD:
1107                 result = 1;
1108                 vcpu->arch.pv.pv_unhalted = 1;
1109                 kvm_make_request(KVM_REQ_EVENT, vcpu);
1110                 kvm_vcpu_kick(vcpu);
1111                 break;
1112
1113         case APIC_DM_SMI:
1114                 result = 1;
1115                 kvm_make_request(KVM_REQ_SMI, vcpu);
1116                 kvm_vcpu_kick(vcpu);
1117                 break;
1118
1119         case APIC_DM_NMI:
1120                 result = 1;
1121                 kvm_inject_nmi(vcpu);
1122                 kvm_vcpu_kick(vcpu);
1123                 break;
1124
1125         case APIC_DM_INIT:
1126                 if (!trig_mode || level) {
1127                         result = 1;
1128                         /* assumes that there are only KVM_APIC_INIT/SIPI */
1129                         apic->pending_events = (1UL << KVM_APIC_INIT);
1130                         kvm_make_request(KVM_REQ_EVENT, vcpu);
1131                         kvm_vcpu_kick(vcpu);
1132                 }
1133                 break;
1134
1135         case APIC_DM_STARTUP:
1136                 result = 1;
1137                 apic->sipi_vector = vector;
1138                 /* make sure sipi_vector is visible for the receiver */
1139                 smp_wmb();
1140                 set_bit(KVM_APIC_SIPI, &apic->pending_events);
1141                 kvm_make_request(KVM_REQ_EVENT, vcpu);
1142                 kvm_vcpu_kick(vcpu);
1143                 break;
1144
1145         case APIC_DM_EXTINT:
1146                 /*
1147                  * Should only be called by kvm_apic_local_deliver() with LVT0,
1148                  * before NMI watchdog was enabled. Already handled by
1149                  * kvm_apic_accept_pic_intr().
1150                  */
1151                 break;
1152
1153         default:
1154                 printk(KERN_ERR "TODO: unsupported delivery mode %x\n",
1155                        delivery_mode);
1156                 break;
1157         }
1158         return result;
1159 }
1160
1161 /*
1162  * This routine identifies the destination vcpus mask meant to receive the
1163  * IOAPIC interrupts. It either uses kvm_apic_map_get_dest_lapic() to find
1164  * out the destination vcpus array and set the bitmap or it traverses to
1165  * each available vcpu to identify the same.
1166  */
1167 void kvm_bitmap_or_dest_vcpus(struct kvm *kvm, struct kvm_lapic_irq *irq,
1168                               unsigned long *vcpu_bitmap)
1169 {
1170         struct kvm_lapic **dest_vcpu = NULL;
1171         struct kvm_lapic *src = NULL;
1172         struct kvm_apic_map *map;
1173         struct kvm_vcpu *vcpu;
1174         unsigned long bitmap;
1175         int i, vcpu_idx;
1176         bool ret;
1177
1178         rcu_read_lock();
1179         map = rcu_dereference(kvm->arch.apic_map);
1180
1181         ret = kvm_apic_map_get_dest_lapic(kvm, &src, irq, map, &dest_vcpu,
1182                                           &bitmap);
1183         if (ret) {
1184                 for_each_set_bit(i, &bitmap, 16) {
1185                         if (!dest_vcpu[i])
1186                                 continue;
1187                         vcpu_idx = dest_vcpu[i]->vcpu->vcpu_idx;
1188                         __set_bit(vcpu_idx, vcpu_bitmap);
1189                 }
1190         } else {
1191                 kvm_for_each_vcpu(i, vcpu, kvm) {
1192                         if (!kvm_apic_present(vcpu))
1193                                 continue;
1194                         if (!kvm_apic_match_dest(vcpu, NULL,
1195                                                  irq->shorthand,
1196                                                  irq->dest_id,
1197                                                  irq->dest_mode))
1198                                 continue;
1199                         __set_bit(i, vcpu_bitmap);
1200                 }
1201         }
1202         rcu_read_unlock();
1203 }
1204
1205 int kvm_apic_compare_prio(struct kvm_vcpu *vcpu1, struct kvm_vcpu *vcpu2)
1206 {
1207         return vcpu1->arch.apic_arb_prio - vcpu2->arch.apic_arb_prio;
1208 }
1209
1210 static bool kvm_ioapic_handles_vector(struct kvm_lapic *apic, int vector)
1211 {
1212         return test_bit(vector, apic->vcpu->arch.ioapic_handled_vectors);
1213 }
1214
1215 static void kvm_ioapic_send_eoi(struct kvm_lapic *apic, int vector)
1216 {
1217         int trigger_mode;
1218
1219         /* Eoi the ioapic only if the ioapic doesn't own the vector. */
1220         if (!kvm_ioapic_handles_vector(apic, vector))
1221                 return;
1222
1223         /* Request a KVM exit to inform the userspace IOAPIC. */
1224         if (irqchip_split(apic->vcpu->kvm)) {
1225                 apic->vcpu->arch.pending_ioapic_eoi = vector;
1226                 kvm_make_request(KVM_REQ_IOAPIC_EOI_EXIT, apic->vcpu);
1227                 return;
1228         }
1229
1230         if (apic_test_vector(vector, apic->regs + APIC_TMR))
1231                 trigger_mode = IOAPIC_LEVEL_TRIG;
1232         else
1233                 trigger_mode = IOAPIC_EDGE_TRIG;
1234
1235         kvm_ioapic_update_eoi(apic->vcpu, vector, trigger_mode);
1236 }
1237
1238 static int apic_set_eoi(struct kvm_lapic *apic)
1239 {
1240         int vector = apic_find_highest_isr(apic);
1241
1242         trace_kvm_eoi(apic, vector);
1243
1244         /*
1245          * Not every write EOI will has corresponding ISR,
1246          * one example is when Kernel check timer on setup_IO_APIC
1247          */
1248         if (vector == -1)
1249                 return vector;
1250
1251         apic_clear_isr(vector, apic);
1252         apic_update_ppr(apic);
1253
1254         if (to_hv_vcpu(apic->vcpu) &&
1255             test_bit(vector, to_hv_synic(apic->vcpu)->vec_bitmap))
1256                 kvm_hv_synic_send_eoi(apic->vcpu, vector);
1257
1258         kvm_ioapic_send_eoi(apic, vector);
1259         kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1260         return vector;
1261 }
1262
1263 /*
1264  * this interface assumes a trap-like exit, which has already finished
1265  * desired side effect including vISR and vPPR update.
1266  */
1267 void kvm_apic_set_eoi_accelerated(struct kvm_vcpu *vcpu, int vector)
1268 {
1269         struct kvm_lapic *apic = vcpu->arch.apic;
1270
1271         trace_kvm_eoi(apic, vector);
1272
1273         kvm_ioapic_send_eoi(apic, vector);
1274         kvm_make_request(KVM_REQ_EVENT, apic->vcpu);
1275 }
1276 EXPORT_SYMBOL_GPL(kvm_apic_set_eoi_accelerated);
1277
1278 void kvm_apic_send_ipi(struct kvm_lapic *apic, u32 icr_low, u32 icr_high)
1279 {
1280         struct kvm_lapic_irq irq;
1281
1282         irq.vector = icr_low & APIC_VECTOR_MASK;
1283         irq.delivery_mode = icr_low & APIC_MODE_MASK;
1284         irq.dest_mode = icr_low & APIC_DEST_MASK;
1285         irq.level = (icr_low & APIC_INT_ASSERT) != 0;
1286         irq.trig_mode = icr_low & APIC_INT_LEVELTRIG;
1287         irq.shorthand = icr_low & APIC_SHORT_MASK;
1288         irq.msi_redir_hint = false;
1289         if (apic_x2apic_mode(apic))
1290                 irq.dest_id = icr_high;
1291         else
1292                 irq.dest_id = GET_APIC_DEST_FIELD(icr_high);
1293
1294         trace_kvm_apic_ipi(icr_low, irq.dest_id);
1295
1296         kvm_irq_delivery_to_apic(apic->vcpu->kvm, apic, &irq, NULL);
1297 }
1298
1299 static u32 apic_get_tmcct(struct kvm_lapic *apic)
1300 {
1301         ktime_t remaining, now;
1302         s64 ns;
1303         u32 tmcct;
1304
1305         ASSERT(apic != NULL);
1306
1307         /* if initial count is 0, current count should also be 0 */
1308         if (kvm_lapic_get_reg(apic, APIC_TMICT) == 0 ||
1309                 apic->lapic_timer.period == 0)
1310                 return 0;
1311
1312         now = ktime_get();
1313         remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
1314         if (ktime_to_ns(remaining) < 0)
1315                 remaining = 0;
1316
1317         ns = mod_64(ktime_to_ns(remaining), apic->lapic_timer.period);
1318         tmcct = div64_u64(ns,
1319                          (APIC_BUS_CYCLE_NS * apic->divide_count));
1320
1321         return tmcct;
1322 }
1323
1324 static void __report_tpr_access(struct kvm_lapic *apic, bool write)
1325 {
1326         struct kvm_vcpu *vcpu = apic->vcpu;
1327         struct kvm_run *run = vcpu->run;
1328
1329         kvm_make_request(KVM_REQ_REPORT_TPR_ACCESS, vcpu);
1330         run->tpr_access.rip = kvm_rip_read(vcpu);
1331         run->tpr_access.is_write = write;
1332 }
1333
1334 static inline void report_tpr_access(struct kvm_lapic *apic, bool write)
1335 {
1336         if (apic->vcpu->arch.tpr_access_reporting)
1337                 __report_tpr_access(apic, write);
1338 }
1339
1340 static u32 __apic_read(struct kvm_lapic *apic, unsigned int offset)
1341 {
1342         u32 val = 0;
1343
1344         if (offset >= LAPIC_MMIO_LENGTH)
1345                 return 0;
1346
1347         switch (offset) {
1348         case APIC_ARBPRI:
1349                 break;
1350
1351         case APIC_TMCCT:        /* Timer CCR */
1352                 if (apic_lvtt_tscdeadline(apic))
1353                         return 0;
1354
1355                 val = apic_get_tmcct(apic);
1356                 break;
1357         case APIC_PROCPRI:
1358                 apic_update_ppr(apic);
1359                 val = kvm_lapic_get_reg(apic, offset);
1360                 break;
1361         case APIC_TASKPRI:
1362                 report_tpr_access(apic, false);
1363                 fallthrough;
1364         default:
1365                 val = kvm_lapic_get_reg(apic, offset);
1366                 break;
1367         }
1368
1369         return val;
1370 }
1371
1372 static inline struct kvm_lapic *to_lapic(struct kvm_io_device *dev)
1373 {
1374         return container_of(dev, struct kvm_lapic, dev);
1375 }
1376
1377 #define APIC_REG_MASK(reg)      (1ull << ((reg) >> 4))
1378 #define APIC_REGS_MASK(first, count) \
1379         (APIC_REG_MASK(first) * ((1ull << (count)) - 1))
1380
1381 int kvm_lapic_reg_read(struct kvm_lapic *apic, u32 offset, int len,
1382                 void *data)
1383 {
1384         unsigned char alignment = offset & 0xf;
1385         u32 result;
1386         /* this bitmask has a bit cleared for each reserved register */
1387         u64 valid_reg_mask =
1388                 APIC_REG_MASK(APIC_ID) |
1389                 APIC_REG_MASK(APIC_LVR) |
1390                 APIC_REG_MASK(APIC_TASKPRI) |
1391                 APIC_REG_MASK(APIC_PROCPRI) |
1392                 APIC_REG_MASK(APIC_LDR) |
1393                 APIC_REG_MASK(APIC_DFR) |
1394                 APIC_REG_MASK(APIC_SPIV) |
1395                 APIC_REGS_MASK(APIC_ISR, APIC_ISR_NR) |
1396                 APIC_REGS_MASK(APIC_TMR, APIC_ISR_NR) |
1397                 APIC_REGS_MASK(APIC_IRR, APIC_ISR_NR) |
1398                 APIC_REG_MASK(APIC_ESR) |
1399                 APIC_REG_MASK(APIC_ICR) |
1400                 APIC_REG_MASK(APIC_ICR2) |
1401                 APIC_REG_MASK(APIC_LVTT) |
1402                 APIC_REG_MASK(APIC_LVTTHMR) |
1403                 APIC_REG_MASK(APIC_LVTPC) |
1404                 APIC_REG_MASK(APIC_LVT0) |
1405                 APIC_REG_MASK(APIC_LVT1) |
1406                 APIC_REG_MASK(APIC_LVTERR) |
1407                 APIC_REG_MASK(APIC_TMICT) |
1408                 APIC_REG_MASK(APIC_TMCCT) |
1409                 APIC_REG_MASK(APIC_TDCR);
1410
1411         /* ARBPRI is not valid on x2APIC */
1412         if (!apic_x2apic_mode(apic))
1413                 valid_reg_mask |= APIC_REG_MASK(APIC_ARBPRI);
1414
1415         if (alignment + len > 4)
1416                 return 1;
1417
1418         if (offset > 0x3f0 || !(valid_reg_mask & APIC_REG_MASK(offset)))
1419                 return 1;
1420
1421         result = __apic_read(apic, offset & ~0xf);
1422
1423         trace_kvm_apic_read(offset, result);
1424
1425         switch (len) {
1426         case 1:
1427         case 2:
1428         case 4:
1429                 memcpy(data, (char *)&result + alignment, len);
1430                 break;
1431         default:
1432                 printk(KERN_ERR "Local APIC read with len = %x, "
1433                        "should be 1,2, or 4 instead\n", len);
1434                 break;
1435         }
1436         return 0;
1437 }
1438 EXPORT_SYMBOL_GPL(kvm_lapic_reg_read);
1439
1440 static int apic_mmio_in_range(struct kvm_lapic *apic, gpa_t addr)
1441 {
1442         return addr >= apic->base_address &&
1443                 addr < apic->base_address + LAPIC_MMIO_LENGTH;
1444 }
1445
1446 static int apic_mmio_read(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
1447                            gpa_t address, int len, void *data)
1448 {
1449         struct kvm_lapic *apic = to_lapic(this);
1450         u32 offset = address - apic->base_address;
1451
1452         if (!apic_mmio_in_range(apic, address))
1453                 return -EOPNOTSUPP;
1454
1455         if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
1456                 if (!kvm_check_has_quirk(vcpu->kvm,
1457                                          KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
1458                         return -EOPNOTSUPP;
1459
1460                 memset(data, 0xff, len);
1461                 return 0;
1462         }
1463
1464         kvm_lapic_reg_read(apic, offset, len, data);
1465
1466         return 0;
1467 }
1468
1469 static void update_divide_count(struct kvm_lapic *apic)
1470 {
1471         u32 tmp1, tmp2, tdcr;
1472
1473         tdcr = kvm_lapic_get_reg(apic, APIC_TDCR);
1474         tmp1 = tdcr & 0xf;
1475         tmp2 = ((tmp1 & 0x3) | ((tmp1 & 0x8) >> 1)) + 1;
1476         apic->divide_count = 0x1 << (tmp2 & 0x7);
1477 }
1478
1479 static void limit_periodic_timer_frequency(struct kvm_lapic *apic)
1480 {
1481         /*
1482          * Do not allow the guest to program periodic timers with small
1483          * interval, since the hrtimers are not throttled by the host
1484          * scheduler.
1485          */
1486         if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
1487                 s64 min_period = min_timer_period_us * 1000LL;
1488
1489                 if (apic->lapic_timer.period < min_period) {
1490                         pr_info_ratelimited(
1491                             "kvm: vcpu %i: requested %lld ns "
1492                             "lapic timer period limited to %lld ns\n",
1493                             apic->vcpu->vcpu_id,
1494                             apic->lapic_timer.period, min_period);
1495                         apic->lapic_timer.period = min_period;
1496                 }
1497         }
1498 }
1499
1500 static void cancel_hv_timer(struct kvm_lapic *apic);
1501
1502 static void cancel_apic_timer(struct kvm_lapic *apic)
1503 {
1504         hrtimer_cancel(&apic->lapic_timer.timer);
1505         preempt_disable();
1506         if (apic->lapic_timer.hv_timer_in_use)
1507                 cancel_hv_timer(apic);
1508         preempt_enable();
1509         atomic_set(&apic->lapic_timer.pending, 0);
1510 }
1511
1512 static void apic_update_lvtt(struct kvm_lapic *apic)
1513 {
1514         u32 timer_mode = kvm_lapic_get_reg(apic, APIC_LVTT) &
1515                         apic->lapic_timer.timer_mode_mask;
1516
1517         if (apic->lapic_timer.timer_mode != timer_mode) {
1518                 if (apic_lvtt_tscdeadline(apic) != (timer_mode ==
1519                                 APIC_LVT_TIMER_TSCDEADLINE)) {
1520                         cancel_apic_timer(apic);
1521                         kvm_lapic_set_reg(apic, APIC_TMICT, 0);
1522                         apic->lapic_timer.period = 0;
1523                         apic->lapic_timer.tscdeadline = 0;
1524                 }
1525                 apic->lapic_timer.timer_mode = timer_mode;
1526                 limit_periodic_timer_frequency(apic);
1527         }
1528 }
1529
1530 /*
1531  * On APICv, this test will cause a busy wait
1532  * during a higher-priority task.
1533  */
1534
1535 static bool lapic_timer_int_injected(struct kvm_vcpu *vcpu)
1536 {
1537         struct kvm_lapic *apic = vcpu->arch.apic;
1538         u32 reg = kvm_lapic_get_reg(apic, APIC_LVTT);
1539
1540         if (kvm_apic_hw_enabled(apic)) {
1541                 int vec = reg & APIC_VECTOR_MASK;
1542                 void *bitmap = apic->regs + APIC_ISR;
1543
1544                 if (vcpu->arch.apicv_active)
1545                         bitmap = apic->regs + APIC_IRR;
1546
1547                 if (apic_test_vector(vec, bitmap))
1548                         return true;
1549         }
1550         return false;
1551 }
1552
1553 static inline void __wait_lapic_expire(struct kvm_vcpu *vcpu, u64 guest_cycles)
1554 {
1555         u64 timer_advance_ns = vcpu->arch.apic->lapic_timer.timer_advance_ns;
1556
1557         /*
1558          * If the guest TSC is running at a different ratio than the host, then
1559          * convert the delay to nanoseconds to achieve an accurate delay.  Note
1560          * that __delay() uses delay_tsc whenever the hardware has TSC, thus
1561          * always for VMX enabled hardware.
1562          */
1563         if (vcpu->arch.tsc_scaling_ratio == kvm_default_tsc_scaling_ratio) {
1564                 __delay(min(guest_cycles,
1565                         nsec_to_cycles(vcpu, timer_advance_ns)));
1566         } else {
1567                 u64 delay_ns = guest_cycles * 1000000ULL;
1568                 do_div(delay_ns, vcpu->arch.virtual_tsc_khz);
1569                 ndelay(min_t(u32, delay_ns, timer_advance_ns));
1570         }
1571 }
1572
1573 static inline void adjust_lapic_timer_advance(struct kvm_vcpu *vcpu,
1574                                               s64 advance_expire_delta)
1575 {
1576         struct kvm_lapic *apic = vcpu->arch.apic;
1577         u32 timer_advance_ns = apic->lapic_timer.timer_advance_ns;
1578         u64 ns;
1579
1580         /* Do not adjust for tiny fluctuations or large random spikes. */
1581         if (abs(advance_expire_delta) > LAPIC_TIMER_ADVANCE_ADJUST_MAX ||
1582             abs(advance_expire_delta) < LAPIC_TIMER_ADVANCE_ADJUST_MIN)
1583                 return;
1584
1585         /* too early */
1586         if (advance_expire_delta < 0) {
1587                 ns = -advance_expire_delta * 1000000ULL;
1588                 do_div(ns, vcpu->arch.virtual_tsc_khz);
1589                 timer_advance_ns -= ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
1590         } else {
1591         /* too late */
1592                 ns = advance_expire_delta * 1000000ULL;
1593                 do_div(ns, vcpu->arch.virtual_tsc_khz);
1594                 timer_advance_ns += ns/LAPIC_TIMER_ADVANCE_ADJUST_STEP;
1595         }
1596
1597         if (unlikely(timer_advance_ns > LAPIC_TIMER_ADVANCE_NS_MAX))
1598                 timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT;
1599         apic->lapic_timer.timer_advance_ns = timer_advance_ns;
1600 }
1601
1602 static void __kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
1603 {
1604         struct kvm_lapic *apic = vcpu->arch.apic;
1605         u64 guest_tsc, tsc_deadline;
1606
1607         tsc_deadline = apic->lapic_timer.expired_tscdeadline;
1608         apic->lapic_timer.expired_tscdeadline = 0;
1609         guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1610         apic->lapic_timer.advance_expire_delta = guest_tsc - tsc_deadline;
1611
1612         if (lapic_timer_advance_dynamic) {
1613                 adjust_lapic_timer_advance(vcpu, apic->lapic_timer.advance_expire_delta);
1614                 /*
1615                  * If the timer fired early, reread the TSC to account for the
1616                  * overhead of the above adjustment to avoid waiting longer
1617                  * than is necessary.
1618                  */
1619                 if (guest_tsc < tsc_deadline)
1620                         guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1621         }
1622
1623         if (guest_tsc < tsc_deadline)
1624                 __wait_lapic_expire(vcpu, tsc_deadline - guest_tsc);
1625 }
1626
1627 void kvm_wait_lapic_expire(struct kvm_vcpu *vcpu)
1628 {
1629         if (lapic_in_kernel(vcpu) &&
1630             vcpu->arch.apic->lapic_timer.expired_tscdeadline &&
1631             vcpu->arch.apic->lapic_timer.timer_advance_ns &&
1632             lapic_timer_int_injected(vcpu))
1633                 __kvm_wait_lapic_expire(vcpu);
1634 }
1635 EXPORT_SYMBOL_GPL(kvm_wait_lapic_expire);
1636
1637 static void kvm_apic_inject_pending_timer_irqs(struct kvm_lapic *apic)
1638 {
1639         struct kvm_timer *ktimer = &apic->lapic_timer;
1640
1641         kvm_apic_local_deliver(apic, APIC_LVTT);
1642         if (apic_lvtt_tscdeadline(apic)) {
1643                 ktimer->tscdeadline = 0;
1644         } else if (apic_lvtt_oneshot(apic)) {
1645                 ktimer->tscdeadline = 0;
1646                 ktimer->target_expiration = 0;
1647         }
1648 }
1649
1650 static void apic_timer_expired(struct kvm_lapic *apic, bool from_timer_fn)
1651 {
1652         struct kvm_vcpu *vcpu = apic->vcpu;
1653         struct kvm_timer *ktimer = &apic->lapic_timer;
1654
1655         if (atomic_read(&apic->lapic_timer.pending))
1656                 return;
1657
1658         if (apic_lvtt_tscdeadline(apic) || ktimer->hv_timer_in_use)
1659                 ktimer->expired_tscdeadline = ktimer->tscdeadline;
1660
1661         if (!from_timer_fn && vcpu->arch.apicv_active) {
1662                 WARN_ON(kvm_get_running_vcpu() != vcpu);
1663                 kvm_apic_inject_pending_timer_irqs(apic);
1664                 return;
1665         }
1666
1667         if (kvm_use_posted_timer_interrupt(apic->vcpu)) {
1668                 /*
1669                  * Ensure the guest's timer has truly expired before posting an
1670                  * interrupt.  Open code the relevant checks to avoid querying
1671                  * lapic_timer_int_injected(), which will be false since the
1672                  * interrupt isn't yet injected.  Waiting until after injecting
1673                  * is not an option since that won't help a posted interrupt.
1674                  */
1675                 if (vcpu->arch.apic->lapic_timer.expired_tscdeadline &&
1676                     vcpu->arch.apic->lapic_timer.timer_advance_ns)
1677                         __kvm_wait_lapic_expire(vcpu);
1678                 kvm_apic_inject_pending_timer_irqs(apic);
1679                 return;
1680         }
1681
1682         atomic_inc(&apic->lapic_timer.pending);
1683         kvm_make_request(KVM_REQ_UNBLOCK, vcpu);
1684         if (from_timer_fn)
1685                 kvm_vcpu_kick(vcpu);
1686 }
1687
1688 static void start_sw_tscdeadline(struct kvm_lapic *apic)
1689 {
1690         struct kvm_timer *ktimer = &apic->lapic_timer;
1691         u64 guest_tsc, tscdeadline = ktimer->tscdeadline;
1692         u64 ns = 0;
1693         ktime_t expire;
1694         struct kvm_vcpu *vcpu = apic->vcpu;
1695         unsigned long this_tsc_khz = vcpu->arch.virtual_tsc_khz;
1696         unsigned long flags;
1697         ktime_t now;
1698
1699         if (unlikely(!tscdeadline || !this_tsc_khz))
1700                 return;
1701
1702         local_irq_save(flags);
1703
1704         now = ktime_get();
1705         guest_tsc = kvm_read_l1_tsc(vcpu, rdtsc());
1706
1707         ns = (tscdeadline - guest_tsc) * 1000000ULL;
1708         do_div(ns, this_tsc_khz);
1709
1710         if (likely(tscdeadline > guest_tsc) &&
1711             likely(ns > apic->lapic_timer.timer_advance_ns)) {
1712                 expire = ktime_add_ns(now, ns);
1713                 expire = ktime_sub_ns(expire, ktimer->timer_advance_ns);
1714                 hrtimer_start(&ktimer->timer, expire, HRTIMER_MODE_ABS_HARD);
1715         } else
1716                 apic_timer_expired(apic, false);
1717
1718         local_irq_restore(flags);
1719 }
1720
1721 static inline u64 tmict_to_ns(struct kvm_lapic *apic, u32 tmict)
1722 {
1723         return (u64)tmict * APIC_BUS_CYCLE_NS * (u64)apic->divide_count;
1724 }
1725
1726 static void update_target_expiration(struct kvm_lapic *apic, uint32_t old_divisor)
1727 {
1728         ktime_t now, remaining;
1729         u64 ns_remaining_old, ns_remaining_new;
1730
1731         apic->lapic_timer.period =
1732                         tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT));
1733         limit_periodic_timer_frequency(apic);
1734
1735         now = ktime_get();
1736         remaining = ktime_sub(apic->lapic_timer.target_expiration, now);
1737         if (ktime_to_ns(remaining) < 0)
1738                 remaining = 0;
1739
1740         ns_remaining_old = ktime_to_ns(remaining);
1741         ns_remaining_new = mul_u64_u32_div(ns_remaining_old,
1742                                            apic->divide_count, old_divisor);
1743
1744         apic->lapic_timer.tscdeadline +=
1745                 nsec_to_cycles(apic->vcpu, ns_remaining_new) -
1746                 nsec_to_cycles(apic->vcpu, ns_remaining_old);
1747         apic->lapic_timer.target_expiration = ktime_add_ns(now, ns_remaining_new);
1748 }
1749
1750 static bool set_target_expiration(struct kvm_lapic *apic, u32 count_reg)
1751 {
1752         ktime_t now;
1753         u64 tscl = rdtsc();
1754         s64 deadline;
1755
1756         now = ktime_get();
1757         apic->lapic_timer.period =
1758                         tmict_to_ns(apic, kvm_lapic_get_reg(apic, APIC_TMICT));
1759
1760         if (!apic->lapic_timer.period) {
1761                 apic->lapic_timer.tscdeadline = 0;
1762                 return false;
1763         }
1764
1765         limit_periodic_timer_frequency(apic);
1766         deadline = apic->lapic_timer.period;
1767
1768         if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic)) {
1769                 if (unlikely(count_reg != APIC_TMICT)) {
1770                         deadline = tmict_to_ns(apic,
1771                                      kvm_lapic_get_reg(apic, count_reg));
1772                         if (unlikely(deadline <= 0))
1773                                 deadline = apic->lapic_timer.period;
1774                         else if (unlikely(deadline > apic->lapic_timer.period)) {
1775                                 pr_info_ratelimited(
1776                                     "kvm: vcpu %i: requested lapic timer restore with "
1777                                     "starting count register %#x=%u (%lld ns) > initial count (%lld ns). "
1778                                     "Using initial count to start timer.\n",
1779                                     apic->vcpu->vcpu_id,
1780                                     count_reg,
1781                                     kvm_lapic_get_reg(apic, count_reg),
1782                                     deadline, apic->lapic_timer.period);
1783                                 kvm_lapic_set_reg(apic, count_reg, 0);
1784                                 deadline = apic->lapic_timer.period;
1785                         }
1786                 }
1787         }
1788
1789         apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
1790                 nsec_to_cycles(apic->vcpu, deadline);
1791         apic->lapic_timer.target_expiration = ktime_add_ns(now, deadline);
1792
1793         return true;
1794 }
1795
1796 static void advance_periodic_target_expiration(struct kvm_lapic *apic)
1797 {
1798         ktime_t now = ktime_get();
1799         u64 tscl = rdtsc();
1800         ktime_t delta;
1801
1802         /*
1803          * Synchronize both deadlines to the same time source or
1804          * differences in the periods (caused by differences in the
1805          * underlying clocks or numerical approximation errors) will
1806          * cause the two to drift apart over time as the errors
1807          * accumulate.
1808          */
1809         apic->lapic_timer.target_expiration =
1810                 ktime_add_ns(apic->lapic_timer.target_expiration,
1811                                 apic->lapic_timer.period);
1812         delta = ktime_sub(apic->lapic_timer.target_expiration, now);
1813         apic->lapic_timer.tscdeadline = kvm_read_l1_tsc(apic->vcpu, tscl) +
1814                 nsec_to_cycles(apic->vcpu, delta);
1815 }
1816
1817 static void start_sw_period(struct kvm_lapic *apic)
1818 {
1819         if (!apic->lapic_timer.period)
1820                 return;
1821
1822         if (ktime_after(ktime_get(),
1823                         apic->lapic_timer.target_expiration)) {
1824                 apic_timer_expired(apic, false);
1825
1826                 if (apic_lvtt_oneshot(apic))
1827                         return;
1828
1829                 advance_periodic_target_expiration(apic);
1830         }
1831
1832         hrtimer_start(&apic->lapic_timer.timer,
1833                 apic->lapic_timer.target_expiration,
1834                 HRTIMER_MODE_ABS_HARD);
1835 }
1836
1837 bool kvm_lapic_hv_timer_in_use(struct kvm_vcpu *vcpu)
1838 {
1839         if (!lapic_in_kernel(vcpu))
1840                 return false;
1841
1842         return vcpu->arch.apic->lapic_timer.hv_timer_in_use;
1843 }
1844 EXPORT_SYMBOL_GPL(kvm_lapic_hv_timer_in_use);
1845
1846 static void cancel_hv_timer(struct kvm_lapic *apic)
1847 {
1848         WARN_ON(preemptible());
1849         WARN_ON(!apic->lapic_timer.hv_timer_in_use);
1850         static_call(kvm_x86_cancel_hv_timer)(apic->vcpu);
1851         apic->lapic_timer.hv_timer_in_use = false;
1852 }
1853
1854 static bool start_hv_timer(struct kvm_lapic *apic)
1855 {
1856         struct kvm_timer *ktimer = &apic->lapic_timer;
1857         struct kvm_vcpu *vcpu = apic->vcpu;
1858         bool expired;
1859
1860         WARN_ON(preemptible());
1861         if (!kvm_can_use_hv_timer(vcpu))
1862                 return false;
1863
1864         if (!ktimer->tscdeadline)
1865                 return false;
1866
1867         if (static_call(kvm_x86_set_hv_timer)(vcpu, ktimer->tscdeadline, &expired))
1868                 return false;
1869
1870         ktimer->hv_timer_in_use = true;
1871         hrtimer_cancel(&ktimer->timer);
1872
1873         /*
1874          * To simplify handling the periodic timer, leave the hv timer running
1875          * even if the deadline timer has expired, i.e. rely on the resulting
1876          * VM-Exit to recompute the periodic timer's target expiration.
1877          */
1878         if (!apic_lvtt_period(apic)) {
1879                 /*
1880                  * Cancel the hv timer if the sw timer fired while the hv timer
1881                  * was being programmed, or if the hv timer itself expired.
1882                  */
1883                 if (atomic_read(&ktimer->pending)) {
1884                         cancel_hv_timer(apic);
1885                 } else if (expired) {
1886                         apic_timer_expired(apic, false);
1887                         cancel_hv_timer(apic);
1888                 }
1889         }
1890
1891         trace_kvm_hv_timer_state(vcpu->vcpu_id, ktimer->hv_timer_in_use);
1892
1893         return true;
1894 }
1895
1896 static void start_sw_timer(struct kvm_lapic *apic)
1897 {
1898         struct kvm_timer *ktimer = &apic->lapic_timer;
1899
1900         WARN_ON(preemptible());
1901         if (apic->lapic_timer.hv_timer_in_use)
1902                 cancel_hv_timer(apic);
1903         if (!apic_lvtt_period(apic) && atomic_read(&ktimer->pending))
1904                 return;
1905
1906         if (apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
1907                 start_sw_period(apic);
1908         else if (apic_lvtt_tscdeadline(apic))
1909                 start_sw_tscdeadline(apic);
1910         trace_kvm_hv_timer_state(apic->vcpu->vcpu_id, false);
1911 }
1912
1913 static void restart_apic_timer(struct kvm_lapic *apic)
1914 {
1915         preempt_disable();
1916
1917         if (!apic_lvtt_period(apic) && atomic_read(&apic->lapic_timer.pending))
1918                 goto out;
1919
1920         if (!start_hv_timer(apic))
1921                 start_sw_timer(apic);
1922 out:
1923         preempt_enable();
1924 }
1925
1926 void kvm_lapic_expired_hv_timer(struct kvm_vcpu *vcpu)
1927 {
1928         struct kvm_lapic *apic = vcpu->arch.apic;
1929
1930         preempt_disable();
1931         /* If the preempt notifier has already run, it also called apic_timer_expired */
1932         if (!apic->lapic_timer.hv_timer_in_use)
1933                 goto out;
1934         WARN_ON(rcuwait_active(&vcpu->wait));
1935         apic_timer_expired(apic, false);
1936         cancel_hv_timer(apic);
1937
1938         if (apic_lvtt_period(apic) && apic->lapic_timer.period) {
1939                 advance_periodic_target_expiration(apic);
1940                 restart_apic_timer(apic);
1941         }
1942 out:
1943         preempt_enable();
1944 }
1945 EXPORT_SYMBOL_GPL(kvm_lapic_expired_hv_timer);
1946
1947 void kvm_lapic_switch_to_hv_timer(struct kvm_vcpu *vcpu)
1948 {
1949         restart_apic_timer(vcpu->arch.apic);
1950 }
1951 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_hv_timer);
1952
1953 void kvm_lapic_switch_to_sw_timer(struct kvm_vcpu *vcpu)
1954 {
1955         struct kvm_lapic *apic = vcpu->arch.apic;
1956
1957         preempt_disable();
1958         /* Possibly the TSC deadline timer is not enabled yet */
1959         if (apic->lapic_timer.hv_timer_in_use)
1960                 start_sw_timer(apic);
1961         preempt_enable();
1962 }
1963 EXPORT_SYMBOL_GPL(kvm_lapic_switch_to_sw_timer);
1964
1965 void kvm_lapic_restart_hv_timer(struct kvm_vcpu *vcpu)
1966 {
1967         struct kvm_lapic *apic = vcpu->arch.apic;
1968
1969         WARN_ON(!apic->lapic_timer.hv_timer_in_use);
1970         restart_apic_timer(apic);
1971 }
1972
1973 static void __start_apic_timer(struct kvm_lapic *apic, u32 count_reg)
1974 {
1975         atomic_set(&apic->lapic_timer.pending, 0);
1976
1977         if ((apic_lvtt_period(apic) || apic_lvtt_oneshot(apic))
1978             && !set_target_expiration(apic, count_reg))
1979                 return;
1980
1981         restart_apic_timer(apic);
1982 }
1983
1984 static void start_apic_timer(struct kvm_lapic *apic)
1985 {
1986         __start_apic_timer(apic, APIC_TMICT);
1987 }
1988
1989 static void apic_manage_nmi_watchdog(struct kvm_lapic *apic, u32 lvt0_val)
1990 {
1991         bool lvt0_in_nmi_mode = apic_lvt_nmi_mode(lvt0_val);
1992
1993         if (apic->lvt0_in_nmi_mode != lvt0_in_nmi_mode) {
1994                 apic->lvt0_in_nmi_mode = lvt0_in_nmi_mode;
1995                 if (lvt0_in_nmi_mode) {
1996                         atomic_inc(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
1997                 } else
1998                         atomic_dec(&apic->vcpu->kvm->arch.vapics_in_nmi_mode);
1999         }
2000 }
2001
2002 int kvm_lapic_reg_write(struct kvm_lapic *apic, u32 reg, u32 val)
2003 {
2004         int ret = 0;
2005
2006         trace_kvm_apic_write(reg, val);
2007
2008         switch (reg) {
2009         case APIC_ID:           /* Local APIC ID */
2010                 if (!apic_x2apic_mode(apic))
2011                         kvm_apic_set_xapic_id(apic, val >> 24);
2012                 else
2013                         ret = 1;
2014                 break;
2015
2016         case APIC_TASKPRI:
2017                 report_tpr_access(apic, true);
2018                 apic_set_tpr(apic, val & 0xff);
2019                 break;
2020
2021         case APIC_EOI:
2022                 apic_set_eoi(apic);
2023                 break;
2024
2025         case APIC_LDR:
2026                 if (!apic_x2apic_mode(apic))
2027                         kvm_apic_set_ldr(apic, val & APIC_LDR_MASK);
2028                 else
2029                         ret = 1;
2030                 break;
2031
2032         case APIC_DFR:
2033                 if (!apic_x2apic_mode(apic))
2034                         kvm_apic_set_dfr(apic, val | 0x0FFFFFFF);
2035                 else
2036                         ret = 1;
2037                 break;
2038
2039         case APIC_SPIV: {
2040                 u32 mask = 0x3ff;
2041                 if (kvm_lapic_get_reg(apic, APIC_LVR) & APIC_LVR_DIRECTED_EOI)
2042                         mask |= APIC_SPIV_DIRECTED_EOI;
2043                 apic_set_spiv(apic, val & mask);
2044                 if (!(val & APIC_SPIV_APIC_ENABLED)) {
2045                         int i;
2046                         u32 lvt_val;
2047
2048                         for (i = 0; i < KVM_APIC_LVT_NUM; i++) {
2049                                 lvt_val = kvm_lapic_get_reg(apic,
2050                                                        APIC_LVTT + 0x10 * i);
2051                                 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i,
2052                                              lvt_val | APIC_LVT_MASKED);
2053                         }
2054                         apic_update_lvtt(apic);
2055                         atomic_set(&apic->lapic_timer.pending, 0);
2056
2057                 }
2058                 break;
2059         }
2060         case APIC_ICR:
2061                 /* No delay here, so we always clear the pending bit */
2062                 val &= ~(1 << 12);
2063                 kvm_apic_send_ipi(apic, val, kvm_lapic_get_reg(apic, APIC_ICR2));
2064                 kvm_lapic_set_reg(apic, APIC_ICR, val);
2065                 break;
2066
2067         case APIC_ICR2:
2068                 if (!apic_x2apic_mode(apic))
2069                         val &= 0xff000000;
2070                 kvm_lapic_set_reg(apic, APIC_ICR2, val);
2071                 break;
2072
2073         case APIC_LVT0:
2074                 apic_manage_nmi_watchdog(apic, val);
2075                 fallthrough;
2076         case APIC_LVTTHMR:
2077         case APIC_LVTPC:
2078         case APIC_LVT1:
2079         case APIC_LVTERR: {
2080                 /* TODO: Check vector */
2081                 size_t size;
2082                 u32 index;
2083
2084                 if (!kvm_apic_sw_enabled(apic))
2085                         val |= APIC_LVT_MASKED;
2086                 size = ARRAY_SIZE(apic_lvt_mask);
2087                 index = array_index_nospec(
2088                                 (reg - APIC_LVTT) >> 4, size);
2089                 val &= apic_lvt_mask[index];
2090                 kvm_lapic_set_reg(apic, reg, val);
2091                 break;
2092         }
2093
2094         case APIC_LVTT:
2095                 if (!kvm_apic_sw_enabled(apic))
2096                         val |= APIC_LVT_MASKED;
2097                 val &= (apic_lvt_mask[0] | apic->lapic_timer.timer_mode_mask);
2098                 kvm_lapic_set_reg(apic, APIC_LVTT, val);
2099                 apic_update_lvtt(apic);
2100                 break;
2101
2102         case APIC_TMICT:
2103                 if (apic_lvtt_tscdeadline(apic))
2104                         break;
2105
2106                 cancel_apic_timer(apic);
2107                 kvm_lapic_set_reg(apic, APIC_TMICT, val);
2108                 start_apic_timer(apic);
2109                 break;
2110
2111         case APIC_TDCR: {
2112                 uint32_t old_divisor = apic->divide_count;
2113
2114                 kvm_lapic_set_reg(apic, APIC_TDCR, val & 0xb);
2115                 update_divide_count(apic);
2116                 if (apic->divide_count != old_divisor &&
2117                                 apic->lapic_timer.period) {
2118                         hrtimer_cancel(&apic->lapic_timer.timer);
2119                         update_target_expiration(apic, old_divisor);
2120                         restart_apic_timer(apic);
2121                 }
2122                 break;
2123         }
2124         case APIC_ESR:
2125                 if (apic_x2apic_mode(apic) && val != 0)
2126                         ret = 1;
2127                 break;
2128
2129         case APIC_SELF_IPI:
2130                 if (apic_x2apic_mode(apic))
2131                         kvm_apic_send_ipi(apic, APIC_DEST_SELF | (val & APIC_VECTOR_MASK), 0);
2132                 else
2133                         ret = 1;
2134                 break;
2135         default:
2136                 ret = 1;
2137                 break;
2138         }
2139
2140         kvm_recalculate_apic_map(apic->vcpu->kvm);
2141
2142         return ret;
2143 }
2144 EXPORT_SYMBOL_GPL(kvm_lapic_reg_write);
2145
2146 static int apic_mmio_write(struct kvm_vcpu *vcpu, struct kvm_io_device *this,
2147                             gpa_t address, int len, const void *data)
2148 {
2149         struct kvm_lapic *apic = to_lapic(this);
2150         unsigned int offset = address - apic->base_address;
2151         u32 val;
2152
2153         if (!apic_mmio_in_range(apic, address))
2154                 return -EOPNOTSUPP;
2155
2156         if (!kvm_apic_hw_enabled(apic) || apic_x2apic_mode(apic)) {
2157                 if (!kvm_check_has_quirk(vcpu->kvm,
2158                                          KVM_X86_QUIRK_LAPIC_MMIO_HOLE))
2159                         return -EOPNOTSUPP;
2160
2161                 return 0;
2162         }
2163
2164         /*
2165          * APIC register must be aligned on 128-bits boundary.
2166          * 32/64/128 bits registers must be accessed thru 32 bits.
2167          * Refer SDM 8.4.1
2168          */
2169         if (len != 4 || (offset & 0xf))
2170                 return 0;
2171
2172         val = *(u32*)data;
2173
2174         kvm_lapic_reg_write(apic, offset & 0xff0, val);
2175
2176         return 0;
2177 }
2178
2179 void kvm_lapic_set_eoi(struct kvm_vcpu *vcpu)
2180 {
2181         kvm_lapic_reg_write(vcpu->arch.apic, APIC_EOI, 0);
2182 }
2183 EXPORT_SYMBOL_GPL(kvm_lapic_set_eoi);
2184
2185 /* emulate APIC access in a trap manner */
2186 void kvm_apic_write_nodecode(struct kvm_vcpu *vcpu, u32 offset)
2187 {
2188         u32 val = 0;
2189
2190         /* hw has done the conditional check and inst decode */
2191         offset &= 0xff0;
2192
2193         kvm_lapic_reg_read(vcpu->arch.apic, offset, 4, &val);
2194
2195         /* TODO: optimize to just emulate side effect w/o one more write */
2196         kvm_lapic_reg_write(vcpu->arch.apic, offset, val);
2197 }
2198 EXPORT_SYMBOL_GPL(kvm_apic_write_nodecode);
2199
2200 void kvm_free_lapic(struct kvm_vcpu *vcpu)
2201 {
2202         struct kvm_lapic *apic = vcpu->arch.apic;
2203
2204         if (!vcpu->arch.apic)
2205                 return;
2206
2207         hrtimer_cancel(&apic->lapic_timer.timer);
2208
2209         if (!(vcpu->arch.apic_base & MSR_IA32_APICBASE_ENABLE))
2210                 static_branch_slow_dec_deferred(&apic_hw_disabled);
2211
2212         if (!apic->sw_enabled)
2213                 static_branch_slow_dec_deferred(&apic_sw_disabled);
2214
2215         if (apic->regs)
2216                 free_page((unsigned long)apic->regs);
2217
2218         kfree(apic);
2219 }
2220
2221 /*
2222  *----------------------------------------------------------------------
2223  * LAPIC interface
2224  *----------------------------------------------------------------------
2225  */
2226 u64 kvm_get_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu)
2227 {
2228         struct kvm_lapic *apic = vcpu->arch.apic;
2229
2230         if (!kvm_apic_present(vcpu) || !apic_lvtt_tscdeadline(apic))
2231                 return 0;
2232
2233         return apic->lapic_timer.tscdeadline;
2234 }
2235
2236 void kvm_set_lapic_tscdeadline_msr(struct kvm_vcpu *vcpu, u64 data)
2237 {
2238         struct kvm_lapic *apic = vcpu->arch.apic;
2239
2240         if (!kvm_apic_present(vcpu) || !apic_lvtt_tscdeadline(apic))
2241                 return;
2242
2243         hrtimer_cancel(&apic->lapic_timer.timer);
2244         apic->lapic_timer.tscdeadline = data;
2245         start_apic_timer(apic);
2246 }
2247
2248 void kvm_lapic_set_tpr(struct kvm_vcpu *vcpu, unsigned long cr8)
2249 {
2250         apic_set_tpr(vcpu->arch.apic, (cr8 & 0x0f) << 4);
2251 }
2252
2253 u64 kvm_lapic_get_cr8(struct kvm_vcpu *vcpu)
2254 {
2255         u64 tpr;
2256
2257         tpr = (u64) kvm_lapic_get_reg(vcpu->arch.apic, APIC_TASKPRI);
2258
2259         return (tpr & 0xf0) >> 4;
2260 }
2261
2262 void kvm_lapic_set_base(struct kvm_vcpu *vcpu, u64 value)
2263 {
2264         u64 old_value = vcpu->arch.apic_base;
2265         struct kvm_lapic *apic = vcpu->arch.apic;
2266
2267         vcpu->arch.apic_base = value;
2268
2269         if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE)
2270                 kvm_update_cpuid_runtime(vcpu);
2271
2272         if (!apic)
2273                 return;
2274
2275         /* update jump label if enable bit changes */
2276         if ((old_value ^ value) & MSR_IA32_APICBASE_ENABLE) {
2277                 if (value & MSR_IA32_APICBASE_ENABLE) {
2278                         kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2279                         static_branch_slow_dec_deferred(&apic_hw_disabled);
2280                         /* Check if there are APF page ready requests pending */
2281                         kvm_make_request(KVM_REQ_APF_READY, vcpu);
2282                 } else {
2283                         static_branch_inc(&apic_hw_disabled.key);
2284                         atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
2285                 }
2286         }
2287
2288         if (((old_value ^ value) & X2APIC_ENABLE) && (value & X2APIC_ENABLE))
2289                 kvm_apic_set_x2apic_id(apic, vcpu->vcpu_id);
2290
2291         if ((old_value ^ value) & (MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE))
2292                 static_call(kvm_x86_set_virtual_apic_mode)(vcpu);
2293
2294         apic->base_address = apic->vcpu->arch.apic_base &
2295                              MSR_IA32_APICBASE_BASE;
2296
2297         if ((value & MSR_IA32_APICBASE_ENABLE) &&
2298              apic->base_address != APIC_DEFAULT_PHYS_BASE)
2299                 pr_warn_once("APIC base relocation is unsupported by KVM");
2300 }
2301
2302 void kvm_apic_update_apicv(struct kvm_vcpu *vcpu)
2303 {
2304         struct kvm_lapic *apic = vcpu->arch.apic;
2305
2306         if (vcpu->arch.apicv_active) {
2307                 /* irr_pending is always true when apicv is activated. */
2308                 apic->irr_pending = true;
2309                 apic->isr_count = 1;
2310         } else {
2311                 apic->irr_pending = (apic_search_irr(apic) != -1);
2312                 apic->isr_count = count_vectors(apic->regs + APIC_ISR);
2313         }
2314 }
2315 EXPORT_SYMBOL_GPL(kvm_apic_update_apicv);
2316
2317 void kvm_lapic_reset(struct kvm_vcpu *vcpu, bool init_event)
2318 {
2319         struct kvm_lapic *apic = vcpu->arch.apic;
2320         u64 msr_val;
2321         int i;
2322
2323         if (!init_event) {
2324                 msr_val = APIC_DEFAULT_PHYS_BASE | MSR_IA32_APICBASE_ENABLE;
2325                 if (kvm_vcpu_is_reset_bsp(vcpu))
2326                         msr_val |= MSR_IA32_APICBASE_BSP;
2327                 kvm_lapic_set_base(vcpu, msr_val);
2328         }
2329
2330         if (!apic)
2331                 return;
2332
2333         /* Stop the timer in case it's a reset to an active apic */
2334         hrtimer_cancel(&apic->lapic_timer.timer);
2335
2336         /* The xAPIC ID is set at RESET even if the APIC was already enabled. */
2337         if (!init_event)
2338                 kvm_apic_set_xapic_id(apic, vcpu->vcpu_id);
2339         kvm_apic_set_version(apic->vcpu);
2340
2341         for (i = 0; i < KVM_APIC_LVT_NUM; i++)
2342                 kvm_lapic_set_reg(apic, APIC_LVTT + 0x10 * i, APIC_LVT_MASKED);
2343         apic_update_lvtt(apic);
2344         if (kvm_vcpu_is_reset_bsp(vcpu) &&
2345             kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_LINT0_REENABLED))
2346                 kvm_lapic_set_reg(apic, APIC_LVT0,
2347                              SET_APIC_DELIVERY_MODE(0, APIC_MODE_EXTINT));
2348         apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
2349
2350         kvm_apic_set_dfr(apic, 0xffffffffU);
2351         apic_set_spiv(apic, 0xff);
2352         kvm_lapic_set_reg(apic, APIC_TASKPRI, 0);
2353         if (!apic_x2apic_mode(apic))
2354                 kvm_apic_set_ldr(apic, 0);
2355         kvm_lapic_set_reg(apic, APIC_ESR, 0);
2356         kvm_lapic_set_reg(apic, APIC_ICR, 0);
2357         kvm_lapic_set_reg(apic, APIC_ICR2, 0);
2358         kvm_lapic_set_reg(apic, APIC_TDCR, 0);
2359         kvm_lapic_set_reg(apic, APIC_TMICT, 0);
2360         for (i = 0; i < 8; i++) {
2361                 kvm_lapic_set_reg(apic, APIC_IRR + 0x10 * i, 0);
2362                 kvm_lapic_set_reg(apic, APIC_ISR + 0x10 * i, 0);
2363                 kvm_lapic_set_reg(apic, APIC_TMR + 0x10 * i, 0);
2364         }
2365         kvm_apic_update_apicv(vcpu);
2366         apic->highest_isr_cache = -1;
2367         update_divide_count(apic);
2368         atomic_set(&apic->lapic_timer.pending, 0);
2369
2370         vcpu->arch.pv_eoi.msr_val = 0;
2371         apic_update_ppr(apic);
2372         if (vcpu->arch.apicv_active) {
2373                 static_call(kvm_x86_apicv_post_state_restore)(vcpu);
2374                 static_call(kvm_x86_hwapic_irr_update)(vcpu, -1);
2375                 static_call(kvm_x86_hwapic_isr_update)(vcpu, -1);
2376         }
2377
2378         vcpu->arch.apic_arb_prio = 0;
2379         vcpu->arch.apic_attention = 0;
2380
2381         kvm_recalculate_apic_map(vcpu->kvm);
2382 }
2383
2384 /*
2385  *----------------------------------------------------------------------
2386  * timer interface
2387  *----------------------------------------------------------------------
2388  */
2389
2390 static bool lapic_is_periodic(struct kvm_lapic *apic)
2391 {
2392         return apic_lvtt_period(apic);
2393 }
2394
2395 int apic_has_pending_timer(struct kvm_vcpu *vcpu)
2396 {
2397         struct kvm_lapic *apic = vcpu->arch.apic;
2398
2399         if (apic_enabled(apic) && apic_lvt_enabled(apic, APIC_LVTT))
2400                 return atomic_read(&apic->lapic_timer.pending);
2401
2402         return 0;
2403 }
2404
2405 int kvm_apic_local_deliver(struct kvm_lapic *apic, int lvt_type)
2406 {
2407         u32 reg = kvm_lapic_get_reg(apic, lvt_type);
2408         int vector, mode, trig_mode;
2409
2410         if (kvm_apic_hw_enabled(apic) && !(reg & APIC_LVT_MASKED)) {
2411                 vector = reg & APIC_VECTOR_MASK;
2412                 mode = reg & APIC_MODE_MASK;
2413                 trig_mode = reg & APIC_LVT_LEVEL_TRIGGER;
2414                 return __apic_accept_irq(apic, mode, vector, 1, trig_mode,
2415                                         NULL);
2416         }
2417         return 0;
2418 }
2419
2420 void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu)
2421 {
2422         struct kvm_lapic *apic = vcpu->arch.apic;
2423
2424         if (apic)
2425                 kvm_apic_local_deliver(apic, APIC_LVT0);
2426 }
2427
2428 static const struct kvm_io_device_ops apic_mmio_ops = {
2429         .read     = apic_mmio_read,
2430         .write    = apic_mmio_write,
2431 };
2432
2433 static enum hrtimer_restart apic_timer_fn(struct hrtimer *data)
2434 {
2435         struct kvm_timer *ktimer = container_of(data, struct kvm_timer, timer);
2436         struct kvm_lapic *apic = container_of(ktimer, struct kvm_lapic, lapic_timer);
2437
2438         apic_timer_expired(apic, true);
2439
2440         if (lapic_is_periodic(apic)) {
2441                 advance_periodic_target_expiration(apic);
2442                 hrtimer_add_expires_ns(&ktimer->timer, ktimer->period);
2443                 return HRTIMER_RESTART;
2444         } else
2445                 return HRTIMER_NORESTART;
2446 }
2447
2448 int kvm_create_lapic(struct kvm_vcpu *vcpu, int timer_advance_ns)
2449 {
2450         struct kvm_lapic *apic;
2451
2452         ASSERT(vcpu != NULL);
2453
2454         apic = kzalloc(sizeof(*apic), GFP_KERNEL_ACCOUNT);
2455         if (!apic)
2456                 goto nomem;
2457
2458         vcpu->arch.apic = apic;
2459
2460         apic->regs = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
2461         if (!apic->regs) {
2462                 printk(KERN_ERR "malloc apic regs error for vcpu %x\n",
2463                        vcpu->vcpu_id);
2464                 goto nomem_free_apic;
2465         }
2466         apic->vcpu = vcpu;
2467
2468         hrtimer_init(&apic->lapic_timer.timer, CLOCK_MONOTONIC,
2469                      HRTIMER_MODE_ABS_HARD);
2470         apic->lapic_timer.timer.function = apic_timer_fn;
2471         if (timer_advance_ns == -1) {
2472                 apic->lapic_timer.timer_advance_ns = LAPIC_TIMER_ADVANCE_NS_INIT;
2473                 lapic_timer_advance_dynamic = true;
2474         } else {
2475                 apic->lapic_timer.timer_advance_ns = timer_advance_ns;
2476                 lapic_timer_advance_dynamic = false;
2477         }
2478
2479         /*
2480          * Stuff the APIC ENABLE bit in lieu of temporarily incrementing
2481          * apic_hw_disabled; the full RESET value is set by kvm_lapic_reset().
2482          */
2483         vcpu->arch.apic_base = MSR_IA32_APICBASE_ENABLE;
2484         static_branch_inc(&apic_sw_disabled.key); /* sw disabled at reset */
2485         kvm_iodevice_init(&apic->dev, &apic_mmio_ops);
2486
2487         return 0;
2488 nomem_free_apic:
2489         kfree(apic);
2490         vcpu->arch.apic = NULL;
2491 nomem:
2492         return -ENOMEM;
2493 }
2494
2495 int kvm_apic_has_interrupt(struct kvm_vcpu *vcpu)
2496 {
2497         struct kvm_lapic *apic = vcpu->arch.apic;
2498         u32 ppr;
2499
2500         if (!kvm_apic_present(vcpu))
2501                 return -1;
2502
2503         __apic_update_ppr(apic, &ppr);
2504         return apic_has_interrupt_for_ppr(apic, ppr);
2505 }
2506 EXPORT_SYMBOL_GPL(kvm_apic_has_interrupt);
2507
2508 int kvm_apic_accept_pic_intr(struct kvm_vcpu *vcpu)
2509 {
2510         u32 lvt0 = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LVT0);
2511
2512         if (!kvm_apic_hw_enabled(vcpu->arch.apic))
2513                 return 1;
2514         if ((lvt0 & APIC_LVT_MASKED) == 0 &&
2515             GET_APIC_DELIVERY_MODE(lvt0) == APIC_MODE_EXTINT)
2516                 return 1;
2517         return 0;
2518 }
2519
2520 void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu)
2521 {
2522         struct kvm_lapic *apic = vcpu->arch.apic;
2523
2524         if (atomic_read(&apic->lapic_timer.pending) > 0) {
2525                 kvm_apic_inject_pending_timer_irqs(apic);
2526                 atomic_set(&apic->lapic_timer.pending, 0);
2527         }
2528 }
2529
2530 int kvm_get_apic_interrupt(struct kvm_vcpu *vcpu)
2531 {
2532         int vector = kvm_apic_has_interrupt(vcpu);
2533         struct kvm_lapic *apic = vcpu->arch.apic;
2534         u32 ppr;
2535
2536         if (vector == -1)
2537                 return -1;
2538
2539         /*
2540          * We get here even with APIC virtualization enabled, if doing
2541          * nested virtualization and L1 runs with the "acknowledge interrupt
2542          * on exit" mode.  Then we cannot inject the interrupt via RVI,
2543          * because the process would deliver it through the IDT.
2544          */
2545
2546         apic_clear_irr(vector, apic);
2547         if (to_hv_vcpu(vcpu) && test_bit(vector, to_hv_synic(vcpu)->auto_eoi_bitmap)) {
2548                 /*
2549                  * For auto-EOI interrupts, there might be another pending
2550                  * interrupt above PPR, so check whether to raise another
2551                  * KVM_REQ_EVENT.
2552                  */
2553                 apic_update_ppr(apic);
2554         } else {
2555                 /*
2556                  * For normal interrupts, PPR has been raised and there cannot
2557                  * be a higher-priority pending interrupt---except if there was
2558                  * a concurrent interrupt injection, but that would have
2559                  * triggered KVM_REQ_EVENT already.
2560                  */
2561                 apic_set_isr(vector, apic);
2562                 __apic_update_ppr(apic, &ppr);
2563         }
2564
2565         return vector;
2566 }
2567
2568 static int kvm_apic_state_fixup(struct kvm_vcpu *vcpu,
2569                 struct kvm_lapic_state *s, bool set)
2570 {
2571         if (apic_x2apic_mode(vcpu->arch.apic)) {
2572                 u32 *id = (u32 *)(s->regs + APIC_ID);
2573                 u32 *ldr = (u32 *)(s->regs + APIC_LDR);
2574
2575                 if (vcpu->kvm->arch.x2apic_format) {
2576                         if (*id != vcpu->vcpu_id)
2577                                 return -EINVAL;
2578                 } else {
2579                         if (set)
2580                                 *id >>= 24;
2581                         else
2582                                 *id <<= 24;
2583                 }
2584
2585                 /* In x2APIC mode, the LDR is fixed and based on the id */
2586                 if (set)
2587                         *ldr = kvm_apic_calc_x2apic_ldr(*id);
2588         }
2589
2590         return 0;
2591 }
2592
2593 int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
2594 {
2595         memcpy(s->regs, vcpu->arch.apic->regs, sizeof(*s));
2596
2597         /*
2598          * Get calculated timer current count for remaining timer period (if
2599          * any) and store it in the returned register set.
2600          */
2601         __kvm_lapic_set_reg(s->regs, APIC_TMCCT,
2602                             __apic_read(vcpu->arch.apic, APIC_TMCCT));
2603
2604         return kvm_apic_state_fixup(vcpu, s, false);
2605 }
2606
2607 int kvm_apic_set_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s)
2608 {
2609         struct kvm_lapic *apic = vcpu->arch.apic;
2610         int r;
2611
2612         kvm_lapic_set_base(vcpu, vcpu->arch.apic_base);
2613         /* set SPIV separately to get count of SW disabled APICs right */
2614         apic_set_spiv(apic, *((u32 *)(s->regs + APIC_SPIV)));
2615
2616         r = kvm_apic_state_fixup(vcpu, s, true);
2617         if (r) {
2618                 kvm_recalculate_apic_map(vcpu->kvm);
2619                 return r;
2620         }
2621         memcpy(vcpu->arch.apic->regs, s->regs, sizeof(*s));
2622
2623         atomic_set_release(&apic->vcpu->kvm->arch.apic_map_dirty, DIRTY);
2624         kvm_recalculate_apic_map(vcpu->kvm);
2625         kvm_apic_set_version(vcpu);
2626
2627         apic_update_ppr(apic);
2628         cancel_apic_timer(apic);
2629         apic->lapic_timer.expired_tscdeadline = 0;
2630         apic_update_lvtt(apic);
2631         apic_manage_nmi_watchdog(apic, kvm_lapic_get_reg(apic, APIC_LVT0));
2632         update_divide_count(apic);
2633         __start_apic_timer(apic, APIC_TMCCT);
2634         kvm_lapic_set_reg(apic, APIC_TMCCT, 0);
2635         kvm_apic_update_apicv(vcpu);
2636         apic->highest_isr_cache = -1;
2637         if (vcpu->arch.apicv_active) {
2638                 static_call(kvm_x86_apicv_post_state_restore)(vcpu);
2639                 static_call(kvm_x86_hwapic_irr_update)(vcpu,
2640                                 apic_find_highest_irr(apic));
2641                 static_call(kvm_x86_hwapic_isr_update)(vcpu,
2642                                 apic_find_highest_isr(apic));
2643         }
2644         kvm_make_request(KVM_REQ_EVENT, vcpu);
2645         if (ioapic_in_kernel(vcpu->kvm))
2646                 kvm_rtc_eoi_tracking_restore_one(vcpu);
2647
2648         vcpu->arch.apic_arb_prio = 0;
2649
2650         return 0;
2651 }
2652
2653 void __kvm_migrate_apic_timer(struct kvm_vcpu *vcpu)
2654 {
2655         struct hrtimer *timer;
2656
2657         if (!lapic_in_kernel(vcpu) ||
2658                 kvm_can_post_timer_interrupt(vcpu))
2659                 return;
2660
2661         timer = &vcpu->arch.apic->lapic_timer.timer;
2662         if (hrtimer_cancel(timer))
2663                 hrtimer_start_expires(timer, HRTIMER_MODE_ABS_HARD);
2664 }
2665
2666 /*
2667  * apic_sync_pv_eoi_from_guest - called on vmexit or cancel interrupt
2668  *
2669  * Detect whether guest triggered PV EOI since the
2670  * last entry. If yes, set EOI on guests's behalf.
2671  * Clear PV EOI in guest memory in any case.
2672  */
2673 static void apic_sync_pv_eoi_from_guest(struct kvm_vcpu *vcpu,
2674                                         struct kvm_lapic *apic)
2675 {
2676         bool pending;
2677         int vector;
2678         /*
2679          * PV EOI state is derived from KVM_APIC_PV_EOI_PENDING in host
2680          * and KVM_PV_EOI_ENABLED in guest memory as follows:
2681          *
2682          * KVM_APIC_PV_EOI_PENDING is unset:
2683          *      -> host disabled PV EOI.
2684          * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is set:
2685          *      -> host enabled PV EOI, guest did not execute EOI yet.
2686          * KVM_APIC_PV_EOI_PENDING is set, KVM_PV_EOI_ENABLED is unset:
2687          *      -> host enabled PV EOI, guest executed EOI.
2688          */
2689         BUG_ON(!pv_eoi_enabled(vcpu));
2690         pending = pv_eoi_get_pending(vcpu);
2691         /*
2692          * Clear pending bit in any case: it will be set again on vmentry.
2693          * While this might not be ideal from performance point of view,
2694          * this makes sure pv eoi is only enabled when we know it's safe.
2695          */
2696         pv_eoi_clr_pending(vcpu);
2697         if (pending)
2698                 return;
2699         vector = apic_set_eoi(apic);
2700         trace_kvm_pv_eoi(apic, vector);
2701 }
2702
2703 void kvm_lapic_sync_from_vapic(struct kvm_vcpu *vcpu)
2704 {
2705         u32 data;
2706
2707         if (test_bit(KVM_APIC_PV_EOI_PENDING, &vcpu->arch.apic_attention))
2708                 apic_sync_pv_eoi_from_guest(vcpu, vcpu->arch.apic);
2709
2710         if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
2711                 return;
2712
2713         if (kvm_read_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2714                                   sizeof(u32)))
2715                 return;
2716
2717         apic_set_tpr(vcpu->arch.apic, data & 0xff);
2718 }
2719
2720 /*
2721  * apic_sync_pv_eoi_to_guest - called before vmentry
2722  *
2723  * Detect whether it's safe to enable PV EOI and
2724  * if yes do so.
2725  */
2726 static void apic_sync_pv_eoi_to_guest(struct kvm_vcpu *vcpu,
2727                                         struct kvm_lapic *apic)
2728 {
2729         if (!pv_eoi_enabled(vcpu) ||
2730             /* IRR set or many bits in ISR: could be nested. */
2731             apic->irr_pending ||
2732             /* Cache not set: could be safe but we don't bother. */
2733             apic->highest_isr_cache == -1 ||
2734             /* Need EOI to update ioapic. */
2735             kvm_ioapic_handles_vector(apic, apic->highest_isr_cache)) {
2736                 /*
2737                  * PV EOI was disabled by apic_sync_pv_eoi_from_guest
2738                  * so we need not do anything here.
2739                  */
2740                 return;
2741         }
2742
2743         pv_eoi_set_pending(apic->vcpu);
2744 }
2745
2746 void kvm_lapic_sync_to_vapic(struct kvm_vcpu *vcpu)
2747 {
2748         u32 data, tpr;
2749         int max_irr, max_isr;
2750         struct kvm_lapic *apic = vcpu->arch.apic;
2751
2752         apic_sync_pv_eoi_to_guest(vcpu, apic);
2753
2754         if (!test_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention))
2755                 return;
2756
2757         tpr = kvm_lapic_get_reg(apic, APIC_TASKPRI) & 0xff;
2758         max_irr = apic_find_highest_irr(apic);
2759         if (max_irr < 0)
2760                 max_irr = 0;
2761         max_isr = apic_find_highest_isr(apic);
2762         if (max_isr < 0)
2763                 max_isr = 0;
2764         data = (tpr & 0xff) | ((max_isr & 0xf0) << 8) | (max_irr << 24);
2765
2766         kvm_write_guest_cached(vcpu->kvm, &vcpu->arch.apic->vapic_cache, &data,
2767                                 sizeof(u32));
2768 }
2769
2770 int kvm_lapic_set_vapic_addr(struct kvm_vcpu *vcpu, gpa_t vapic_addr)
2771 {
2772         if (vapic_addr) {
2773                 if (kvm_gfn_to_hva_cache_init(vcpu->kvm,
2774                                         &vcpu->arch.apic->vapic_cache,
2775                                         vapic_addr, sizeof(u32)))
2776                         return -EINVAL;
2777                 __set_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
2778         } else {
2779                 __clear_bit(KVM_APIC_CHECK_VAPIC, &vcpu->arch.apic_attention);
2780         }
2781
2782         vcpu->arch.apic->vapic_addr = vapic_addr;
2783         return 0;
2784 }
2785
2786 int kvm_x2apic_msr_write(struct kvm_vcpu *vcpu, u32 msr, u64 data)
2787 {
2788         struct kvm_lapic *apic = vcpu->arch.apic;
2789         u32 reg = (msr - APIC_BASE_MSR) << 4;
2790
2791         if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
2792                 return 1;
2793
2794         if (reg == APIC_ICR2)
2795                 return 1;
2796
2797         /* if this is ICR write vector before command */
2798         if (reg == APIC_ICR)
2799                 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2800         return kvm_lapic_reg_write(apic, reg, (u32)data);
2801 }
2802
2803 int kvm_x2apic_msr_read(struct kvm_vcpu *vcpu, u32 msr, u64 *data)
2804 {
2805         struct kvm_lapic *apic = vcpu->arch.apic;
2806         u32 reg = (msr - APIC_BASE_MSR) << 4, low, high = 0;
2807
2808         if (!lapic_in_kernel(vcpu) || !apic_x2apic_mode(apic))
2809                 return 1;
2810
2811         if (reg == APIC_DFR || reg == APIC_ICR2)
2812                 return 1;
2813
2814         if (kvm_lapic_reg_read(apic, reg, 4, &low))
2815                 return 1;
2816         if (reg == APIC_ICR)
2817                 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
2818
2819         *data = (((u64)high) << 32) | low;
2820
2821         return 0;
2822 }
2823
2824 int kvm_hv_vapic_msr_write(struct kvm_vcpu *vcpu, u32 reg, u64 data)
2825 {
2826         struct kvm_lapic *apic = vcpu->arch.apic;
2827
2828         if (!lapic_in_kernel(vcpu))
2829                 return 1;
2830
2831         /* if this is ICR write vector before command */
2832         if (reg == APIC_ICR)
2833                 kvm_lapic_reg_write(apic, APIC_ICR2, (u32)(data >> 32));
2834         return kvm_lapic_reg_write(apic, reg, (u32)data);
2835 }
2836
2837 int kvm_hv_vapic_msr_read(struct kvm_vcpu *vcpu, u32 reg, u64 *data)
2838 {
2839         struct kvm_lapic *apic = vcpu->arch.apic;
2840         u32 low, high = 0;
2841
2842         if (!lapic_in_kernel(vcpu))
2843                 return 1;
2844
2845         if (kvm_lapic_reg_read(apic, reg, 4, &low))
2846                 return 1;
2847         if (reg == APIC_ICR)
2848                 kvm_lapic_reg_read(apic, APIC_ICR2, 4, &high);
2849
2850         *data = (((u64)high) << 32) | low;
2851
2852         return 0;
2853 }
2854
2855 int kvm_lapic_enable_pv_eoi(struct kvm_vcpu *vcpu, u64 data, unsigned long len)
2856 {
2857         u64 addr = data & ~KVM_MSR_ENABLED;
2858         struct gfn_to_hva_cache *ghc = &vcpu->arch.pv_eoi.data;
2859         unsigned long new_len;
2860
2861         if (!IS_ALIGNED(addr, 4))
2862                 return 1;
2863
2864         vcpu->arch.pv_eoi.msr_val = data;
2865         if (!pv_eoi_enabled(vcpu))
2866                 return 0;
2867
2868         if (addr == ghc->gpa && len <= ghc->len)
2869                 new_len = ghc->len;
2870         else
2871                 new_len = len;
2872
2873         return kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, addr, new_len);
2874 }
2875
2876 int kvm_apic_accept_events(struct kvm_vcpu *vcpu)
2877 {
2878         struct kvm_lapic *apic = vcpu->arch.apic;
2879         u8 sipi_vector;
2880         int r;
2881         unsigned long pe;
2882
2883         if (!lapic_in_kernel(vcpu))
2884                 return 0;
2885
2886         /*
2887          * Read pending events before calling the check_events
2888          * callback.
2889          */
2890         pe = smp_load_acquire(&apic->pending_events);
2891         if (!pe)
2892                 return 0;
2893
2894         if (is_guest_mode(vcpu)) {
2895                 r = kvm_check_nested_events(vcpu);
2896                 if (r < 0)
2897                         return r == -EBUSY ? 0 : r;
2898                 /*
2899                  * If an event has happened and caused a vmexit,
2900                  * we know INITs are latched and therefore
2901                  * we will not incorrectly deliver an APIC
2902                  * event instead of a vmexit.
2903                  */
2904         }
2905
2906         /*
2907          * INITs are latched while CPU is in specific states
2908          * (SMM, VMX root mode, SVM with GIF=0).
2909          * Because a CPU cannot be in these states immediately
2910          * after it has processed an INIT signal (and thus in
2911          * KVM_MP_STATE_INIT_RECEIVED state), just eat SIPIs
2912          * and leave the INIT pending.
2913          */
2914         if (kvm_vcpu_latch_init(vcpu)) {
2915                 WARN_ON_ONCE(vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED);
2916                 if (test_bit(KVM_APIC_SIPI, &pe))
2917                         clear_bit(KVM_APIC_SIPI, &apic->pending_events);
2918                 return 0;
2919         }
2920
2921         if (test_bit(KVM_APIC_INIT, &pe)) {
2922                 clear_bit(KVM_APIC_INIT, &apic->pending_events);
2923                 kvm_vcpu_reset(vcpu, true);
2924                 if (kvm_vcpu_is_bsp(apic->vcpu))
2925                         vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2926                 else
2927                         vcpu->arch.mp_state = KVM_MP_STATE_INIT_RECEIVED;
2928         }
2929         if (test_bit(KVM_APIC_SIPI, &pe)) {
2930                 clear_bit(KVM_APIC_SIPI, &apic->pending_events);
2931                 if (vcpu->arch.mp_state == KVM_MP_STATE_INIT_RECEIVED) {
2932                         /* evaluate pending_events before reading the vector */
2933                         smp_rmb();
2934                         sipi_vector = apic->sipi_vector;
2935                         kvm_x86_ops.vcpu_deliver_sipi_vector(vcpu, sipi_vector);
2936                         vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
2937                 }
2938         }
2939         return 0;
2940 }
2941
2942 void kvm_lapic_exit(void)
2943 {
2944         static_key_deferred_flush(&apic_hw_disabled);
2945         WARN_ON(static_branch_unlikely(&apic_hw_disabled.key));
2946         static_key_deferred_flush(&apic_sw_disabled);
2947         WARN_ON(static_branch_unlikely(&apic_sw_disabled.key));
2948 }