arch/x86/kvm/vmx/nested.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include <linux/frame.h>
   4 #include <linux/percpu.h>
   5
   6 #include <asm/debugreg.h>
   7 #include <asm/mmu_context.h>
   8
   9 #include "cpuid.h"
  10 #include "hyperv.h"
  11 #include "mmu.h"
  12 #include "nested.h"
  13 #include "trace.h"
  14 #include "vmx.h"
  15 #include "x86.h"
  16
  17 static bool __read_mostly enable_shadow_vmcs = 1;
  18 module_param_named(enable_shadow_vmcs, enable_shadow_vmcs, bool, S_IRUGO);
  19
  20 static bool __read_mostly nested_early_check = 0;
  21 module_param(nested_early_check, bool, S_IRUGO);
  22
  23 #define CC(consistency_check)                                           \
  24 ({                                                                      \
  25         bool failed = (consistency_check);                              \
  26         if (failed)                                                     \
  27                 trace_kvm_nested_vmenter_failed(#consistency_check, 0); \
  28         failed;                                                         \
  29 })
  30
  31 /*
  32  * Hyper-V requires all of these, so mark them as supported even though
  33  * they are just treated the same as all-context.
  34  */
  35 #define VMX_VPID_EXTENT_SUPPORTED_MASK          \
  36         (VMX_VPID_EXTENT_INDIVIDUAL_ADDR_BIT |  \
  37         VMX_VPID_EXTENT_SINGLE_CONTEXT_BIT |    \
  38         VMX_VPID_EXTENT_GLOBAL_CONTEXT_BIT |    \
  39         VMX_VPID_EXTENT_SINGLE_NON_GLOBAL_BIT)
  40
  41 #define VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE 5
  42
  43 enum {
  44         VMX_VMREAD_BITMAP,
  45         VMX_VMWRITE_BITMAP,
  46         VMX_BITMAP_NR
  47 };
  48 static unsigned long *vmx_bitmap[VMX_BITMAP_NR];
  49
  50 #define vmx_vmread_bitmap                    (vmx_bitmap[VMX_VMREAD_BITMAP])
  51 #define vmx_vmwrite_bitmap                   (vmx_bitmap[VMX_VMWRITE_BITMAP])
  52
  53 struct shadow_vmcs_field {
  54         u16     encoding;
  55         u16     offset;
  56 };
  57 static struct shadow_vmcs_field shadow_read_only_fields[] = {
  58 #define SHADOW_FIELD_RO(x, y) { x, offsetof(struct vmcs12, y) },
  59 #include "vmcs_shadow_fields.h"
  60 };
  61 static int max_shadow_read_only_fields =
  62         ARRAY_SIZE(shadow_read_only_fields);
  63
  64 static struct shadow_vmcs_field shadow_read_write_fields[] = {
  65 #define SHADOW_FIELD_RW(x, y) { x, offsetof(struct vmcs12, y) },
  66 #include "vmcs_shadow_fields.h"
  67 };
  68 static int max_shadow_read_write_fields =
  69         ARRAY_SIZE(shadow_read_write_fields);
  70
  71 static void init_vmcs_shadow_fields(void)
  72 {
  73         int i, j;
  74
  75         memset(vmx_vmread_bitmap, 0xff, PAGE_SIZE);
  76         memset(vmx_vmwrite_bitmap, 0xff, PAGE_SIZE);
  77
  78         for (i = j = 0; i < max_shadow_read_only_fields; i++) {
  79                 struct shadow_vmcs_field entry = shadow_read_only_fields[i];
  80                 u16 field = entry.encoding;
  81
  82                 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 &&
  83                     (i + 1 == max_shadow_read_only_fields ||
  84                      shadow_read_only_fields[i + 1].encoding != field + 1))
  85                         pr_err("Missing field from shadow_read_only_field %x\n",
  86                                field + 1);
  87
  88                 clear_bit(field, vmx_vmread_bitmap);
  89                 if (field & 1)
  90 #ifdef CONFIG_X86_64
  91                         continue;
  92 #else
  93                         entry.offset += sizeof(u32);
  94 #endif
  95                 shadow_read_only_fields[j++] = entry;
  96         }
  97         max_shadow_read_only_fields = j;
  98
  99         for (i = j = 0; i < max_shadow_read_write_fields; i++) {
 100                 struct shadow_vmcs_field entry = shadow_read_write_fields[i];
 101                 u16 field = entry.encoding;
 102
 103                 if (vmcs_field_width(field) == VMCS_FIELD_WIDTH_U64 &&
 104                     (i + 1 == max_shadow_read_write_fields ||
 105                      shadow_read_write_fields[i + 1].encoding != field + 1))
 106                         pr_err("Missing field from shadow_read_write_field %x\n",
 107                                field + 1);
 108
 109                 WARN_ONCE(field >= GUEST_ES_AR_BYTES &&
 110                           field <= GUEST_TR_AR_BYTES,
 111                           "Update vmcs12_write_any() to drop reserved bits from AR_BYTES");
 112
 113                 /*
 114                  * PML and the preemption timer can be emulated, but the
 115                  * processor cannot vmwrite to fields that don't exist
 116                  * on bare metal.
 117                  */
 118                 switch (field) {
 119                 case GUEST_PML_INDEX:
 120                         if (!cpu_has_vmx_pml())
 121                                 continue;
 122                         break;
 123                 case VMX_PREEMPTION_TIMER_VALUE:
 124                         if (!cpu_has_vmx_preemption_timer())
 125                                 continue;
 126                         break;
 127                 case GUEST_INTR_STATUS:
 128                         if (!cpu_has_vmx_apicv())
 129                                 continue;
 130                         break;
 131                 default:
 132                         break;
 133                 }
 134
 135                 clear_bit(field, vmx_vmwrite_bitmap);
 136                 clear_bit(field, vmx_vmread_bitmap);
 137                 if (field & 1)
 138 #ifdef CONFIG_X86_64
 139                         continue;
 140 #else
 141                         entry.offset += sizeof(u32);
 142 #endif
 143                 shadow_read_write_fields[j++] = entry;
 144         }
 145         max_shadow_read_write_fields = j;
 146 }
 147
 148 /*
 149  * The following 3 functions, nested_vmx_succeed()/failValid()/failInvalid(),
 150  * set the success or error code of an emulated VMX instruction (as specified
 151  * by Vol 2B, VMX Instruction Reference, "Conventions"), and skip the emulated
 152  * instruction.
 153  */
 154 static int nested_vmx_succeed(struct kvm_vcpu *vcpu)
 155 {
 156         vmx_set_rflags(vcpu, vmx_get_rflags(vcpu)
 157                         & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
 158                             X86_EFLAGS_ZF | X86_EFLAGS_SF | X86_EFLAGS_OF));
 159         return kvm_skip_emulated_instruction(vcpu);
 160 }
 161
 162 static int nested_vmx_failInvalid(struct kvm_vcpu *vcpu)
 163 {
 164         vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
 165                         & ~(X86_EFLAGS_PF | X86_EFLAGS_AF | X86_EFLAGS_ZF |
 166                             X86_EFLAGS_SF | X86_EFLAGS_OF))
 167                         | X86_EFLAGS_CF);
 168         return kvm_skip_emulated_instruction(vcpu);
 169 }
 170
 171 static int nested_vmx_failValid(struct kvm_vcpu *vcpu,
 172                                 u32 vm_instruction_error)
 173 {
 174         struct vcpu_vmx *vmx = to_vmx(vcpu);
 175
 176         /*
 177          * failValid writes the error number to the current VMCS, which
 178          * can't be done if there isn't a current VMCS.
 179          */
 180         if (vmx->nested.current_vmptr == -1ull && !vmx->nested.hv_evmcs)
 181                 return nested_vmx_failInvalid(vcpu);
 182
 183         vmx_set_rflags(vcpu, (vmx_get_rflags(vcpu)
 184                         & ~(X86_EFLAGS_CF | X86_EFLAGS_PF | X86_EFLAGS_AF |
 185                             X86_EFLAGS_SF | X86_EFLAGS_OF))
 186                         | X86_EFLAGS_ZF);
 187         get_vmcs12(vcpu)->vm_instruction_error = vm_instruction_error;
 188         /*
 189          * We don't need to force a shadow sync because
 190          * VM_INSTRUCTION_ERROR is not shadowed
 191          */
 192         return kvm_skip_emulated_instruction(vcpu);
 193 }
 194
 195 static void nested_vmx_abort(struct kvm_vcpu *vcpu, u32 indicator)
 196 {
 197         /* TODO: not to reset guest simply here. */
 198         kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
 199         pr_debug_ratelimited("kvm: nested vmx abort, indicator %d\n", indicator);
 200 }
 201
 202 static inline bool vmx_control_verify(u32 control, u32 low, u32 high)
 203 {
 204         return fixed_bits_valid(control, low, high);
 205 }
 206
 207 static inline u64 vmx_control_msr(u32 low, u32 high)
 208 {
 209         return low | ((u64)high << 32);
 210 }
 211
 212 static void vmx_disable_shadow_vmcs(struct vcpu_vmx *vmx)
 213 {
 214         secondary_exec_controls_clearbit(vmx, SECONDARY_EXEC_SHADOW_VMCS);
 215         vmcs_write64(VMCS_LINK_POINTER, -1ull);
 216         vmx->nested.need_vmcs12_to_shadow_sync = false;
 217 }
 218
 219 static inline void nested_release_evmcs(struct kvm_vcpu *vcpu)
 220 {
 221         struct vcpu_vmx *vmx = to_vmx(vcpu);
 222
 223         if (!vmx->nested.hv_evmcs)
 224                 return;
 225
 226         kvm_vcpu_unmap(vcpu, &vmx->nested.hv_evmcs_map, true);
 227         vmx->nested.hv_evmcs_vmptr = 0;
 228         vmx->nested.hv_evmcs = NULL;
 229 }
 230
 231 /*
 232  * Free whatever needs to be freed from vmx->nested when L1 goes down, or
 233  * just stops using VMX.
 234  */
 235 static void free_nested(struct kvm_vcpu *vcpu)
 236 {
 237         struct vcpu_vmx *vmx = to_vmx(vcpu);
 238
 239         if (!vmx->nested.vmxon && !vmx->nested.smm.vmxon)
 240                 return;
 241
 242         kvm_clear_request(KVM_REQ_GET_VMCS12_PAGES, vcpu);
 243
 244         vmx->nested.vmxon = false;
 245         vmx->nested.smm.vmxon = false;
 246         free_vpid(vmx->nested.vpid02);
 247         vmx->nested.posted_intr_nv = -1;
 248         vmx->nested.current_vmptr = -1ull;
 249         if (enable_shadow_vmcs) {
 250                 vmx_disable_shadow_vmcs(vmx);
 251                 vmcs_clear(vmx->vmcs01.shadow_vmcs);
 252                 free_vmcs(vmx->vmcs01.shadow_vmcs);
 253                 vmx->vmcs01.shadow_vmcs = NULL;
 254         }
 255         kfree(vmx->nested.cached_vmcs12);
 256         vmx->nested.cached_vmcs12 = NULL;
 257         kfree(vmx->nested.cached_shadow_vmcs12);
 258         vmx->nested.cached_shadow_vmcs12 = NULL;
 259         /* Unpin physical memory we referred to in the vmcs02 */
 260         if (vmx->nested.apic_access_page) {
 261                 kvm_release_page_dirty(vmx->nested.apic_access_page);
 262                 vmx->nested.apic_access_page = NULL;
 263         }
 264         kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true);
 265         kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true);
 266         vmx->nested.pi_desc = NULL;
 267
 268         kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
 269
 270         nested_release_evmcs(vcpu);
 271
 272         free_loaded_vmcs(&vmx->nested.vmcs02);
 273 }
 274
 275 static void vmx_sync_vmcs_host_state(struct vcpu_vmx *vmx,
 276                                      struct loaded_vmcs *prev)
 277 {
 278         struct vmcs_host_state *dest, *src;
 279
 280         if (unlikely(!vmx->guest_state_loaded))
 281                 return;
 282
 283         src = &prev->host_state;
 284         dest = &vmx->loaded_vmcs->host_state;
 285
 286         vmx_set_host_fs_gs(dest, src->fs_sel, src->gs_sel, src->fs_base, src->gs_base);
 287         dest->ldt_sel = src->ldt_sel;
 288 #ifdef CONFIG_X86_64
 289         dest->ds_sel = src->ds_sel;
 290         dest->es_sel = src->es_sel;
 291 #endif
 292 }
 293
 294 static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs)
 295 {
 296         struct vcpu_vmx *vmx = to_vmx(vcpu);
 297         struct loaded_vmcs *prev;
 298         int cpu;
 299
 300         if (vmx->loaded_vmcs == vmcs)
 301                 return;
 302
 303         cpu = get_cpu();
 304         prev = vmx->loaded_vmcs;
 305         vmx->loaded_vmcs = vmcs;
 306         vmx_vcpu_load_vmcs(vcpu, cpu, prev);
 307         vmx_sync_vmcs_host_state(vmx, prev);
 308         put_cpu();
 309
 310         vmx_segment_cache_clear(vmx);
 311 }
 312
 313 /*
 314  * Ensure that the current vmcs of the logical processor is the
 315  * vmcs01 of the vcpu before calling free_nested().
 316  */
 317 void nested_vmx_free_vcpu(struct kvm_vcpu *vcpu)
 318 {
 319         vcpu_load(vcpu);
 320         vmx_leave_nested(vcpu);
 321         vmx_switch_vmcs(vcpu, &to_vmx(vcpu)->vmcs01);
 322         free_nested(vcpu);
 323         vcpu_put(vcpu);
 324 }
 325
 326 static void nested_ept_inject_page_fault(struct kvm_vcpu *vcpu,
 327                 struct x86_exception *fault)
 328 {
 329         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
 330         struct vcpu_vmx *vmx = to_vmx(vcpu);
 331         u32 exit_reason;
 332         unsigned long exit_qualification = vcpu->arch.exit_qualification;
 333
 334         if (vmx->nested.pml_full) {
 335                 exit_reason = EXIT_REASON_PML_FULL;
 336                 vmx->nested.pml_full = false;
 337                 exit_qualification &= INTR_INFO_UNBLOCK_NMI;
 338         } else if (fault->error_code & PFERR_RSVD_MASK)
 339                 exit_reason = EXIT_REASON_EPT_MISCONFIG;
 340         else
 341                 exit_reason = EXIT_REASON_EPT_VIOLATION;
 342
 343         nested_vmx_vmexit(vcpu, exit_reason, 0, exit_qualification);
 344         vmcs12->guest_physical_address = fault->address;
 345 }
 346
 347 static void nested_ept_init_mmu_context(struct kvm_vcpu *vcpu)
 348 {
 349         WARN_ON(mmu_is_nested(vcpu));
 350
 351         vcpu->arch.mmu = &vcpu->arch.guest_mmu;
 352         kvm_init_shadow_ept_mmu(vcpu,
 353                         to_vmx(vcpu)->nested.msrs.ept_caps &
 354                         VMX_EPT_EXECUTE_ONLY_BIT,
 355                         nested_ept_ad_enabled(vcpu),
 356                         nested_ept_get_cr3(vcpu));
 357         vcpu->arch.mmu->set_cr3           = vmx_set_cr3;
 358         vcpu->arch.mmu->get_cr3           = nested_ept_get_cr3;
 359         vcpu->arch.mmu->inject_page_fault = nested_ept_inject_page_fault;
 360         vcpu->arch.mmu->get_pdptr         = kvm_pdptr_read;
 361
 362         vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
 363 }
 364
 365 static void nested_ept_uninit_mmu_context(struct kvm_vcpu *vcpu)
 366 {
 367         vcpu->arch.mmu = &vcpu->arch.root_mmu;
 368         vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
 369 }
 370
 371 static bool nested_vmx_is_page_fault_vmexit(struct vmcs12 *vmcs12,
 372                                             u16 error_code)
 373 {
 374         bool inequality, bit;
 375
 376         bit = (vmcs12->exception_bitmap & (1u << PF_VECTOR)) != 0;
 377         inequality =
 378                 (error_code & vmcs12->page_fault_error_code_mask) !=
 379                  vmcs12->page_fault_error_code_match;
 380         return inequality ^ bit;
 381 }
 382
 383
 384 /*
 385  * KVM wants to inject page-faults which it got to the guest. This function
 386  * checks whether in a nested guest, we need to inject them to L1 or L2.
 387  */
 388 static int nested_vmx_check_exception(struct kvm_vcpu *vcpu, unsigned long *exit_qual)
 389 {
 390         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
 391         unsigned int nr = vcpu->arch.exception.nr;
 392         bool has_payload = vcpu->arch.exception.has_payload;
 393         unsigned long payload = vcpu->arch.exception.payload;
 394
 395         if (nr == PF_VECTOR) {
 396                 if (vcpu->arch.exception.nested_apf) {
 397                         *exit_qual = vcpu->arch.apf.nested_apf_token;
 398                         return 1;
 399                 }
 400                 if (nested_vmx_is_page_fault_vmexit(vmcs12,
 401                                                     vcpu->arch.exception.error_code)) {
 402                         *exit_qual = has_payload ? payload : vcpu->arch.cr2;
 403                         return 1;
 404                 }
 405         } else if (vmcs12->exception_bitmap & (1u << nr)) {
 406                 if (nr == DB_VECTOR) {
 407                         if (!has_payload) {
 408                                 payload = vcpu->arch.dr6;
 409                                 payload &= ~(DR6_FIXED_1 | DR6_BT);
 410                                 payload ^= DR6_RTM;
 411                         }
 412                         *exit_qual = payload;
 413                 } else
 414                         *exit_qual = 0;
 415                 return 1;
 416         }
 417
 418         return 0;
 419 }
 420
 421
 422 static void vmx_inject_page_fault_nested(struct kvm_vcpu *vcpu,
 423                 struct x86_exception *fault)
 424 {
 425         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
 426
 427         WARN_ON(!is_guest_mode(vcpu));
 428
 429         if (nested_vmx_is_page_fault_vmexit(vmcs12, fault->error_code) &&
 430                 !to_vmx(vcpu)->nested.nested_run_pending) {
 431                 vmcs12->vm_exit_intr_error_code = fault->error_code;
 432                 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
 433                                   PF_VECTOR | INTR_TYPE_HARD_EXCEPTION |
 434                                   INTR_INFO_DELIVER_CODE_MASK | INTR_INFO_VALID_MASK,
 435                                   fault->address);
 436         } else {
 437                 kvm_inject_page_fault(vcpu, fault);
 438         }
 439 }
 440
 441 static bool page_address_valid(struct kvm_vcpu *vcpu, gpa_t gpa)
 442 {
 443         return PAGE_ALIGNED(gpa) && !(gpa >> cpuid_maxphyaddr(vcpu));
 444 }
 445
 446 static int nested_vmx_check_io_bitmap_controls(struct kvm_vcpu *vcpu,
 447                                                struct vmcs12 *vmcs12)
 448 {
 449         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
 450                 return 0;
 451
 452         if (CC(!page_address_valid(vcpu, vmcs12->io_bitmap_a)) ||
 453             CC(!page_address_valid(vcpu, vmcs12->io_bitmap_b)))
 454                 return -EINVAL;
 455
 456         return 0;
 457 }
 458
 459 static int nested_vmx_check_msr_bitmap_controls(struct kvm_vcpu *vcpu,
 460                                                 struct vmcs12 *vmcs12)
 461 {
 462         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
 463                 return 0;
 464
 465         if (CC(!page_address_valid(vcpu, vmcs12->msr_bitmap)))
 466                 return -EINVAL;
 467
 468         return 0;
 469 }
 470
 471 static int nested_vmx_check_tpr_shadow_controls(struct kvm_vcpu *vcpu,
 472                                                 struct vmcs12 *vmcs12)
 473 {
 474         if (!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW))
 475                 return 0;
 476
 477         if (CC(!page_address_valid(vcpu, vmcs12->virtual_apic_page_addr)))
 478                 return -EINVAL;
 479
 480         return 0;
 481 }
 482
 483 /*
 484  * Check if MSR is intercepted for L01 MSR bitmap.
 485  */
 486 static bool msr_write_intercepted_l01(struct kvm_vcpu *vcpu, u32 msr)
 487 {
 488         unsigned long *msr_bitmap;
 489         int f = sizeof(unsigned long);
 490
 491         if (!cpu_has_vmx_msr_bitmap())
 492                 return true;
 493
 494         msr_bitmap = to_vmx(vcpu)->vmcs01.msr_bitmap;
 495
 496         if (msr <= 0x1fff) {
 497                 return !!test_bit(msr, msr_bitmap + 0x800 / f);
 498         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
 499                 msr &= 0x1fff;
 500                 return !!test_bit(msr, msr_bitmap + 0xc00 / f);
 501         }
 502
 503         return true;
 504 }
 505
 506 /*
 507  * If a msr is allowed by L0, we should check whether it is allowed by L1.
 508  * The corresponding bit will be cleared unless both of L0 and L1 allow it.
 509  */
 510 static void nested_vmx_disable_intercept_for_msr(unsigned long *msr_bitmap_l1,
 511                                                unsigned long *msr_bitmap_nested,
 512                                                u32 msr, int type)
 513 {
 514         int f = sizeof(unsigned long);
 515
 516         /*
 517          * See Intel PRM Vol. 3, 20.6.9 (MSR-Bitmap Address). Early manuals
 518          * have the write-low and read-high bitmap offsets the wrong way round.
 519          * We can control MSRs 0x00000000-0x00001fff and 0xc0000000-0xc0001fff.
 520          */
 521         if (msr <= 0x1fff) {
 522                 if (type & MSR_TYPE_R &&
 523                    !test_bit(msr, msr_bitmap_l1 + 0x000 / f))
 524                         /* read-low */
 525                         __clear_bit(msr, msr_bitmap_nested + 0x000 / f);
 526
 527                 if (type & MSR_TYPE_W &&
 528                    !test_bit(msr, msr_bitmap_l1 + 0x800 / f))
 529                         /* write-low */
 530                         __clear_bit(msr, msr_bitmap_nested + 0x800 / f);
 531
 532         } else if ((msr >= 0xc0000000) && (msr <= 0xc0001fff)) {
 533                 msr &= 0x1fff;
 534                 if (type & MSR_TYPE_R &&
 535                    !test_bit(msr, msr_bitmap_l1 + 0x400 / f))
 536                         /* read-high */
 537                         __clear_bit(msr, msr_bitmap_nested + 0x400 / f);
 538
 539                 if (type & MSR_TYPE_W &&
 540                    !test_bit(msr, msr_bitmap_l1 + 0xc00 / f))
 541                         /* write-high */
 542                         __clear_bit(msr, msr_bitmap_nested + 0xc00 / f);
 543
 544         }
 545 }
 546
 547 static inline void enable_x2apic_msr_intercepts(unsigned long *msr_bitmap) {
 548         int msr;
 549
 550         for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
 551                 unsigned word = msr / BITS_PER_LONG;
 552
 553                 msr_bitmap[word] = ~0;
 554                 msr_bitmap[word + (0x800 / sizeof(long))] = ~0;
 555         }
 556 }
 557
 558 /*
 559  * Merge L0's and L1's MSR bitmap, return false to indicate that
 560  * we do not use the hardware.
 561  */
 562 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu,
 563                                                  struct vmcs12 *vmcs12)
 564 {
 565         int msr;
 566         unsigned long *msr_bitmap_l1;
 567         unsigned long *msr_bitmap_l0 = to_vmx(vcpu)->nested.vmcs02.msr_bitmap;
 568         struct kvm_host_map *map = &to_vmx(vcpu)->nested.msr_bitmap_map;
 569
 570         /* Nothing to do if the MSR bitmap is not in use.  */
 571         if (!cpu_has_vmx_msr_bitmap() ||
 572             !nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
 573                 return false;
 574
 575         if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->msr_bitmap), map))
 576                 return false;
 577
 578         msr_bitmap_l1 = (unsigned long *)map->hva;
 579
 580         /*
 581          * To keep the control flow simple, pay eight 8-byte writes (sixteen
 582          * 4-byte writes on 32-bit systems) up front to enable intercepts for
 583          * the x2APIC MSR range and selectively disable them below.
 584          */
 585         enable_x2apic_msr_intercepts(msr_bitmap_l0);
 586
 587         if (nested_cpu_has_virt_x2apic_mode(vmcs12)) {
 588                 if (nested_cpu_has_apic_reg_virt(vmcs12)) {
 589                         /*
 590                          * L0 need not intercept reads for MSRs between 0x800
 591                          * and 0x8ff, it just lets the processor take the value
 592                          * from the virtual-APIC page; take those 256 bits
 593                          * directly from the L1 bitmap.
 594                          */
 595                         for (msr = 0x800; msr <= 0x8ff; msr += BITS_PER_LONG) {
 596                                 unsigned word = msr / BITS_PER_LONG;
 597
 598                                 msr_bitmap_l0[word] = msr_bitmap_l1[word];
 599                         }
 600                 }
 601
 602                 nested_vmx_disable_intercept_for_msr(
 603                         msr_bitmap_l1, msr_bitmap_l0,
 604                         X2APIC_MSR(APIC_TASKPRI),
 605                         MSR_TYPE_R | MSR_TYPE_W);
 606
 607                 if (nested_cpu_has_vid(vmcs12)) {
 608                         nested_vmx_disable_intercept_for_msr(
 609                                 msr_bitmap_l1, msr_bitmap_l0,
 610                                 X2APIC_MSR(APIC_EOI),
 611                                 MSR_TYPE_W);
 612                         nested_vmx_disable_intercept_for_msr(
 613                                 msr_bitmap_l1, msr_bitmap_l0,
 614                                 X2APIC_MSR(APIC_SELF_IPI),
 615                                 MSR_TYPE_W);
 616                 }
 617         }
 618
 619         /* KVM unconditionally exposes the FS/GS base MSRs to L1. */
 620         nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0,
 621                                              MSR_FS_BASE, MSR_TYPE_RW);
 622
 623         nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0,
 624                                              MSR_GS_BASE, MSR_TYPE_RW);
 625
 626         nested_vmx_disable_intercept_for_msr(msr_bitmap_l1, msr_bitmap_l0,
 627                                              MSR_KERNEL_GS_BASE, MSR_TYPE_RW);
 628
 629         /*
 630          * Checking the L0->L1 bitmap is trying to verify two things:
 631          *
 632          * 1. L0 gave a permission to L1 to actually passthrough the MSR. This
 633          *    ensures that we do not accidentally generate an L02 MSR bitmap
 634          *    from the L12 MSR bitmap that is too permissive.
 635          * 2. That L1 or L2s have actually used the MSR. This avoids
 636          *    unnecessarily merging of the bitmap if the MSR is unused. This
 637          *    works properly because we only update the L01 MSR bitmap lazily.
 638          *    So even if L0 should pass L1 these MSRs, the L01 bitmap is only
 639          *    updated to reflect this when L1 (or its L2s) actually write to
 640          *    the MSR.
 641          */
 642         if (!msr_write_intercepted_l01(vcpu, MSR_IA32_SPEC_CTRL))
 643                 nested_vmx_disable_intercept_for_msr(
 644                                         msr_bitmap_l1, msr_bitmap_l0,
 645                                         MSR_IA32_SPEC_CTRL,
 646                                         MSR_TYPE_R | MSR_TYPE_W);
 647
 648         if (!msr_write_intercepted_l01(vcpu, MSR_IA32_PRED_CMD))
 649                 nested_vmx_disable_intercept_for_msr(
 650                                         msr_bitmap_l1, msr_bitmap_l0,
 651                                         MSR_IA32_PRED_CMD,
 652                                         MSR_TYPE_W);
 653
 654         kvm_vcpu_unmap(vcpu, &to_vmx(vcpu)->nested.msr_bitmap_map, false);
 655
 656         return true;
 657 }
 658
 659 static void nested_cache_shadow_vmcs12(struct kvm_vcpu *vcpu,
 660                                        struct vmcs12 *vmcs12)
 661 {
 662         struct kvm_host_map map;
 663         struct vmcs12 *shadow;
 664
 665         if (!nested_cpu_has_shadow_vmcs(vmcs12) ||
 666             vmcs12->vmcs_link_pointer == -1ull)
 667                 return;
 668
 669         shadow = get_shadow_vmcs12(vcpu);
 670
 671         if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->vmcs_link_pointer), &map))
 672                 return;
 673
 674         memcpy(shadow, map.hva, VMCS12_SIZE);
 675         kvm_vcpu_unmap(vcpu, &map, false);
 676 }
 677
 678 static void nested_flush_cached_shadow_vmcs12(struct kvm_vcpu *vcpu,
 679                                               struct vmcs12 *vmcs12)
 680 {
 681         struct vcpu_vmx *vmx = to_vmx(vcpu);
 682
 683         if (!nested_cpu_has_shadow_vmcs(vmcs12) ||
 684             vmcs12->vmcs_link_pointer == -1ull)
 685                 return;
 686
 687         kvm_write_guest(vmx->vcpu.kvm, vmcs12->vmcs_link_pointer,
 688                         get_shadow_vmcs12(vcpu), VMCS12_SIZE);
 689 }
 690
 691 /*
 692  * In nested virtualization, check if L1 has set
 693  * VM_EXIT_ACK_INTR_ON_EXIT
 694  */
 695 static bool nested_exit_intr_ack_set(struct kvm_vcpu *vcpu)
 696 {
 697         return get_vmcs12(vcpu)->vm_exit_controls &
 698                 VM_EXIT_ACK_INTR_ON_EXIT;
 699 }
 700
 701 static bool nested_exit_on_nmi(struct kvm_vcpu *vcpu)
 702 {
 703         return nested_cpu_has_nmi_exiting(get_vmcs12(vcpu));
 704 }
 705
 706 static int nested_vmx_check_apic_access_controls(struct kvm_vcpu *vcpu,
 707                                           struct vmcs12 *vmcs12)
 708 {
 709         if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES) &&
 710             CC(!page_address_valid(vcpu, vmcs12->apic_access_addr)))
 711                 return -EINVAL;
 712         else
 713                 return 0;
 714 }
 715
 716 static int nested_vmx_check_apicv_controls(struct kvm_vcpu *vcpu,
 717                                            struct vmcs12 *vmcs12)
 718 {
 719         if (!nested_cpu_has_virt_x2apic_mode(vmcs12) &&
 720             !nested_cpu_has_apic_reg_virt(vmcs12) &&
 721             !nested_cpu_has_vid(vmcs12) &&
 722             !nested_cpu_has_posted_intr(vmcs12))
 723                 return 0;
 724
 725         /*
 726          * If virtualize x2apic mode is enabled,
 727          * virtualize apic access must be disabled.
 728          */
 729         if (CC(nested_cpu_has_virt_x2apic_mode(vmcs12) &&
 730                nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)))
 731                 return -EINVAL;
 732
 733         /*
 734          * If virtual interrupt delivery is enabled,
 735          * we must exit on external interrupts.
 736          */
 737         if (CC(nested_cpu_has_vid(vmcs12) && !nested_exit_on_intr(vcpu)))
 738                 return -EINVAL;
 739
 740         /*
 741          * bits 15:8 should be zero in posted_intr_nv,
 742          * the descriptor address has been already checked
 743          * in nested_get_vmcs12_pages.
 744          *
 745          * bits 5:0 of posted_intr_desc_addr should be zero.
 746          */
 747         if (nested_cpu_has_posted_intr(vmcs12) &&
 748            (CC(!nested_cpu_has_vid(vmcs12)) ||
 749             CC(!nested_exit_intr_ack_set(vcpu)) ||
 750             CC((vmcs12->posted_intr_nv & 0xff00)) ||
 751             CC((vmcs12->posted_intr_desc_addr & 0x3f)) ||
 752             CC((vmcs12->posted_intr_desc_addr >> cpuid_maxphyaddr(vcpu)))))
 753                 return -EINVAL;
 754
 755         /* tpr shadow is needed by all apicv features. */
 756         if (CC(!nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)))
 757                 return -EINVAL;
 758
 759         return 0;
 760 }
 761
 762 static int nested_vmx_check_msr_switch(struct kvm_vcpu *vcpu,
 763                                        u32 count, u64 addr)
 764 {
 765         int maxphyaddr;
 766
 767         if (count == 0)
 768                 return 0;
 769         maxphyaddr = cpuid_maxphyaddr(vcpu);
 770         if (!IS_ALIGNED(addr, 16) || addr >> maxphyaddr ||
 771             (addr + count * sizeof(struct vmx_msr_entry) - 1) >> maxphyaddr)
 772                 return -EINVAL;
 773
 774         return 0;
 775 }
 776
 777 static int nested_vmx_check_exit_msr_switch_controls(struct kvm_vcpu *vcpu,
 778                                                      struct vmcs12 *vmcs12)
 779 {
 780         if (CC(nested_vmx_check_msr_switch(vcpu,
 781                                            vmcs12->vm_exit_msr_load_count,
 782                                            vmcs12->vm_exit_msr_load_addr)) ||
 783             CC(nested_vmx_check_msr_switch(vcpu,
 784                                            vmcs12->vm_exit_msr_store_count,
 785                                            vmcs12->vm_exit_msr_store_addr)))
 786                 return -EINVAL;
 787
 788         return 0;
 789 }
 790
 791 static int nested_vmx_check_entry_msr_switch_controls(struct kvm_vcpu *vcpu,
 792                                                       struct vmcs12 *vmcs12)
 793 {
 794         if (CC(nested_vmx_check_msr_switch(vcpu,
 795                                            vmcs12->vm_entry_msr_load_count,
 796                                            vmcs12->vm_entry_msr_load_addr)))
 797                 return -EINVAL;
 798
 799         return 0;
 800 }
 801
 802 static int nested_vmx_check_pml_controls(struct kvm_vcpu *vcpu,
 803                                          struct vmcs12 *vmcs12)
 804 {
 805         if (!nested_cpu_has_pml(vmcs12))
 806                 return 0;
 807
 808         if (CC(!nested_cpu_has_ept(vmcs12)) ||
 809             CC(!page_address_valid(vcpu, vmcs12->pml_address)))
 810                 return -EINVAL;
 811
 812         return 0;
 813 }
 814
 815 static int nested_vmx_check_unrestricted_guest_controls(struct kvm_vcpu *vcpu,
 816                                                         struct vmcs12 *vmcs12)
 817 {
 818         if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_UNRESTRICTED_GUEST) &&
 819                !nested_cpu_has_ept(vmcs12)))
 820                 return -EINVAL;
 821         return 0;
 822 }
 823
 824 static int nested_vmx_check_mode_based_ept_exec_controls(struct kvm_vcpu *vcpu,
 825                                                          struct vmcs12 *vmcs12)
 826 {
 827         if (CC(nested_cpu_has2(vmcs12, SECONDARY_EXEC_MODE_BASED_EPT_EXEC) &&
 828                !nested_cpu_has_ept(vmcs12)))
 829                 return -EINVAL;
 830         return 0;
 831 }
 832
 833 static int nested_vmx_check_shadow_vmcs_controls(struct kvm_vcpu *vcpu,
 834                                                  struct vmcs12 *vmcs12)
 835 {
 836         if (!nested_cpu_has_shadow_vmcs(vmcs12))
 837                 return 0;
 838
 839         if (CC(!page_address_valid(vcpu, vmcs12->vmread_bitmap)) ||
 840             CC(!page_address_valid(vcpu, vmcs12->vmwrite_bitmap)))
 841                 return -EINVAL;
 842
 843         return 0;
 844 }
 845
 846 static int nested_vmx_msr_check_common(struct kvm_vcpu *vcpu,
 847                                        struct vmx_msr_entry *e)
 848 {
 849         /* x2APIC MSR accesses are not allowed */
 850         if (CC(vcpu->arch.apic_base & X2APIC_ENABLE && e->index >> 8 == 0x8))
 851                 return -EINVAL;
 852         if (CC(e->index == MSR_IA32_UCODE_WRITE) || /* SDM Table 35-2 */
 853             CC(e->index == MSR_IA32_UCODE_REV))
 854                 return -EINVAL;
 855         if (CC(e->reserved != 0))
 856                 return -EINVAL;
 857         return 0;
 858 }
 859
 860 static int nested_vmx_load_msr_check(struct kvm_vcpu *vcpu,
 861                                      struct vmx_msr_entry *e)
 862 {
 863         if (CC(e->index == MSR_FS_BASE) ||
 864             CC(e->index == MSR_GS_BASE) ||
 865             CC(e->index == MSR_IA32_SMM_MONITOR_CTL) || /* SMM is not supported */
 866             nested_vmx_msr_check_common(vcpu, e))
 867                 return -EINVAL;
 868         return 0;
 869 }
 870
 871 static int nested_vmx_store_msr_check(struct kvm_vcpu *vcpu,
 872                                       struct vmx_msr_entry *e)
 873 {
 874         if (CC(e->index == MSR_IA32_SMBASE) || /* SMM is not supported */
 875             nested_vmx_msr_check_common(vcpu, e))
 876                 return -EINVAL;
 877         return 0;
 878 }
 879
 880 static u32 nested_vmx_max_atomic_switch_msrs(struct kvm_vcpu *vcpu)
 881 {
 882         struct vcpu_vmx *vmx = to_vmx(vcpu);
 883         u64 vmx_misc = vmx_control_msr(vmx->nested.msrs.misc_low,
 884                                        vmx->nested.msrs.misc_high);
 885
 886         return (vmx_misc_max_msr(vmx_misc) + 1) * VMX_MISC_MSR_LIST_MULTIPLIER;
 887 }
 888
 889 /*
 890  * Load guest's/host's msr at nested entry/exit.
 891  * return 0 for success, entry index for failure.
 892  *
 893  * One of the failure modes for MSR load/store is when a list exceeds the
 894  * virtual hardware's capacity. To maintain compatibility with hardware inasmuch
 895  * as possible, process all valid entries before failing rather than precheck
 896  * for a capacity violation.
 897  */
 898 static u32 nested_vmx_load_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
 899 {
 900         u32 i;
 901         struct vmx_msr_entry e;
 902         u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu);
 903
 904         for (i = 0; i < count; i++) {
 905                 if (unlikely(i >= max_msr_list_size))
 906                         goto fail;
 907
 908                 if (kvm_vcpu_read_guest(vcpu, gpa + i * sizeof(e),
 909                                         &e, sizeof(e))) {
 910                         pr_debug_ratelimited(
 911                                 "%s cannot read MSR entry (%u, 0x%08llx)\n",
 912                                 __func__, i, gpa + i * sizeof(e));
 913                         goto fail;
 914                 }
 915                 if (nested_vmx_load_msr_check(vcpu, &e)) {
 916                         pr_debug_ratelimited(
 917                                 "%s check failed (%u, 0x%x, 0x%x)\n",
 918                                 __func__, i, e.index, e.reserved);
 919                         goto fail;
 920                 }
 921                 if (kvm_set_msr(vcpu, e.index, e.value)) {
 922                         pr_debug_ratelimited(
 923                                 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
 924                                 __func__, i, e.index, e.value);
 925                         goto fail;
 926                 }
 927         }
 928         return 0;
 929 fail:
 930         return i + 1;
 931 }
 932
 933 static int nested_vmx_store_msr(struct kvm_vcpu *vcpu, u64 gpa, u32 count)
 934 {
 935         u64 data;
 936         u32 i;
 937         struct vmx_msr_entry e;
 938         u32 max_msr_list_size = nested_vmx_max_atomic_switch_msrs(vcpu);
 939
 940         for (i = 0; i < count; i++) {
 941                 if (unlikely(i >= max_msr_list_size))
 942                         return -EINVAL;
 943
 944                 if (kvm_vcpu_read_guest(vcpu,
 945                                         gpa + i * sizeof(e),
 946                                         &e, 2 * sizeof(u32))) {
 947                         pr_debug_ratelimited(
 948                                 "%s cannot read MSR entry (%u, 0x%08llx)\n",
 949                                 __func__, i, gpa + i * sizeof(e));
 950                         return -EINVAL;
 951                 }
 952                 if (nested_vmx_store_msr_check(vcpu, &e)) {
 953                         pr_debug_ratelimited(
 954                                 "%s check failed (%u, 0x%x, 0x%x)\n",
 955                                 __func__, i, e.index, e.reserved);
 956                         return -EINVAL;
 957                 }
 958                 if (kvm_get_msr(vcpu, e.index, &data)) {
 959                         pr_debug_ratelimited(
 960                                 "%s cannot read MSR (%u, 0x%x)\n",
 961                                 __func__, i, e.index);
 962                         return -EINVAL;
 963                 }
 964                 if (kvm_vcpu_write_guest(vcpu,
 965                                          gpa + i * sizeof(e) +
 966                                              offsetof(struct vmx_msr_entry, value),
 967                                          &data, sizeof(data))) {
 968                         pr_debug_ratelimited(
 969                                 "%s cannot write MSR (%u, 0x%x, 0x%llx)\n",
 970                                 __func__, i, e.index, data);
 971                         return -EINVAL;
 972                 }
 973         }
 974         return 0;
 975 }
 976
 977 static bool nested_cr3_valid(struct kvm_vcpu *vcpu, unsigned long val)
 978 {
 979         unsigned long invalid_mask;
 980
 981         invalid_mask = (~0ULL) << cpuid_maxphyaddr(vcpu);
 982         return (val & invalid_mask) == 0;
 983 }
 984
 985 /*
 986  * Load guest's/host's cr3 at nested entry/exit. nested_ept is true if we are
 987  * emulating VM entry into a guest with EPT enabled.
 988  * Returns 0 on success, 1 on failure. Invalid state exit qualification code
 989  * is assigned to entry_failure_code on failure.
 990  */
 991 static int nested_vmx_load_cr3(struct kvm_vcpu *vcpu, unsigned long cr3, bool nested_ept,
 992                                u32 *entry_failure_code)
 993 {
 994         if (cr3 != kvm_read_cr3(vcpu) || (!nested_ept && pdptrs_changed(vcpu))) {
 995                 if (CC(!nested_cr3_valid(vcpu, cr3))) {
 996                         *entry_failure_code = ENTRY_FAIL_DEFAULT;
 997                         return -EINVAL;
 998                 }
 999
1000                 /*
1001                  * If PAE paging and EPT are both on, CR3 is not used by the CPU and
1002                  * must not be dereferenced.
1003                  */
1004                 if (is_pae_paging(vcpu) && !nested_ept) {
1005                         if (CC(!load_pdptrs(vcpu, vcpu->arch.walk_mmu, cr3))) {
1006                                 *entry_failure_code = ENTRY_FAIL_PDPTE;
1007                                 return -EINVAL;
1008                         }
1009                 }
1010         }
1011
1012         if (!nested_ept)
1013                 kvm_mmu_new_cr3(vcpu, cr3, false);
1014
1015         vcpu->arch.cr3 = cr3;
1016         __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
1017
1018         kvm_init_mmu(vcpu, false);
1019
1020         return 0;
1021 }
1022
1023 /*
1024  * Returns if KVM is able to config CPU to tag TLB entries
1025  * populated by L2 differently than TLB entries populated
1026  * by L1.
1027  *
1028  * If L1 uses EPT, then TLB entries are tagged with different EPTP.
1029  *
1030  * If L1 uses VPID and we allocated a vpid02, TLB entries are tagged
1031  * with different VPID (L1 entries are tagged with vmx->vpid
1032  * while L2 entries are tagged with vmx->nested.vpid02).
1033  */
1034 static bool nested_has_guest_tlb_tag(struct kvm_vcpu *vcpu)
1035 {
1036         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1037
1038         return nested_cpu_has_ept(vmcs12) ||
1039                (nested_cpu_has_vpid(vmcs12) && to_vmx(vcpu)->nested.vpid02);
1040 }
1041
1042 static u16 nested_get_vpid02(struct kvm_vcpu *vcpu)
1043 {
1044         struct vcpu_vmx *vmx = to_vmx(vcpu);
1045
1046         return vmx->nested.vpid02 ? vmx->nested.vpid02 : vmx->vpid;
1047 }
1048
1049 static bool is_bitwise_subset(u64 superset, u64 subset, u64 mask)
1050 {
1051         superset &= mask;
1052         subset &= mask;
1053
1054         return (superset | subset) == superset;
1055 }
1056
1057 static int vmx_restore_vmx_basic(struct vcpu_vmx *vmx, u64 data)
1058 {
1059         const u64 feature_and_reserved =
1060                 /* feature (except bit 48; see below) */
1061                 BIT_ULL(49) | BIT_ULL(54) | BIT_ULL(55) |
1062                 /* reserved */
1063                 BIT_ULL(31) | GENMASK_ULL(47, 45) | GENMASK_ULL(63, 56);
1064         u64 vmx_basic = vmcs_config.nested.basic;
1065
1066         if (!is_bitwise_subset(vmx_basic, data, feature_and_reserved))
1067                 return -EINVAL;
1068
1069         /*
1070          * KVM does not emulate a version of VMX that constrains physical
1071          * addresses of VMX structures (e.g. VMCS) to 32-bits.
1072          */
1073         if (data & BIT_ULL(48))
1074                 return -EINVAL;
1075
1076         if (vmx_basic_vmcs_revision_id(vmx_basic) !=
1077             vmx_basic_vmcs_revision_id(data))
1078                 return -EINVAL;
1079
1080         if (vmx_basic_vmcs_size(vmx_basic) > vmx_basic_vmcs_size(data))
1081                 return -EINVAL;
1082
1083         vmx->nested.msrs.basic = data;
1084         return 0;
1085 }
1086
1087 static void vmx_get_control_msr(struct nested_vmx_msrs *msrs, u32 msr_index,
1088                                 u32 **low, u32 **high)
1089 {
1090         switch (msr_index) {
1091         case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
1092                 *low = &msrs->pinbased_ctls_low;
1093                 *high = &msrs->pinbased_ctls_high;
1094                 break;
1095         case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
1096                 *low = &msrs->procbased_ctls_low;
1097                 *high = &msrs->procbased_ctls_high;
1098                 break;
1099         case MSR_IA32_VMX_TRUE_EXIT_CTLS:
1100                 *low = &msrs->exit_ctls_low;
1101                 *high = &msrs->exit_ctls_high;
1102                 break;
1103         case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
1104                 *low = &msrs->entry_ctls_low;
1105                 *high = &msrs->entry_ctls_high;
1106                 break;
1107         case MSR_IA32_VMX_PROCBASED_CTLS2:
1108                 *low = &msrs->secondary_ctls_low;
1109                 *high = &msrs->secondary_ctls_high;
1110                 break;
1111         default:
1112                 BUG();
1113         }
1114 }
1115
1116 static int
1117 vmx_restore_control_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
1118 {
1119         u32 *lowp, *highp;
1120         u64 supported;
1121
1122         vmx_get_control_msr(&vmcs_config.nested, msr_index, &lowp, &highp);
1123
1124         supported = vmx_control_msr(*lowp, *highp);
1125
1126         /* Check must-be-1 bits are still 1. */
1127         if (!is_bitwise_subset(data, supported, GENMASK_ULL(31, 0)))
1128                 return -EINVAL;
1129
1130         /* Check must-be-0 bits are still 0. */
1131         if (!is_bitwise_subset(supported, data, GENMASK_ULL(63, 32)))
1132                 return -EINVAL;
1133
1134         vmx_get_control_msr(&vmx->nested.msrs, msr_index, &lowp, &highp);
1135         *lowp = data;
1136         *highp = data >> 32;
1137         return 0;
1138 }
1139
1140 static int vmx_restore_vmx_misc(struct vcpu_vmx *vmx, u64 data)
1141 {
1142         const u64 feature_and_reserved_bits =
1143                 /* feature */
1144                 BIT_ULL(5) | GENMASK_ULL(8, 6) | BIT_ULL(14) | BIT_ULL(15) |
1145                 BIT_ULL(28) | BIT_ULL(29) | BIT_ULL(30) |
1146                 /* reserved */
1147                 GENMASK_ULL(13, 9) | BIT_ULL(31);
1148         u64 vmx_misc = vmx_control_msr(vmcs_config.nested.misc_low,
1149                                        vmcs_config.nested.misc_high);
1150
1151         if (!is_bitwise_subset(vmx_misc, data, feature_and_reserved_bits))
1152                 return -EINVAL;
1153
1154         if ((vmx->nested.msrs.pinbased_ctls_high &
1155              PIN_BASED_VMX_PREEMPTION_TIMER) &&
1156             vmx_misc_preemption_timer_rate(data) !=
1157             vmx_misc_preemption_timer_rate(vmx_misc))
1158                 return -EINVAL;
1159
1160         if (vmx_misc_cr3_count(data) > vmx_misc_cr3_count(vmx_misc))
1161                 return -EINVAL;
1162
1163         if (vmx_misc_max_msr(data) > vmx_misc_max_msr(vmx_misc))
1164                 return -EINVAL;
1165
1166         if (vmx_misc_mseg_revid(data) != vmx_misc_mseg_revid(vmx_misc))
1167                 return -EINVAL;
1168
1169         vmx->nested.msrs.misc_low = data;
1170         vmx->nested.msrs.misc_high = data >> 32;
1171
1172         return 0;
1173 }
1174
1175 static int vmx_restore_vmx_ept_vpid_cap(struct vcpu_vmx *vmx, u64 data)
1176 {
1177         u64 vmx_ept_vpid_cap = vmx_control_msr(vmcs_config.nested.ept_caps,
1178                                                vmcs_config.nested.vpid_caps);
1179
1180         /* Every bit is either reserved or a feature bit. */
1181         if (!is_bitwise_subset(vmx_ept_vpid_cap, data, -1ULL))
1182                 return -EINVAL;
1183
1184         vmx->nested.msrs.ept_caps = data;
1185         vmx->nested.msrs.vpid_caps = data >> 32;
1186         return 0;
1187 }
1188
1189 static u64 *vmx_get_fixed0_msr(struct nested_vmx_msrs *msrs, u32 msr_index)
1190 {
1191         switch (msr_index) {
1192         case MSR_IA32_VMX_CR0_FIXED0:
1193                 return &msrs->cr0_fixed0;
1194         case MSR_IA32_VMX_CR4_FIXED0:
1195                 return &msrs->cr4_fixed0;
1196         default:
1197                 BUG();
1198         }
1199 }
1200
1201 static int vmx_restore_fixed0_msr(struct vcpu_vmx *vmx, u32 msr_index, u64 data)
1202 {
1203         const u64 *msr = vmx_get_fixed0_msr(&vmcs_config.nested, msr_index);
1204
1205         /*
1206          * 1 bits (which indicates bits which "must-be-1" during VMX operation)
1207          * must be 1 in the restored value.
1208          */
1209         if (!is_bitwise_subset(data, *msr, -1ULL))
1210                 return -EINVAL;
1211
1212         *vmx_get_fixed0_msr(&vmx->nested.msrs, msr_index) = data;
1213         return 0;
1214 }
1215
1216 /*
1217  * Called when userspace is restoring VMX MSRs.
1218  *
1219  * Returns 0 on success, non-0 otherwise.
1220  */
1221 int vmx_set_vmx_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 data)
1222 {
1223         struct vcpu_vmx *vmx = to_vmx(vcpu);
1224
1225         /*
1226          * Don't allow changes to the VMX capability MSRs while the vCPU
1227          * is in VMX operation.
1228          */
1229         if (vmx->nested.vmxon)
1230                 return -EBUSY;
1231
1232         switch (msr_index) {
1233         case MSR_IA32_VMX_BASIC:
1234                 return vmx_restore_vmx_basic(vmx, data);
1235         case MSR_IA32_VMX_PINBASED_CTLS:
1236         case MSR_IA32_VMX_PROCBASED_CTLS:
1237         case MSR_IA32_VMX_EXIT_CTLS:
1238         case MSR_IA32_VMX_ENTRY_CTLS:
1239                 /*
1240                  * The "non-true" VMX capability MSRs are generated from the
1241                  * "true" MSRs, so we do not support restoring them directly.
1242                  *
1243                  * If userspace wants to emulate VMX_BASIC[55]=0, userspace
1244                  * should restore the "true" MSRs with the must-be-1 bits
1245                  * set according to the SDM Vol 3. A.2 "RESERVED CONTROLS AND
1246                  * DEFAULT SETTINGS".
1247                  */
1248                 return -EINVAL;
1249         case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
1250         case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
1251         case MSR_IA32_VMX_TRUE_EXIT_CTLS:
1252         case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
1253         case MSR_IA32_VMX_PROCBASED_CTLS2:
1254                 return vmx_restore_control_msr(vmx, msr_index, data);
1255         case MSR_IA32_VMX_MISC:
1256                 return vmx_restore_vmx_misc(vmx, data);
1257         case MSR_IA32_VMX_CR0_FIXED0:
1258         case MSR_IA32_VMX_CR4_FIXED0:
1259                 return vmx_restore_fixed0_msr(vmx, msr_index, data);
1260         case MSR_IA32_VMX_CR0_FIXED1:
1261         case MSR_IA32_VMX_CR4_FIXED1:
1262                 /*
1263                  * These MSRs are generated based on the vCPU's CPUID, so we
1264                  * do not support restoring them directly.
1265                  */
1266                 return -EINVAL;
1267         case MSR_IA32_VMX_EPT_VPID_CAP:
1268                 return vmx_restore_vmx_ept_vpid_cap(vmx, data);
1269         case MSR_IA32_VMX_VMCS_ENUM:
1270                 vmx->nested.msrs.vmcs_enum = data;
1271                 return 0;
1272         case MSR_IA32_VMX_VMFUNC:
1273                 if (data & ~vmcs_config.nested.vmfunc_controls)
1274                         return -EINVAL;
1275                 vmx->nested.msrs.vmfunc_controls = data;
1276                 return 0;
1277         default:
1278                 /*
1279                  * The rest of the VMX capability MSRs do not support restore.
1280                  */
1281                 return -EINVAL;
1282         }
1283 }
1284
1285 /* Returns 0 on success, non-0 otherwise. */
1286 int vmx_get_vmx_msr(struct nested_vmx_msrs *msrs, u32 msr_index, u64 *pdata)
1287 {
1288         switch (msr_index) {
1289         case MSR_IA32_VMX_BASIC:
1290                 *pdata = msrs->basic;
1291                 break;
1292         case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
1293         case MSR_IA32_VMX_PINBASED_CTLS:
1294                 *pdata = vmx_control_msr(
1295                         msrs->pinbased_ctls_low,
1296                         msrs->pinbased_ctls_high);
1297                 if (msr_index == MSR_IA32_VMX_PINBASED_CTLS)
1298                         *pdata |= PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
1299                 break;
1300         case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
1301         case MSR_IA32_VMX_PROCBASED_CTLS:
1302                 *pdata = vmx_control_msr(
1303                         msrs->procbased_ctls_low,
1304                         msrs->procbased_ctls_high);
1305                 if (msr_index == MSR_IA32_VMX_PROCBASED_CTLS)
1306                         *pdata |= CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
1307                 break;
1308         case MSR_IA32_VMX_TRUE_EXIT_CTLS:
1309         case MSR_IA32_VMX_EXIT_CTLS:
1310                 *pdata = vmx_control_msr(
1311                         msrs->exit_ctls_low,
1312                         msrs->exit_ctls_high);
1313                 if (msr_index == MSR_IA32_VMX_EXIT_CTLS)
1314                         *pdata |= VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
1315                 break;
1316         case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
1317         case MSR_IA32_VMX_ENTRY_CTLS:
1318                 *pdata = vmx_control_msr(
1319                         msrs->entry_ctls_low,
1320                         msrs->entry_ctls_high);
1321                 if (msr_index == MSR_IA32_VMX_ENTRY_CTLS)
1322                         *pdata |= VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
1323                 break;
1324         case MSR_IA32_VMX_MISC:
1325                 *pdata = vmx_control_msr(
1326                         msrs->misc_low,
1327                         msrs->misc_high);
1328                 break;
1329         case MSR_IA32_VMX_CR0_FIXED0:
1330                 *pdata = msrs->cr0_fixed0;
1331                 break;
1332         case MSR_IA32_VMX_CR0_FIXED1:
1333                 *pdata = msrs->cr0_fixed1;
1334                 break;
1335         case MSR_IA32_VMX_CR4_FIXED0:
1336                 *pdata = msrs->cr4_fixed0;
1337                 break;
1338         case MSR_IA32_VMX_CR4_FIXED1:
1339                 *pdata = msrs->cr4_fixed1;
1340                 break;
1341         case MSR_IA32_VMX_VMCS_ENUM:
1342                 *pdata = msrs->vmcs_enum;
1343                 break;
1344         case MSR_IA32_VMX_PROCBASED_CTLS2:
1345                 *pdata = vmx_control_msr(
1346                         msrs->secondary_ctls_low,
1347                         msrs->secondary_ctls_high);
1348                 break;
1349         case MSR_IA32_VMX_EPT_VPID_CAP:
1350                 *pdata = msrs->ept_caps |
1351                         ((u64)msrs->vpid_caps << 32);
1352                 break;
1353         case MSR_IA32_VMX_VMFUNC:
1354                 *pdata = msrs->vmfunc_controls;
1355                 break;
1356         default:
1357                 return 1;
1358         }
1359
1360         return 0;
1361 }
1362
1363 /*
1364  * Copy the writable VMCS shadow fields back to the VMCS12, in case they have
1365  * been modified by the L1 guest.  Note, "writable" in this context means
1366  * "writable by the guest", i.e. tagged SHADOW_FIELD_RW; the set of
1367  * fields tagged SHADOW_FIELD_RO may or may not align with the "read-only"
1368  * VM-exit information fields (which are actually writable if the vCPU is
1369  * configured to support "VMWRITE to any supported field in the VMCS").
1370  */
1371 static void copy_shadow_to_vmcs12(struct vcpu_vmx *vmx)
1372 {
1373         struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
1374         struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu);
1375         struct shadow_vmcs_field field;
1376         unsigned long val;
1377         int i;
1378
1379         if (WARN_ON(!shadow_vmcs))
1380                 return;
1381
1382         preempt_disable();
1383
1384         vmcs_load(shadow_vmcs);
1385
1386         for (i = 0; i < max_shadow_read_write_fields; i++) {
1387                 field = shadow_read_write_fields[i];
1388                 val = __vmcs_readl(field.encoding);
1389                 vmcs12_write_any(vmcs12, field.encoding, field.offset, val);
1390         }
1391
1392         vmcs_clear(shadow_vmcs);
1393         vmcs_load(vmx->loaded_vmcs->vmcs);
1394
1395         preempt_enable();
1396 }
1397
1398 static void copy_vmcs12_to_shadow(struct vcpu_vmx *vmx)
1399 {
1400         const struct shadow_vmcs_field *fields[] = {
1401                 shadow_read_write_fields,
1402                 shadow_read_only_fields
1403         };
1404         const int max_fields[] = {
1405                 max_shadow_read_write_fields,
1406                 max_shadow_read_only_fields
1407         };
1408         struct vmcs *shadow_vmcs = vmx->vmcs01.shadow_vmcs;
1409         struct vmcs12 *vmcs12 = get_vmcs12(&vmx->vcpu);
1410         struct shadow_vmcs_field field;
1411         unsigned long val;
1412         int i, q;
1413
1414         if (WARN_ON(!shadow_vmcs))
1415                 return;
1416
1417         vmcs_load(shadow_vmcs);
1418
1419         for (q = 0; q < ARRAY_SIZE(fields); q++) {
1420                 for (i = 0; i < max_fields[q]; i++) {
1421                         field = fields[q][i];
1422                         val = vmcs12_read_any(vmcs12, field.encoding,
1423                                               field.offset);
1424                         __vmcs_writel(field.encoding, val);
1425                 }
1426         }
1427
1428         vmcs_clear(shadow_vmcs);
1429         vmcs_load(vmx->loaded_vmcs->vmcs);
1430 }
1431
1432 static int copy_enlightened_to_vmcs12(struct vcpu_vmx *vmx)
1433 {
1434         struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12;
1435         struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs;
1436
1437         /* HV_VMX_ENLIGHTENED_CLEAN_FIELD_NONE */
1438         vmcs12->tpr_threshold = evmcs->tpr_threshold;
1439         vmcs12->guest_rip = evmcs->guest_rip;
1440
1441         if (unlikely(!(evmcs->hv_clean_fields &
1442                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_BASIC))) {
1443                 vmcs12->guest_rsp = evmcs->guest_rsp;
1444                 vmcs12->guest_rflags = evmcs->guest_rflags;
1445                 vmcs12->guest_interruptibility_info =
1446                         evmcs->guest_interruptibility_info;
1447         }
1448
1449         if (unlikely(!(evmcs->hv_clean_fields &
1450                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_PROC))) {
1451                 vmcs12->cpu_based_vm_exec_control =
1452                         evmcs->cpu_based_vm_exec_control;
1453         }
1454
1455         if (unlikely(!(evmcs->hv_clean_fields &
1456                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EXCPN))) {
1457                 vmcs12->exception_bitmap = evmcs->exception_bitmap;
1458         }
1459
1460         if (unlikely(!(evmcs->hv_clean_fields &
1461                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_ENTRY))) {
1462                 vmcs12->vm_entry_controls = evmcs->vm_entry_controls;
1463         }
1464
1465         if (unlikely(!(evmcs->hv_clean_fields &
1466                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_EVENT))) {
1467                 vmcs12->vm_entry_intr_info_field =
1468                         evmcs->vm_entry_intr_info_field;
1469                 vmcs12->vm_entry_exception_error_code =
1470                         evmcs->vm_entry_exception_error_code;
1471                 vmcs12->vm_entry_instruction_len =
1472                         evmcs->vm_entry_instruction_len;
1473         }
1474
1475         if (unlikely(!(evmcs->hv_clean_fields &
1476                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_GRP1))) {
1477                 vmcs12->host_ia32_pat = evmcs->host_ia32_pat;
1478                 vmcs12->host_ia32_efer = evmcs->host_ia32_efer;
1479                 vmcs12->host_cr0 = evmcs->host_cr0;
1480                 vmcs12->host_cr3 = evmcs->host_cr3;
1481                 vmcs12->host_cr4 = evmcs->host_cr4;
1482                 vmcs12->host_ia32_sysenter_esp = evmcs->host_ia32_sysenter_esp;
1483                 vmcs12->host_ia32_sysenter_eip = evmcs->host_ia32_sysenter_eip;
1484                 vmcs12->host_rip = evmcs->host_rip;
1485                 vmcs12->host_ia32_sysenter_cs = evmcs->host_ia32_sysenter_cs;
1486                 vmcs12->host_es_selector = evmcs->host_es_selector;
1487                 vmcs12->host_cs_selector = evmcs->host_cs_selector;
1488                 vmcs12->host_ss_selector = evmcs->host_ss_selector;
1489                 vmcs12->host_ds_selector = evmcs->host_ds_selector;
1490                 vmcs12->host_fs_selector = evmcs->host_fs_selector;
1491                 vmcs12->host_gs_selector = evmcs->host_gs_selector;
1492                 vmcs12->host_tr_selector = evmcs->host_tr_selector;
1493         }
1494
1495         if (unlikely(!(evmcs->hv_clean_fields &
1496                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP1))) {
1497                 vmcs12->pin_based_vm_exec_control =
1498                         evmcs->pin_based_vm_exec_control;
1499                 vmcs12->vm_exit_controls = evmcs->vm_exit_controls;
1500                 vmcs12->secondary_vm_exec_control =
1501                         evmcs->secondary_vm_exec_control;
1502         }
1503
1504         if (unlikely(!(evmcs->hv_clean_fields &
1505                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_IO_BITMAP))) {
1506                 vmcs12->io_bitmap_a = evmcs->io_bitmap_a;
1507                 vmcs12->io_bitmap_b = evmcs->io_bitmap_b;
1508         }
1509
1510         if (unlikely(!(evmcs->hv_clean_fields &
1511                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_MSR_BITMAP))) {
1512                 vmcs12->msr_bitmap = evmcs->msr_bitmap;
1513         }
1514
1515         if (unlikely(!(evmcs->hv_clean_fields &
1516                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2))) {
1517                 vmcs12->guest_es_base = evmcs->guest_es_base;
1518                 vmcs12->guest_cs_base = evmcs->guest_cs_base;
1519                 vmcs12->guest_ss_base = evmcs->guest_ss_base;
1520                 vmcs12->guest_ds_base = evmcs->guest_ds_base;
1521                 vmcs12->guest_fs_base = evmcs->guest_fs_base;
1522                 vmcs12->guest_gs_base = evmcs->guest_gs_base;
1523                 vmcs12->guest_ldtr_base = evmcs->guest_ldtr_base;
1524                 vmcs12->guest_tr_base = evmcs->guest_tr_base;
1525                 vmcs12->guest_gdtr_base = evmcs->guest_gdtr_base;
1526                 vmcs12->guest_idtr_base = evmcs->guest_idtr_base;
1527                 vmcs12->guest_es_limit = evmcs->guest_es_limit;
1528                 vmcs12->guest_cs_limit = evmcs->guest_cs_limit;
1529                 vmcs12->guest_ss_limit = evmcs->guest_ss_limit;
1530                 vmcs12->guest_ds_limit = evmcs->guest_ds_limit;
1531                 vmcs12->guest_fs_limit = evmcs->guest_fs_limit;
1532                 vmcs12->guest_gs_limit = evmcs->guest_gs_limit;
1533                 vmcs12->guest_ldtr_limit = evmcs->guest_ldtr_limit;
1534                 vmcs12->guest_tr_limit = evmcs->guest_tr_limit;
1535                 vmcs12->guest_gdtr_limit = evmcs->guest_gdtr_limit;
1536                 vmcs12->guest_idtr_limit = evmcs->guest_idtr_limit;
1537                 vmcs12->guest_es_ar_bytes = evmcs->guest_es_ar_bytes;
1538                 vmcs12->guest_cs_ar_bytes = evmcs->guest_cs_ar_bytes;
1539                 vmcs12->guest_ss_ar_bytes = evmcs->guest_ss_ar_bytes;
1540                 vmcs12->guest_ds_ar_bytes = evmcs->guest_ds_ar_bytes;
1541                 vmcs12->guest_fs_ar_bytes = evmcs->guest_fs_ar_bytes;
1542                 vmcs12->guest_gs_ar_bytes = evmcs->guest_gs_ar_bytes;
1543                 vmcs12->guest_ldtr_ar_bytes = evmcs->guest_ldtr_ar_bytes;
1544                 vmcs12->guest_tr_ar_bytes = evmcs->guest_tr_ar_bytes;
1545                 vmcs12->guest_es_selector = evmcs->guest_es_selector;
1546                 vmcs12->guest_cs_selector = evmcs->guest_cs_selector;
1547                 vmcs12->guest_ss_selector = evmcs->guest_ss_selector;
1548                 vmcs12->guest_ds_selector = evmcs->guest_ds_selector;
1549                 vmcs12->guest_fs_selector = evmcs->guest_fs_selector;
1550                 vmcs12->guest_gs_selector = evmcs->guest_gs_selector;
1551                 vmcs12->guest_ldtr_selector = evmcs->guest_ldtr_selector;
1552                 vmcs12->guest_tr_selector = evmcs->guest_tr_selector;
1553         }
1554
1555         if (unlikely(!(evmcs->hv_clean_fields &
1556                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_GRP2))) {
1557                 vmcs12->tsc_offset = evmcs->tsc_offset;
1558                 vmcs12->virtual_apic_page_addr = evmcs->virtual_apic_page_addr;
1559                 vmcs12->xss_exit_bitmap = evmcs->xss_exit_bitmap;
1560         }
1561
1562         if (unlikely(!(evmcs->hv_clean_fields &
1563                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_CRDR))) {
1564                 vmcs12->cr0_guest_host_mask = evmcs->cr0_guest_host_mask;
1565                 vmcs12->cr4_guest_host_mask = evmcs->cr4_guest_host_mask;
1566                 vmcs12->cr0_read_shadow = evmcs->cr0_read_shadow;
1567                 vmcs12->cr4_read_shadow = evmcs->cr4_read_shadow;
1568                 vmcs12->guest_cr0 = evmcs->guest_cr0;
1569                 vmcs12->guest_cr3 = evmcs->guest_cr3;
1570                 vmcs12->guest_cr4 = evmcs->guest_cr4;
1571                 vmcs12->guest_dr7 = evmcs->guest_dr7;
1572         }
1573
1574         if (unlikely(!(evmcs->hv_clean_fields &
1575                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_HOST_POINTER))) {
1576                 vmcs12->host_fs_base = evmcs->host_fs_base;
1577                 vmcs12->host_gs_base = evmcs->host_gs_base;
1578                 vmcs12->host_tr_base = evmcs->host_tr_base;
1579                 vmcs12->host_gdtr_base = evmcs->host_gdtr_base;
1580                 vmcs12->host_idtr_base = evmcs->host_idtr_base;
1581                 vmcs12->host_rsp = evmcs->host_rsp;
1582         }
1583
1584         if (unlikely(!(evmcs->hv_clean_fields &
1585                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_CONTROL_XLAT))) {
1586                 vmcs12->ept_pointer = evmcs->ept_pointer;
1587                 vmcs12->virtual_processor_id = evmcs->virtual_processor_id;
1588         }
1589
1590         if (unlikely(!(evmcs->hv_clean_fields &
1591                        HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1))) {
1592                 vmcs12->vmcs_link_pointer = evmcs->vmcs_link_pointer;
1593                 vmcs12->guest_ia32_debugctl = evmcs->guest_ia32_debugctl;
1594                 vmcs12->guest_ia32_pat = evmcs->guest_ia32_pat;
1595                 vmcs12->guest_ia32_efer = evmcs->guest_ia32_efer;
1596                 vmcs12->guest_pdptr0 = evmcs->guest_pdptr0;
1597                 vmcs12->guest_pdptr1 = evmcs->guest_pdptr1;
1598                 vmcs12->guest_pdptr2 = evmcs->guest_pdptr2;
1599                 vmcs12->guest_pdptr3 = evmcs->guest_pdptr3;
1600                 vmcs12->guest_pending_dbg_exceptions =
1601                         evmcs->guest_pending_dbg_exceptions;
1602                 vmcs12->guest_sysenter_esp = evmcs->guest_sysenter_esp;
1603                 vmcs12->guest_sysenter_eip = evmcs->guest_sysenter_eip;
1604                 vmcs12->guest_bndcfgs = evmcs->guest_bndcfgs;
1605                 vmcs12->guest_activity_state = evmcs->guest_activity_state;
1606                 vmcs12->guest_sysenter_cs = evmcs->guest_sysenter_cs;
1607         }
1608
1609         /*
1610          * Not used?
1611          * vmcs12->vm_exit_msr_store_addr = evmcs->vm_exit_msr_store_addr;
1612          * vmcs12->vm_exit_msr_load_addr = evmcs->vm_exit_msr_load_addr;
1613          * vmcs12->vm_entry_msr_load_addr = evmcs->vm_entry_msr_load_addr;
1614          * vmcs12->cr3_target_value0 = evmcs->cr3_target_value0;
1615          * vmcs12->cr3_target_value1 = evmcs->cr3_target_value1;
1616          * vmcs12->cr3_target_value2 = evmcs->cr3_target_value2;
1617          * vmcs12->cr3_target_value3 = evmcs->cr3_target_value3;
1618          * vmcs12->page_fault_error_code_mask =
1619          *              evmcs->page_fault_error_code_mask;
1620          * vmcs12->page_fault_error_code_match =
1621          *              evmcs->page_fault_error_code_match;
1622          * vmcs12->cr3_target_count = evmcs->cr3_target_count;
1623          * vmcs12->vm_exit_msr_store_count = evmcs->vm_exit_msr_store_count;
1624          * vmcs12->vm_exit_msr_load_count = evmcs->vm_exit_msr_load_count;
1625          * vmcs12->vm_entry_msr_load_count = evmcs->vm_entry_msr_load_count;
1626          */
1627
1628         /*
1629          * Read only fields:
1630          * vmcs12->guest_physical_address = evmcs->guest_physical_address;
1631          * vmcs12->vm_instruction_error = evmcs->vm_instruction_error;
1632          * vmcs12->vm_exit_reason = evmcs->vm_exit_reason;
1633          * vmcs12->vm_exit_intr_info = evmcs->vm_exit_intr_info;
1634          * vmcs12->vm_exit_intr_error_code = evmcs->vm_exit_intr_error_code;
1635          * vmcs12->idt_vectoring_info_field = evmcs->idt_vectoring_info_field;
1636          * vmcs12->idt_vectoring_error_code = evmcs->idt_vectoring_error_code;
1637          * vmcs12->vm_exit_instruction_len = evmcs->vm_exit_instruction_len;
1638          * vmcs12->vmx_instruction_info = evmcs->vmx_instruction_info;
1639          * vmcs12->exit_qualification = evmcs->exit_qualification;
1640          * vmcs12->guest_linear_address = evmcs->guest_linear_address;
1641          *
1642          * Not present in struct vmcs12:
1643          * vmcs12->exit_io_instruction_ecx = evmcs->exit_io_instruction_ecx;
1644          * vmcs12->exit_io_instruction_esi = evmcs->exit_io_instruction_esi;
1645          * vmcs12->exit_io_instruction_edi = evmcs->exit_io_instruction_edi;
1646          * vmcs12->exit_io_instruction_eip = evmcs->exit_io_instruction_eip;
1647          */
1648
1649         return 0;
1650 }
1651
1652 static int copy_vmcs12_to_enlightened(struct vcpu_vmx *vmx)
1653 {
1654         struct vmcs12 *vmcs12 = vmx->nested.cached_vmcs12;
1655         struct hv_enlightened_vmcs *evmcs = vmx->nested.hv_evmcs;
1656
1657         /*
1658          * Should not be changed by KVM:
1659          *
1660          * evmcs->host_es_selector = vmcs12->host_es_selector;
1661          * evmcs->host_cs_selector = vmcs12->host_cs_selector;
1662          * evmcs->host_ss_selector = vmcs12->host_ss_selector;
1663          * evmcs->host_ds_selector = vmcs12->host_ds_selector;
1664          * evmcs->host_fs_selector = vmcs12->host_fs_selector;
1665          * evmcs->host_gs_selector = vmcs12->host_gs_selector;
1666          * evmcs->host_tr_selector = vmcs12->host_tr_selector;
1667          * evmcs->host_ia32_pat = vmcs12->host_ia32_pat;
1668          * evmcs->host_ia32_efer = vmcs12->host_ia32_efer;
1669          * evmcs->host_cr0 = vmcs12->host_cr0;
1670          * evmcs->host_cr3 = vmcs12->host_cr3;
1671          * evmcs->host_cr4 = vmcs12->host_cr4;
1672          * evmcs->host_ia32_sysenter_esp = vmcs12->host_ia32_sysenter_esp;
1673          * evmcs->host_ia32_sysenter_eip = vmcs12->host_ia32_sysenter_eip;
1674          * evmcs->host_rip = vmcs12->host_rip;
1675          * evmcs->host_ia32_sysenter_cs = vmcs12->host_ia32_sysenter_cs;
1676          * evmcs->host_fs_base = vmcs12->host_fs_base;
1677          * evmcs->host_gs_base = vmcs12->host_gs_base;
1678          * evmcs->host_tr_base = vmcs12->host_tr_base;
1679          * evmcs->host_gdtr_base = vmcs12->host_gdtr_base;
1680          * evmcs->host_idtr_base = vmcs12->host_idtr_base;
1681          * evmcs->host_rsp = vmcs12->host_rsp;
1682          * sync_vmcs02_to_vmcs12() doesn't read these:
1683          * evmcs->io_bitmap_a = vmcs12->io_bitmap_a;
1684          * evmcs->io_bitmap_b = vmcs12->io_bitmap_b;
1685          * evmcs->msr_bitmap = vmcs12->msr_bitmap;
1686          * evmcs->ept_pointer = vmcs12->ept_pointer;
1687          * evmcs->xss_exit_bitmap = vmcs12->xss_exit_bitmap;
1688          * evmcs->vm_exit_msr_store_addr = vmcs12->vm_exit_msr_store_addr;
1689          * evmcs->vm_exit_msr_load_addr = vmcs12->vm_exit_msr_load_addr;
1690          * evmcs->vm_entry_msr_load_addr = vmcs12->vm_entry_msr_load_addr;
1691          * evmcs->cr3_target_value0 = vmcs12->cr3_target_value0;
1692          * evmcs->cr3_target_value1 = vmcs12->cr3_target_value1;
1693          * evmcs->cr3_target_value2 = vmcs12->cr3_target_value2;
1694          * evmcs->cr3_target_value3 = vmcs12->cr3_target_value3;
1695          * evmcs->tpr_threshold = vmcs12->tpr_threshold;
1696          * evmcs->virtual_processor_id = vmcs12->virtual_processor_id;
1697          * evmcs->exception_bitmap = vmcs12->exception_bitmap;
1698          * evmcs->vmcs_link_pointer = vmcs12->vmcs_link_pointer;
1699          * evmcs->pin_based_vm_exec_control = vmcs12->pin_based_vm_exec_control;
1700          * evmcs->vm_exit_controls = vmcs12->vm_exit_controls;
1701          * evmcs->secondary_vm_exec_control = vmcs12->secondary_vm_exec_control;
1702          * evmcs->page_fault_error_code_mask =
1703          *              vmcs12->page_fault_error_code_mask;
1704          * evmcs->page_fault_error_code_match =
1705          *              vmcs12->page_fault_error_code_match;
1706          * evmcs->cr3_target_count = vmcs12->cr3_target_count;
1707          * evmcs->virtual_apic_page_addr = vmcs12->virtual_apic_page_addr;
1708          * evmcs->tsc_offset = vmcs12->tsc_offset;
1709          * evmcs->guest_ia32_debugctl = vmcs12->guest_ia32_debugctl;
1710          * evmcs->cr0_guest_host_mask = vmcs12->cr0_guest_host_mask;
1711          * evmcs->cr4_guest_host_mask = vmcs12->cr4_guest_host_mask;
1712          * evmcs->cr0_read_shadow = vmcs12->cr0_read_shadow;
1713          * evmcs->cr4_read_shadow = vmcs12->cr4_read_shadow;
1714          * evmcs->vm_exit_msr_store_count = vmcs12->vm_exit_msr_store_count;
1715          * evmcs->vm_exit_msr_load_count = vmcs12->vm_exit_msr_load_count;
1716          * evmcs->vm_entry_msr_load_count = vmcs12->vm_entry_msr_load_count;
1717          *
1718          * Not present in struct vmcs12:
1719          * evmcs->exit_io_instruction_ecx = vmcs12->exit_io_instruction_ecx;
1720          * evmcs->exit_io_instruction_esi = vmcs12->exit_io_instruction_esi;
1721          * evmcs->exit_io_instruction_edi = vmcs12->exit_io_instruction_edi;
1722          * evmcs->exit_io_instruction_eip = vmcs12->exit_io_instruction_eip;
1723          */
1724
1725         evmcs->guest_es_selector = vmcs12->guest_es_selector;
1726         evmcs->guest_cs_selector = vmcs12->guest_cs_selector;
1727         evmcs->guest_ss_selector = vmcs12->guest_ss_selector;
1728         evmcs->guest_ds_selector = vmcs12->guest_ds_selector;
1729         evmcs->guest_fs_selector = vmcs12->guest_fs_selector;
1730         evmcs->guest_gs_selector = vmcs12->guest_gs_selector;
1731         evmcs->guest_ldtr_selector = vmcs12->guest_ldtr_selector;
1732         evmcs->guest_tr_selector = vmcs12->guest_tr_selector;
1733
1734         evmcs->guest_es_limit = vmcs12->guest_es_limit;
1735         evmcs->guest_cs_limit = vmcs12->guest_cs_limit;
1736         evmcs->guest_ss_limit = vmcs12->guest_ss_limit;
1737         evmcs->guest_ds_limit = vmcs12->guest_ds_limit;
1738         evmcs->guest_fs_limit = vmcs12->guest_fs_limit;
1739         evmcs->guest_gs_limit = vmcs12->guest_gs_limit;
1740         evmcs->guest_ldtr_limit = vmcs12->guest_ldtr_limit;
1741         evmcs->guest_tr_limit = vmcs12->guest_tr_limit;
1742         evmcs->guest_gdtr_limit = vmcs12->guest_gdtr_limit;
1743         evmcs->guest_idtr_limit = vmcs12->guest_idtr_limit;
1744
1745         evmcs->guest_es_ar_bytes = vmcs12->guest_es_ar_bytes;
1746         evmcs->guest_cs_ar_bytes = vmcs12->guest_cs_ar_bytes;
1747         evmcs->guest_ss_ar_bytes = vmcs12->guest_ss_ar_bytes;
1748         evmcs->guest_ds_ar_bytes = vmcs12->guest_ds_ar_bytes;
1749         evmcs->guest_fs_ar_bytes = vmcs12->guest_fs_ar_bytes;
1750         evmcs->guest_gs_ar_bytes = vmcs12->guest_gs_ar_bytes;
1751         evmcs->guest_ldtr_ar_bytes = vmcs12->guest_ldtr_ar_bytes;
1752         evmcs->guest_tr_ar_bytes = vmcs12->guest_tr_ar_bytes;
1753
1754         evmcs->guest_es_base = vmcs12->guest_es_base;
1755         evmcs->guest_cs_base = vmcs12->guest_cs_base;
1756         evmcs->guest_ss_base = vmcs12->guest_ss_base;
1757         evmcs->guest_ds_base = vmcs12->guest_ds_base;
1758         evmcs->guest_fs_base = vmcs12->guest_fs_base;
1759         evmcs->guest_gs_base = vmcs12->guest_gs_base;
1760         evmcs->guest_ldtr_base = vmcs12->guest_ldtr_base;
1761         evmcs->guest_tr_base = vmcs12->guest_tr_base;
1762         evmcs->guest_gdtr_base = vmcs12->guest_gdtr_base;
1763         evmcs->guest_idtr_base = vmcs12->guest_idtr_base;
1764
1765         evmcs->guest_ia32_pat = vmcs12->guest_ia32_pat;
1766         evmcs->guest_ia32_efer = vmcs12->guest_ia32_efer;
1767
1768         evmcs->guest_pdptr0 = vmcs12->guest_pdptr0;
1769         evmcs->guest_pdptr1 = vmcs12->guest_pdptr1;
1770         evmcs->guest_pdptr2 = vmcs12->guest_pdptr2;
1771         evmcs->guest_pdptr3 = vmcs12->guest_pdptr3;
1772
1773         evmcs->guest_pending_dbg_exceptions =
1774                 vmcs12->guest_pending_dbg_exceptions;
1775         evmcs->guest_sysenter_esp = vmcs12->guest_sysenter_esp;
1776         evmcs->guest_sysenter_eip = vmcs12->guest_sysenter_eip;
1777
1778         evmcs->guest_activity_state = vmcs12->guest_activity_state;
1779         evmcs->guest_sysenter_cs = vmcs12->guest_sysenter_cs;
1780
1781         evmcs->guest_cr0 = vmcs12->guest_cr0;
1782         evmcs->guest_cr3 = vmcs12->guest_cr3;
1783         evmcs->guest_cr4 = vmcs12->guest_cr4;
1784         evmcs->guest_dr7 = vmcs12->guest_dr7;
1785
1786         evmcs->guest_physical_address = vmcs12->guest_physical_address;
1787
1788         evmcs->vm_instruction_error = vmcs12->vm_instruction_error;
1789         evmcs->vm_exit_reason = vmcs12->vm_exit_reason;
1790         evmcs->vm_exit_intr_info = vmcs12->vm_exit_intr_info;
1791         evmcs->vm_exit_intr_error_code = vmcs12->vm_exit_intr_error_code;
1792         evmcs->idt_vectoring_info_field = vmcs12->idt_vectoring_info_field;
1793         evmcs->idt_vectoring_error_code = vmcs12->idt_vectoring_error_code;
1794         evmcs->vm_exit_instruction_len = vmcs12->vm_exit_instruction_len;
1795         evmcs->vmx_instruction_info = vmcs12->vmx_instruction_info;
1796
1797         evmcs->exit_qualification = vmcs12->exit_qualification;
1798
1799         evmcs->guest_linear_address = vmcs12->guest_linear_address;
1800         evmcs->guest_rsp = vmcs12->guest_rsp;
1801         evmcs->guest_rflags = vmcs12->guest_rflags;
1802
1803         evmcs->guest_interruptibility_info =
1804                 vmcs12->guest_interruptibility_info;
1805         evmcs->cpu_based_vm_exec_control = vmcs12->cpu_based_vm_exec_control;
1806         evmcs->vm_entry_controls = vmcs12->vm_entry_controls;
1807         evmcs->vm_entry_intr_info_field = vmcs12->vm_entry_intr_info_field;
1808         evmcs->vm_entry_exception_error_code =
1809                 vmcs12->vm_entry_exception_error_code;
1810         evmcs->vm_entry_instruction_len = vmcs12->vm_entry_instruction_len;
1811
1812         evmcs->guest_rip = vmcs12->guest_rip;
1813
1814         evmcs->guest_bndcfgs = vmcs12->guest_bndcfgs;
1815
1816         return 0;
1817 }
1818
1819 /*
1820  * This is an equivalent of the nested hypervisor executing the vmptrld
1821  * instruction.
1822  */
1823 static int nested_vmx_handle_enlightened_vmptrld(struct kvm_vcpu *vcpu,
1824                                                  bool from_launch)
1825 {
1826         struct vcpu_vmx *vmx = to_vmx(vcpu);
1827         bool evmcs_gpa_changed = false;
1828         u64 evmcs_gpa;
1829
1830         if (likely(!vmx->nested.enlightened_vmcs_enabled))
1831                 return 1;
1832
1833         if (!nested_enlightened_vmentry(vcpu, &evmcs_gpa))
1834                 return 1;
1835
1836         if (unlikely(!vmx->nested.hv_evmcs ||
1837                      evmcs_gpa != vmx->nested.hv_evmcs_vmptr)) {
1838                 if (!vmx->nested.hv_evmcs)
1839                         vmx->nested.current_vmptr = -1ull;
1840
1841                 nested_release_evmcs(vcpu);
1842
1843                 if (kvm_vcpu_map(vcpu, gpa_to_gfn(evmcs_gpa),
1844                                  &vmx->nested.hv_evmcs_map))
1845                         return 0;
1846
1847                 vmx->nested.hv_evmcs = vmx->nested.hv_evmcs_map.hva;
1848
1849                 /*
1850                  * Currently, KVM only supports eVMCS version 1
1851                  * (== KVM_EVMCS_VERSION) and thus we expect guest to set this
1852                  * value to first u32 field of eVMCS which should specify eVMCS
1853                  * VersionNumber.
1854                  *
1855                  * Guest should be aware of supported eVMCS versions by host by
1856                  * examining CPUID.0x4000000A.EAX[0:15]. Host userspace VMM is
1857                  * expected to set this CPUID leaf according to the value
1858                  * returned in vmcs_version from nested_enable_evmcs().
1859                  *
1860                  * However, it turns out that Microsoft Hyper-V fails to comply
1861                  * to their own invented interface: When Hyper-V use eVMCS, it
1862                  * just sets first u32 field of eVMCS to revision_id specified
1863                  * in MSR_IA32_VMX_BASIC. Instead of used eVMCS version number
1864                  * which is one of the supported versions specified in
1865                  * CPUID.0x4000000A.EAX[0:15].
1866                  *
1867                  * To overcome Hyper-V bug, we accept here either a supported
1868                  * eVMCS version or VMCS12 revision_id as valid values for first
1869                  * u32 field of eVMCS.
1870                  */
1871                 if ((vmx->nested.hv_evmcs->revision_id != KVM_EVMCS_VERSION) &&
1872                     (vmx->nested.hv_evmcs->revision_id != VMCS12_REVISION)) {
1873                         nested_release_evmcs(vcpu);
1874                         return 0;
1875                 }
1876
1877                 vmx->nested.dirty_vmcs12 = true;
1878                 vmx->nested.hv_evmcs_vmptr = evmcs_gpa;
1879
1880                 evmcs_gpa_changed = true;
1881                 /*
1882                  * Unlike normal vmcs12, enlightened vmcs12 is not fully
1883                  * reloaded from guest's memory (read only fields, fields not
1884                  * present in struct hv_enlightened_vmcs, ...). Make sure there
1885                  * are no leftovers.
1886                  */
1887                 if (from_launch) {
1888                         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
1889                         memset(vmcs12, 0, sizeof(*vmcs12));
1890                         vmcs12->hdr.revision_id = VMCS12_REVISION;
1891                 }
1892
1893         }
1894
1895         /*
1896          * Clean fields data can't de used on VMLAUNCH and when we switch
1897          * between different L2 guests as KVM keeps a single VMCS12 per L1.
1898          */
1899         if (from_launch || evmcs_gpa_changed)
1900                 vmx->nested.hv_evmcs->hv_clean_fields &=
1901                         ~HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL;
1902
1903         return 1;
1904 }
1905
1906 void nested_sync_vmcs12_to_shadow(struct kvm_vcpu *vcpu)
1907 {
1908         struct vcpu_vmx *vmx = to_vmx(vcpu);
1909
1910         /*
1911          * hv_evmcs may end up being not mapped after migration (when
1912          * L2 was running), map it here to make sure vmcs12 changes are
1913          * properly reflected.
1914          */
1915         if (vmx->nested.enlightened_vmcs_enabled && !vmx->nested.hv_evmcs)
1916                 nested_vmx_handle_enlightened_vmptrld(vcpu, false);
1917
1918         if (vmx->nested.hv_evmcs) {
1919                 copy_vmcs12_to_enlightened(vmx);
1920                 /* All fields are clean */
1921                 vmx->nested.hv_evmcs->hv_clean_fields |=
1922                         HV_VMX_ENLIGHTENED_CLEAN_FIELD_ALL;
1923         } else {
1924                 copy_vmcs12_to_shadow(vmx);
1925         }
1926
1927         vmx->nested.need_vmcs12_to_shadow_sync = false;
1928 }
1929
1930 static enum hrtimer_restart vmx_preemption_timer_fn(struct hrtimer *timer)
1931 {
1932         struct vcpu_vmx *vmx =
1933                 container_of(timer, struct vcpu_vmx, nested.preemption_timer);
1934
1935         vmx->nested.preemption_timer_expired = true;
1936         kvm_make_request(KVM_REQ_EVENT, &vmx->vcpu);
1937         kvm_vcpu_kick(&vmx->vcpu);
1938
1939         return HRTIMER_NORESTART;
1940 }
1941
1942 static void vmx_start_preemption_timer(struct kvm_vcpu *vcpu)
1943 {
1944         u64 preemption_timeout = get_vmcs12(vcpu)->vmx_preemption_timer_value;
1945         struct vcpu_vmx *vmx = to_vmx(vcpu);
1946
1947         /*
1948          * A timer value of zero is architecturally guaranteed to cause
1949          * a VMExit prior to executing any instructions in the guest.
1950          */
1951         if (preemption_timeout == 0) {
1952                 vmx_preemption_timer_fn(&vmx->nested.preemption_timer);
1953                 return;
1954         }
1955
1956         if (vcpu->arch.virtual_tsc_khz == 0)
1957                 return;
1958
1959         preemption_timeout <<= VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
1960         preemption_timeout *= 1000000;
1961         do_div(preemption_timeout, vcpu->arch.virtual_tsc_khz);
1962         hrtimer_start(&vmx->nested.preemption_timer,
1963                       ns_to_ktime(preemption_timeout), HRTIMER_MODE_REL);
1964 }
1965
1966 static u64 nested_vmx_calc_efer(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12)
1967 {
1968         if (vmx->nested.nested_run_pending &&
1969             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER))
1970                 return vmcs12->guest_ia32_efer;
1971         else if (vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE)
1972                 return vmx->vcpu.arch.efer | (EFER_LMA | EFER_LME);
1973         else
1974                 return vmx->vcpu.arch.efer & ~(EFER_LMA | EFER_LME);
1975 }
1976
1977 static void prepare_vmcs02_constant_state(struct vcpu_vmx *vmx)
1978 {
1979         /*
1980          * If vmcs02 hasn't been initialized, set the constant vmcs02 state
1981          * according to L0's settings (vmcs12 is irrelevant here).  Host
1982          * fields that come from L0 and are not constant, e.g. HOST_CR3,
1983          * will be set as needed prior to VMLAUNCH/VMRESUME.
1984          */
1985         if (vmx->nested.vmcs02_initialized)
1986                 return;
1987         vmx->nested.vmcs02_initialized = true;
1988
1989         /*
1990          * We don't care what the EPTP value is we just need to guarantee
1991          * it's valid so we don't get a false positive when doing early
1992          * consistency checks.
1993          */
1994         if (enable_ept && nested_early_check)
1995                 vmcs_write64(EPT_POINTER, construct_eptp(&vmx->vcpu, 0));
1996
1997         /* All VMFUNCs are currently emulated through L0 vmexits.  */
1998         if (cpu_has_vmx_vmfunc())
1999                 vmcs_write64(VM_FUNCTION_CONTROL, 0);
2000
2001         if (cpu_has_vmx_posted_intr())
2002                 vmcs_write16(POSTED_INTR_NV, POSTED_INTR_NESTED_VECTOR);
2003
2004         if (cpu_has_vmx_msr_bitmap())
2005                 vmcs_write64(MSR_BITMAP, __pa(vmx->nested.vmcs02.msr_bitmap));
2006
2007         /*
2008          * The PML address never changes, so it is constant in vmcs02.
2009          * Conceptually we want to copy the PML index from vmcs01 here,
2010          * and then back to vmcs01 on nested vmexit.  But since we flush
2011          * the log and reset GUEST_PML_INDEX on each vmexit, the PML
2012          * index is also effectively constant in vmcs02.
2013          */
2014         if (enable_pml) {
2015                 vmcs_write64(PML_ADDRESS, page_to_phys(vmx->pml_pg));
2016                 vmcs_write16(GUEST_PML_INDEX, PML_ENTITY_NUM - 1);
2017         }
2018
2019         if (cpu_has_vmx_encls_vmexit())
2020                 vmcs_write64(ENCLS_EXITING_BITMAP, -1ull);
2021
2022         /*
2023          * Set the MSR load/store lists to match L0's settings.  Only the
2024          * addresses are constant (for vmcs02), the counts can change based
2025          * on L2's behavior, e.g. switching to/from long mode.
2026          */
2027         vmcs_write32(VM_EXIT_MSR_STORE_COUNT, 0);
2028         vmcs_write64(VM_EXIT_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.host.val));
2029         vmcs_write64(VM_ENTRY_MSR_LOAD_ADDR, __pa(vmx->msr_autoload.guest.val));
2030
2031         vmx_set_constant_host_state(vmx);
2032 }
2033
2034 static void prepare_vmcs02_early_rare(struct vcpu_vmx *vmx,
2035                                       struct vmcs12 *vmcs12)
2036 {
2037         prepare_vmcs02_constant_state(vmx);
2038
2039         vmcs_write64(VMCS_LINK_POINTER, -1ull);
2040
2041         if (enable_vpid) {
2042                 if (nested_cpu_has_vpid(vmcs12) && vmx->nested.vpid02)
2043                         vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->nested.vpid02);
2044                 else
2045                         vmcs_write16(VIRTUAL_PROCESSOR_ID, vmx->vpid);
2046         }
2047 }
2048
2049 static void prepare_vmcs02_early(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12)
2050 {
2051         u32 exec_control, vmcs12_exec_ctrl;
2052         u64 guest_efer = nested_vmx_calc_efer(vmx, vmcs12);
2053
2054         if (vmx->nested.dirty_vmcs12 || vmx->nested.hv_evmcs)
2055                 prepare_vmcs02_early_rare(vmx, vmcs12);
2056
2057         /*
2058          * PIN CONTROLS
2059          */
2060         exec_control = vmx_pin_based_exec_ctrl(vmx);
2061         exec_control |= (vmcs12->pin_based_vm_exec_control &
2062                          ~PIN_BASED_VMX_PREEMPTION_TIMER);
2063
2064         /* Posted interrupts setting is only taken from vmcs12.  */
2065         vmx->nested.pi_pending = false;
2066         if (nested_cpu_has_posted_intr(vmcs12))
2067                 vmx->nested.posted_intr_nv = vmcs12->posted_intr_nv;
2068         else
2069                 exec_control &= ~PIN_BASED_POSTED_INTR;
2070         pin_controls_set(vmx, exec_control);
2071
2072         /*
2073          * EXEC CONTROLS
2074          */
2075         exec_control = vmx_exec_control(vmx); /* L0's desires */
2076         exec_control &= ~CPU_BASED_INTR_WINDOW_EXITING;
2077         exec_control &= ~CPU_BASED_NMI_WINDOW_EXITING;
2078         exec_control &= ~CPU_BASED_TPR_SHADOW;
2079         exec_control |= vmcs12->cpu_based_vm_exec_control;
2080
2081         if (exec_control & CPU_BASED_TPR_SHADOW)
2082                 vmcs_write32(TPR_THRESHOLD, vmcs12->tpr_threshold);
2083 #ifdef CONFIG_X86_64
2084         else
2085                 exec_control |= CPU_BASED_CR8_LOAD_EXITING |
2086                                 CPU_BASED_CR8_STORE_EXITING;
2087 #endif
2088
2089         /*
2090          * A vmexit (to either L1 hypervisor or L0 userspace) is always needed
2091          * for I/O port accesses.
2092          */
2093         exec_control |= CPU_BASED_UNCOND_IO_EXITING;
2094         exec_control &= ~CPU_BASED_USE_IO_BITMAPS;
2095
2096         /*
2097          * This bit will be computed in nested_get_vmcs12_pages, because
2098          * we do not have access to L1's MSR bitmap yet.  For now, keep
2099          * the same bit as before, hoping to avoid multiple VMWRITEs that
2100          * only set/clear this bit.
2101          */
2102         exec_control &= ~CPU_BASED_USE_MSR_BITMAPS;
2103         exec_control |= exec_controls_get(vmx) & CPU_BASED_USE_MSR_BITMAPS;
2104
2105         exec_controls_set(vmx, exec_control);
2106
2107         /*
2108          * SECONDARY EXEC CONTROLS
2109          */
2110         if (cpu_has_secondary_exec_ctrls()) {
2111                 exec_control = vmx->secondary_exec_control;
2112
2113                 /* Take the following fields only from vmcs12 */
2114                 exec_control &= ~(SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES |
2115                                   SECONDARY_EXEC_ENABLE_INVPCID |
2116                                   SECONDARY_EXEC_RDTSCP |
2117                                   SECONDARY_EXEC_XSAVES |
2118                                   SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE |
2119                                   SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
2120                                   SECONDARY_EXEC_APIC_REGISTER_VIRT |
2121                                   SECONDARY_EXEC_ENABLE_VMFUNC);
2122                 if (nested_cpu_has(vmcs12,
2123                                    CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)) {
2124                         vmcs12_exec_ctrl = vmcs12->secondary_vm_exec_control &
2125                                 ~SECONDARY_EXEC_ENABLE_PML;
2126                         exec_control |= vmcs12_exec_ctrl;
2127                 }
2128
2129                 /* VMCS shadowing for L2 is emulated for now */
2130                 exec_control &= ~SECONDARY_EXEC_SHADOW_VMCS;
2131
2132                 /*
2133                  * Preset *DT exiting when emulating UMIP, so that vmx_set_cr4()
2134                  * will not have to rewrite the controls just for this bit.
2135                  */
2136                 if (!boot_cpu_has(X86_FEATURE_UMIP) && vmx_umip_emulated() &&
2137                     (vmcs12->guest_cr4 & X86_CR4_UMIP))
2138                         exec_control |= SECONDARY_EXEC_DESC;
2139
2140                 if (exec_control & SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY)
2141                         vmcs_write16(GUEST_INTR_STATUS,
2142                                 vmcs12->guest_intr_status);
2143
2144                 secondary_exec_controls_set(vmx, exec_control);
2145         }
2146
2147         /*
2148          * ENTRY CONTROLS
2149          *
2150          * vmcs12's VM_{ENTRY,EXIT}_LOAD_IA32_EFER and VM_ENTRY_IA32E_MODE
2151          * are emulated by vmx_set_efer() in prepare_vmcs02(), but speculate
2152          * on the related bits (if supported by the CPU) in the hope that
2153          * we can avoid VMWrites during vmx_set_efer().
2154          */
2155         exec_control = (vmcs12->vm_entry_controls | vmx_vmentry_ctrl()) &
2156                         ~VM_ENTRY_IA32E_MODE & ~VM_ENTRY_LOAD_IA32_EFER;
2157         if (cpu_has_load_ia32_efer()) {
2158                 if (guest_efer & EFER_LMA)
2159                         exec_control |= VM_ENTRY_IA32E_MODE;
2160                 if (guest_efer != host_efer)
2161                         exec_control |= VM_ENTRY_LOAD_IA32_EFER;
2162         }
2163         vm_entry_controls_set(vmx, exec_control);
2164
2165         /*
2166          * EXIT CONTROLS
2167          *
2168          * L2->L1 exit controls are emulated - the hardware exit is to L0 so
2169          * we should use its exit controls. Note that VM_EXIT_LOAD_IA32_EFER
2170          * bits may be modified by vmx_set_efer() in prepare_vmcs02().
2171          */
2172         exec_control = vmx_vmexit_ctrl();
2173         if (cpu_has_load_ia32_efer() && guest_efer != host_efer)
2174                 exec_control |= VM_EXIT_LOAD_IA32_EFER;
2175         vm_exit_controls_set(vmx, exec_control);
2176
2177         /*
2178          * Interrupt/Exception Fields
2179          */
2180         if (vmx->nested.nested_run_pending) {
2181                 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD,
2182                              vmcs12->vm_entry_intr_info_field);
2183                 vmcs_write32(VM_ENTRY_EXCEPTION_ERROR_CODE,
2184                              vmcs12->vm_entry_exception_error_code);
2185                 vmcs_write32(VM_ENTRY_INSTRUCTION_LEN,
2186                              vmcs12->vm_entry_instruction_len);
2187                 vmcs_write32(GUEST_INTERRUPTIBILITY_INFO,
2188                              vmcs12->guest_interruptibility_info);
2189                 vmx->loaded_vmcs->nmi_known_unmasked =
2190                         !(vmcs12->guest_interruptibility_info & GUEST_INTR_STATE_NMI);
2191         } else {
2192                 vmcs_write32(VM_ENTRY_INTR_INFO_FIELD, 0);
2193         }
2194 }
2195
2196 static void prepare_vmcs02_rare(struct vcpu_vmx *vmx, struct vmcs12 *vmcs12)
2197 {
2198         struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs;
2199
2200         if (!hv_evmcs || !(hv_evmcs->hv_clean_fields &
2201                            HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP2)) {
2202                 vmcs_write16(GUEST_ES_SELECTOR, vmcs12->guest_es_selector);
2203                 vmcs_write16(GUEST_CS_SELECTOR, vmcs12->guest_cs_selector);
2204                 vmcs_write16(GUEST_SS_SELECTOR, vmcs12->guest_ss_selector);
2205                 vmcs_write16(GUEST_DS_SELECTOR, vmcs12->guest_ds_selector);
2206                 vmcs_write16(GUEST_FS_SELECTOR, vmcs12->guest_fs_selector);
2207                 vmcs_write16(GUEST_GS_SELECTOR, vmcs12->guest_gs_selector);
2208                 vmcs_write16(GUEST_LDTR_SELECTOR, vmcs12->guest_ldtr_selector);
2209                 vmcs_write16(GUEST_TR_SELECTOR, vmcs12->guest_tr_selector);
2210                 vmcs_write32(GUEST_ES_LIMIT, vmcs12->guest_es_limit);
2211                 vmcs_write32(GUEST_CS_LIMIT, vmcs12->guest_cs_limit);
2212                 vmcs_write32(GUEST_SS_LIMIT, vmcs12->guest_ss_limit);
2213                 vmcs_write32(GUEST_DS_LIMIT, vmcs12->guest_ds_limit);
2214                 vmcs_write32(GUEST_FS_LIMIT, vmcs12->guest_fs_limit);
2215                 vmcs_write32(GUEST_GS_LIMIT, vmcs12->guest_gs_limit);
2216                 vmcs_write32(GUEST_LDTR_LIMIT, vmcs12->guest_ldtr_limit);
2217                 vmcs_write32(GUEST_TR_LIMIT, vmcs12->guest_tr_limit);
2218                 vmcs_write32(GUEST_GDTR_LIMIT, vmcs12->guest_gdtr_limit);
2219                 vmcs_write32(GUEST_IDTR_LIMIT, vmcs12->guest_idtr_limit);
2220                 vmcs_write32(GUEST_CS_AR_BYTES, vmcs12->guest_cs_ar_bytes);
2221                 vmcs_write32(GUEST_SS_AR_BYTES, vmcs12->guest_ss_ar_bytes);
2222                 vmcs_write32(GUEST_ES_AR_BYTES, vmcs12->guest_es_ar_bytes);
2223                 vmcs_write32(GUEST_DS_AR_BYTES, vmcs12->guest_ds_ar_bytes);
2224                 vmcs_write32(GUEST_FS_AR_BYTES, vmcs12->guest_fs_ar_bytes);
2225                 vmcs_write32(GUEST_GS_AR_BYTES, vmcs12->guest_gs_ar_bytes);
2226                 vmcs_write32(GUEST_LDTR_AR_BYTES, vmcs12->guest_ldtr_ar_bytes);
2227                 vmcs_write32(GUEST_TR_AR_BYTES, vmcs12->guest_tr_ar_bytes);
2228                 vmcs_writel(GUEST_ES_BASE, vmcs12->guest_es_base);
2229                 vmcs_writel(GUEST_CS_BASE, vmcs12->guest_cs_base);
2230                 vmcs_writel(GUEST_SS_BASE, vmcs12->guest_ss_base);
2231                 vmcs_writel(GUEST_DS_BASE, vmcs12->guest_ds_base);
2232                 vmcs_writel(GUEST_FS_BASE, vmcs12->guest_fs_base);
2233                 vmcs_writel(GUEST_GS_BASE, vmcs12->guest_gs_base);
2234                 vmcs_writel(GUEST_LDTR_BASE, vmcs12->guest_ldtr_base);
2235                 vmcs_writel(GUEST_TR_BASE, vmcs12->guest_tr_base);
2236                 vmcs_writel(GUEST_GDTR_BASE, vmcs12->guest_gdtr_base);
2237                 vmcs_writel(GUEST_IDTR_BASE, vmcs12->guest_idtr_base);
2238
2239                 vmx->segment_cache.bitmask = 0;
2240         }
2241
2242         if (!hv_evmcs || !(hv_evmcs->hv_clean_fields &
2243                            HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1)) {
2244                 vmcs_write32(GUEST_SYSENTER_CS, vmcs12->guest_sysenter_cs);
2245                 vmcs_writel(GUEST_PENDING_DBG_EXCEPTIONS,
2246                             vmcs12->guest_pending_dbg_exceptions);
2247                 vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->guest_sysenter_esp);
2248                 vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->guest_sysenter_eip);
2249
2250                 /*
2251                  * L1 may access the L2's PDPTR, so save them to construct
2252                  * vmcs12
2253                  */
2254                 if (enable_ept) {
2255                         vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0);
2256                         vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1);
2257                         vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2);
2258                         vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3);
2259                 }
2260
2261                 if (kvm_mpx_supported() && vmx->nested.nested_run_pending &&
2262                     (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS))
2263                         vmcs_write64(GUEST_BNDCFGS, vmcs12->guest_bndcfgs);
2264         }
2265
2266         if (nested_cpu_has_xsaves(vmcs12))
2267                 vmcs_write64(XSS_EXIT_BITMAP, vmcs12->xss_exit_bitmap);
2268
2269         /*
2270          * Whether page-faults are trapped is determined by a combination of
2271          * 3 settings: PFEC_MASK, PFEC_MATCH and EXCEPTION_BITMAP.PF.
2272          * If enable_ept, L0 doesn't care about page faults and we should
2273          * set all of these to L1's desires. However, if !enable_ept, L0 does
2274          * care about (at least some) page faults, and because it is not easy
2275          * (if at all possible?) to merge L0 and L1's desires, we simply ask
2276          * to exit on each and every L2 page fault. This is done by setting
2277          * MASK=MATCH=0 and (see below) EB.PF=1.
2278          * Note that below we don't need special code to set EB.PF beyond the
2279          * "or"ing of the EB of vmcs01 and vmcs12, because when enable_ept,
2280          * vmcs01's EB.PF is 0 so the "or" will take vmcs12's value, and when
2281          * !enable_ept, EB.PF is 1, so the "or" will always be 1.
2282          */
2283         vmcs_write32(PAGE_FAULT_ERROR_CODE_MASK,
2284                 enable_ept ? vmcs12->page_fault_error_code_mask : 0);
2285         vmcs_write32(PAGE_FAULT_ERROR_CODE_MATCH,
2286                 enable_ept ? vmcs12->page_fault_error_code_match : 0);
2287
2288         if (cpu_has_vmx_apicv()) {
2289                 vmcs_write64(EOI_EXIT_BITMAP0, vmcs12->eoi_exit_bitmap0);
2290                 vmcs_write64(EOI_EXIT_BITMAP1, vmcs12->eoi_exit_bitmap1);
2291                 vmcs_write64(EOI_EXIT_BITMAP2, vmcs12->eoi_exit_bitmap2);
2292                 vmcs_write64(EOI_EXIT_BITMAP3, vmcs12->eoi_exit_bitmap3);
2293         }
2294
2295         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
2296         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
2297
2298         set_cr4_guest_host_mask(vmx);
2299 }
2300
2301 /*
2302  * prepare_vmcs02 is called when the L1 guest hypervisor runs its nested
2303  * L2 guest. L1 has a vmcs for L2 (vmcs12), and this function "merges" it
2304  * with L0's requirements for its guest (a.k.a. vmcs01), so we can run the L2
2305  * guest in a way that will both be appropriate to L1's requests, and our
2306  * needs. In addition to modifying the active vmcs (which is vmcs02), this
2307  * function also has additional necessary side-effects, like setting various
2308  * vcpu->arch fields.
2309  * Returns 0 on success, 1 on failure. Invalid state exit qualification code
2310  * is assigned to entry_failure_code on failure.
2311  */
2312 static int prepare_vmcs02(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
2313                           u32 *entry_failure_code)
2314 {
2315         struct vcpu_vmx *vmx = to_vmx(vcpu);
2316         struct hv_enlightened_vmcs *hv_evmcs = vmx->nested.hv_evmcs;
2317         bool load_guest_pdptrs_vmcs12 = false;
2318
2319         if (vmx->nested.dirty_vmcs12 || hv_evmcs) {
2320                 prepare_vmcs02_rare(vmx, vmcs12);
2321                 vmx->nested.dirty_vmcs12 = false;
2322
2323                 load_guest_pdptrs_vmcs12 = !hv_evmcs ||
2324                         !(hv_evmcs->hv_clean_fields &
2325                           HV_VMX_ENLIGHTENED_CLEAN_FIELD_GUEST_GRP1);
2326         }
2327
2328         if (vmx->nested.nested_run_pending &&
2329             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS)) {
2330                 kvm_set_dr(vcpu, 7, vmcs12->guest_dr7);
2331                 vmcs_write64(GUEST_IA32_DEBUGCTL, vmcs12->guest_ia32_debugctl);
2332         } else {
2333                 kvm_set_dr(vcpu, 7, vcpu->arch.dr7);
2334                 vmcs_write64(GUEST_IA32_DEBUGCTL, vmx->nested.vmcs01_debugctl);
2335         }
2336         if (kvm_mpx_supported() && (!vmx->nested.nested_run_pending ||
2337             !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)))
2338                 vmcs_write64(GUEST_BNDCFGS, vmx->nested.vmcs01_guest_bndcfgs);
2339         vmx_set_rflags(vcpu, vmcs12->guest_rflags);
2340
2341         /* EXCEPTION_BITMAP and CR0_GUEST_HOST_MASK should basically be the
2342          * bitwise-or of what L1 wants to trap for L2, and what we want to
2343          * trap. Note that CR0.TS also needs updating - we do this later.
2344          */
2345         update_exception_bitmap(vcpu);
2346         vcpu->arch.cr0_guest_owned_bits &= ~vmcs12->cr0_guest_host_mask;
2347         vmcs_writel(CR0_GUEST_HOST_MASK, ~vcpu->arch.cr0_guest_owned_bits);
2348
2349         if (vmx->nested.nested_run_pending &&
2350             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT)) {
2351                 vmcs_write64(GUEST_IA32_PAT, vmcs12->guest_ia32_pat);
2352                 vcpu->arch.pat = vmcs12->guest_ia32_pat;
2353         } else if (vmcs_config.vmentry_ctrl & VM_ENTRY_LOAD_IA32_PAT) {
2354                 vmcs_write64(GUEST_IA32_PAT, vmx->vcpu.arch.pat);
2355         }
2356
2357         vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
2358
2359         if (kvm_has_tsc_control)
2360                 decache_tsc_multiplier(vmx);
2361
2362         if (enable_vpid) {
2363                 /*
2364                  * There is no direct mapping between vpid02 and vpid12, the
2365                  * vpid02 is per-vCPU for L0 and reused while the value of
2366                  * vpid12 is changed w/ one invvpid during nested vmentry.
2367                  * The vpid12 is allocated by L1 for L2, so it will not
2368                  * influence global bitmap(for vpid01 and vpid02 allocation)
2369                  * even if spawn a lot of nested vCPUs.
2370                  */
2371                 if (nested_cpu_has_vpid(vmcs12) && nested_has_guest_tlb_tag(vcpu)) {
2372                         if (vmcs12->virtual_processor_id != vmx->nested.last_vpid) {
2373                                 vmx->nested.last_vpid = vmcs12->virtual_processor_id;
2374                                 __vmx_flush_tlb(vcpu, nested_get_vpid02(vcpu), false);
2375                         }
2376                 } else {
2377                         /*
2378                          * If L1 use EPT, then L0 needs to execute INVEPT on
2379                          * EPTP02 instead of EPTP01. Therefore, delay TLB
2380                          * flush until vmcs02->eptp is fully updated by
2381                          * KVM_REQ_LOAD_CR3. Note that this assumes
2382                          * KVM_REQ_TLB_FLUSH is evaluated after
2383                          * KVM_REQ_LOAD_CR3 in vcpu_enter_guest().
2384                          */
2385                         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
2386                 }
2387         }
2388
2389         if (nested_cpu_has_ept(vmcs12))
2390                 nested_ept_init_mmu_context(vcpu);
2391         else if (nested_cpu_has2(vmcs12,
2392                                  SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES))
2393                 vmx_flush_tlb(vcpu, true);
2394
2395         /*
2396          * This sets GUEST_CR0 to vmcs12->guest_cr0, possibly modifying those
2397          * bits which we consider mandatory enabled.
2398          * The CR0_READ_SHADOW is what L2 should have expected to read given
2399          * the specifications by L1; It's not enough to take
2400          * vmcs12->cr0_read_shadow because on our cr0_guest_host_mask we we
2401          * have more bits than L1 expected.
2402          */
2403         vmx_set_cr0(vcpu, vmcs12->guest_cr0);
2404         vmcs_writel(CR0_READ_SHADOW, nested_read_cr0(vmcs12));
2405
2406         vmx_set_cr4(vcpu, vmcs12->guest_cr4);
2407         vmcs_writel(CR4_READ_SHADOW, nested_read_cr4(vmcs12));
2408
2409         vcpu->arch.efer = nested_vmx_calc_efer(vmx, vmcs12);
2410         /* Note: may modify VM_ENTRY/EXIT_CONTROLS and GUEST/HOST_IA32_EFER */
2411         vmx_set_efer(vcpu, vcpu->arch.efer);
2412
2413         /*
2414          * Guest state is invalid and unrestricted guest is disabled,
2415          * which means L1 attempted VMEntry to L2 with invalid state.
2416          * Fail the VMEntry.
2417          */
2418         if (vmx->emulation_required) {
2419                 *entry_failure_code = ENTRY_FAIL_DEFAULT;
2420                 return -EINVAL;
2421         }
2422
2423         /* Shadow page tables on either EPT or shadow page tables. */
2424         if (nested_vmx_load_cr3(vcpu, vmcs12->guest_cr3, nested_cpu_has_ept(vmcs12),
2425                                 entry_failure_code))
2426                 return -EINVAL;
2427
2428         /*
2429          * Immediately write vmcs02.GUEST_CR3.  It will be propagated to vmcs12
2430          * on nested VM-Exit, which can occur without actually running L2 and
2431          * thus without hitting vmx_set_cr3(), e.g. if L1 is entering L2 with
2432          * vmcs12.GUEST_ACTIVITYSTATE=HLT, in which case KVM will intercept the
2433          * transition to HLT instead of running L2.
2434          */
2435         if (enable_ept)
2436                 vmcs_writel(GUEST_CR3, vmcs12->guest_cr3);
2437
2438         /* Late preparation of GUEST_PDPTRs now that EFER and CRs are set. */
2439         if (load_guest_pdptrs_vmcs12 && nested_cpu_has_ept(vmcs12) &&
2440             is_pae_paging(vcpu)) {
2441                 vmcs_write64(GUEST_PDPTR0, vmcs12->guest_pdptr0);
2442                 vmcs_write64(GUEST_PDPTR1, vmcs12->guest_pdptr1);
2443                 vmcs_write64(GUEST_PDPTR2, vmcs12->guest_pdptr2);
2444                 vmcs_write64(GUEST_PDPTR3, vmcs12->guest_pdptr3);
2445         }
2446
2447         if (!enable_ept)
2448                 vcpu->arch.walk_mmu->inject_page_fault = vmx_inject_page_fault_nested;
2449
2450         kvm_rsp_write(vcpu, vmcs12->guest_rsp);
2451         kvm_rip_write(vcpu, vmcs12->guest_rip);
2452         return 0;
2453 }
2454
2455 static int nested_vmx_check_nmi_controls(struct vmcs12 *vmcs12)
2456 {
2457         if (CC(!nested_cpu_has_nmi_exiting(vmcs12) &&
2458                nested_cpu_has_virtual_nmis(vmcs12)))
2459                 return -EINVAL;
2460
2461         if (CC(!nested_cpu_has_virtual_nmis(vmcs12) &&
2462                nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING)))
2463                 return -EINVAL;
2464
2465         return 0;
2466 }
2467
2468 static bool valid_ept_address(struct kvm_vcpu *vcpu, u64 address)
2469 {
2470         struct vcpu_vmx *vmx = to_vmx(vcpu);
2471         int maxphyaddr = cpuid_maxphyaddr(vcpu);
2472
2473         /* Check for memory type validity */
2474         switch (address & VMX_EPTP_MT_MASK) {
2475         case VMX_EPTP_MT_UC:
2476                 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_UC_BIT)))
2477                         return false;
2478                 break;
2479         case VMX_EPTP_MT_WB:
2480                 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPTP_WB_BIT)))
2481                         return false;
2482                 break;
2483         default:
2484                 return false;
2485         }
2486
2487         /* only 4 levels page-walk length are valid */
2488         if (CC((address & VMX_EPTP_PWL_MASK) != VMX_EPTP_PWL_4))
2489                 return false;
2490
2491         /* Reserved bits should not be set */
2492         if (CC(address >> maxphyaddr || ((address >> 7) & 0x1f)))
2493                 return false;
2494
2495         /* AD, if set, should be supported */
2496         if (address & VMX_EPTP_AD_ENABLE_BIT) {
2497                 if (CC(!(vmx->nested.msrs.ept_caps & VMX_EPT_AD_BIT)))
2498                         return false;
2499         }
2500
2501         return true;
2502 }
2503
2504 /*
2505  * Checks related to VM-Execution Control Fields
2506  */
2507 static int nested_check_vm_execution_controls(struct kvm_vcpu *vcpu,
2508                                               struct vmcs12 *vmcs12)
2509 {
2510         struct vcpu_vmx *vmx = to_vmx(vcpu);
2511
2512         if (CC(!vmx_control_verify(vmcs12->pin_based_vm_exec_control,
2513                                    vmx->nested.msrs.pinbased_ctls_low,
2514                                    vmx->nested.msrs.pinbased_ctls_high)) ||
2515             CC(!vmx_control_verify(vmcs12->cpu_based_vm_exec_control,
2516                                    vmx->nested.msrs.procbased_ctls_low,
2517                                    vmx->nested.msrs.procbased_ctls_high)))
2518                 return -EINVAL;
2519
2520         if (nested_cpu_has(vmcs12, CPU_BASED_ACTIVATE_SECONDARY_CONTROLS) &&
2521             CC(!vmx_control_verify(vmcs12->secondary_vm_exec_control,
2522                                    vmx->nested.msrs.secondary_ctls_low,
2523                                    vmx->nested.msrs.secondary_ctls_high)))
2524                 return -EINVAL;
2525
2526         if (CC(vmcs12->cr3_target_count > nested_cpu_vmx_misc_cr3_count(vcpu)) ||
2527             nested_vmx_check_io_bitmap_controls(vcpu, vmcs12) ||
2528             nested_vmx_check_msr_bitmap_controls(vcpu, vmcs12) ||
2529             nested_vmx_check_tpr_shadow_controls(vcpu, vmcs12) ||
2530             nested_vmx_check_apic_access_controls(vcpu, vmcs12) ||
2531             nested_vmx_check_apicv_controls(vcpu, vmcs12) ||
2532             nested_vmx_check_nmi_controls(vmcs12) ||
2533             nested_vmx_check_pml_controls(vcpu, vmcs12) ||
2534             nested_vmx_check_unrestricted_guest_controls(vcpu, vmcs12) ||
2535             nested_vmx_check_mode_based_ept_exec_controls(vcpu, vmcs12) ||
2536             nested_vmx_check_shadow_vmcs_controls(vcpu, vmcs12) ||
2537             CC(nested_cpu_has_vpid(vmcs12) && !vmcs12->virtual_processor_id))
2538                 return -EINVAL;
2539
2540         if (!nested_cpu_has_preemption_timer(vmcs12) &&
2541             nested_cpu_has_save_preemption_timer(vmcs12))
2542                 return -EINVAL;
2543
2544         if (nested_cpu_has_ept(vmcs12) &&
2545             CC(!valid_ept_address(vcpu, vmcs12->ept_pointer)))
2546                 return -EINVAL;
2547
2548         if (nested_cpu_has_vmfunc(vmcs12)) {
2549                 if (CC(vmcs12->vm_function_control &
2550                        ~vmx->nested.msrs.vmfunc_controls))
2551                         return -EINVAL;
2552
2553                 if (nested_cpu_has_eptp_switching(vmcs12)) {
2554                         if (CC(!nested_cpu_has_ept(vmcs12)) ||
2555                             CC(!page_address_valid(vcpu, vmcs12->eptp_list_address)))
2556                                 return -EINVAL;
2557                 }
2558         }
2559
2560         return 0;
2561 }
2562
2563 /*
2564  * Checks related to VM-Exit Control Fields
2565  */
2566 static int nested_check_vm_exit_controls(struct kvm_vcpu *vcpu,
2567                                          struct vmcs12 *vmcs12)
2568 {
2569         struct vcpu_vmx *vmx = to_vmx(vcpu);
2570
2571         if (CC(!vmx_control_verify(vmcs12->vm_exit_controls,
2572                                     vmx->nested.msrs.exit_ctls_low,
2573                                     vmx->nested.msrs.exit_ctls_high)) ||
2574             CC(nested_vmx_check_exit_msr_switch_controls(vcpu, vmcs12)))
2575                 return -EINVAL;
2576
2577         return 0;
2578 }
2579
2580 /*
2581  * Checks related to VM-Entry Control Fields
2582  */
2583 static int nested_check_vm_entry_controls(struct kvm_vcpu *vcpu,
2584                                           struct vmcs12 *vmcs12)
2585 {
2586         struct vcpu_vmx *vmx = to_vmx(vcpu);
2587
2588         if (CC(!vmx_control_verify(vmcs12->vm_entry_controls,
2589                                     vmx->nested.msrs.entry_ctls_low,
2590                                     vmx->nested.msrs.entry_ctls_high)))
2591                 return -EINVAL;
2592
2593         /*
2594          * From the Intel SDM, volume 3:
2595          * Fields relevant to VM-entry event injection must be set properly.
2596          * These fields are the VM-entry interruption-information field, the
2597          * VM-entry exception error code, and the VM-entry instruction length.
2598          */
2599         if (vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) {
2600                 u32 intr_info = vmcs12->vm_entry_intr_info_field;
2601                 u8 vector = intr_info & INTR_INFO_VECTOR_MASK;
2602                 u32 intr_type = intr_info & INTR_INFO_INTR_TYPE_MASK;
2603                 bool has_error_code = intr_info & INTR_INFO_DELIVER_CODE_MASK;
2604                 bool should_have_error_code;
2605                 bool urg = nested_cpu_has2(vmcs12,
2606                                            SECONDARY_EXEC_UNRESTRICTED_GUEST);
2607                 bool prot_mode = !urg || vmcs12->guest_cr0 & X86_CR0_PE;
2608
2609                 /* VM-entry interruption-info field: interruption type */
2610                 if (CC(intr_type == INTR_TYPE_RESERVED) ||
2611                     CC(intr_type == INTR_TYPE_OTHER_EVENT &&
2612                        !nested_cpu_supports_monitor_trap_flag(vcpu)))
2613                         return -EINVAL;
2614
2615                 /* VM-entry interruption-info field: vector */
2616                 if (CC(intr_type == INTR_TYPE_NMI_INTR && vector != NMI_VECTOR) ||
2617                     CC(intr_type == INTR_TYPE_HARD_EXCEPTION && vector > 31) ||
2618                     CC(intr_type == INTR_TYPE_OTHER_EVENT && vector != 0))
2619                         return -EINVAL;
2620
2621                 /* VM-entry interruption-info field: deliver error code */
2622                 should_have_error_code =
2623                         intr_type == INTR_TYPE_HARD_EXCEPTION && prot_mode &&
2624                         x86_exception_has_error_code(vector);
2625                 if (CC(has_error_code != should_have_error_code))
2626                         return -EINVAL;
2627
2628                 /* VM-entry exception error code */
2629                 if (CC(has_error_code &&
2630                        vmcs12->vm_entry_exception_error_code & GENMASK(31, 16)))
2631                         return -EINVAL;
2632
2633                 /* VM-entry interruption-info field: reserved bits */
2634                 if (CC(intr_info & INTR_INFO_RESVD_BITS_MASK))
2635                         return -EINVAL;
2636
2637                 /* VM-entry instruction length */
2638                 switch (intr_type) {
2639                 case INTR_TYPE_SOFT_EXCEPTION:
2640                 case INTR_TYPE_SOFT_INTR:
2641                 case INTR_TYPE_PRIV_SW_EXCEPTION:
2642                         if (CC(vmcs12->vm_entry_instruction_len > 15) ||
2643                             CC(vmcs12->vm_entry_instruction_len == 0 &&
2644                             CC(!nested_cpu_has_zero_length_injection(vcpu))))
2645                                 return -EINVAL;
2646                 }
2647         }
2648
2649         if (nested_vmx_check_entry_msr_switch_controls(vcpu, vmcs12))
2650                 return -EINVAL;
2651
2652         return 0;
2653 }
2654
2655 static int nested_vmx_check_controls(struct kvm_vcpu *vcpu,
2656                                      struct vmcs12 *vmcs12)
2657 {
2658         if (nested_check_vm_execution_controls(vcpu, vmcs12) ||
2659             nested_check_vm_exit_controls(vcpu, vmcs12) ||
2660             nested_check_vm_entry_controls(vcpu, vmcs12))
2661                 return -EINVAL;
2662
2663         return 0;
2664 }
2665
2666 static int nested_vmx_check_host_state(struct kvm_vcpu *vcpu,
2667                                        struct vmcs12 *vmcs12)
2668 {
2669         bool ia32e;
2670
2671         if (CC(!nested_host_cr0_valid(vcpu, vmcs12->host_cr0)) ||
2672             CC(!nested_host_cr4_valid(vcpu, vmcs12->host_cr4)) ||
2673             CC(!nested_cr3_valid(vcpu, vmcs12->host_cr3)))
2674                 return -EINVAL;
2675
2676         if (CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_esp, vcpu)) ||
2677             CC(is_noncanonical_address(vmcs12->host_ia32_sysenter_eip, vcpu)))
2678                 return -EINVAL;
2679
2680         if ((vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) &&
2681             CC(!kvm_pat_valid(vmcs12->host_ia32_pat)))
2682                 return -EINVAL;
2683
2684 #ifdef CONFIG_X86_64
2685         ia32e = !!(vcpu->arch.efer & EFER_LMA);
2686 #else
2687         ia32e = false;
2688 #endif
2689
2690         if (ia32e) {
2691                 if (CC(!(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)) ||
2692                     CC(!(vmcs12->host_cr4 & X86_CR4_PAE)))
2693                         return -EINVAL;
2694         } else {
2695                 if (CC(vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE) ||
2696                     CC(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE) ||
2697                     CC(vmcs12->host_cr4 & X86_CR4_PCIDE) ||
2698                     CC((vmcs12->host_rip) >> 32))
2699                         return -EINVAL;
2700         }
2701
2702         if (CC(vmcs12->host_cs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
2703             CC(vmcs12->host_ss_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
2704             CC(vmcs12->host_ds_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
2705             CC(vmcs12->host_es_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
2706             CC(vmcs12->host_fs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
2707             CC(vmcs12->host_gs_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
2708             CC(vmcs12->host_tr_selector & (SEGMENT_RPL_MASK | SEGMENT_TI_MASK)) ||
2709             CC(vmcs12->host_cs_selector == 0) ||
2710             CC(vmcs12->host_tr_selector == 0) ||
2711             CC(vmcs12->host_ss_selector == 0 && !ia32e))
2712                 return -EINVAL;
2713
2714 #ifdef CONFIG_X86_64
2715         if (CC(is_noncanonical_address(vmcs12->host_fs_base, vcpu)) ||
2716             CC(is_noncanonical_address(vmcs12->host_gs_base, vcpu)) ||
2717             CC(is_noncanonical_address(vmcs12->host_gdtr_base, vcpu)) ||
2718             CC(is_noncanonical_address(vmcs12->host_idtr_base, vcpu)) ||
2719             CC(is_noncanonical_address(vmcs12->host_tr_base, vcpu)) ||
2720             CC(is_noncanonical_address(vmcs12->host_rip, vcpu)))
2721                 return -EINVAL;
2722 #endif
2723
2724         /*
2725          * If the load IA32_EFER VM-exit control is 1, bits reserved in the
2726          * IA32_EFER MSR must be 0 in the field for that register. In addition,
2727          * the values of the LMA and LME bits in the field must each be that of
2728          * the host address-space size VM-exit control.
2729          */
2730         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER) {
2731                 if (CC(!kvm_valid_efer(vcpu, vmcs12->host_ia32_efer)) ||
2732                     CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LMA)) ||
2733                     CC(ia32e != !!(vmcs12->host_ia32_efer & EFER_LME)))
2734                         return -EINVAL;
2735         }
2736
2737         return 0;
2738 }
2739
2740 static int nested_vmx_check_vmcs_link_ptr(struct kvm_vcpu *vcpu,
2741                                           struct vmcs12 *vmcs12)
2742 {
2743         int r = 0;
2744         struct vmcs12 *shadow;
2745         struct kvm_host_map map;
2746
2747         if (vmcs12->vmcs_link_pointer == -1ull)
2748                 return 0;
2749
2750         if (CC(!page_address_valid(vcpu, vmcs12->vmcs_link_pointer)))
2751                 return -EINVAL;
2752
2753         if (CC(kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->vmcs_link_pointer), &map)))
2754                 return -EINVAL;
2755
2756         shadow = map.hva;
2757
2758         if (CC(shadow->hdr.revision_id != VMCS12_REVISION) ||
2759             CC(shadow->hdr.shadow_vmcs != nested_cpu_has_shadow_vmcs(vmcs12)))
2760                 r = -EINVAL;
2761
2762         kvm_vcpu_unmap(vcpu, &map, false);
2763         return r;
2764 }
2765
2766 /*
2767  * Checks related to Guest Non-register State
2768  */
2769 static int nested_check_guest_non_reg_state(struct vmcs12 *vmcs12)
2770 {
2771         if (CC(vmcs12->guest_activity_state != GUEST_ACTIVITY_ACTIVE &&
2772                vmcs12->guest_activity_state != GUEST_ACTIVITY_HLT))
2773                 return -EINVAL;
2774
2775         return 0;
2776 }
2777
2778 static int nested_vmx_check_guest_state(struct kvm_vcpu *vcpu,
2779                                         struct vmcs12 *vmcs12,
2780                                         u32 *exit_qual)
2781 {
2782         bool ia32e = !!(vmcs12->vm_entry_controls & VM_ENTRY_IA32E_MODE);
2783
2784         *exit_qual = ENTRY_FAIL_DEFAULT;
2785
2786         if (CC(!nested_guest_cr0_valid(vcpu, vmcs12->guest_cr0)) ||
2787             CC(!nested_guest_cr4_valid(vcpu, vmcs12->guest_cr4)))
2788                 return -EINVAL;
2789
2790         if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_PAT) &&
2791             CC(!kvm_pat_valid(vmcs12->guest_ia32_pat)))
2792                 return -EINVAL;
2793
2794         if (nested_vmx_check_vmcs_link_ptr(vcpu, vmcs12)) {
2795                 *exit_qual = ENTRY_FAIL_VMCS_LINK_PTR;
2796                 return -EINVAL;
2797         }
2798
2799         if (CC((vmcs12->guest_cr0 & (X86_CR0_PG | X86_CR0_PE)) == X86_CR0_PG))
2800                 return -EINVAL;
2801
2802         if (CC(ia32e && !(vmcs12->guest_cr4 & X86_CR4_PAE)) ||
2803             CC(ia32e && !(vmcs12->guest_cr0 & X86_CR0_PG)))
2804                 return -EINVAL;
2805
2806         /*
2807          * If the load IA32_EFER VM-entry control is 1, the following checks
2808          * are performed on the field for the IA32_EFER MSR:
2809          * - Bits reserved in the IA32_EFER MSR must be 0.
2810          * - Bit 10 (corresponding to IA32_EFER.LMA) must equal the value of
2811          *   the IA-32e mode guest VM-exit control. It must also be identical
2812          *   to bit 8 (LME) if bit 31 in the CR0 field (corresponding to
2813          *   CR0.PG) is 1.
2814          */
2815         if (to_vmx(vcpu)->nested.nested_run_pending &&
2816             (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_IA32_EFER)) {
2817                 if (CC(!kvm_valid_efer(vcpu, vmcs12->guest_ia32_efer)) ||
2818                     CC(ia32e != !!(vmcs12->guest_ia32_efer & EFER_LMA)) ||
2819                     CC(((vmcs12->guest_cr0 & X86_CR0_PG) &&
2820                      ia32e != !!(vmcs12->guest_ia32_efer & EFER_LME))))
2821                         return -EINVAL;
2822         }
2823
2824         if ((vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS) &&
2825             (CC(is_noncanonical_address(vmcs12->guest_bndcfgs & PAGE_MASK, vcpu)) ||
2826              CC((vmcs12->guest_bndcfgs & MSR_IA32_BNDCFGS_RSVD))))
2827                 return -EINVAL;
2828
2829         if (nested_check_guest_non_reg_state(vmcs12))
2830                 return -EINVAL;
2831
2832         return 0;
2833 }
2834
2835 static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu)
2836 {
2837         struct vcpu_vmx *vmx = to_vmx(vcpu);
2838         unsigned long cr3, cr4;
2839         bool vm_fail;
2840
2841         if (!nested_early_check)
2842                 return 0;
2843
2844         if (vmx->msr_autoload.host.nr)
2845                 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, 0);
2846         if (vmx->msr_autoload.guest.nr)
2847                 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, 0);
2848
2849         preempt_disable();
2850
2851         vmx_prepare_switch_to_guest(vcpu);
2852
2853         /*
2854          * Induce a consistency check VMExit by clearing bit 1 in GUEST_RFLAGS,
2855          * which is reserved to '1' by hardware.  GUEST_RFLAGS is guaranteed to
2856          * be written (by preparve_vmcs02()) before the "real" VMEnter, i.e.
2857          * there is no need to preserve other bits or save/restore the field.
2858          */
2859         vmcs_writel(GUEST_RFLAGS, 0);
2860
2861         cr3 = __get_current_cr3_fast();
2862         if (unlikely(cr3 != vmx->loaded_vmcs->host_state.cr3)) {
2863                 vmcs_writel(HOST_CR3, cr3);
2864                 vmx->loaded_vmcs->host_state.cr3 = cr3;
2865         }
2866
2867         cr4 = cr4_read_shadow();
2868         if (unlikely(cr4 != vmx->loaded_vmcs->host_state.cr4)) {
2869                 vmcs_writel(HOST_CR4, cr4);
2870                 vmx->loaded_vmcs->host_state.cr4 = cr4;
2871         }
2872
2873         vm_fail = __vmx_vcpu_run(vmx, (unsigned long *)&vcpu->arch.regs,
2874                                  __vmx_vcpu_run_flags(vmx));
2875
2876         if (vmx->msr_autoload.host.nr)
2877                 vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
2878         if (vmx->msr_autoload.guest.nr)
2879                 vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
2880
2881         if (vm_fail) {
2882                 u32 error = vmcs_read32(VM_INSTRUCTION_ERROR);
2883
2884                 preempt_enable();
2885
2886                 trace_kvm_nested_vmenter_failed(
2887                         "early hardware check VM-instruction error: ", error);
2888                 WARN_ON_ONCE(error != VMXERR_ENTRY_INVALID_CONTROL_FIELD);
2889                 return 1;
2890         }
2891
2892         /*
2893          * VMExit clears RFLAGS.IF and DR7, even on a consistency check.
2894          */
2895         local_irq_enable();
2896         if (hw_breakpoint_active())
2897                 set_debugreg(__this_cpu_read(cpu_dr7), 7);
2898         preempt_enable();
2899
2900         /*
2901          * A non-failing VMEntry means we somehow entered guest mode with
2902          * an illegal RIP, and that's just the tip of the iceberg.  There
2903          * is no telling what memory has been modified or what state has
2904          * been exposed to unknown code.  Hitting this all but guarantees
2905          * a (very critical) hardware issue.
2906          */
2907         WARN_ON(!(vmcs_read32(VM_EXIT_REASON) &
2908                 VMX_EXIT_REASONS_FAILED_VMENTRY));
2909
2910         return 0;
2911 }
2912
2913 static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu,
2914                                                  struct vmcs12 *vmcs12);
2915
2916 static bool nested_get_vmcs12_pages(struct kvm_vcpu *vcpu)
2917 {
2918         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
2919         struct vcpu_vmx *vmx = to_vmx(vcpu);
2920         struct kvm_host_map *map;
2921         struct page *page;
2922         u64 hpa;
2923
2924         if (nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
2925                 /*
2926                  * Translate L1 physical address to host physical
2927                  * address for vmcs02. Keep the page pinned, so this
2928                  * physical address remains valid. We keep a reference
2929                  * to it so we can release it later.
2930                  */
2931                 if (vmx->nested.apic_access_page) { /* shouldn't happen */
2932                         kvm_release_page_dirty(vmx->nested.apic_access_page);
2933                         vmx->nested.apic_access_page = NULL;
2934                 }
2935                 page = kvm_vcpu_gpa_to_page(vcpu, vmcs12->apic_access_addr);
2936                 if (!is_error_page(page)) {
2937                         vmx->nested.apic_access_page = page;
2938                         hpa = page_to_phys(vmx->nested.apic_access_page);
2939                         vmcs_write64(APIC_ACCESS_ADDR, hpa);
2940                 } else {
2941                         pr_debug_ratelimited("%s: no backing 'struct page' for APIC-access address in vmcs12\n",
2942                                              __func__);
2943                         vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
2944                         vcpu->run->internal.suberror =
2945                                 KVM_INTERNAL_ERROR_EMULATION;
2946                         vcpu->run->internal.ndata = 0;
2947                         return false;
2948                 }
2949         }
2950
2951         if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
2952                 map = &vmx->nested.virtual_apic_map;
2953
2954                 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->virtual_apic_page_addr), map)) {
2955                         vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, pfn_to_hpa(map->pfn));
2956                 } else if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING) &&
2957                            nested_cpu_has(vmcs12, CPU_BASED_CR8_STORE_EXITING) &&
2958                            !nested_cpu_has2(vmcs12, SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
2959                         /*
2960                          * The processor will never use the TPR shadow, simply
2961                          * clear the bit from the execution control.  Such a
2962                          * configuration is useless, but it happens in tests.
2963                          * For any other configuration, failing the vm entry is
2964                          * _not_ what the processor does but it's basically the
2965                          * only possibility we have.
2966                          */
2967                         exec_controls_clearbit(vmx, CPU_BASED_TPR_SHADOW);
2968                 } else {
2969                         /*
2970                          * Write an illegal value to VIRTUAL_APIC_PAGE_ADDR to
2971                          * force VM-Entry to fail.
2972                          */
2973                         vmcs_write64(VIRTUAL_APIC_PAGE_ADDR, -1ull);
2974                 }
2975         }
2976
2977         if (nested_cpu_has_posted_intr(vmcs12)) {
2978                 map = &vmx->nested.pi_desc_map;
2979
2980                 if (!kvm_vcpu_map(vcpu, gpa_to_gfn(vmcs12->posted_intr_desc_addr), map)) {
2981                         vmx->nested.pi_desc =
2982                                 (struct pi_desc *)(((void *)map->hva) +
2983                                 offset_in_page(vmcs12->posted_intr_desc_addr));
2984                         vmcs_write64(POSTED_INTR_DESC_ADDR,
2985                                      pfn_to_hpa(map->pfn) + offset_in_page(vmcs12->posted_intr_desc_addr));
2986                 }
2987         }
2988         if (nested_vmx_prepare_msr_bitmap(vcpu, vmcs12))
2989                 exec_controls_setbit(vmx, CPU_BASED_USE_MSR_BITMAPS);
2990         else
2991                 exec_controls_clearbit(vmx, CPU_BASED_USE_MSR_BITMAPS);
2992         return true;
2993 }
2994
2995 /*
2996  * Intel's VMX Instruction Reference specifies a common set of prerequisites
2997  * for running VMX instructions (except VMXON, whose prerequisites are
2998  * slightly different). It also specifies what exception to inject otherwise.
2999  * Note that many of these exceptions have priority over VM exits, so they
3000  * don't have to be checked again here.
3001  */
3002 static int nested_vmx_check_permission(struct kvm_vcpu *vcpu)
3003 {
3004         if (!to_vmx(vcpu)->nested.vmxon) {
3005                 kvm_queue_exception(vcpu, UD_VECTOR);
3006                 return 0;
3007         }
3008
3009         if (vmx_get_cpl(vcpu)) {
3010                 kvm_inject_gp(vcpu, 0);
3011                 return 0;
3012         }
3013
3014         return 1;
3015 }
3016
3017 static u8 vmx_has_apicv_interrupt(struct kvm_vcpu *vcpu)
3018 {
3019         u8 rvi = vmx_get_rvi();
3020         u8 vppr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_PROCPRI);
3021
3022         return ((rvi & 0xf0) > (vppr & 0xf0));
3023 }
3024
3025 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
3026                                    struct vmcs12 *vmcs12);
3027
3028 /*
3029  * If from_vmentry is false, this is being called from state restore (either RSM
3030  * or KVM_SET_NESTED_STATE).  Otherwise it's called from vmlaunch/vmresume.
3031  *
3032  * Returns:
3033  *      NVMX_ENTRY_SUCCESS: Entered VMX non-root mode
3034  *      NVMX_ENTRY_VMFAIL:  Consistency check VMFail
3035  *      NVMX_ENTRY_VMEXIT:  Consistency check VMExit
3036  *      NVMX_ENTRY_KVM_INTERNAL_ERROR: KVM internal error
3037  */
3038 enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu,
3039                                                         bool from_vmentry)
3040 {
3041         struct vcpu_vmx *vmx = to_vmx(vcpu);
3042         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3043         bool evaluate_pending_interrupts;
3044         u32 exit_reason = EXIT_REASON_INVALID_STATE;
3045         u32 exit_qual;
3046
3047         evaluate_pending_interrupts = exec_controls_get(vmx) &
3048                 (CPU_BASED_INTR_WINDOW_EXITING | CPU_BASED_NMI_WINDOW_EXITING);
3049         if (likely(!evaluate_pending_interrupts) && kvm_vcpu_apicv_active(vcpu))
3050                 evaluate_pending_interrupts |= vmx_has_apicv_interrupt(vcpu);
3051
3052         if (!vmx->nested.nested_run_pending ||
3053             !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS))
3054                 vmx->nested.vmcs01_debugctl = vmcs_read64(GUEST_IA32_DEBUGCTL);
3055         if (kvm_mpx_supported() &&
3056             (!vmx->nested.nested_run_pending ||
3057              !(vmcs12->vm_entry_controls & VM_ENTRY_LOAD_BNDCFGS)))
3058                 vmx->nested.vmcs01_guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS);
3059
3060         /*
3061          * Overwrite vmcs01.GUEST_CR3 with L1's CR3 if EPT is disabled *and*
3062          * nested early checks are disabled.  In the event of a "late" VM-Fail,
3063          * i.e. a VM-Fail detected by hardware but not KVM, KVM must unwind its
3064          * software model to the pre-VMEntry host state.  When EPT is disabled,
3065          * GUEST_CR3 holds KVM's shadow CR3, not L1's "real" CR3, which causes
3066          * nested_vmx_restore_host_state() to corrupt vcpu->arch.cr3.  Stuffing
3067          * vmcs01.GUEST_CR3 results in the unwind naturally setting arch.cr3 to
3068          * the correct value.  Smashing vmcs01.GUEST_CR3 is safe because nested
3069          * VM-Exits, and the unwind, reset KVM's MMU, i.e. vmcs01.GUEST_CR3 is
3070          * guaranteed to be overwritten with a shadow CR3 prior to re-entering
3071          * L1.  Don't stuff vmcs01.GUEST_CR3 when using nested early checks as
3072          * KVM modifies vcpu->arch.cr3 if and only if the early hardware checks
3073          * pass, and early VM-Fails do not reset KVM's MMU, i.e. the VM-Fail
3074          * path would need to manually save/restore vmcs01.GUEST_CR3.
3075          */
3076         if (!enable_ept && !nested_early_check)
3077                 vmcs_writel(GUEST_CR3, vcpu->arch.cr3);
3078
3079         vmx_switch_vmcs(vcpu, &vmx->nested.vmcs02);
3080
3081         prepare_vmcs02_early(vmx, vmcs12);
3082
3083         if (from_vmentry) {
3084                 if (unlikely(!nested_get_vmcs12_pages(vcpu))) {
3085                         vmx_switch_vmcs(vcpu, &vmx->vmcs01);
3086                         return NVMX_VMENTRY_KVM_INTERNAL_ERROR;
3087                 }
3088
3089                 if (nested_vmx_check_vmentry_hw(vcpu)) {
3090                         vmx_switch_vmcs(vcpu, &vmx->vmcs01);
3091                         return NVMX_VMENTRY_VMFAIL;
3092                 }
3093
3094                 if (nested_vmx_check_guest_state(vcpu, vmcs12, &exit_qual))
3095                         goto vmentry_fail_vmexit;
3096         }
3097
3098         enter_guest_mode(vcpu);
3099         if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING)
3100                 vcpu->arch.tsc_offset += vmcs12->tsc_offset;
3101
3102         if (prepare_vmcs02(vcpu, vmcs12, &exit_qual))
3103                 goto vmentry_fail_vmexit_guest_mode;
3104
3105         if (from_vmentry) {
3106                 exit_reason = EXIT_REASON_MSR_LOAD_FAIL;
3107                 exit_qual = nested_vmx_load_msr(vcpu,
3108                                                 vmcs12->vm_entry_msr_load_addr,
3109                                                 vmcs12->vm_entry_msr_load_count);
3110                 if (exit_qual)
3111                         goto vmentry_fail_vmexit_guest_mode;
3112         } else {
3113                 /*
3114                  * The MMU is not initialized to point at the right entities yet and
3115                  * "get pages" would need to read data from the guest (i.e. we will
3116                  * need to perform gpa to hpa translation). Request a call
3117                  * to nested_get_vmcs12_pages before the next VM-entry.  The MSRs
3118                  * have already been set at vmentry time and should not be reset.
3119                  */
3120                 kvm_make_request(KVM_REQ_GET_VMCS12_PAGES, vcpu);
3121         }
3122
3123         /*
3124          * If L1 had a pending IRQ/NMI until it executed
3125          * VMLAUNCH/VMRESUME which wasn't delivered because it was
3126          * disallowed (e.g. interrupts disabled), L0 needs to
3127          * evaluate if this pending event should cause an exit from L2
3128          * to L1 or delivered directly to L2 (e.g. In case L1 don't
3129          * intercept EXTERNAL_INTERRUPT).
3130          *
3131          * Usually this would be handled by the processor noticing an
3132          * IRQ/NMI window request, or checking RVI during evaluation of
3133          * pending virtual interrupts.  However, this setting was done
3134          * on VMCS01 and now VMCS02 is active instead. Thus, we force L0
3135          * to perform pending event evaluation by requesting a KVM_REQ_EVENT.
3136          */
3137         if (unlikely(evaluate_pending_interrupts))
3138                 kvm_make_request(KVM_REQ_EVENT, vcpu);
3139
3140         /*
3141          * Do not start the preemption timer hrtimer until after we know
3142          * we are successful, so that only nested_vmx_vmexit needs to cancel
3143          * the timer.
3144          */
3145         vmx->nested.preemption_timer_expired = false;
3146         if (nested_cpu_has_preemption_timer(vmcs12))
3147                 vmx_start_preemption_timer(vcpu);
3148
3149         /*
3150          * Note no nested_vmx_succeed or nested_vmx_fail here. At this point
3151          * we are no longer running L1, and VMLAUNCH/VMRESUME has not yet
3152          * returned as far as L1 is concerned. It will only return (and set
3153          * the success flag) when L2 exits (see nested_vmx_vmexit()).
3154          */
3155         return NVMX_VMENTRY_SUCCESS;
3156
3157         /*
3158          * A failed consistency check that leads to a VMExit during L1's
3159          * VMEnter to L2 is a variation of a normal VMexit, as explained in
3160          * 26.7 "VM-entry failures during or after loading guest state".
3161          */
3162 vmentry_fail_vmexit_guest_mode:
3163         if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING)
3164                 vcpu->arch.tsc_offset -= vmcs12->tsc_offset;
3165         leave_guest_mode(vcpu);
3166
3167 vmentry_fail_vmexit:
3168         vmx_switch_vmcs(vcpu, &vmx->vmcs01);
3169
3170         if (!from_vmentry)
3171                 return NVMX_VMENTRY_VMEXIT;
3172
3173         load_vmcs12_host_state(vcpu, vmcs12);
3174         vmcs12->vm_exit_reason = exit_reason | VMX_EXIT_REASONS_FAILED_VMENTRY;
3175         vmcs12->exit_qualification = exit_qual;
3176         if (enable_shadow_vmcs || vmx->nested.hv_evmcs)
3177                 vmx->nested.need_vmcs12_to_shadow_sync = true;
3178         return NVMX_VMENTRY_VMEXIT;
3179 }
3180
3181 /*
3182  * nested_vmx_run() handles a nested entry, i.e., a VMLAUNCH or VMRESUME on L1
3183  * for running an L2 nested guest.
3184  */
3185 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch)
3186 {
3187         struct vmcs12 *vmcs12;
3188         enum nvmx_vmentry_status status;
3189         struct vcpu_vmx *vmx = to_vmx(vcpu);
3190         u32 interrupt_shadow = vmx_get_interrupt_shadow(vcpu);
3191
3192         if (!nested_vmx_check_permission(vcpu))
3193                 return 1;
3194
3195         if (!nested_vmx_handle_enlightened_vmptrld(vcpu, launch))
3196                 return 1;
3197
3198         if (!vmx->nested.hv_evmcs && vmx->nested.current_vmptr == -1ull)
3199                 return nested_vmx_failInvalid(vcpu);
3200
3201         vmcs12 = get_vmcs12(vcpu);
3202
3203         /*
3204          * Can't VMLAUNCH or VMRESUME a shadow VMCS. Despite the fact
3205          * that there *is* a valid VMCS pointer, RFLAGS.CF is set
3206          * rather than RFLAGS.ZF, and no error number is stored to the
3207          * VM-instruction error field.
3208          */
3209         if (vmcs12->hdr.shadow_vmcs)
3210                 return nested_vmx_failInvalid(vcpu);
3211
3212         if (vmx->nested.hv_evmcs) {
3213                 copy_enlightened_to_vmcs12(vmx);
3214                 /* Enlightened VMCS doesn't have launch state */
3215                 vmcs12->launch_state = !launch;
3216         } else if (enable_shadow_vmcs) {
3217                 copy_shadow_to_vmcs12(vmx);
3218         }
3219
3220         /*
3221          * The nested entry process starts with enforcing various prerequisites
3222          * on vmcs12 as required by the Intel SDM, and act appropriately when
3223          * they fail: As the SDM explains, some conditions should cause the
3224          * instruction to fail, while others will cause the instruction to seem
3225          * to succeed, but return an EXIT_REASON_INVALID_STATE.
3226          * To speed up the normal (success) code path, we should avoid checking
3227          * for misconfigurations which will anyway be caught by the processor
3228          * when using the merged vmcs02.
3229          */
3230         if (interrupt_shadow & KVM_X86_SHADOW_INT_MOV_SS)
3231                 return nested_vmx_failValid(vcpu,
3232                         VMXERR_ENTRY_EVENTS_BLOCKED_BY_MOV_SS);
3233
3234         if (vmcs12->launch_state == launch)
3235                 return nested_vmx_failValid(vcpu,
3236                         launch ? VMXERR_VMLAUNCH_NONCLEAR_VMCS
3237                                : VMXERR_VMRESUME_NONLAUNCHED_VMCS);
3238
3239         if (nested_vmx_check_controls(vcpu, vmcs12))
3240                 return nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
3241
3242         if (nested_vmx_check_host_state(vcpu, vmcs12))
3243                 return nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_HOST_STATE_FIELD);
3244
3245         /*
3246          * We're finally done with prerequisite checking, and can start with
3247          * the nested entry.
3248          */
3249         vmx->nested.nested_run_pending = 1;
3250         status = nested_vmx_enter_non_root_mode(vcpu, true);
3251         if (unlikely(status != NVMX_VMENTRY_SUCCESS))
3252                 goto vmentry_failed;
3253
3254         /* Hide L1D cache contents from the nested guest.  */
3255         vmx->vcpu.arch.l1tf_flush_l1d = true;
3256
3257         /*
3258          * Must happen outside of nested_vmx_enter_non_root_mode() as it will
3259          * also be used as part of restoring nVMX state for
3260          * snapshot restore (migration).
3261          *
3262          * In this flow, it is assumed that vmcs12 cache was
3263          * trasferred as part of captured nVMX state and should
3264          * therefore not be read from guest memory (which may not
3265          * exist on destination host yet).
3266          */
3267         nested_cache_shadow_vmcs12(vcpu, vmcs12);
3268
3269         /*
3270          * If we're entering a halted L2 vcpu and the L2 vcpu won't be
3271          * awakened by event injection or by an NMI-window VM-exit or
3272          * by an interrupt-window VM-exit, halt the vcpu.
3273          */
3274         if ((vmcs12->guest_activity_state == GUEST_ACTIVITY_HLT) &&
3275             !(vmcs12->vm_entry_intr_info_field & INTR_INFO_VALID_MASK) &&
3276             !(vmcs12->cpu_based_vm_exec_control & CPU_BASED_NMI_WINDOW_EXITING) &&
3277             !((vmcs12->cpu_based_vm_exec_control & CPU_BASED_INTR_WINDOW_EXITING) &&
3278               (vmcs12->guest_rflags & X86_EFLAGS_IF))) {
3279                 vmx->nested.nested_run_pending = 0;
3280                 return kvm_vcpu_halt(vcpu);
3281         }
3282         return 1;
3283
3284 vmentry_failed:
3285         vmx->nested.nested_run_pending = 0;
3286         if (status == NVMX_VMENTRY_KVM_INTERNAL_ERROR)
3287                 return 0;
3288         if (status == NVMX_VMENTRY_VMEXIT)
3289                 return 1;
3290         WARN_ON_ONCE(status != NVMX_VMENTRY_VMFAIL);
3291         return nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
3292 }
3293
3294 /*
3295  * On a nested exit from L2 to L1, vmcs12.guest_cr0 might not be up-to-date
3296  * because L2 may have changed some cr0 bits directly (CRO_GUEST_HOST_MASK).
3297  * This function returns the new value we should put in vmcs12.guest_cr0.
3298  * It's not enough to just return the vmcs02 GUEST_CR0. Rather,
3299  *  1. Bits that neither L0 nor L1 trapped, were set directly by L2 and are now
3300  *     available in vmcs02 GUEST_CR0. (Note: It's enough to check that L0
3301  *     didn't trap the bit, because if L1 did, so would L0).
3302  *  2. Bits that L1 asked to trap (and therefore L0 also did) could not have
3303  *     been modified by L2, and L1 knows it. So just leave the old value of
3304  *     the bit from vmcs12.guest_cr0. Note that the bit from vmcs02 GUEST_CR0
3305  *     isn't relevant, because if L0 traps this bit it can set it to anything.
3306  *  3. Bits that L1 didn't trap, but L0 did. L1 believes the guest could have
3307  *     changed these bits, and therefore they need to be updated, but L0
3308  *     didn't necessarily allow them to be changed in GUEST_CR0 - and rather
3309  *     put them in vmcs02 CR0_READ_SHADOW. So take these bits from there.
3310  */
3311 static inline unsigned long
3312 vmcs12_guest_cr0(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
3313 {
3314         return
3315         /*1*/   (vmcs_readl(GUEST_CR0) & vcpu->arch.cr0_guest_owned_bits) |
3316         /*2*/   (vmcs12->guest_cr0 & vmcs12->cr0_guest_host_mask) |
3317         /*3*/   (vmcs_readl(CR0_READ_SHADOW) & ~(vmcs12->cr0_guest_host_mask |
3318                         vcpu->arch.cr0_guest_owned_bits));
3319 }
3320
3321 static inline unsigned long
3322 vmcs12_guest_cr4(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
3323 {
3324         return
3325         /*1*/   (vmcs_readl(GUEST_CR4) & vcpu->arch.cr4_guest_owned_bits) |
3326         /*2*/   (vmcs12->guest_cr4 & vmcs12->cr4_guest_host_mask) |
3327         /*3*/   (vmcs_readl(CR4_READ_SHADOW) & ~(vmcs12->cr4_guest_host_mask |
3328                         vcpu->arch.cr4_guest_owned_bits));
3329 }
3330
3331 static void vmcs12_save_pending_event(struct kvm_vcpu *vcpu,
3332                                       struct vmcs12 *vmcs12)
3333 {
3334         u32 idt_vectoring;
3335         unsigned int nr;
3336
3337         if (vcpu->arch.exception.injected) {
3338                 nr = vcpu->arch.exception.nr;
3339                 idt_vectoring = nr | VECTORING_INFO_VALID_MASK;
3340
3341                 if (kvm_exception_is_soft(nr)) {
3342                         vmcs12->vm_exit_instruction_len =
3343                                 vcpu->arch.event_exit_inst_len;
3344                         idt_vectoring |= INTR_TYPE_SOFT_EXCEPTION;
3345                 } else
3346                         idt_vectoring |= INTR_TYPE_HARD_EXCEPTION;
3347
3348                 if (vcpu->arch.exception.has_error_code) {
3349                         idt_vectoring |= VECTORING_INFO_DELIVER_CODE_MASK;
3350                         vmcs12->idt_vectoring_error_code =
3351                                 vcpu->arch.exception.error_code;
3352                 }
3353
3354                 vmcs12->idt_vectoring_info_field = idt_vectoring;
3355         } else if (vcpu->arch.nmi_injected) {
3356                 vmcs12->idt_vectoring_info_field =
3357                         INTR_TYPE_NMI_INTR | INTR_INFO_VALID_MASK | NMI_VECTOR;
3358         } else if (vcpu->arch.interrupt.injected) {
3359                 nr = vcpu->arch.interrupt.nr;
3360                 idt_vectoring = nr | VECTORING_INFO_VALID_MASK;
3361
3362                 if (vcpu->arch.interrupt.soft) {
3363                         idt_vectoring |= INTR_TYPE_SOFT_INTR;
3364                         vmcs12->vm_entry_instruction_len =
3365                                 vcpu->arch.event_exit_inst_len;
3366                 } else
3367                         idt_vectoring |= INTR_TYPE_EXT_INTR;
3368
3369                 vmcs12->idt_vectoring_info_field = idt_vectoring;
3370         }
3371 }
3372
3373
3374 static void nested_mark_vmcs12_pages_dirty(struct kvm_vcpu *vcpu)
3375 {
3376         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3377         gfn_t gfn;
3378
3379         /*
3380          * Don't need to mark the APIC access page dirty; it is never
3381          * written to by the CPU during APIC virtualization.
3382          */
3383
3384         if (nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW)) {
3385                 gfn = vmcs12->virtual_apic_page_addr >> PAGE_SHIFT;
3386                 kvm_vcpu_mark_page_dirty(vcpu, gfn);
3387         }
3388
3389         if (nested_cpu_has_posted_intr(vmcs12)) {
3390                 gfn = vmcs12->posted_intr_desc_addr >> PAGE_SHIFT;
3391                 kvm_vcpu_mark_page_dirty(vcpu, gfn);
3392         }
3393 }
3394
3395 static void vmx_complete_nested_posted_interrupt(struct kvm_vcpu *vcpu)
3396 {
3397         struct vcpu_vmx *vmx = to_vmx(vcpu);
3398         int max_irr;
3399         void *vapic_page;
3400         u16 status;
3401
3402         if (!vmx->nested.pi_desc || !vmx->nested.pi_pending)
3403                 return;
3404
3405         vmx->nested.pi_pending = false;
3406         if (!pi_test_and_clear_on(vmx->nested.pi_desc))
3407                 return;
3408
3409         max_irr = find_last_bit((unsigned long *)vmx->nested.pi_desc->pir, 256);
3410         if (max_irr != 256) {
3411                 vapic_page = vmx->nested.virtual_apic_map.hva;
3412                 if (!vapic_page)
3413                         return;
3414
3415                 __kvm_apic_update_irr(vmx->nested.pi_desc->pir,
3416                         vapic_page, &max_irr);
3417                 status = vmcs_read16(GUEST_INTR_STATUS);
3418                 if ((u8)max_irr > ((u8)status & 0xff)) {
3419                         status &= ~0xff;
3420                         status |= (u8)max_irr;
3421                         vmcs_write16(GUEST_INTR_STATUS, status);
3422                 }
3423         }
3424
3425         nested_mark_vmcs12_pages_dirty(vcpu);
3426 }
3427
3428 static void nested_vmx_inject_exception_vmexit(struct kvm_vcpu *vcpu,
3429                                                unsigned long exit_qual)
3430 {
3431         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3432         unsigned int nr = vcpu->arch.exception.nr;
3433         u32 intr_info = nr | INTR_INFO_VALID_MASK;
3434
3435         if (vcpu->arch.exception.has_error_code) {
3436                 /*
3437                  * Intel CPUs do not generate error codes with bits 31:16 set,
3438                  * and more importantly VMX disallows setting bits 31:16 in the
3439                  * injected error code for VM-Entry.  Drop the bits to mimic
3440                  * hardware and avoid inducing failure on nested VM-Entry if L1
3441                  * chooses to inject the exception back to L2.  AMD CPUs _do_
3442                  * generate "full" 32-bit error codes, so KVM allows userspace
3443                  * to inject exception error codes with bits 31:16 set.
3444                  */
3445                 vmcs12->vm_exit_intr_error_code = (u16)vcpu->arch.exception.error_code;
3446                 intr_info |= INTR_INFO_DELIVER_CODE_MASK;
3447         }
3448
3449         if (kvm_exception_is_soft(nr))
3450                 intr_info |= INTR_TYPE_SOFT_EXCEPTION;
3451         else
3452                 intr_info |= INTR_TYPE_HARD_EXCEPTION;
3453
3454         if (!(vmcs12->idt_vectoring_info_field & VECTORING_INFO_VALID_MASK) &&
3455             vmx_get_nmi_mask(vcpu))
3456                 intr_info |= INTR_INFO_UNBLOCK_NMI;
3457
3458         nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI, intr_info, exit_qual);
3459 }
3460
3461 static int vmx_check_nested_events(struct kvm_vcpu *vcpu)
3462 {
3463         struct vcpu_vmx *vmx = to_vmx(vcpu);
3464         unsigned long exit_qual;
3465         bool block_nested_events =
3466             vmx->nested.nested_run_pending || kvm_event_needs_reinjection(vcpu);
3467         struct kvm_lapic *apic = vcpu->arch.apic;
3468
3469         if (lapic_in_kernel(vcpu) &&
3470                 test_bit(KVM_APIC_INIT, &apic->pending_events)) {
3471                 if (block_nested_events)
3472                         return -EBUSY;
3473                 nested_vmx_vmexit(vcpu, EXIT_REASON_INIT_SIGNAL, 0, 0);
3474                 return 0;
3475         }
3476
3477         if (vcpu->arch.exception.pending &&
3478                 nested_vmx_check_exception(vcpu, &exit_qual)) {
3479                 if (block_nested_events)
3480                         return -EBUSY;
3481                 nested_vmx_inject_exception_vmexit(vcpu, exit_qual);
3482                 return 0;
3483         }
3484
3485         if (nested_cpu_has_preemption_timer(get_vmcs12(vcpu)) &&
3486             vmx->nested.preemption_timer_expired) {
3487                 if (block_nested_events)
3488                         return -EBUSY;
3489                 nested_vmx_vmexit(vcpu, EXIT_REASON_PREEMPTION_TIMER, 0, 0);
3490                 return 0;
3491         }
3492
3493         if (vcpu->arch.nmi_pending && nested_exit_on_nmi(vcpu)) {
3494                 if (block_nested_events)
3495                         return -EBUSY;
3496                 nested_vmx_vmexit(vcpu, EXIT_REASON_EXCEPTION_NMI,
3497                                   NMI_VECTOR | INTR_TYPE_NMI_INTR |
3498                                   INTR_INFO_VALID_MASK, 0);
3499                 /*
3500                  * The NMI-triggered VM exit counts as injection:
3501                  * clear this one and block further NMIs.
3502                  */
3503                 vcpu->arch.nmi_pending = 0;
3504                 vmx_set_nmi_mask(vcpu, true);
3505                 return 0;
3506         }
3507
3508         if (kvm_cpu_has_interrupt(vcpu) && nested_exit_on_intr(vcpu)) {
3509                 if (block_nested_events)
3510                         return -EBUSY;
3511                 nested_vmx_vmexit(vcpu, EXIT_REASON_EXTERNAL_INTERRUPT, 0, 0);
3512                 return 0;
3513         }
3514
3515         vmx_complete_nested_posted_interrupt(vcpu);
3516         return 0;
3517 }
3518
3519 static u32 vmx_get_preemption_timer_value(struct kvm_vcpu *vcpu)
3520 {
3521         ktime_t remaining =
3522                 hrtimer_get_remaining(&to_vmx(vcpu)->nested.preemption_timer);
3523         u64 value;
3524
3525         if (ktime_to_ns(remaining) <= 0)
3526                 return 0;
3527
3528         value = ktime_to_ns(remaining) * vcpu->arch.virtual_tsc_khz;
3529         do_div(value, 1000000);
3530         return value >> VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE;
3531 }
3532
3533 static bool is_vmcs12_ext_field(unsigned long field)
3534 {
3535         switch (field) {
3536         case GUEST_ES_SELECTOR:
3537         case GUEST_CS_SELECTOR:
3538         case GUEST_SS_SELECTOR:
3539         case GUEST_DS_SELECTOR:
3540         case GUEST_FS_SELECTOR:
3541         case GUEST_GS_SELECTOR:
3542         case GUEST_LDTR_SELECTOR:
3543         case GUEST_TR_SELECTOR:
3544         case GUEST_ES_LIMIT:
3545         case GUEST_CS_LIMIT:
3546         case GUEST_SS_LIMIT:
3547         case GUEST_DS_LIMIT:
3548         case GUEST_FS_LIMIT:
3549         case GUEST_GS_LIMIT:
3550         case GUEST_LDTR_LIMIT:
3551         case GUEST_TR_LIMIT:
3552         case GUEST_GDTR_LIMIT:
3553         case GUEST_IDTR_LIMIT:
3554         case GUEST_ES_AR_BYTES:
3555         case GUEST_DS_AR_BYTES:
3556         case GUEST_FS_AR_BYTES:
3557         case GUEST_GS_AR_BYTES:
3558         case GUEST_LDTR_AR_BYTES:
3559         case GUEST_TR_AR_BYTES:
3560         case GUEST_ES_BASE:
3561         case GUEST_CS_BASE:
3562         case GUEST_SS_BASE:
3563         case GUEST_DS_BASE:
3564         case GUEST_FS_BASE:
3565         case GUEST_GS_BASE:
3566         case GUEST_LDTR_BASE:
3567         case GUEST_TR_BASE:
3568         case GUEST_GDTR_BASE:
3569         case GUEST_IDTR_BASE:
3570         case GUEST_PENDING_DBG_EXCEPTIONS:
3571         case GUEST_BNDCFGS:
3572                 return true;
3573         default:
3574                 break;
3575         }
3576
3577         return false;
3578 }
3579
3580 static void sync_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu,
3581                                        struct vmcs12 *vmcs12)
3582 {
3583         struct vcpu_vmx *vmx = to_vmx(vcpu);
3584
3585         vmcs12->guest_es_selector = vmcs_read16(GUEST_ES_SELECTOR);
3586         vmcs12->guest_cs_selector = vmcs_read16(GUEST_CS_SELECTOR);
3587         vmcs12->guest_ss_selector = vmcs_read16(GUEST_SS_SELECTOR);
3588         vmcs12->guest_ds_selector = vmcs_read16(GUEST_DS_SELECTOR);
3589         vmcs12->guest_fs_selector = vmcs_read16(GUEST_FS_SELECTOR);
3590         vmcs12->guest_gs_selector = vmcs_read16(GUEST_GS_SELECTOR);
3591         vmcs12->guest_ldtr_selector = vmcs_read16(GUEST_LDTR_SELECTOR);
3592         vmcs12->guest_tr_selector = vmcs_read16(GUEST_TR_SELECTOR);
3593         vmcs12->guest_es_limit = vmcs_read32(GUEST_ES_LIMIT);
3594         vmcs12->guest_cs_limit = vmcs_read32(GUEST_CS_LIMIT);
3595         vmcs12->guest_ss_limit = vmcs_read32(GUEST_SS_LIMIT);
3596         vmcs12->guest_ds_limit = vmcs_read32(GUEST_DS_LIMIT);
3597         vmcs12->guest_fs_limit = vmcs_read32(GUEST_FS_LIMIT);
3598         vmcs12->guest_gs_limit = vmcs_read32(GUEST_GS_LIMIT);
3599         vmcs12->guest_ldtr_limit = vmcs_read32(GUEST_LDTR_LIMIT);
3600         vmcs12->guest_tr_limit = vmcs_read32(GUEST_TR_LIMIT);
3601         vmcs12->guest_gdtr_limit = vmcs_read32(GUEST_GDTR_LIMIT);
3602         vmcs12->guest_idtr_limit = vmcs_read32(GUEST_IDTR_LIMIT);
3603         vmcs12->guest_es_ar_bytes = vmcs_read32(GUEST_ES_AR_BYTES);
3604         vmcs12->guest_ds_ar_bytes = vmcs_read32(GUEST_DS_AR_BYTES);
3605         vmcs12->guest_fs_ar_bytes = vmcs_read32(GUEST_FS_AR_BYTES);
3606         vmcs12->guest_gs_ar_bytes = vmcs_read32(GUEST_GS_AR_BYTES);
3607         vmcs12->guest_ldtr_ar_bytes = vmcs_read32(GUEST_LDTR_AR_BYTES);
3608         vmcs12->guest_tr_ar_bytes = vmcs_read32(GUEST_TR_AR_BYTES);
3609         vmcs12->guest_es_base = vmcs_readl(GUEST_ES_BASE);
3610         vmcs12->guest_cs_base = vmcs_readl(GUEST_CS_BASE);
3611         vmcs12->guest_ss_base = vmcs_readl(GUEST_SS_BASE);
3612         vmcs12->guest_ds_base = vmcs_readl(GUEST_DS_BASE);
3613         vmcs12->guest_fs_base = vmcs_readl(GUEST_FS_BASE);
3614         vmcs12->guest_gs_base = vmcs_readl(GUEST_GS_BASE);
3615         vmcs12->guest_ldtr_base = vmcs_readl(GUEST_LDTR_BASE);
3616         vmcs12->guest_tr_base = vmcs_readl(GUEST_TR_BASE);
3617         vmcs12->guest_gdtr_base = vmcs_readl(GUEST_GDTR_BASE);
3618         vmcs12->guest_idtr_base = vmcs_readl(GUEST_IDTR_BASE);
3619         vmcs12->guest_pending_dbg_exceptions =
3620                 vmcs_readl(GUEST_PENDING_DBG_EXCEPTIONS);
3621         if (kvm_mpx_supported())
3622                 vmcs12->guest_bndcfgs = vmcs_read64(GUEST_BNDCFGS);
3623
3624         vmx->nested.need_sync_vmcs02_to_vmcs12_rare = false;
3625 }
3626
3627 static void copy_vmcs02_to_vmcs12_rare(struct kvm_vcpu *vcpu,
3628                                        struct vmcs12 *vmcs12)
3629 {
3630         struct vcpu_vmx *vmx = to_vmx(vcpu);
3631         int cpu;
3632
3633         if (!vmx->nested.need_sync_vmcs02_to_vmcs12_rare)
3634                 return;
3635
3636
3637         WARN_ON_ONCE(vmx->loaded_vmcs != &vmx->vmcs01);
3638
3639         cpu = get_cpu();
3640         vmx->loaded_vmcs = &vmx->nested.vmcs02;
3641         vmx_vcpu_load(&vmx->vcpu, cpu);
3642
3643         sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
3644
3645         vmx->loaded_vmcs = &vmx->vmcs01;
3646         vmx_vcpu_load(&vmx->vcpu, cpu);
3647         put_cpu();
3648 }
3649
3650 /*
3651  * Update the guest state fields of vmcs12 to reflect changes that
3652  * occurred while L2 was running. (The "IA-32e mode guest" bit of the
3653  * VM-entry controls is also updated, since this is really a guest
3654  * state bit.)
3655  */
3656 static void sync_vmcs02_to_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12)
3657 {
3658         struct vcpu_vmx *vmx = to_vmx(vcpu);
3659
3660         if (vmx->nested.hv_evmcs)
3661                 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
3662
3663         vmx->nested.need_sync_vmcs02_to_vmcs12_rare = !vmx->nested.hv_evmcs;
3664
3665         vmcs12->guest_cr0 = vmcs12_guest_cr0(vcpu, vmcs12);
3666         vmcs12->guest_cr4 = vmcs12_guest_cr4(vcpu, vmcs12);
3667
3668         vmcs12->guest_rsp = kvm_rsp_read(vcpu);
3669         vmcs12->guest_rip = kvm_rip_read(vcpu);
3670         vmcs12->guest_rflags = vmcs_readl(GUEST_RFLAGS);
3671
3672         vmcs12->guest_cs_ar_bytes = vmcs_read32(GUEST_CS_AR_BYTES);
3673         vmcs12->guest_ss_ar_bytes = vmcs_read32(GUEST_SS_AR_BYTES);
3674
3675         vmcs12->guest_sysenter_cs = vmcs_read32(GUEST_SYSENTER_CS);
3676         vmcs12->guest_sysenter_esp = vmcs_readl(GUEST_SYSENTER_ESP);
3677         vmcs12->guest_sysenter_eip = vmcs_readl(GUEST_SYSENTER_EIP);
3678
3679         vmcs12->guest_interruptibility_info =
3680                 vmcs_read32(GUEST_INTERRUPTIBILITY_INFO);
3681
3682         if (vcpu->arch.mp_state == KVM_MP_STATE_HALTED)
3683                 vmcs12->guest_activity_state = GUEST_ACTIVITY_HLT;
3684         else
3685                 vmcs12->guest_activity_state = GUEST_ACTIVITY_ACTIVE;
3686
3687         if (nested_cpu_has_preemption_timer(vmcs12) &&
3688             vmcs12->vm_exit_controls & VM_EXIT_SAVE_VMX_PREEMPTION_TIMER)
3689                         vmcs12->vmx_preemption_timer_value =
3690                                 vmx_get_preemption_timer_value(vcpu);
3691
3692         /*
3693          * In some cases (usually, nested EPT), L2 is allowed to change its
3694          * own CR3 without exiting. If it has changed it, we must keep it.
3695          * Of course, if L0 is using shadow page tables, GUEST_CR3 was defined
3696          * by L0, not L1 or L2, so we mustn't unconditionally copy it to vmcs12.
3697          *
3698          * Additionally, restore L2's PDPTR to vmcs12.
3699          */
3700         if (enable_ept) {
3701                 vmcs12->guest_cr3 = vmcs_readl(GUEST_CR3);
3702                 if (nested_cpu_has_ept(vmcs12) && is_pae_paging(vcpu)) {
3703                         vmcs12->guest_pdptr0 = vmcs_read64(GUEST_PDPTR0);
3704                         vmcs12->guest_pdptr1 = vmcs_read64(GUEST_PDPTR1);
3705                         vmcs12->guest_pdptr2 = vmcs_read64(GUEST_PDPTR2);
3706                         vmcs12->guest_pdptr3 = vmcs_read64(GUEST_PDPTR3);
3707                 }
3708         }
3709
3710         vmcs12->guest_linear_address = vmcs_readl(GUEST_LINEAR_ADDRESS);
3711
3712         if (nested_cpu_has_vid(vmcs12))
3713                 vmcs12->guest_intr_status = vmcs_read16(GUEST_INTR_STATUS);
3714
3715         vmcs12->vm_entry_controls =
3716                 (vmcs12->vm_entry_controls & ~VM_ENTRY_IA32E_MODE) |
3717                 (vm_entry_controls_get(to_vmx(vcpu)) & VM_ENTRY_IA32E_MODE);
3718
3719         if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_DEBUG_CONTROLS)
3720                 kvm_get_dr(vcpu, 7, (unsigned long *)&vmcs12->guest_dr7);
3721
3722         if (vmcs12->vm_exit_controls & VM_EXIT_SAVE_IA32_EFER)
3723                 vmcs12->guest_ia32_efer = vcpu->arch.efer;
3724 }
3725
3726 /*
3727  * prepare_vmcs12 is part of what we need to do when the nested L2 guest exits
3728  * and we want to prepare to run its L1 parent. L1 keeps a vmcs for L2 (vmcs12),
3729  * and this function updates it to reflect the changes to the guest state while
3730  * L2 was running (and perhaps made some exits which were handled directly by L0
3731  * without going back to L1), and to reflect the exit reason.
3732  * Note that we do not have to copy here all VMCS fields, just those that
3733  * could have changed by the L2 guest or the exit - i.e., the guest-state and
3734  * exit-information fields only. Other fields are modified by L1 with VMWRITE,
3735  * which already writes to vmcs12 directly.
3736  */
3737 static void prepare_vmcs12(struct kvm_vcpu *vcpu, struct vmcs12 *vmcs12,
3738                            u32 exit_reason, u32 exit_intr_info,
3739                            unsigned long exit_qualification)
3740 {
3741         /* update exit information fields: */
3742         vmcs12->vm_exit_reason = exit_reason;
3743         vmcs12->exit_qualification = exit_qualification;
3744
3745         /*
3746          * On VM-Exit due to a failed VM-Entry, the VMCS isn't marked launched
3747          * and only EXIT_REASON and EXIT_QUALIFICATION are updated, all other
3748          * exit info fields are unmodified.
3749          */
3750         if (!(vmcs12->vm_exit_reason & VMX_EXIT_REASONS_FAILED_VMENTRY)) {
3751                 vmcs12->launch_state = 1;
3752
3753                 /* vm_entry_intr_info_field is cleared on exit. Emulate this
3754                  * instead of reading the real value. */
3755                 vmcs12->vm_entry_intr_info_field &= ~INTR_INFO_VALID_MASK;
3756
3757                 /*
3758                  * Transfer the event that L0 or L1 may wanted to inject into
3759                  * L2 to IDT_VECTORING_INFO_FIELD.
3760                  */
3761                 vmcs12->idt_vectoring_info_field = 0;
3762                 vmcs12_save_pending_event(vcpu, vmcs12);
3763
3764                 vmcs12->vm_exit_intr_info = exit_intr_info;
3765                 vmcs12->vm_exit_instruction_len = vmcs_read32(VM_EXIT_INSTRUCTION_LEN);
3766                 vmcs12->vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
3767
3768                 /*
3769                  * According to spec, there's no need to store the guest's
3770                  * MSRs if the exit is due to a VM-entry failure that occurs
3771                  * during or after loading the guest state. Since this exit
3772                  * does not fall in that category, we need to save the MSRs.
3773                  */
3774                 if (nested_vmx_store_msr(vcpu,
3775                                          vmcs12->vm_exit_msr_store_addr,
3776                                          vmcs12->vm_exit_msr_store_count))
3777                         nested_vmx_abort(vcpu,
3778                                          VMX_ABORT_SAVE_GUEST_MSR_FAIL);
3779         }
3780 }
3781
3782 /*
3783  * A part of what we need to when the nested L2 guest exits and we want to
3784  * run its L1 parent, is to reset L1's guest state to the host state specified
3785  * in vmcs12.
3786  * This function is to be called not only on normal nested exit, but also on
3787  * a nested entry failure, as explained in Intel's spec, 3B.23.7 ("VM-Entry
3788  * Failures During or After Loading Guest State").
3789  * This function should be called when the active VMCS is L1's (vmcs01).
3790  */
3791 static void load_vmcs12_host_state(struct kvm_vcpu *vcpu,
3792                                    struct vmcs12 *vmcs12)
3793 {
3794         struct kvm_segment seg;
3795         u32 entry_failure_code;
3796
3797         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_EFER)
3798                 vcpu->arch.efer = vmcs12->host_ia32_efer;
3799         else if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
3800                 vcpu->arch.efer |= (EFER_LMA | EFER_LME);
3801         else
3802                 vcpu->arch.efer &= ~(EFER_LMA | EFER_LME);
3803         vmx_set_efer(vcpu, vcpu->arch.efer);
3804
3805         kvm_rsp_write(vcpu, vmcs12->host_rsp);
3806         kvm_rip_write(vcpu, vmcs12->host_rip);
3807         vmx_set_rflags(vcpu, X86_EFLAGS_FIXED);
3808         vmx_set_interrupt_shadow(vcpu, 0);
3809
3810         /*
3811          * Note that calling vmx_set_cr0 is important, even if cr0 hasn't
3812          * actually changed, because vmx_set_cr0 refers to efer set above.
3813          *
3814          * CR0_GUEST_HOST_MASK is already set in the original vmcs01
3815          * (KVM doesn't change it);
3816          */
3817         vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
3818         vmx_set_cr0(vcpu, vmcs12->host_cr0);
3819
3820         /* Same as above - no reason to call set_cr4_guest_host_mask().  */
3821         vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK);
3822         vmx_set_cr4(vcpu, vmcs12->host_cr4);
3823
3824         nested_ept_uninit_mmu_context(vcpu);
3825
3826         /*
3827          * Only PDPTE load can fail as the value of cr3 was checked on entry and
3828          * couldn't have changed.
3829          */
3830         if (nested_vmx_load_cr3(vcpu, vmcs12->host_cr3, false, &entry_failure_code))
3831                 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_PDPTE_FAIL);
3832
3833         if (!enable_ept)
3834                 vcpu->arch.walk_mmu->inject_page_fault = kvm_inject_page_fault;
3835
3836         /*
3837          * If vmcs01 doesn't use VPID, CPU flushes TLB on every
3838          * VMEntry/VMExit. Thus, no need to flush TLB.
3839          *
3840          * If vmcs12 doesn't use VPID, L1 expects TLB to be
3841          * flushed on every VMEntry/VMExit.
3842          *
3843          * Otherwise, we can preserve TLB entries as long as we are
3844          * able to tag L1 TLB entries differently than L2 TLB entries.
3845          *
3846          * If vmcs12 uses EPT, we need to execute this flush on EPTP01
3847          * and therefore we request the TLB flush to happen only after VMCS EPTP
3848          * has been set by KVM_REQ_LOAD_CR3.
3849          */
3850         if (enable_vpid &&
3851             (!nested_cpu_has_vpid(vmcs12) || !nested_has_guest_tlb_tag(vcpu))) {
3852                 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
3853         }
3854
3855         vmcs_write32(GUEST_SYSENTER_CS, vmcs12->host_ia32_sysenter_cs);
3856         vmcs_writel(GUEST_SYSENTER_ESP, vmcs12->host_ia32_sysenter_esp);
3857         vmcs_writel(GUEST_SYSENTER_EIP, vmcs12->host_ia32_sysenter_eip);
3858         vmcs_writel(GUEST_IDTR_BASE, vmcs12->host_idtr_base);
3859         vmcs_writel(GUEST_GDTR_BASE, vmcs12->host_gdtr_base);
3860         vmcs_write32(GUEST_IDTR_LIMIT, 0xFFFF);
3861         vmcs_write32(GUEST_GDTR_LIMIT, 0xFFFF);
3862
3863         /* If not VM_EXIT_CLEAR_BNDCFGS, the L2 value propagates to L1.  */
3864         if (vmcs12->vm_exit_controls & VM_EXIT_CLEAR_BNDCFGS)
3865                 vmcs_write64(GUEST_BNDCFGS, 0);
3866
3867         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PAT) {
3868                 vmcs_write64(GUEST_IA32_PAT, vmcs12->host_ia32_pat);
3869                 vcpu->arch.pat = vmcs12->host_ia32_pat;
3870         }
3871         if (vmcs12->vm_exit_controls & VM_EXIT_LOAD_IA32_PERF_GLOBAL_CTRL)
3872                 vmcs_write64(GUEST_IA32_PERF_GLOBAL_CTRL,
3873                         vmcs12->host_ia32_perf_global_ctrl);
3874
3875         /* Set L1 segment info according to Intel SDM
3876             27.5.2 Loading Host Segment and Descriptor-Table Registers */
3877         seg = (struct kvm_segment) {
3878                 .base = 0,
3879                 .limit = 0xFFFFFFFF,
3880                 .selector = vmcs12->host_cs_selector,
3881                 .type = 11,
3882                 .present = 1,
3883                 .s = 1,
3884                 .g = 1
3885         };
3886         if (vmcs12->vm_exit_controls & VM_EXIT_HOST_ADDR_SPACE_SIZE)
3887                 seg.l = 1;
3888         else
3889                 seg.db = 1;
3890         vmx_set_segment(vcpu, &seg, VCPU_SREG_CS);
3891         seg = (struct kvm_segment) {
3892                 .base = 0,
3893                 .limit = 0xFFFFFFFF,
3894                 .type = 3,
3895                 .present = 1,
3896                 .s = 1,
3897                 .db = 1,
3898                 .g = 1
3899         };
3900         seg.selector = vmcs12->host_ds_selector;
3901         vmx_set_segment(vcpu, &seg, VCPU_SREG_DS);
3902         seg.selector = vmcs12->host_es_selector;
3903         vmx_set_segment(vcpu, &seg, VCPU_SREG_ES);
3904         seg.selector = vmcs12->host_ss_selector;
3905         vmx_set_segment(vcpu, &seg, VCPU_SREG_SS);
3906         seg.selector = vmcs12->host_fs_selector;
3907         seg.base = vmcs12->host_fs_base;
3908         vmx_set_segment(vcpu, &seg, VCPU_SREG_FS);
3909         seg.selector = vmcs12->host_gs_selector;
3910         seg.base = vmcs12->host_gs_base;
3911         vmx_set_segment(vcpu, &seg, VCPU_SREG_GS);
3912         seg = (struct kvm_segment) {
3913                 .base = vmcs12->host_tr_base,
3914                 .limit = 0x67,
3915                 .selector = vmcs12->host_tr_selector,
3916                 .type = 11,
3917                 .present = 1
3918         };
3919         vmx_set_segment(vcpu, &seg, VCPU_SREG_TR);
3920
3921         kvm_set_dr(vcpu, 7, 0x400);
3922         vmcs_write64(GUEST_IA32_DEBUGCTL, 0);
3923
3924         if (cpu_has_vmx_msr_bitmap())
3925                 vmx_update_msr_bitmap(vcpu);
3926
3927         if (nested_vmx_load_msr(vcpu, vmcs12->vm_exit_msr_load_addr,
3928                                 vmcs12->vm_exit_msr_load_count))
3929                 nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL);
3930 }
3931
3932 static inline u64 nested_vmx_get_vmcs01_guest_efer(struct vcpu_vmx *vmx)
3933 {
3934         struct shared_msr_entry *efer_msr;
3935         unsigned int i;
3936
3937         if (vm_entry_controls_get(vmx) & VM_ENTRY_LOAD_IA32_EFER)
3938                 return vmcs_read64(GUEST_IA32_EFER);
3939
3940         if (cpu_has_load_ia32_efer())
3941                 return host_efer;
3942
3943         for (i = 0; i < vmx->msr_autoload.guest.nr; ++i) {
3944                 if (vmx->msr_autoload.guest.val[i].index == MSR_EFER)
3945                         return vmx->msr_autoload.guest.val[i].value;
3946         }
3947
3948         efer_msr = find_msr_entry(vmx, MSR_EFER);
3949         if (efer_msr)
3950                 return efer_msr->data;
3951
3952         return host_efer;
3953 }
3954
3955 static void nested_vmx_restore_host_state(struct kvm_vcpu *vcpu)
3956 {
3957         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
3958         struct vcpu_vmx *vmx = to_vmx(vcpu);
3959         struct vmx_msr_entry g, h;
3960         gpa_t gpa;
3961         u32 i, j;
3962
3963         vcpu->arch.pat = vmcs_read64(GUEST_IA32_PAT);
3964
3965         if (vmcs12->vm_entry_controls & VM_ENTRY_LOAD_DEBUG_CONTROLS) {
3966                 /*
3967                  * L1's host DR7 is lost if KVM_GUESTDBG_USE_HW_BP is set
3968                  * as vmcs01.GUEST_DR7 contains a userspace defined value
3969                  * and vcpu->arch.dr7 is not squirreled away before the
3970                  * nested VMENTER (not worth adding a variable in nested_vmx).
3971                  */
3972                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_HW_BP)
3973                         kvm_set_dr(vcpu, 7, DR7_FIXED_1);
3974                 else
3975                         WARN_ON(kvm_set_dr(vcpu, 7, vmcs_readl(GUEST_DR7)));
3976         }
3977
3978         /*
3979          * Note that calling vmx_set_{efer,cr0,cr4} is important as they
3980          * handle a variety of side effects to KVM's software model.
3981          */
3982         vmx_set_efer(vcpu, nested_vmx_get_vmcs01_guest_efer(vmx));
3983
3984         vcpu->arch.cr0_guest_owned_bits = X86_CR0_TS;
3985         vmx_set_cr0(vcpu, vmcs_readl(CR0_READ_SHADOW));
3986
3987         vcpu->arch.cr4_guest_owned_bits = ~vmcs_readl(CR4_GUEST_HOST_MASK);
3988         vmx_set_cr4(vcpu, vmcs_readl(CR4_READ_SHADOW));
3989
3990         nested_ept_uninit_mmu_context(vcpu);
3991         vcpu->arch.cr3 = vmcs_readl(GUEST_CR3);
3992         __set_bit(VCPU_EXREG_CR3, (ulong *)&vcpu->arch.regs_avail);
3993
3994         /*
3995          * Use ept_save_pdptrs(vcpu) to load the MMU's cached PDPTRs
3996          * from vmcs01 (if necessary).  The PDPTRs are not loaded on
3997          * VMFail, like everything else we just need to ensure our
3998          * software model is up-to-date.
3999          */
4000         if (enable_ept)
4001                 ept_save_pdptrs(vcpu);
4002
4003         kvm_mmu_reset_context(vcpu);
4004
4005         if (cpu_has_vmx_msr_bitmap())
4006                 vmx_update_msr_bitmap(vcpu);
4007
4008         /*
4009          * This nasty bit of open coding is a compromise between blindly
4010          * loading L1's MSRs using the exit load lists (incorrect emulation
4011          * of VMFail), leaving the nested VM's MSRs in the software model
4012          * (incorrect behavior) and snapshotting the modified MSRs (too
4013          * expensive since the lists are unbound by hardware).  For each
4014          * MSR that was (prematurely) loaded from the nested VMEntry load
4015          * list, reload it from the exit load list if it exists and differs
4016          * from the guest value.  The intent is to stuff host state as
4017          * silently as possible, not to fully process the exit load list.
4018          */
4019         for (i = 0; i < vmcs12->vm_entry_msr_load_count; i++) {
4020                 gpa = vmcs12->vm_entry_msr_load_addr + (i * sizeof(g));
4021                 if (kvm_vcpu_read_guest(vcpu, gpa, &g, sizeof(g))) {
4022                         pr_debug_ratelimited(
4023                                 "%s read MSR index failed (%u, 0x%08llx)\n",
4024                                 __func__, i, gpa);
4025                         goto vmabort;
4026                 }
4027
4028                 for (j = 0; j < vmcs12->vm_exit_msr_load_count; j++) {
4029                         gpa = vmcs12->vm_exit_msr_load_addr + (j * sizeof(h));
4030                         if (kvm_vcpu_read_guest(vcpu, gpa, &h, sizeof(h))) {
4031                                 pr_debug_ratelimited(
4032                                         "%s read MSR failed (%u, 0x%08llx)\n",
4033                                         __func__, j, gpa);
4034                                 goto vmabort;
4035                         }
4036                         if (h.index != g.index)
4037                                 continue;
4038                         if (h.value == g.value)
4039                                 break;
4040
4041                         if (nested_vmx_load_msr_check(vcpu, &h)) {
4042                                 pr_debug_ratelimited(
4043                                         "%s check failed (%u, 0x%x, 0x%x)\n",
4044                                         __func__, j, h.index, h.reserved);
4045                                 goto vmabort;
4046                         }
4047
4048                         if (kvm_set_msr(vcpu, h.index, h.value)) {
4049                                 pr_debug_ratelimited(
4050                                         "%s WRMSR failed (%u, 0x%x, 0x%llx)\n",
4051                                         __func__, j, h.index, h.value);
4052                                 goto vmabort;
4053                         }
4054                 }
4055         }
4056
4057         return;
4058
4059 vmabort:
4060         nested_vmx_abort(vcpu, VMX_ABORT_LOAD_HOST_MSR_FAIL);
4061 }
4062
4063 /*
4064  * Emulate an exit from nested guest (L2) to L1, i.e., prepare to run L1
4065  * and modify vmcs12 to make it see what it would expect to see there if
4066  * L2 was its real guest. Must only be called when in L2 (is_guest_mode())
4067  */
4068 void nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 exit_reason,
4069                        u32 exit_intr_info, unsigned long exit_qualification)
4070 {
4071         struct vcpu_vmx *vmx = to_vmx(vcpu);
4072         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
4073
4074         /* trying to cancel vmlaunch/vmresume is a bug */
4075         WARN_ON_ONCE(vmx->nested.nested_run_pending);
4076
4077         leave_guest_mode(vcpu);
4078
4079         if (nested_cpu_has_preemption_timer(vmcs12))
4080                 hrtimer_cancel(&to_vmx(vcpu)->nested.preemption_timer);
4081
4082         if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING)
4083                 vcpu->arch.tsc_offset -= vmcs12->tsc_offset;
4084
4085         if (likely(!vmx->fail)) {
4086                 sync_vmcs02_to_vmcs12(vcpu, vmcs12);
4087
4088                 if (exit_reason != -1)
4089                         prepare_vmcs12(vcpu, vmcs12, exit_reason, exit_intr_info,
4090                                        exit_qualification);
4091
4092                 /*
4093                  * Must happen outside of sync_vmcs02_to_vmcs12() as it will
4094                  * also be used to capture vmcs12 cache as part of
4095                  * capturing nVMX state for snapshot (migration).
4096                  *
4097                  * Otherwise, this flush will dirty guest memory at a
4098                  * point it is already assumed by user-space to be
4099                  * immutable.
4100                  */
4101                 nested_flush_cached_shadow_vmcs12(vcpu, vmcs12);
4102         } else {
4103                 /*
4104                  * The only expected VM-instruction error is "VM entry with
4105                  * invalid control field(s)." Anything else indicates a
4106                  * problem with L0.  And we should never get here with a
4107                  * VMFail of any type if early consistency checks are enabled.
4108                  */
4109                 WARN_ON_ONCE(vmcs_read32(VM_INSTRUCTION_ERROR) !=
4110                              VMXERR_ENTRY_INVALID_CONTROL_FIELD);
4111                 WARN_ON_ONCE(nested_early_check);
4112         }
4113
4114         /*
4115          * Drop events/exceptions that were queued for re-injection to L2
4116          * (picked up via vmx_complete_interrupts()), as well as exceptions
4117          * that were pending for L2.  Note, this must NOT be hoisted above
4118          * prepare_vmcs12(), events/exceptions queued for re-injection need to
4119          * be captured in vmcs12 (see vmcs12_save_pending_event()).
4120          */
4121         vcpu->arch.nmi_injected = false;
4122         kvm_clear_exception_queue(vcpu);
4123         kvm_clear_interrupt_queue(vcpu);
4124
4125         vmx_switch_vmcs(vcpu, &vmx->vmcs01);
4126
4127         /*
4128          * If IBRS is advertised to the vCPU, KVM must flush the indirect
4129          * branch predictors when transitioning from L2 to L1, as L1 expects
4130          * hardware (KVM in this case) to provide separate predictor modes.
4131          * Bare metal isolates VMX root (host) from VMX non-root (guest), but
4132          * doesn't isolate different VMCSs, i.e. in this case, doesn't provide
4133          * separate modes for L2 vs L1.
4134          */
4135         if (guest_cpuid_has(vcpu, X86_FEATURE_SPEC_CTRL))
4136                 indirect_branch_prediction_barrier();
4137
4138         /* Update any VMCS fields that might have changed while L2 ran */
4139         vmcs_write32(VM_EXIT_MSR_LOAD_COUNT, vmx->msr_autoload.host.nr);
4140         vmcs_write32(VM_ENTRY_MSR_LOAD_COUNT, vmx->msr_autoload.guest.nr);
4141         vmcs_write64(TSC_OFFSET, vcpu->arch.tsc_offset);
4142
4143         if (kvm_has_tsc_control)
4144                 decache_tsc_multiplier(vmx);
4145
4146         if (vmx->nested.change_vmcs01_virtual_apic_mode) {
4147                 vmx->nested.change_vmcs01_virtual_apic_mode = false;
4148                 vmx_set_virtual_apic_mode(vcpu);
4149         } else if (!nested_cpu_has_ept(vmcs12) &&
4150                    nested_cpu_has2(vmcs12,
4151                                    SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES)) {
4152                 vmx_flush_tlb(vcpu, true);
4153         }
4154
4155         /* Unpin physical memory we referred to in vmcs02 */
4156         if (vmx->nested.apic_access_page) {
4157                 kvm_release_page_dirty(vmx->nested.apic_access_page);
4158                 vmx->nested.apic_access_page = NULL;
4159         }
4160         kvm_vcpu_unmap(vcpu, &vmx->nested.virtual_apic_map, true);
4161         kvm_vcpu_unmap(vcpu, &vmx->nested.pi_desc_map, true);
4162         vmx->nested.pi_desc = NULL;
4163
4164         /*
4165          * We are now running in L2, mmu_notifier will force to reload the
4166          * page's hpa for L2 vmcs. Need to reload it for L1 before entering L1.
4167          */
4168         kvm_make_request(KVM_REQ_APIC_PAGE_RELOAD, vcpu);
4169
4170         if ((exit_reason != -1) && (enable_shadow_vmcs || vmx->nested.hv_evmcs))
4171                 vmx->nested.need_vmcs12_to_shadow_sync = true;
4172
4173         /* in case we halted in L2 */
4174         vcpu->arch.mp_state = KVM_MP_STATE_RUNNABLE;
4175
4176         if (likely(!vmx->fail)) {
4177                 if (exit_reason == EXIT_REASON_EXTERNAL_INTERRUPT &&
4178                     nested_exit_intr_ack_set(vcpu)) {
4179                         int irq = kvm_cpu_get_interrupt(vcpu);
4180                         WARN_ON(irq < 0);
4181                         vmcs12->vm_exit_intr_info = irq |
4182                                 INTR_INFO_VALID_MASK | INTR_TYPE_EXT_INTR;
4183                 }
4184
4185                 if (exit_reason != -1)
4186                         trace_kvm_nested_vmexit_inject(vmcs12->vm_exit_reason,
4187                                                        vmcs12->exit_qualification,
4188                                                        vmcs12->idt_vectoring_info_field,
4189                                                        vmcs12->vm_exit_intr_info,
4190                                                        vmcs12->vm_exit_intr_error_code,
4191                                                        KVM_ISA_VMX);
4192
4193                 load_vmcs12_host_state(vcpu, vmcs12);
4194
4195                 return;
4196         }
4197
4198         /*
4199          * After an early L2 VM-entry failure, we're now back
4200          * in L1 which thinks it just finished a VMLAUNCH or
4201          * VMRESUME instruction, so we need to set the failure
4202          * flag and the VM-instruction error field of the VMCS
4203          * accordingly, and skip the emulated instruction.
4204          */
4205         (void)nested_vmx_failValid(vcpu, VMXERR_ENTRY_INVALID_CONTROL_FIELD);
4206
4207         /*
4208          * Restore L1's host state to KVM's software model.  We're here
4209          * because a consistency check was caught by hardware, which
4210          * means some amount of guest state has been propagated to KVM's
4211          * model and needs to be unwound to the host's state.
4212          */
4213         nested_vmx_restore_host_state(vcpu);
4214
4215         vmx->fail = 0;
4216 }
4217
4218 /*
4219  * Decode the memory-address operand of a vmx instruction, as recorded on an
4220  * exit caused by such an instruction (run by a guest hypervisor).
4221  * On success, returns 0. When the operand is invalid, returns 1 and throws
4222  * #UD or #GP.
4223  */
4224 int get_vmx_mem_address(struct kvm_vcpu *vcpu, unsigned long exit_qualification,
4225                         u32 vmx_instruction_info, bool wr, int len, gva_t *ret)
4226 {
4227         gva_t off;
4228         bool exn;
4229         struct kvm_segment s;
4230
4231         /*
4232          * According to Vol. 3B, "Information for VM Exits Due to Instruction
4233          * Execution", on an exit, vmx_instruction_info holds most of the
4234          * addressing components of the operand. Only the displacement part
4235          * is put in exit_qualification (see 3B, "Basic VM-Exit Information").
4236          * For how an actual address is calculated from all these components,
4237          * refer to Vol. 1, "Operand Addressing".
4238          */
4239         int  scaling = vmx_instruction_info & 3;
4240         int  addr_size = (vmx_instruction_info >> 7) & 7;
4241         bool is_reg = vmx_instruction_info & (1u << 10);
4242         int  seg_reg = (vmx_instruction_info >> 15) & 7;
4243         int  index_reg = (vmx_instruction_info >> 18) & 0xf;
4244         bool index_is_valid = !(vmx_instruction_info & (1u << 22));
4245         int  base_reg       = (vmx_instruction_info >> 23) & 0xf;
4246         bool base_is_valid  = !(vmx_instruction_info & (1u << 27));
4247
4248         if (is_reg) {
4249                 kvm_queue_exception(vcpu, UD_VECTOR);
4250                 return 1;
4251         }
4252
4253         /* Addr = segment_base + offset */
4254         /* offset = base + [index * scale] + displacement */
4255         off = exit_qualification; /* holds the displacement */
4256         if (addr_size == 1)
4257                 off = (gva_t)sign_extend64(off, 31);
4258         else if (addr_size == 0)
4259                 off = (gva_t)sign_extend64(off, 15);
4260         if (base_is_valid)
4261                 off += kvm_register_read(vcpu, base_reg);
4262         if (index_is_valid)
4263                 off += kvm_register_read(vcpu, index_reg)<<scaling;
4264         vmx_get_segment(vcpu, &s, seg_reg);
4265
4266         /*
4267          * The effective address, i.e. @off, of a memory operand is truncated
4268          * based on the address size of the instruction.  Note that this is
4269          * the *effective address*, i.e. the address prior to accounting for
4270          * the segment's base.
4271          */
4272         if (addr_size == 1) /* 32 bit */
4273                 off &= 0xffffffff;
4274         else if (addr_size == 0) /* 16 bit */
4275                 off &= 0xffff;
4276
4277         /* Checks for #GP/#SS exceptions. */
4278         exn = false;
4279         if (is_long_mode(vcpu)) {
4280                 /*
4281                  * The virtual/linear address is never truncated in 64-bit
4282                  * mode, e.g. a 32-bit address size can yield a 64-bit virtual
4283                  * address when using FS/GS with a non-zero base.
4284                  */
4285                 if (seg_reg == VCPU_SREG_FS || seg_reg == VCPU_SREG_GS)
4286                         *ret = s.base + off;
4287                 else
4288                         *ret = off;
4289
4290                 /* Long mode: #GP(0)/#SS(0) if the memory address is in a
4291                  * non-canonical form. This is the only check on the memory
4292                  * destination for long mode!
4293                  */
4294                 exn = is_noncanonical_address(*ret, vcpu);
4295         } else {
4296                 /*
4297                  * When not in long mode, the virtual/linear address is
4298                  * unconditionally truncated to 32 bits regardless of the
4299                  * address size.
4300                  */
4301                 *ret = (s.base + off) & 0xffffffff;
4302
4303                 /* Protected mode: apply checks for segment validity in the
4304                  * following order:
4305                  * - segment type check (#GP(0) may be thrown)
4306                  * - usability check (#GP(0)/#SS(0))
4307                  * - limit check (#GP(0)/#SS(0))
4308                  */
4309                 if (wr)
4310                         /* #GP(0) if the destination operand is located in a
4311                          * read-only data segment or any code segment.
4312                          */
4313                         exn = ((s.type & 0xa) == 0 || (s.type & 8));
4314                 else
4315                         /* #GP(0) if the source operand is located in an
4316                          * execute-only code segment
4317                          */
4318                         exn = ((s.type & 0xa) == 8);
4319                 if (exn) {
4320                         kvm_queue_exception_e(vcpu, GP_VECTOR, 0);
4321                         return 1;
4322                 }
4323                 /* Protected mode: #GP(0)/#SS(0) if the segment is unusable.
4324                  */
4325                 exn = (s.unusable != 0);
4326
4327                 /*
4328                  * Protected mode: #GP(0)/#SS(0) if the memory operand is
4329                  * outside the segment limit.  All CPUs that support VMX ignore
4330                  * limit checks for flat segments, i.e. segments with base==0,
4331                  * limit==0xffffffff and of type expand-up data or code.
4332                  */
4333                 if (!(s.base == 0 && s.limit == 0xffffffff &&
4334                      ((s.type & 8) || !(s.type & 4))))
4335                         exn = exn || ((u64)off + len - 1 > s.limit);
4336         }
4337         if (exn) {
4338                 kvm_queue_exception_e(vcpu,
4339                                       seg_reg == VCPU_SREG_SS ?
4340                                                 SS_VECTOR : GP_VECTOR,
4341                                       0);
4342                 return 1;
4343         }
4344
4345         return 0;
4346 }
4347
4348 static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer)
4349 {
4350         gva_t gva;
4351         struct x86_exception e;
4352
4353         if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
4354                                 vmcs_read32(VMX_INSTRUCTION_INFO), false,
4355                                 sizeof(*vmpointer), &gva))
4356                 return 1;
4357
4358         if (kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e)) {
4359                 kvm_inject_page_fault(vcpu, &e);
4360                 return 1;
4361         }
4362
4363         return 0;
4364 }
4365
4366 /*
4367  * Allocate a shadow VMCS and associate it with the currently loaded
4368  * VMCS, unless such a shadow VMCS already exists. The newly allocated
4369  * VMCS is also VMCLEARed, so that it is ready for use.
4370  */
4371 static struct vmcs *alloc_shadow_vmcs(struct kvm_vcpu *vcpu)
4372 {
4373         struct vcpu_vmx *vmx = to_vmx(vcpu);
4374         struct loaded_vmcs *loaded_vmcs = vmx->loaded_vmcs;
4375
4376         /*
4377          * We should allocate a shadow vmcs for vmcs01 only when L1
4378          * executes VMXON and free it when L1 executes VMXOFF.
4379          * As it is invalid to execute VMXON twice, we shouldn't reach
4380          * here when vmcs01 already have an allocated shadow vmcs.
4381          */
4382         WARN_ON(loaded_vmcs == &vmx->vmcs01 && loaded_vmcs->shadow_vmcs);
4383
4384         if (!loaded_vmcs->shadow_vmcs) {
4385                 loaded_vmcs->shadow_vmcs = alloc_vmcs(true);
4386                 if (loaded_vmcs->shadow_vmcs)
4387                         vmcs_clear(loaded_vmcs->shadow_vmcs);
4388         }
4389         return loaded_vmcs->shadow_vmcs;
4390 }
4391
4392 static int enter_vmx_operation(struct kvm_vcpu *vcpu)
4393 {
4394         struct vcpu_vmx *vmx = to_vmx(vcpu);
4395         int r;
4396
4397         r = alloc_loaded_vmcs(&vmx->nested.vmcs02);
4398         if (r < 0)
4399                 goto out_vmcs02;
4400
4401         vmx->nested.cached_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT);
4402         if (!vmx->nested.cached_vmcs12)
4403                 goto out_cached_vmcs12;
4404
4405         vmx->nested.cached_shadow_vmcs12 = kzalloc(VMCS12_SIZE, GFP_KERNEL_ACCOUNT);
4406         if (!vmx->nested.cached_shadow_vmcs12)
4407                 goto out_cached_shadow_vmcs12;
4408
4409         if (enable_shadow_vmcs && !alloc_shadow_vmcs(vcpu))
4410                 goto out_shadow_vmcs;
4411
4412         hrtimer_init(&vmx->nested.preemption_timer, CLOCK_MONOTONIC,
4413                      HRTIMER_MODE_REL_PINNED);
4414         vmx->nested.preemption_timer.function = vmx_preemption_timer_fn;
4415
4416         vmx->nested.vpid02 = allocate_vpid();
4417
4418         vmx->nested.vmcs02_initialized = false;
4419         vmx->nested.vmxon = true;
4420
4421         if (pt_mode == PT_MODE_HOST_GUEST) {
4422                 vmx->pt_desc.guest.ctl = 0;
4423                 pt_update_intercept_for_msr(vmx);
4424         }
4425
4426         return 0;
4427
4428 out_shadow_vmcs:
4429         kfree(vmx->nested.cached_shadow_vmcs12);
4430
4431 out_cached_shadow_vmcs12:
4432         kfree(vmx->nested.cached_vmcs12);
4433
4434 out_cached_vmcs12:
4435         free_loaded_vmcs(&vmx->nested.vmcs02);
4436
4437 out_vmcs02:
4438         return -ENOMEM;
4439 }
4440
4441 /*
4442  * Emulate the VMXON instruction.
4443  * Currently, we just remember that VMX is active, and do not save or even
4444  * inspect the argument to VMXON (the so-called "VMXON pointer") because we
4445  * do not currently need to store anything in that guest-allocated memory
4446  * region. Consequently, VMCLEAR and VMPTRLD also do not verify that the their
4447  * argument is different from the VMXON pointer (which the spec says they do).
4448  */
4449 static int handle_vmon(struct kvm_vcpu *vcpu)
4450 {
4451         int ret;
4452         gpa_t vmptr;
4453         uint32_t revision;
4454         struct vcpu_vmx *vmx = to_vmx(vcpu);
4455         const u64 VMXON_NEEDED_FEATURES = FEATURE_CONTROL_LOCKED
4456                 | FEATURE_CONTROL_VMXON_ENABLED_OUTSIDE_SMX;
4457
4458         /*
4459          * The Intel VMX Instruction Reference lists a bunch of bits that are
4460          * prerequisite to running VMXON, most notably cr4.VMXE must be set to
4461          * 1 (see vmx_set_cr4() for when we allow the guest to set this).
4462          * Otherwise, we should fail with #UD.  But most faulting conditions
4463          * have already been checked by hardware, prior to the VM-exit for
4464          * VMXON.  We do test guest cr4.VMXE because processor CR4 always has
4465          * that bit set to 1 in non-root mode.
4466          */
4467         if (!kvm_read_cr4_bits(vcpu, X86_CR4_VMXE)) {
4468                 kvm_queue_exception(vcpu, UD_VECTOR);
4469                 return 1;
4470         }
4471
4472         /* CPL=0 must be checked manually. */
4473         if (vmx_get_cpl(vcpu)) {
4474                 kvm_inject_gp(vcpu, 0);
4475                 return 1;
4476         }
4477
4478         if (vmx->nested.vmxon)
4479                 return nested_vmx_failValid(vcpu,
4480                         VMXERR_VMXON_IN_VMX_ROOT_OPERATION);
4481
4482         if ((vmx->msr_ia32_feature_control & VMXON_NEEDED_FEATURES)
4483                         != VMXON_NEEDED_FEATURES) {
4484                 kvm_inject_gp(vcpu, 0);
4485                 return 1;
4486         }
4487
4488         if (nested_vmx_get_vmptr(vcpu, &vmptr))
4489                 return 1;
4490
4491         /*
4492          * SDM 3: 24.11.5
4493          * The first 4 bytes of VMXON region contain the supported
4494          * VMCS revision identifier
4495          *
4496          * Note - IA32_VMX_BASIC[48] will never be 1 for the nested case;
4497          * which replaces physical address width with 32
4498          */
4499         if (!page_address_valid(vcpu, vmptr))
4500                 return nested_vmx_failInvalid(vcpu);
4501
4502         if (kvm_read_guest(vcpu->kvm, vmptr, &revision, sizeof(revision)) ||
4503             revision != VMCS12_REVISION)
4504                 return nested_vmx_failInvalid(vcpu);
4505
4506         vmx->nested.vmxon_ptr = vmptr;
4507         ret = enter_vmx_operation(vcpu);
4508         if (ret)
4509                 return ret;
4510
4511         return nested_vmx_succeed(vcpu);
4512 }
4513
4514 static inline void nested_release_vmcs12(struct kvm_vcpu *vcpu)
4515 {
4516         struct vcpu_vmx *vmx = to_vmx(vcpu);
4517
4518         if (vmx->nested.current_vmptr == -1ull)
4519                 return;
4520
4521         copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu));
4522
4523         if (enable_shadow_vmcs) {
4524                 /* copy to memory all shadowed fields in case
4525                    they were modified */
4526                 copy_shadow_to_vmcs12(vmx);
4527                 vmx_disable_shadow_vmcs(vmx);
4528         }
4529         vmx->nested.posted_intr_nv = -1;
4530
4531         /* Flush VMCS12 to guest memory */
4532         kvm_vcpu_write_guest_page(vcpu,
4533                                   vmx->nested.current_vmptr >> PAGE_SHIFT,
4534                                   vmx->nested.cached_vmcs12, 0, VMCS12_SIZE);
4535
4536         kvm_mmu_free_roots(vcpu, &vcpu->arch.guest_mmu, KVM_MMU_ROOTS_ALL);
4537
4538         vmx->nested.current_vmptr = -1ull;
4539 }
4540
4541 /* Emulate the VMXOFF instruction */
4542 static int handle_vmoff(struct kvm_vcpu *vcpu)
4543 {
4544         if (!nested_vmx_check_permission(vcpu))
4545                 return 1;
4546
4547         free_nested(vcpu);
4548
4549         /* Process a latched INIT during time CPU was in VMX operation */
4550         kvm_make_request(KVM_REQ_EVENT, vcpu);
4551
4552         return nested_vmx_succeed(vcpu);
4553 }
4554
4555 /* Emulate the VMCLEAR instruction */
4556 static int handle_vmclear(struct kvm_vcpu *vcpu)
4557 {
4558         struct vcpu_vmx *vmx = to_vmx(vcpu);
4559         u32 zero = 0;
4560         gpa_t vmptr;
4561         u64 evmcs_gpa;
4562
4563         if (!nested_vmx_check_permission(vcpu))
4564                 return 1;
4565
4566         if (nested_vmx_get_vmptr(vcpu, &vmptr))
4567                 return 1;
4568
4569         if (!page_address_valid(vcpu, vmptr))
4570                 return nested_vmx_failValid(vcpu,
4571                         VMXERR_VMCLEAR_INVALID_ADDRESS);
4572
4573         if (vmptr == vmx->nested.vmxon_ptr)
4574                 return nested_vmx_failValid(vcpu,
4575                         VMXERR_VMCLEAR_VMXON_POINTER);
4576
4577         /*
4578          * When Enlightened VMEntry is enabled on the calling CPU we treat
4579          * memory area pointer by vmptr as Enlightened VMCS (as there's no good
4580          * way to distinguish it from VMCS12) and we must not corrupt it by
4581          * writing to the non-existent 'launch_state' field. The area doesn't
4582          * have to be the currently active EVMCS on the calling CPU and there's
4583          * nothing KVM has to do to transition it from 'active' to 'non-active'
4584          * state. It is possible that the area will stay mapped as
4585          * vmx->nested.hv_evmcs but this shouldn't be a problem.
4586          */
4587         if (likely(!vmx->nested.enlightened_vmcs_enabled ||
4588                    !nested_enlightened_vmentry(vcpu, &evmcs_gpa))) {
4589                 if (vmptr == vmx->nested.current_vmptr)
4590                         nested_release_vmcs12(vcpu);
4591
4592                 kvm_vcpu_write_guest(vcpu,
4593                                      vmptr + offsetof(struct vmcs12,
4594                                                       launch_state),
4595                                      &zero, sizeof(zero));
4596         }
4597
4598         return nested_vmx_succeed(vcpu);
4599 }
4600
4601 static int nested_vmx_run(struct kvm_vcpu *vcpu, bool launch);
4602
4603 /* Emulate the VMLAUNCH instruction */
4604 static int handle_vmlaunch(struct kvm_vcpu *vcpu)
4605 {
4606         return nested_vmx_run(vcpu, true);
4607 }
4608
4609 /* Emulate the VMRESUME instruction */
4610 static int handle_vmresume(struct kvm_vcpu *vcpu)
4611 {
4612
4613         return nested_vmx_run(vcpu, false);
4614 }
4615
4616 static int handle_vmread(struct kvm_vcpu *vcpu)
4617 {
4618         unsigned long field;
4619         u64 field_value;
4620         struct vcpu_vmx *vmx = to_vmx(vcpu);
4621         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
4622         u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
4623         int len;
4624         gva_t gva = 0;
4625         struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu)
4626                                                     : get_vmcs12(vcpu);
4627         struct x86_exception e;
4628         short offset;
4629
4630         if (!nested_vmx_check_permission(vcpu))
4631                 return 1;
4632
4633         /*
4634          * In VMX non-root operation, when the VMCS-link pointer is -1ull,
4635          * any VMREAD sets the ALU flags for VMfailInvalid.
4636          */
4637         if (vmx->nested.current_vmptr == -1ull ||
4638             (is_guest_mode(vcpu) &&
4639              get_vmcs12(vcpu)->vmcs_link_pointer == -1ull))
4640                 return nested_vmx_failInvalid(vcpu);
4641
4642         /* Decode instruction info and find the field to read */
4643         field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
4644
4645         offset = vmcs_field_to_offset(field);
4646         if (offset < 0)
4647                 return nested_vmx_failValid(vcpu,
4648                         VMXERR_UNSUPPORTED_VMCS_COMPONENT);
4649
4650         if (!is_guest_mode(vcpu) && is_vmcs12_ext_field(field))
4651                 copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
4652
4653         /* Read the field, zero-extended to a u64 field_value */
4654         field_value = vmcs12_read_any(vmcs12, field, offset);
4655
4656         /*
4657          * Now copy part of this value to register or memory, as requested.
4658          * Note that the number of bits actually copied is 32 or 64 depending
4659          * on the guest's mode (32 or 64 bit), not on the given field's length.
4660          */
4661         if (vmx_instruction_info & (1u << 10)) {
4662                 kvm_register_writel(vcpu, (((vmx_instruction_info) >> 3) & 0xf),
4663                         field_value);
4664         } else {
4665                 len = is_64_bit_mode(vcpu) ? 8 : 4;
4666                 if (get_vmx_mem_address(vcpu, exit_qualification,
4667                                 vmx_instruction_info, true, len, &gva))
4668                         return 1;
4669                 /* _system ok, nested_vmx_check_permission has verified cpl=0 */
4670                 if (kvm_write_guest_virt_system(vcpu, gva, &field_value, len, &e)) {
4671                         kvm_inject_page_fault(vcpu, &e);
4672                         return 1;
4673                 }
4674         }
4675
4676         return nested_vmx_succeed(vcpu);
4677 }
4678
4679 static bool is_shadow_field_rw(unsigned long field)
4680 {
4681         switch (field) {
4682 #define SHADOW_FIELD_RW(x, y) case x:
4683 #include "vmcs_shadow_fields.h"
4684                 return true;
4685         default:
4686                 break;
4687         }
4688         return false;
4689 }
4690
4691 static bool is_shadow_field_ro(unsigned long field)
4692 {
4693         switch (field) {
4694 #define SHADOW_FIELD_RO(x, y) case x:
4695 #include "vmcs_shadow_fields.h"
4696                 return true;
4697         default:
4698                 break;
4699         }
4700         return false;
4701 }
4702
4703 static int handle_vmwrite(struct kvm_vcpu *vcpu)
4704 {
4705         unsigned long field;
4706         int len;
4707         gva_t gva;
4708         struct vcpu_vmx *vmx = to_vmx(vcpu);
4709         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
4710         u32 vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
4711
4712         /* The value to write might be 32 or 64 bits, depending on L1's long
4713          * mode, and eventually we need to write that into a field of several
4714          * possible lengths. The code below first zero-extends the value to 64
4715          * bit (field_value), and then copies only the appropriate number of
4716          * bits into the vmcs12 field.
4717          */
4718         u64 field_value = 0;
4719         struct x86_exception e;
4720         struct vmcs12 *vmcs12 = is_guest_mode(vcpu) ? get_shadow_vmcs12(vcpu)
4721                                                     : get_vmcs12(vcpu);
4722         short offset;
4723
4724         if (!nested_vmx_check_permission(vcpu))
4725                 return 1;
4726
4727         /*
4728          * In VMX non-root operation, when the VMCS-link pointer is -1ull,
4729          * any VMWRITE sets the ALU flags for VMfailInvalid.
4730          */
4731         if (vmx->nested.current_vmptr == -1ull ||
4732             (is_guest_mode(vcpu) &&
4733              get_vmcs12(vcpu)->vmcs_link_pointer == -1ull))
4734                 return nested_vmx_failInvalid(vcpu);
4735
4736         if (vmx_instruction_info & (1u << 10))
4737                 field_value = kvm_register_readl(vcpu,
4738                         (((vmx_instruction_info) >> 3) & 0xf));
4739         else {
4740                 len = is_64_bit_mode(vcpu) ? 8 : 4;
4741                 if (get_vmx_mem_address(vcpu, exit_qualification,
4742                                 vmx_instruction_info, false, len, &gva))
4743                         return 1;
4744                 if (kvm_read_guest_virt(vcpu, gva, &field_value, len, &e)) {
4745                         kvm_inject_page_fault(vcpu, &e);
4746                         return 1;
4747                 }
4748         }
4749
4750
4751         field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
4752
4753         offset = vmcs_field_to_offset(field);
4754         if (offset < 0)
4755                 return nested_vmx_failValid(vcpu,
4756                         VMXERR_UNSUPPORTED_VMCS_COMPONENT);
4757
4758         /*
4759          * If the vCPU supports "VMWRITE to any supported field in the
4760          * VMCS," then the "read-only" fields are actually read/write.
4761          */
4762         if (vmcs_field_readonly(field) &&
4763             !nested_cpu_has_vmwrite_any_field(vcpu))
4764                 return nested_vmx_failValid(vcpu,
4765                         VMXERR_VMWRITE_READ_ONLY_VMCS_COMPONENT);
4766
4767         /*
4768          * Ensure vmcs12 is up-to-date before any VMWRITE that dirties
4769          * vmcs12, else we may crush a field or consume a stale value.
4770          */
4771         if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field))
4772                 copy_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
4773
4774         /*
4775          * Some Intel CPUs intentionally drop the reserved bits of the AR byte
4776          * fields on VMWRITE.  Emulate this behavior to ensure consistent KVM
4777          * behavior regardless of the underlying hardware, e.g. if an AR_BYTE
4778          * field is intercepted for VMWRITE but not VMREAD (in L1), then VMREAD
4779          * from L1 will return a different value than VMREAD from L2 (L1 sees
4780          * the stripped down value, L2 sees the full value as stored by KVM).
4781          */
4782         if (field >= GUEST_ES_AR_BYTES && field <= GUEST_TR_AR_BYTES)
4783                 field_value &= 0x1f0ff;
4784
4785         vmcs12_write_any(vmcs12, field, offset, field_value);
4786
4787         /*
4788          * Do not track vmcs12 dirty-state if in guest-mode as we actually
4789          * dirty shadow vmcs12 instead of vmcs12.  Fields that can be updated
4790          * by L1 without a vmexit are always updated in the vmcs02, i.e. don't
4791          * "dirty" vmcs12, all others go down the prepare_vmcs02() slow path.
4792          */
4793         if (!is_guest_mode(vcpu) && !is_shadow_field_rw(field)) {
4794                 /*
4795                  * L1 can read these fields without exiting, ensure the
4796                  * shadow VMCS is up-to-date.
4797                  */
4798                 if (enable_shadow_vmcs && is_shadow_field_ro(field)) {
4799                         preempt_disable();
4800                         vmcs_load(vmx->vmcs01.shadow_vmcs);
4801
4802                         __vmcs_writel(field, field_value);
4803
4804                         vmcs_clear(vmx->vmcs01.shadow_vmcs);
4805                         vmcs_load(vmx->loaded_vmcs->vmcs);
4806                         preempt_enable();
4807                 }
4808                 vmx->nested.dirty_vmcs12 = true;
4809         }
4810
4811         return nested_vmx_succeed(vcpu);
4812 }
4813
4814 static void set_current_vmptr(struct vcpu_vmx *vmx, gpa_t vmptr)
4815 {
4816         vmx->nested.current_vmptr = vmptr;
4817         if (enable_shadow_vmcs) {
4818                 secondary_exec_controls_setbit(vmx, SECONDARY_EXEC_SHADOW_VMCS);
4819                 vmcs_write64(VMCS_LINK_POINTER,
4820                              __pa(vmx->vmcs01.shadow_vmcs));
4821                 vmx->nested.need_vmcs12_to_shadow_sync = true;
4822         }
4823         vmx->nested.dirty_vmcs12 = true;
4824 }
4825
4826 /* Emulate the VMPTRLD instruction */
4827 static int handle_vmptrld(struct kvm_vcpu *vcpu)
4828 {
4829         struct vcpu_vmx *vmx = to_vmx(vcpu);
4830         gpa_t vmptr;
4831
4832         if (!nested_vmx_check_permission(vcpu))
4833                 return 1;
4834
4835         if (nested_vmx_get_vmptr(vcpu, &vmptr))
4836                 return 1;
4837
4838         if (!page_address_valid(vcpu, vmptr))
4839                 return nested_vmx_failValid(vcpu,
4840                         VMXERR_VMPTRLD_INVALID_ADDRESS);
4841
4842         if (vmptr == vmx->nested.vmxon_ptr)
4843                 return nested_vmx_failValid(vcpu,
4844                         VMXERR_VMPTRLD_VMXON_POINTER);
4845
4846         /* Forbid normal VMPTRLD if Enlightened version was used */
4847         if (vmx->nested.hv_evmcs)
4848                 return 1;
4849
4850         if (vmx->nested.current_vmptr != vmptr) {
4851                 struct kvm_host_map map;
4852                 struct vmcs12 *new_vmcs12;
4853
4854                 if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmptr), &map)) {
4855                         /*
4856                          * Reads from an unbacked page return all 1s,
4857                          * which means that the 32 bits located at the
4858                          * given physical address won't match the required
4859                          * VMCS12_REVISION identifier.
4860                          */
4861                         return nested_vmx_failValid(vcpu,
4862                                 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
4863                 }
4864
4865                 new_vmcs12 = map.hva;
4866
4867                 if (new_vmcs12->hdr.revision_id != VMCS12_REVISION ||
4868                     (new_vmcs12->hdr.shadow_vmcs &&
4869                      !nested_cpu_has_vmx_shadow_vmcs(vcpu))) {
4870                         kvm_vcpu_unmap(vcpu, &map, false);
4871                         return nested_vmx_failValid(vcpu,
4872                                 VMXERR_VMPTRLD_INCORRECT_VMCS_REVISION_ID);
4873                 }
4874
4875                 nested_release_vmcs12(vcpu);
4876
4877                 /*
4878                  * Load VMCS12 from guest memory since it is not already
4879                  * cached.
4880                  */
4881                 memcpy(vmx->nested.cached_vmcs12, new_vmcs12, VMCS12_SIZE);
4882                 kvm_vcpu_unmap(vcpu, &map, false);
4883
4884                 set_current_vmptr(vmx, vmptr);
4885         }
4886
4887         return nested_vmx_succeed(vcpu);
4888 }
4889
4890 /* Emulate the VMPTRST instruction */
4891 static int handle_vmptrst(struct kvm_vcpu *vcpu)
4892 {
4893         unsigned long exit_qual = vmcs_readl(EXIT_QUALIFICATION);
4894         u32 instr_info = vmcs_read32(VMX_INSTRUCTION_INFO);
4895         gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr;
4896         struct x86_exception e;
4897         gva_t gva;
4898
4899         if (!nested_vmx_check_permission(vcpu))
4900                 return 1;
4901
4902         if (unlikely(to_vmx(vcpu)->nested.hv_evmcs))
4903                 return 1;
4904
4905         if (get_vmx_mem_address(vcpu, exit_qual, instr_info,
4906                                 true, sizeof(gpa_t), &gva))
4907                 return 1;
4908         /* *_system ok, nested_vmx_check_permission has verified cpl=0 */
4909         if (kvm_write_guest_virt_system(vcpu, gva, (void *)&current_vmptr,
4910                                         sizeof(gpa_t), &e)) {
4911                 kvm_inject_page_fault(vcpu, &e);
4912                 return 1;
4913         }
4914         return nested_vmx_succeed(vcpu);
4915 }
4916
4917 /* Emulate the INVEPT instruction */
4918 static int handle_invept(struct kvm_vcpu *vcpu)
4919 {
4920         struct vcpu_vmx *vmx = to_vmx(vcpu);
4921         u32 vmx_instruction_info, types;
4922         unsigned long type;
4923         gva_t gva;
4924         struct x86_exception e;
4925         struct {
4926                 u64 eptp, gpa;
4927         } operand;
4928
4929         if (!(vmx->nested.msrs.secondary_ctls_high &
4930               SECONDARY_EXEC_ENABLE_EPT) ||
4931             !(vmx->nested.msrs.ept_caps & VMX_EPT_INVEPT_BIT)) {
4932                 kvm_queue_exception(vcpu, UD_VECTOR);
4933                 return 1;
4934         }
4935
4936         if (!nested_vmx_check_permission(vcpu))
4937                 return 1;
4938
4939         vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
4940         type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
4941
4942         types = (vmx->nested.msrs.ept_caps >> VMX_EPT_EXTENT_SHIFT) & 6;
4943
4944         if (type >= 32 || !(types & (1 << type)))
4945                 return nested_vmx_failValid(vcpu,
4946                                 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
4947
4948         /* According to the Intel VMX instruction reference, the memory
4949          * operand is read even if it isn't needed (e.g., for type==global)
4950          */
4951         if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
4952                         vmx_instruction_info, false, sizeof(operand), &gva))
4953                 return 1;
4954         if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) {
4955                 kvm_inject_page_fault(vcpu, &e);
4956                 return 1;
4957         }
4958
4959         switch (type) {
4960         case VMX_EPT_EXTENT_GLOBAL:
4961         case VMX_EPT_EXTENT_CONTEXT:
4962         /*
4963          * TODO: Sync the necessary shadow EPT roots here, rather than
4964          * at the next emulated VM-entry.
4965          */
4966                 break;
4967         default:
4968                 BUG_ON(1);
4969                 break;
4970         }
4971
4972         return nested_vmx_succeed(vcpu);
4973 }
4974
4975 static int handle_invvpid(struct kvm_vcpu *vcpu)
4976 {
4977         struct vcpu_vmx *vmx = to_vmx(vcpu);
4978         u32 vmx_instruction_info;
4979         unsigned long type, types;
4980         gva_t gva;
4981         struct x86_exception e;
4982         struct {
4983                 u64 vpid;
4984                 u64 gla;
4985         } operand;
4986         u16 vpid02;
4987
4988         if (!(vmx->nested.msrs.secondary_ctls_high &
4989               SECONDARY_EXEC_ENABLE_VPID) ||
4990                         !(vmx->nested.msrs.vpid_caps & VMX_VPID_INVVPID_BIT)) {
4991                 kvm_queue_exception(vcpu, UD_VECTOR);
4992                 return 1;
4993         }
4994
4995         if (!nested_vmx_check_permission(vcpu))
4996                 return 1;
4997
4998         vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
4999         type = kvm_register_readl(vcpu, (vmx_instruction_info >> 28) & 0xf);
5000
5001         types = (vmx->nested.msrs.vpid_caps &
5002                         VMX_VPID_EXTENT_SUPPORTED_MASK) >> 8;
5003
5004         if (type >= 32 || !(types & (1 << type)))
5005                 return nested_vmx_failValid(vcpu,
5006                         VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
5007
5008         /* according to the intel vmx instruction reference, the memory
5009          * operand is read even if it isn't needed (e.g., for type==global)
5010          */
5011         if (get_vmx_mem_address(vcpu, vmcs_readl(EXIT_QUALIFICATION),
5012                         vmx_instruction_info, false, sizeof(operand), &gva))
5013                 return 1;
5014         if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) {
5015                 kvm_inject_page_fault(vcpu, &e);
5016                 return 1;
5017         }
5018         if (operand.vpid >> 16)
5019                 return nested_vmx_failValid(vcpu,
5020                         VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
5021
5022         vpid02 = nested_get_vpid02(vcpu);
5023         switch (type) {
5024         case VMX_VPID_EXTENT_INDIVIDUAL_ADDR:
5025                 if (!operand.vpid ||
5026                     is_noncanonical_address(operand.gla, vcpu))
5027                         return nested_vmx_failValid(vcpu,
5028                                 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
5029                 if (cpu_has_vmx_invvpid_individual_addr()) {
5030                         __invvpid(VMX_VPID_EXTENT_INDIVIDUAL_ADDR,
5031                                 vpid02, operand.gla);
5032                 } else
5033                         __vmx_flush_tlb(vcpu, vpid02, false);
5034                 break;
5035         case VMX_VPID_EXTENT_SINGLE_CONTEXT:
5036         case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL:
5037                 if (!operand.vpid)
5038                         return nested_vmx_failValid(vcpu,
5039                                 VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID);
5040                 __vmx_flush_tlb(vcpu, vpid02, false);
5041                 break;
5042         case VMX_VPID_EXTENT_ALL_CONTEXT:
5043                 __vmx_flush_tlb(vcpu, vpid02, false);
5044                 break;
5045         default:
5046                 WARN_ON_ONCE(1);
5047                 return kvm_skip_emulated_instruction(vcpu);
5048         }
5049
5050         return nested_vmx_succeed(vcpu);
5051 }
5052
5053 static int nested_vmx_eptp_switching(struct kvm_vcpu *vcpu,
5054                                      struct vmcs12 *vmcs12)
5055 {
5056         u32 index = kvm_rcx_read(vcpu);
5057         u64 address;
5058         bool accessed_dirty;
5059         struct kvm_mmu *mmu = vcpu->arch.walk_mmu;
5060
5061         if (!nested_cpu_has_eptp_switching(vmcs12) ||
5062             !nested_cpu_has_ept(vmcs12))
5063                 return 1;
5064
5065         if (index >= VMFUNC_EPTP_ENTRIES)
5066                 return 1;
5067
5068
5069         if (kvm_vcpu_read_guest_page(vcpu, vmcs12->eptp_list_address >> PAGE_SHIFT,
5070                                      &address, index * 8, 8))
5071                 return 1;
5072
5073         accessed_dirty = !!(address & VMX_EPTP_AD_ENABLE_BIT);
5074
5075         /*
5076          * If the (L2) guest does a vmfunc to the currently
5077          * active ept pointer, we don't have to do anything else
5078          */
5079         if (vmcs12->ept_pointer != address) {
5080                 if (!valid_ept_address(vcpu, address))
5081                         return 1;
5082
5083                 kvm_mmu_unload(vcpu);
5084                 mmu->ept_ad = accessed_dirty;
5085                 mmu->mmu_role.base.ad_disabled = !accessed_dirty;
5086                 vmcs12->ept_pointer = address;
5087                 /*
5088                  * TODO: Check what's the correct approach in case
5089                  * mmu reload fails. Currently, we just let the next
5090                  * reload potentially fail
5091                  */
5092                 kvm_mmu_reload(vcpu);
5093         }
5094
5095         return 0;
5096 }
5097
5098 static int handle_vmfunc(struct kvm_vcpu *vcpu)
5099 {
5100         struct vcpu_vmx *vmx = to_vmx(vcpu);
5101         struct vmcs12 *vmcs12;
5102         u32 function = kvm_rax_read(vcpu);
5103
5104         /*
5105          * VMFUNC is only supported for nested guests, but we always enable the
5106          * secondary control for simplicity; for non-nested mode, fake that we
5107          * didn't by injecting #UD.
5108          */
5109         if (!is_guest_mode(vcpu)) {
5110                 kvm_queue_exception(vcpu, UD_VECTOR);
5111                 return 1;
5112         }
5113
5114         vmcs12 = get_vmcs12(vcpu);
5115         if (!(vmcs12->vm_function_control & BIT_ULL(function)))
5116                 goto fail;
5117
5118         switch (function) {
5119         case 0:
5120                 if (nested_vmx_eptp_switching(vcpu, vmcs12))
5121                         goto fail;
5122                 break;
5123         default:
5124                 goto fail;
5125         }
5126         return kvm_skip_emulated_instruction(vcpu);
5127
5128 fail:
5129         nested_vmx_vmexit(vcpu, vmx->exit_reason,
5130                           vmcs_read32(VM_EXIT_INTR_INFO),
5131                           vmcs_readl(EXIT_QUALIFICATION));
5132         return 1;
5133 }
5134
5135 /*
5136  * Return true if an IO instruction with the specified port and size should cause
5137  * a VM-exit into L1.
5138  */
5139 bool nested_vmx_check_io_bitmaps(struct kvm_vcpu *vcpu, unsigned int port,
5140                                  int size)
5141 {
5142         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
5143         gpa_t bitmap, last_bitmap;
5144         u8 b;
5145
5146         last_bitmap = (gpa_t)-1;
5147         b = -1;
5148
5149         while (size > 0) {
5150                 if (port < 0x8000)
5151                         bitmap = vmcs12->io_bitmap_a;
5152                 else if (port < 0x10000)
5153                         bitmap = vmcs12->io_bitmap_b;
5154                 else
5155                         return true;
5156                 bitmap += (port & 0x7fff) / 8;
5157
5158                 if (last_bitmap != bitmap)
5159                         if (kvm_vcpu_read_guest(vcpu, bitmap, &b, 1))
5160                                 return true;
5161                 if (b & (1 << (port & 7)))
5162                         return true;
5163
5164                 port++;
5165                 size--;
5166                 last_bitmap = bitmap;
5167         }
5168
5169         return false;
5170 }
5171
5172 static bool nested_vmx_exit_handled_io(struct kvm_vcpu *vcpu,
5173                                        struct vmcs12 *vmcs12)
5174 {
5175         unsigned long exit_qualification;
5176         unsigned short port;
5177         int size;
5178
5179         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_IO_BITMAPS))
5180                 return nested_cpu_has(vmcs12, CPU_BASED_UNCOND_IO_EXITING);
5181
5182         exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5183
5184         port = exit_qualification >> 16;
5185         size = (exit_qualification & 7) + 1;
5186
5187         return nested_vmx_check_io_bitmaps(vcpu, port, size);
5188 }
5189
5190 /*
5191  * Return 1 if we should exit from L2 to L1 to handle an MSR access access,
5192  * rather than handle it ourselves in L0. I.e., check whether L1 expressed
5193  * disinterest in the current event (read or write a specific MSR) by using an
5194  * MSR bitmap. This may be the case even when L0 doesn't use MSR bitmaps.
5195  */
5196 static bool nested_vmx_exit_handled_msr(struct kvm_vcpu *vcpu,
5197         struct vmcs12 *vmcs12, u32 exit_reason)
5198 {
5199         u32 msr_index = kvm_rcx_read(vcpu);
5200         gpa_t bitmap;
5201
5202         if (!nested_cpu_has(vmcs12, CPU_BASED_USE_MSR_BITMAPS))
5203                 return true;
5204
5205         /*
5206          * The MSR_BITMAP page is divided into four 1024-byte bitmaps,
5207          * for the four combinations of read/write and low/high MSR numbers.
5208          * First we need to figure out which of the four to use:
5209          */
5210         bitmap = vmcs12->msr_bitmap;
5211         if (exit_reason == EXIT_REASON_MSR_WRITE)
5212                 bitmap += 2048;
5213         if (msr_index >= 0xc0000000) {
5214                 msr_index -= 0xc0000000;
5215                 bitmap += 1024;
5216         }
5217
5218         /* Then read the msr_index'th bit from this bitmap: */
5219         if (msr_index < 1024*8) {
5220                 unsigned char b;
5221                 if (kvm_vcpu_read_guest(vcpu, bitmap + msr_index/8, &b, 1))
5222                         return true;
5223                 return 1 & (b >> (msr_index & 7));
5224         } else
5225                 return true; /* let L1 handle the wrong parameter */
5226 }
5227
5228 /*
5229  * Return 1 if we should exit from L2 to L1 to handle a CR access exit,
5230  * rather than handle it ourselves in L0. I.e., check if L1 wanted to
5231  * intercept (via guest_host_mask etc.) the current event.
5232  */
5233 static bool nested_vmx_exit_handled_cr(struct kvm_vcpu *vcpu,
5234         struct vmcs12 *vmcs12)
5235 {
5236         unsigned long exit_qualification = vmcs_readl(EXIT_QUALIFICATION);
5237         int cr = exit_qualification & 15;
5238         int reg;
5239         unsigned long val;
5240
5241         switch ((exit_qualification >> 4) & 3) {
5242         case 0: /* mov to cr */
5243                 reg = (exit_qualification >> 8) & 15;
5244                 val = kvm_register_readl(vcpu, reg);
5245                 switch (cr) {
5246                 case 0:
5247                         if (vmcs12->cr0_guest_host_mask &
5248                             (val ^ vmcs12->cr0_read_shadow))
5249                                 return true;
5250                         break;
5251                 case 3:
5252                         if ((vmcs12->cr3_target_count >= 1 &&
5253                                         vmcs12->cr3_target_value0 == val) ||
5254                                 (vmcs12->cr3_target_count >= 2 &&
5255                                         vmcs12->cr3_target_value1 == val) ||
5256                                 (vmcs12->cr3_target_count >= 3 &&
5257                                         vmcs12->cr3_target_value2 == val) ||
5258                                 (vmcs12->cr3_target_count >= 4 &&
5259                                         vmcs12->cr3_target_value3 == val))
5260                                 return false;
5261                         if (nested_cpu_has(vmcs12, CPU_BASED_CR3_LOAD_EXITING))
5262                                 return true;
5263                         break;
5264                 case 4:
5265                         if (vmcs12->cr4_guest_host_mask &
5266                             (vmcs12->cr4_read_shadow ^ val))
5267                                 return true;
5268                         break;
5269                 case 8:
5270                         if (nested_cpu_has(vmcs12, CPU_BASED_CR8_LOAD_EXITING))
5271                                 return true;
5272                         break;
5273                 }
5274                 break;
5275         case 2: /* clts */
5276                 if ((vmcs12->cr0_guest_host_mask & X86_CR0_TS) &&
5277                     (vmcs12->cr0_read_shadow & X86_CR0_TS))
5278                         return true;
5279                 break;
5280         case 1: /* mov from cr */
5281                 switch (cr) {
5282                 case 3:
5283                         if (vmcs12->cpu_based_vm_exec_control &
5284                             CPU_BASED_CR3_STORE_EXITING)
5285                                 return true;
5286                         break;
5287                 case 8:
5288                         if (vmcs12->cpu_based_vm_exec_control &
5289                             CPU_BASED_CR8_STORE_EXITING)
5290                                 return true;
5291                         break;
5292                 }
5293                 break;
5294         case 3: /* lmsw */
5295                 /*
5296                  * lmsw can change bits 1..3 of cr0, and only set bit 0 of
5297                  * cr0. Other attempted changes are ignored, with no exit.
5298                  */
5299                 val = (exit_qualification >> LMSW_SOURCE_DATA_SHIFT) & 0x0f;
5300                 if (vmcs12->cr0_guest_host_mask & 0xe &
5301                     (val ^ vmcs12->cr0_read_shadow))
5302                         return true;
5303                 if ((vmcs12->cr0_guest_host_mask & 0x1) &&
5304                     !(vmcs12->cr0_read_shadow & 0x1) &&
5305                     (val & 0x1))
5306                         return true;
5307                 break;
5308         }
5309         return false;
5310 }
5311
5312 static bool nested_vmx_exit_handled_vmcs_access(struct kvm_vcpu *vcpu,
5313         struct vmcs12 *vmcs12, gpa_t bitmap)
5314 {
5315         u32 vmx_instruction_info;
5316         unsigned long field;
5317         u8 b;
5318
5319         if (!nested_cpu_has_shadow_vmcs(vmcs12))
5320                 return true;
5321
5322         /* Decode instruction info and find the field to access */
5323         vmx_instruction_info = vmcs_read32(VMX_INSTRUCTION_INFO);
5324         field = kvm_register_readl(vcpu, (((vmx_instruction_info) >> 28) & 0xf));
5325
5326         /* Out-of-range fields always cause a VM exit from L2 to L1 */
5327         if (field >> 15)
5328                 return true;
5329
5330         if (kvm_vcpu_read_guest(vcpu, bitmap + field/8, &b, 1))
5331                 return true;
5332
5333         return 1 & (b >> (field & 7));
5334 }
5335
5336 /*
5337  * Return 1 if we should exit from L2 to L1 to handle an exit, or 0 if we
5338  * should handle it ourselves in L0 (and then continue L2). Only call this
5339  * when in is_guest_mode (L2).
5340  */
5341 bool nested_vmx_exit_reflected(struct kvm_vcpu *vcpu, u32 exit_reason)
5342 {
5343         u32 intr_info = vmcs_read32(VM_EXIT_INTR_INFO);
5344         struct vcpu_vmx *vmx = to_vmx(vcpu);
5345         struct vmcs12 *vmcs12 = get_vmcs12(vcpu);
5346
5347         if (vmx->nested.nested_run_pending)
5348                 return false;
5349
5350         if (unlikely(vmx->fail)) {
5351                 trace_kvm_nested_vmenter_failed(
5352                         "hardware VM-instruction error: ",
5353                         vmcs_read32(VM_INSTRUCTION_ERROR));
5354                 return true;
5355         }
5356
5357         /*
5358          * The host physical addresses of some pages of guest memory
5359          * are loaded into the vmcs02 (e.g. vmcs12's Virtual APIC
5360          * Page). The CPU may write to these pages via their host
5361          * physical address while L2 is running, bypassing any
5362          * address-translation-based dirty tracking (e.g. EPT write
5363          * protection).
5364          *
5365          * Mark them dirty on every exit from L2 to prevent them from
5366          * getting out of sync with dirty tracking.
5367          */
5368         nested_mark_vmcs12_pages_dirty(vcpu);
5369
5370         trace_kvm_nested_vmexit(kvm_rip_read(vcpu), exit_reason,
5371                                 vmcs_readl(EXIT_QUALIFICATION),
5372                                 vmx->idt_vectoring_info,
5373                                 intr_info,
5374                                 vmcs_read32(VM_EXIT_INTR_ERROR_CODE),
5375                                 KVM_ISA_VMX);
5376
5377         switch ((u16)exit_reason) {
5378         case EXIT_REASON_EXCEPTION_NMI:
5379                 if (is_nmi(intr_info))
5380                         return false;
5381                 else if (is_page_fault(intr_info))
5382                         return !vmx->vcpu.arch.apf.host_apf_reason && enable_ept;
5383                 else if (is_debug(intr_info) &&
5384                          vcpu->guest_debug &
5385                          (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP))
5386                         return false;
5387                 else if (is_breakpoint(intr_info) &&
5388                          vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
5389                         return false;
5390                 return vmcs12->exception_bitmap &
5391                                 (1u << (intr_info & INTR_INFO_VECTOR_MASK));
5392         case EXIT_REASON_EXTERNAL_INTERRUPT:
5393                 return false;
5394         case EXIT_REASON_TRIPLE_FAULT:
5395                 return true;
5396         case EXIT_REASON_INTERRUPT_WINDOW:
5397                 return nested_cpu_has(vmcs12, CPU_BASED_INTR_WINDOW_EXITING);
5398         case EXIT_REASON_NMI_WINDOW:
5399                 return nested_cpu_has(vmcs12, CPU_BASED_NMI_WINDOW_EXITING);
5400         case EXIT_REASON_TASK_SWITCH:
5401                 return true;
5402         case EXIT_REASON_CPUID:
5403                 return true;
5404         case EXIT_REASON_HLT:
5405                 return nested_cpu_has(vmcs12, CPU_BASED_HLT_EXITING);
5406         case EXIT_REASON_INVD:
5407                 return true;
5408         case EXIT_REASON_INVLPG:
5409                 return nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
5410         case EXIT_REASON_RDPMC:
5411                 return nested_cpu_has(vmcs12, CPU_BASED_RDPMC_EXITING);
5412         case EXIT_REASON_RDRAND:
5413                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDRAND_EXITING);
5414         case EXIT_REASON_RDSEED:
5415                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_RDSEED_EXITING);
5416         case EXIT_REASON_RDTSC: case EXIT_REASON_RDTSCP:
5417                 return nested_cpu_has(vmcs12, CPU_BASED_RDTSC_EXITING);
5418         case EXIT_REASON_VMREAD:
5419                 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12,
5420                         vmcs12->vmread_bitmap);
5421         case EXIT_REASON_VMWRITE:
5422                 return nested_vmx_exit_handled_vmcs_access(vcpu, vmcs12,
5423                         vmcs12->vmwrite_bitmap);
5424         case EXIT_REASON_VMCALL: case EXIT_REASON_VMCLEAR:
5425         case EXIT_REASON_VMLAUNCH: case EXIT_REASON_VMPTRLD:
5426         case EXIT_REASON_VMPTRST: case EXIT_REASON_VMRESUME:
5427         case EXIT_REASON_VMOFF: case EXIT_REASON_VMON:
5428         case EXIT_REASON_INVEPT: case EXIT_REASON_INVVPID:
5429                 /*
5430                  * VMX instructions trap unconditionally. This allows L1 to
5431                  * emulate them for its L2 guest, i.e., allows 3-level nesting!
5432                  */
5433                 return true;
5434         case EXIT_REASON_CR_ACCESS:
5435                 return nested_vmx_exit_handled_cr(vcpu, vmcs12);
5436         case EXIT_REASON_DR_ACCESS:
5437                 return nested_cpu_has(vmcs12, CPU_BASED_MOV_DR_EXITING);
5438         case EXIT_REASON_IO_INSTRUCTION:
5439                 return nested_vmx_exit_handled_io(vcpu, vmcs12);
5440         case EXIT_REASON_GDTR_IDTR: case EXIT_REASON_LDTR_TR:
5441                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_DESC);
5442         case EXIT_REASON_MSR_READ:
5443         case EXIT_REASON_MSR_WRITE:
5444                 return nested_vmx_exit_handled_msr(vcpu, vmcs12, exit_reason);
5445         case EXIT_REASON_INVALID_STATE:
5446                 return true;
5447         case EXIT_REASON_MWAIT_INSTRUCTION:
5448                 return nested_cpu_has(vmcs12, CPU_BASED_MWAIT_EXITING);
5449         case EXIT_REASON_MONITOR_TRAP_FLAG:
5450                 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_TRAP_FLAG);
5451         case EXIT_REASON_MONITOR_INSTRUCTION:
5452                 return nested_cpu_has(vmcs12, CPU_BASED_MONITOR_EXITING);
5453         case EXIT_REASON_PAUSE_INSTRUCTION:
5454                 return nested_cpu_has(vmcs12, CPU_BASED_PAUSE_EXITING) ||
5455                         nested_cpu_has2(vmcs12,
5456                                 SECONDARY_EXEC_PAUSE_LOOP_EXITING);
5457         case EXIT_REASON_MCE_DURING_VMENTRY:
5458                 return false;
5459         case EXIT_REASON_TPR_BELOW_THRESHOLD:
5460                 return nested_cpu_has(vmcs12, CPU_BASED_TPR_SHADOW);
5461         case EXIT_REASON_APIC_ACCESS:
5462         case EXIT_REASON_APIC_WRITE:
5463         case EXIT_REASON_EOI_INDUCED:
5464                 /*
5465                  * The controls for "virtualize APIC accesses," "APIC-
5466                  * register virtualization," and "virtual-interrupt
5467                  * delivery" only come from vmcs12.
5468                  */
5469                 return true;
5470         case EXIT_REASON_EPT_VIOLATION:
5471                 /*
5472                  * L0 always deals with the EPT violation. If nested EPT is
5473                  * used, and the nested mmu code discovers that the address is
5474                  * missing in the guest EPT table (EPT12), the EPT violation
5475                  * will be injected with nested_ept_inject_page_fault()
5476                  */
5477                 return false;
5478         case EXIT_REASON_EPT_MISCONFIG:
5479                 /*
5480                  * L2 never uses directly L1's EPT, but rather L0's own EPT
5481                  * table (shadow on EPT) or a merged EPT table that L0 built
5482                  * (EPT on EPT). So any problems with the structure of the
5483                  * table is L0's fault.
5484                  */
5485                 return false;
5486         case EXIT_REASON_INVPCID:
5487                 return
5488                         nested_cpu_has2(vmcs12, SECONDARY_EXEC_ENABLE_INVPCID) &&
5489                         nested_cpu_has(vmcs12, CPU_BASED_INVLPG_EXITING);
5490         case EXIT_REASON_WBINVD:
5491                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_WBINVD_EXITING);
5492         case EXIT_REASON_XSETBV:
5493                 return true;
5494         case EXIT_REASON_XSAVES: case EXIT_REASON_XRSTORS:
5495                 /*
5496                  * This should never happen, since it is not possible to
5497                  * set XSS to a non-zero value---neither in L1 nor in L2.
5498                  * If if it were, XSS would have to be checked against
5499                  * the XSS exit bitmap in vmcs12.
5500                  */
5501                 return nested_cpu_has2(vmcs12, SECONDARY_EXEC_XSAVES);
5502         case EXIT_REASON_PREEMPTION_TIMER:
5503                 return false;
5504         case EXIT_REASON_PML_FULL:
5505                 /* We emulate PML support to L1. */
5506                 return false;
5507         case EXIT_REASON_VMFUNC:
5508                 /* VM functions are emulated through L2->L0 vmexits. */
5509                 return false;
5510         case EXIT_REASON_ENCLS:
5511                 /* SGX is never exposed to L1 */
5512                 return false;
5513         case EXIT_REASON_UMWAIT:
5514         case EXIT_REASON_TPAUSE:
5515                 return nested_cpu_has2(vmcs12,
5516                         SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE);
5517         default:
5518                 return true;
5519         }
5520 }
5521
5522
5523 static int vmx_get_nested_state(struct kvm_vcpu *vcpu,
5524                                 struct kvm_nested_state __user *user_kvm_nested_state,
5525                                 u32 user_data_size)
5526 {
5527         struct vcpu_vmx *vmx;
5528         struct vmcs12 *vmcs12;
5529         struct kvm_nested_state kvm_state = {
5530                 .flags = 0,
5531                 .format = KVM_STATE_NESTED_FORMAT_VMX,
5532                 .size = sizeof(kvm_state),
5533                 .hdr.vmx.vmxon_pa = -1ull,
5534                 .hdr.vmx.vmcs12_pa = -1ull,
5535         };
5536         struct kvm_vmx_nested_state_data __user *user_vmx_nested_state =
5537                 &user_kvm_nested_state->data.vmx[0];
5538
5539         if (!vcpu)
5540                 return kvm_state.size + sizeof(*user_vmx_nested_state);
5541
5542         vmx = to_vmx(vcpu);
5543         vmcs12 = get_vmcs12(vcpu);
5544
5545         if (nested_vmx_allowed(vcpu) &&
5546             (vmx->nested.vmxon || vmx->nested.smm.vmxon)) {
5547                 kvm_state.hdr.vmx.vmxon_pa = vmx->nested.vmxon_ptr;
5548                 kvm_state.hdr.vmx.vmcs12_pa = vmx->nested.current_vmptr;
5549
5550                 if (vmx_has_valid_vmcs12(vcpu)) {
5551                         kvm_state.size += sizeof(user_vmx_nested_state->vmcs12);
5552
5553                         if (vmx->nested.hv_evmcs)
5554                                 kvm_state.flags |= KVM_STATE_NESTED_EVMCS;
5555
5556                         if (is_guest_mode(vcpu) &&
5557                             nested_cpu_has_shadow_vmcs(vmcs12) &&
5558                             vmcs12->vmcs_link_pointer != -1ull)
5559                                 kvm_state.size += sizeof(user_vmx_nested_state->shadow_vmcs12);
5560                 }
5561
5562                 if (vmx->nested.smm.vmxon)
5563                         kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_VMXON;
5564
5565                 if (vmx->nested.smm.guest_mode)
5566                         kvm_state.hdr.vmx.smm.flags |= KVM_STATE_NESTED_SMM_GUEST_MODE;
5567
5568                 if (is_guest_mode(vcpu)) {
5569                         kvm_state.flags |= KVM_STATE_NESTED_GUEST_MODE;
5570
5571                         if (vmx->nested.nested_run_pending)
5572                                 kvm_state.flags |= KVM_STATE_NESTED_RUN_PENDING;
5573                 }
5574         }
5575
5576         if (user_data_size < kvm_state.size)
5577                 goto out;
5578
5579         if (copy_to_user(user_kvm_nested_state, &kvm_state, sizeof(kvm_state)))
5580                 return -EFAULT;
5581
5582         if (!vmx_has_valid_vmcs12(vcpu))
5583                 goto out;
5584
5585         /*
5586          * When running L2, the authoritative vmcs12 state is in the
5587          * vmcs02. When running L1, the authoritative vmcs12 state is
5588          * in the shadow or enlightened vmcs linked to vmcs01, unless
5589          * need_vmcs12_to_shadow_sync is set, in which case, the authoritative
5590          * vmcs12 state is in the vmcs12 already.
5591          */
5592         if (is_guest_mode(vcpu)) {
5593                 sync_vmcs02_to_vmcs12(vcpu, vmcs12);
5594                 sync_vmcs02_to_vmcs12_rare(vcpu, vmcs12);
5595         } else  {
5596                 copy_vmcs02_to_vmcs12_rare(vcpu, get_vmcs12(vcpu));
5597                 if (!vmx->nested.need_vmcs12_to_shadow_sync) {
5598                         if (vmx->nested.hv_evmcs)
5599                                 copy_enlightened_to_vmcs12(vmx);
5600                         else if (enable_shadow_vmcs)
5601                                 copy_shadow_to_vmcs12(vmx);
5602                 }
5603         }
5604
5605         BUILD_BUG_ON(sizeof(user_vmx_nested_state->vmcs12) < VMCS12_SIZE);
5606         BUILD_BUG_ON(sizeof(user_vmx_nested_state->shadow_vmcs12) < VMCS12_SIZE);
5607
5608         /*
5609          * Copy over the full allocated size of vmcs12 rather than just the size
5610          * of the struct.
5611          */
5612         if (copy_to_user(user_vmx_nested_state->vmcs12, vmcs12, VMCS12_SIZE))
5613                 return -EFAULT;
5614
5615         if (nested_cpu_has_shadow_vmcs(vmcs12) &&
5616             vmcs12->vmcs_link_pointer != -1ull) {
5617                 if (copy_to_user(user_vmx_nested_state->shadow_vmcs12,
5618                                  get_shadow_vmcs12(vcpu), VMCS12_SIZE))
5619                         return -EFAULT;
5620         }
5621
5622 out:
5623         return kvm_state.size;
5624 }
5625
5626 /*
5627  * Forcibly leave nested mode in order to be able to reset the VCPU later on.
5628  */
5629 void vmx_leave_nested(struct kvm_vcpu *vcpu)
5630 {
5631         if (is_guest_mode(vcpu)) {
5632                 to_vmx(vcpu)->nested.nested_run_pending = 0;
5633                 nested_vmx_vmexit(vcpu, -1, 0, 0);
5634         }
5635         free_nested(vcpu);
5636 }
5637
5638 static int vmx_set_nested_state(struct kvm_vcpu *vcpu,
5639                                 struct kvm_nested_state __user *user_kvm_nested_state,
5640                                 struct kvm_nested_state *kvm_state)
5641 {
5642         struct vcpu_vmx *vmx = to_vmx(vcpu);
5643         struct vmcs12 *vmcs12;
5644         u32 exit_qual;
5645         struct kvm_vmx_nested_state_data __user *user_vmx_nested_state =
5646                 &user_kvm_nested_state->data.vmx[0];
5647         int ret;
5648
5649         if (kvm_state->format != KVM_STATE_NESTED_FORMAT_VMX)
5650                 return -EINVAL;
5651
5652         if (kvm_state->hdr.vmx.vmxon_pa == -1ull) {
5653                 if (kvm_state->hdr.vmx.smm.flags)
5654                         return -EINVAL;
5655
5656                 if (kvm_state->hdr.vmx.vmcs12_pa != -1ull)
5657                         return -EINVAL;
5658
5659                 /*
5660                  * KVM_STATE_NESTED_EVMCS used to signal that KVM should
5661                  * enable eVMCS capability on vCPU. However, since then
5662                  * code was changed such that flag signals vmcs12 should
5663                  * be copied into eVMCS in guest memory.
5664                  *
5665                  * To preserve backwards compatability, allow user
5666                  * to set this flag even when there is no VMXON region.
5667                  */
5668                 if (kvm_state->flags & ~KVM_STATE_NESTED_EVMCS)
5669                         return -EINVAL;
5670         } else {
5671                 if (!nested_vmx_allowed(vcpu))
5672                         return -EINVAL;
5673
5674                 if (!page_address_valid(vcpu, kvm_state->hdr.vmx.vmxon_pa))
5675                         return -EINVAL;
5676         }
5677
5678         if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) &&
5679             (kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE))
5680                 return -EINVAL;
5681
5682         if (kvm_state->hdr.vmx.smm.flags &
5683             ~(KVM_STATE_NESTED_SMM_GUEST_MODE | KVM_STATE_NESTED_SMM_VMXON))
5684                 return -EINVAL;
5685
5686         /*
5687          * SMM temporarily disables VMX, so we cannot be in guest mode,
5688          * nor can VMLAUNCH/VMRESUME be pending.  Outside SMM, SMM flags
5689          * must be zero.
5690          */
5691         if (is_smm(vcpu) ?
5692                 (kvm_state->flags &
5693                  (KVM_STATE_NESTED_GUEST_MODE | KVM_STATE_NESTED_RUN_PENDING))
5694                 : kvm_state->hdr.vmx.smm.flags)
5695                 return -EINVAL;
5696
5697         if ((kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE) &&
5698             !(kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON))
5699                 return -EINVAL;
5700
5701         if ((kvm_state->flags & KVM_STATE_NESTED_EVMCS) &&
5702                 (!nested_vmx_allowed(vcpu) || !vmx->nested.enlightened_vmcs_enabled))
5703                         return -EINVAL;
5704
5705         vmx_leave_nested(vcpu);
5706
5707         if (kvm_state->hdr.vmx.vmxon_pa == -1ull)
5708                 return 0;
5709
5710         vmx->nested.vmxon_ptr = kvm_state->hdr.vmx.vmxon_pa;
5711         ret = enter_vmx_operation(vcpu);
5712         if (ret)
5713                 return ret;
5714
5715         /* Empty 'VMXON' state is permitted */
5716         if (kvm_state->size < sizeof(*kvm_state) + sizeof(*vmcs12))
5717                 return 0;
5718
5719         if (kvm_state->hdr.vmx.vmcs12_pa != -1ull) {
5720                 if (kvm_state->hdr.vmx.vmcs12_pa == kvm_state->hdr.vmx.vmxon_pa ||
5721                     !page_address_valid(vcpu, kvm_state->hdr.vmx.vmcs12_pa))
5722                         return -EINVAL;
5723
5724                 set_current_vmptr(vmx, kvm_state->hdr.vmx.vmcs12_pa);
5725         } else if (kvm_state->flags & KVM_STATE_NESTED_EVMCS) {
5726                 /*
5727                  * Sync eVMCS upon entry as we may not have
5728                  * HV_X64_MSR_VP_ASSIST_PAGE set up yet.
5729                  */
5730                 vmx->nested.need_vmcs12_to_shadow_sync = true;
5731         } else {
5732                 return -EINVAL;
5733         }
5734
5735         if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_VMXON) {
5736                 vmx->nested.smm.vmxon = true;
5737                 vmx->nested.vmxon = false;
5738
5739                 if (kvm_state->hdr.vmx.smm.flags & KVM_STATE_NESTED_SMM_GUEST_MODE)
5740                         vmx->nested.smm.guest_mode = true;
5741         }
5742
5743         vmcs12 = get_vmcs12(vcpu);
5744         if (copy_from_user(vmcs12, user_vmx_nested_state->vmcs12, sizeof(*vmcs12)))
5745                 return -EFAULT;
5746
5747         if (vmcs12->hdr.revision_id != VMCS12_REVISION)
5748                 return -EINVAL;
5749
5750         if (!(kvm_state->flags & KVM_STATE_NESTED_GUEST_MODE))
5751                 return 0;
5752
5753         vmx->nested.nested_run_pending =
5754                 !!(kvm_state->flags & KVM_STATE_NESTED_RUN_PENDING);
5755
5756         ret = -EINVAL;
5757         if (nested_cpu_has_shadow_vmcs(vmcs12) &&
5758             vmcs12->vmcs_link_pointer != -1ull) {
5759                 struct vmcs12 *shadow_vmcs12 = get_shadow_vmcs12(vcpu);
5760
5761                 if (kvm_state->size <
5762                     sizeof(*kvm_state) +
5763                     sizeof(user_vmx_nested_state->vmcs12) + sizeof(*shadow_vmcs12))
5764                         goto error_guest_mode;
5765
5766                 if (copy_from_user(shadow_vmcs12,
5767                                    user_vmx_nested_state->shadow_vmcs12,
5768                                    sizeof(*shadow_vmcs12))) {
5769                         ret = -EFAULT;
5770                         goto error_guest_mode;
5771                 }
5772
5773                 if (shadow_vmcs12->hdr.revision_id != VMCS12_REVISION ||
5774                     !shadow_vmcs12->hdr.shadow_vmcs)
5775                         goto error_guest_mode;
5776         }
5777
5778         if (nested_vmx_check_controls(vcpu, vmcs12) ||
5779             nested_vmx_check_host_state(vcpu, vmcs12) ||
5780             nested_vmx_check_guest_state(vcpu, vmcs12, &exit_qual))
5781                 goto error_guest_mode;
5782
5783         vmx->nested.dirty_vmcs12 = true;
5784         ret = nested_vmx_enter_non_root_mode(vcpu, false);
5785         if (ret)
5786                 goto error_guest_mode;
5787
5788         return 0;
5789
5790 error_guest_mode:
5791         vmx->nested.nested_run_pending = 0;
5792         return ret;
5793 }
5794
5795 void nested_vmx_vcpu_setup(void)
5796 {
5797         if (enable_shadow_vmcs) {
5798                 vmcs_write64(VMREAD_BITMAP, __pa(vmx_vmread_bitmap));
5799                 vmcs_write64(VMWRITE_BITMAP, __pa(vmx_vmwrite_bitmap));
5800         }
5801 }
5802
5803 /*
5804  * nested_vmx_setup_ctls_msrs() sets up variables containing the values to be
5805  * returned for the various VMX controls MSRs when nested VMX is enabled.
5806  * The same values should also be used to verify that vmcs12 control fields are
5807  * valid during nested entry from L1 to L2.
5808  * Each of these control msrs has a low and high 32-bit half: A low bit is on
5809  * if the corresponding bit in the (32-bit) control field *must* be on, and a
5810  * bit in the high half is on if the corresponding bit in the control field
5811  * may be on. See also vmx_control_verify().
5812  */
5813 void nested_vmx_setup_ctls_msrs(struct nested_vmx_msrs *msrs, u32 ept_caps)
5814 {
5815         /*
5816          * Note that as a general rule, the high half of the MSRs (bits in
5817          * the control fields which may be 1) should be initialized by the
5818          * intersection of the underlying hardware's MSR (i.e., features which
5819          * can be supported) and the list of features we want to expose -
5820          * because they are known to be properly supported in our code.
5821          * Also, usually, the low half of the MSRs (bits which must be 1) can
5822          * be set to 0, meaning that L1 may turn off any of these bits. The
5823          * reason is that if one of these bits is necessary, it will appear
5824          * in vmcs01 and prepare_vmcs02, when it bitwise-or's the control
5825          * fields of vmcs01 and vmcs02, will turn these bits off - and
5826          * nested_vmx_exit_reflected() will not pass related exits to L1.
5827          * These rules have exceptions below.
5828          */
5829
5830         /* pin-based controls */
5831         rdmsr(MSR_IA32_VMX_PINBASED_CTLS,
5832                 msrs->pinbased_ctls_low,
5833                 msrs->pinbased_ctls_high);
5834         msrs->pinbased_ctls_low |=
5835                 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
5836         msrs->pinbased_ctls_high &=
5837                 PIN_BASED_EXT_INTR_MASK |
5838                 PIN_BASED_NMI_EXITING |
5839                 PIN_BASED_VIRTUAL_NMIS |
5840                 (enable_apicv ? PIN_BASED_POSTED_INTR : 0);
5841         msrs->pinbased_ctls_high |=
5842                 PIN_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
5843                 PIN_BASED_VMX_PREEMPTION_TIMER;
5844
5845         /* exit controls */
5846         rdmsr(MSR_IA32_VMX_EXIT_CTLS,
5847                 msrs->exit_ctls_low,
5848                 msrs->exit_ctls_high);
5849         msrs->exit_ctls_low =
5850                 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR;
5851
5852         msrs->exit_ctls_high &=
5853 #ifdef CONFIG_X86_64
5854                 VM_EXIT_HOST_ADDR_SPACE_SIZE |
5855 #endif
5856                 VM_EXIT_LOAD_IA32_PAT | VM_EXIT_SAVE_IA32_PAT;
5857         msrs->exit_ctls_high |=
5858                 VM_EXIT_ALWAYSON_WITHOUT_TRUE_MSR |
5859                 VM_EXIT_LOAD_IA32_EFER | VM_EXIT_SAVE_IA32_EFER |
5860                 VM_EXIT_SAVE_VMX_PREEMPTION_TIMER | VM_EXIT_ACK_INTR_ON_EXIT;
5861
5862         /* We support free control of debug control saving. */
5863         msrs->exit_ctls_low &= ~VM_EXIT_SAVE_DEBUG_CONTROLS;
5864
5865         /* entry controls */
5866         rdmsr(MSR_IA32_VMX_ENTRY_CTLS,
5867                 msrs->entry_ctls_low,
5868                 msrs->entry_ctls_high);
5869         msrs->entry_ctls_low =
5870                 VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR;
5871         msrs->entry_ctls_high &=
5872 #ifdef CONFIG_X86_64
5873                 VM_ENTRY_IA32E_MODE |
5874 #endif
5875                 VM_ENTRY_LOAD_IA32_PAT;
5876         msrs->entry_ctls_high |=
5877                 (VM_ENTRY_ALWAYSON_WITHOUT_TRUE_MSR | VM_ENTRY_LOAD_IA32_EFER);
5878
5879         /* We support free control of debug control loading. */
5880         msrs->entry_ctls_low &= ~VM_ENTRY_LOAD_DEBUG_CONTROLS;
5881
5882         /* cpu-based controls */
5883         rdmsr(MSR_IA32_VMX_PROCBASED_CTLS,
5884                 msrs->procbased_ctls_low,
5885                 msrs->procbased_ctls_high);
5886         msrs->procbased_ctls_low =
5887                 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR;
5888         msrs->procbased_ctls_high &=
5889                 CPU_BASED_INTR_WINDOW_EXITING |
5890                 CPU_BASED_NMI_WINDOW_EXITING | CPU_BASED_USE_TSC_OFFSETTING |
5891                 CPU_BASED_HLT_EXITING | CPU_BASED_INVLPG_EXITING |
5892                 CPU_BASED_MWAIT_EXITING | CPU_BASED_CR3_LOAD_EXITING |
5893                 CPU_BASED_CR3_STORE_EXITING |
5894 #ifdef CONFIG_X86_64
5895                 CPU_BASED_CR8_LOAD_EXITING | CPU_BASED_CR8_STORE_EXITING |
5896 #endif
5897                 CPU_BASED_MOV_DR_EXITING | CPU_BASED_UNCOND_IO_EXITING |
5898                 CPU_BASED_USE_IO_BITMAPS | CPU_BASED_MONITOR_TRAP_FLAG |
5899                 CPU_BASED_MONITOR_EXITING | CPU_BASED_RDPMC_EXITING |
5900                 CPU_BASED_RDTSC_EXITING | CPU_BASED_PAUSE_EXITING |
5901                 CPU_BASED_TPR_SHADOW | CPU_BASED_ACTIVATE_SECONDARY_CONTROLS;
5902         /*
5903          * We can allow some features even when not supported by the
5904          * hardware. For example, L1 can specify an MSR bitmap - and we
5905          * can use it to avoid exits to L1 - even when L0 runs L2
5906          * without MSR bitmaps.
5907          */
5908         msrs->procbased_ctls_high |=
5909                 CPU_BASED_ALWAYSON_WITHOUT_TRUE_MSR |
5910                 CPU_BASED_USE_MSR_BITMAPS;
5911
5912         /* We support free control of CR3 access interception. */
5913         msrs->procbased_ctls_low &=
5914                 ~(CPU_BASED_CR3_LOAD_EXITING | CPU_BASED_CR3_STORE_EXITING);
5915
5916         /*
5917          * secondary cpu-based controls.  Do not include those that
5918          * depend on CPUID bits, they are added later by vmx_cpuid_update.
5919          */
5920         if (msrs->procbased_ctls_high & CPU_BASED_ACTIVATE_SECONDARY_CONTROLS)
5921                 rdmsr(MSR_IA32_VMX_PROCBASED_CTLS2,
5922                       msrs->secondary_ctls_low,
5923                       msrs->secondary_ctls_high);
5924
5925         msrs->secondary_ctls_low = 0;
5926         msrs->secondary_ctls_high &=
5927                 SECONDARY_EXEC_DESC |
5928                 SECONDARY_EXEC_RDTSCP |
5929                 SECONDARY_EXEC_VIRTUALIZE_X2APIC_MODE |
5930                 SECONDARY_EXEC_WBINVD_EXITING |
5931                 SECONDARY_EXEC_APIC_REGISTER_VIRT |
5932                 SECONDARY_EXEC_VIRTUAL_INTR_DELIVERY |
5933                 SECONDARY_EXEC_RDRAND_EXITING |
5934                 SECONDARY_EXEC_ENABLE_INVPCID |
5935                 SECONDARY_EXEC_RDSEED_EXITING |
5936                 SECONDARY_EXEC_XSAVES |
5937                 SECONDARY_EXEC_ENABLE_USR_WAIT_PAUSE;
5938
5939         /*
5940          * We can emulate "VMCS shadowing," even if the hardware
5941          * doesn't support it.
5942          */
5943         msrs->secondary_ctls_high |=
5944                 SECONDARY_EXEC_SHADOW_VMCS;
5945
5946         if (enable_ept) {
5947                 /* nested EPT: emulate EPT also to L1 */
5948                 msrs->secondary_ctls_high |=
5949                         SECONDARY_EXEC_ENABLE_EPT;
5950                 msrs->ept_caps = VMX_EPT_PAGE_WALK_4_BIT |
5951                          VMX_EPTP_WB_BIT | VMX_EPT_INVEPT_BIT;
5952                 if (cpu_has_vmx_ept_execute_only())
5953                         msrs->ept_caps |=
5954                                 VMX_EPT_EXECUTE_ONLY_BIT;
5955                 msrs->ept_caps &= ept_caps;
5956                 msrs->ept_caps |= VMX_EPT_EXTENT_GLOBAL_BIT |
5957                         VMX_EPT_EXTENT_CONTEXT_BIT | VMX_EPT_2MB_PAGE_BIT |
5958                         VMX_EPT_1GB_PAGE_BIT;
5959                 if (enable_ept_ad_bits) {
5960                         msrs->secondary_ctls_high |=
5961                                 SECONDARY_EXEC_ENABLE_PML;
5962                         msrs->ept_caps |= VMX_EPT_AD_BIT;
5963                 }
5964         }
5965
5966         if (cpu_has_vmx_vmfunc()) {
5967                 msrs->secondary_ctls_high |=
5968                         SECONDARY_EXEC_ENABLE_VMFUNC;
5969                 /*
5970                  * Advertise EPTP switching unconditionally
5971                  * since we emulate it
5972                  */
5973                 if (enable_ept)
5974                         msrs->vmfunc_controls =
5975                                 VMX_VMFUNC_EPTP_SWITCHING;
5976         }
5977
5978         /*
5979          * Old versions of KVM use the single-context version without
5980          * checking for support, so declare that it is supported even
5981          * though it is treated as global context.  The alternative is
5982          * not failing the single-context invvpid, and it is worse.
5983          */
5984         if (enable_vpid) {
5985                 msrs->secondary_ctls_high |=
5986                         SECONDARY_EXEC_ENABLE_VPID;
5987                 msrs->vpid_caps = VMX_VPID_INVVPID_BIT |
5988                         VMX_VPID_EXTENT_SUPPORTED_MASK;
5989         }
5990
5991         if (enable_unrestricted_guest)
5992                 msrs->secondary_ctls_high |=
5993                         SECONDARY_EXEC_UNRESTRICTED_GUEST;
5994
5995         if (flexpriority_enabled)
5996                 msrs->secondary_ctls_high |=
5997                         SECONDARY_EXEC_VIRTUALIZE_APIC_ACCESSES;
5998
5999         /* miscellaneous data */
6000         rdmsr(MSR_IA32_VMX_MISC,
6001                 msrs->misc_low,
6002                 msrs->misc_high);
6003         msrs->misc_low &= VMX_MISC_SAVE_EFER_LMA;
6004         msrs->misc_low |=
6005                 MSR_IA32_VMX_MISC_VMWRITE_SHADOW_RO_FIELDS |
6006                 VMX_MISC_EMULATED_PREEMPTION_TIMER_RATE |
6007                 VMX_MISC_ACTIVITY_HLT;
6008         msrs->misc_high = 0;
6009
6010         /*
6011          * This MSR reports some information about VMX support. We
6012          * should return information about the VMX we emulate for the
6013          * guest, and the VMCS structure we give it - not about the
6014          * VMX support of the underlying hardware.
6015          */
6016         msrs->basic =
6017                 VMCS12_REVISION |
6018                 VMX_BASIC_TRUE_CTLS |
6019                 ((u64)VMCS12_SIZE << VMX_BASIC_VMCS_SIZE_SHIFT) |
6020                 (VMX_BASIC_MEM_TYPE_WB << VMX_BASIC_MEM_TYPE_SHIFT);
6021
6022         if (cpu_has_vmx_basic_inout())
6023                 msrs->basic |= VMX_BASIC_INOUT;
6024
6025         /*
6026          * These MSRs specify bits which the guest must keep fixed on
6027          * while L1 is in VMXON mode (in L1's root mode, or running an L2).
6028          * We picked the standard core2 setting.
6029          */
6030 #define VMXON_CR0_ALWAYSON     (X86_CR0_PE | X86_CR0_PG | X86_CR0_NE)
6031 #define VMXON_CR4_ALWAYSON     X86_CR4_VMXE
6032         msrs->cr0_fixed0 = VMXON_CR0_ALWAYSON;
6033         msrs->cr4_fixed0 = VMXON_CR4_ALWAYSON;
6034
6035         /* These MSRs specify bits which the guest must keep fixed off. */
6036         rdmsrl(MSR_IA32_VMX_CR0_FIXED1, msrs->cr0_fixed1);
6037         rdmsrl(MSR_IA32_VMX_CR4_FIXED1, msrs->cr4_fixed1);
6038
6039         /* highest index: VMX_PREEMPTION_TIMER_VALUE */
6040         msrs->vmcs_enum = VMCS12_MAX_FIELD_INDEX << 1;
6041 }
6042
6043 void nested_vmx_hardware_unsetup(void)
6044 {
6045         int i;
6046
6047         if (enable_shadow_vmcs) {
6048                 for (i = 0; i < VMX_BITMAP_NR; i++)
6049                         free_page((unsigned long)vmx_bitmap[i]);
6050         }
6051 }
6052
6053 __init int nested_vmx_hardware_setup(int (*exit_handlers[])(struct kvm_vcpu *))
6054 {
6055         int i;
6056
6057         if (!cpu_has_vmx_shadow_vmcs())
6058                 enable_shadow_vmcs = 0;
6059         if (enable_shadow_vmcs) {
6060                 for (i = 0; i < VMX_BITMAP_NR; i++) {
6061                         /*
6062                          * The vmx_bitmap is not tied to a VM and so should
6063                          * not be charged to a memcg.
6064                          */
6065                         vmx_bitmap[i] = (unsigned long *)
6066                                 __get_free_page(GFP_KERNEL);
6067                         if (!vmx_bitmap[i]) {
6068                                 nested_vmx_hardware_unsetup();
6069                                 return -ENOMEM;
6070                         }
6071                 }
6072
6073                 init_vmcs_shadow_fields();
6074         }
6075
6076         exit_handlers[EXIT_REASON_VMCLEAR]      = handle_vmclear,
6077         exit_handlers[EXIT_REASON_VMLAUNCH]     = handle_vmlaunch,
6078         exit_handlers[EXIT_REASON_VMPTRLD]      = handle_vmptrld,
6079         exit_handlers[EXIT_REASON_VMPTRST]      = handle_vmptrst,
6080         exit_handlers[EXIT_REASON_VMREAD]       = handle_vmread,
6081         exit_handlers[EXIT_REASON_VMRESUME]     = handle_vmresume,
6082         exit_handlers[EXIT_REASON_VMWRITE]      = handle_vmwrite,
6083         exit_handlers[EXIT_REASON_VMOFF]        = handle_vmoff,
6084         exit_handlers[EXIT_REASON_VMON]         = handle_vmon,
6085         exit_handlers[EXIT_REASON_INVEPT]       = handle_invept,
6086         exit_handlers[EXIT_REASON_INVVPID]      = handle_invvpid,
6087         exit_handlers[EXIT_REASON_VMFUNC]       = handle_vmfunc,
6088
6089         kvm_x86_ops->check_nested_events = vmx_check_nested_events;
6090         kvm_x86_ops->get_nested_state = vmx_get_nested_state;
6091         kvm_x86_ops->set_nested_state = vmx_set_nested_state;
6092         kvm_x86_ops->get_vmcs12_pages = nested_get_vmcs12_pages,
6093         kvm_x86_ops->nested_enable_evmcs = nested_enable_evmcs;
6094         kvm_x86_ops->nested_get_evmcs_version = nested_get_evmcs_version;
6095
6096         return 0;
6097 }