arch/powerpc/kvm/powerpc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  *
   4  * Copyright IBM Corp. 2007
   5  *
   6  * Authors: Hollis Blanchard <hollisb@us.ibm.com>
   7  *          Christian Ehrhardt <ehrhardt@linux.vnet.ibm.com>
   8  */
   9
  10 #include <linux/errno.h>
  11 #include <linux/err.h>
  12 #include <linux/kvm_host.h>
  13 #include <linux/vmalloc.h>
  14 #include <linux/hrtimer.h>
  15 #include <linux/sched/signal.h>
  16 #include <linux/fs.h>
  17 #include <linux/slab.h>
  18 #include <linux/file.h>
  19 #include <linux/module.h>
  20 #include <linux/irqbypass.h>
  21 #include <linux/kvm_irqfd.h>
  22 #include <linux/of.h>
  23 #include <asm/cputable.h>
  24 #include <linux/uaccess.h>
  25 #include <asm/kvm_ppc.h>
  26 #include <asm/cputhreads.h>
  27 #include <asm/irqflags.h>
  28 #include <asm/iommu.h>
  29 #include <asm/switch_to.h>
  30 #include <asm/xive.h>
  31 #ifdef CONFIG_PPC_PSERIES
  32 #include <asm/hvcall.h>
  33 #include <asm/plpar_wrappers.h>
  34 #endif
  35 #include <asm/ultravisor.h>
  36 #include <asm/setup.h>
  37
  38 #include "timing.h"
  39 #include "../mm/mmu_decl.h"
  40
  41 #define CREATE_TRACE_POINTS
  42 #include "trace.h"
  43
  44 struct kvmppc_ops *kvmppc_hv_ops;
  45 EXPORT_SYMBOL_GPL(kvmppc_hv_ops);
  46 struct kvmppc_ops *kvmppc_pr_ops;
  47 EXPORT_SYMBOL_GPL(kvmppc_pr_ops);
  48
  49
  50 int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
  51 {
  52         return !!(v->arch.pending_exceptions) || kvm_request_pending(v);
  53 }
  54
  55 bool kvm_arch_dy_runnable(struct kvm_vcpu *vcpu)
  56 {
  57         return kvm_arch_vcpu_runnable(vcpu);
  58 }
  59
  60 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
  61 {
  62         return false;
  63 }
  64
  65 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
  66 {
  67         return 1;
  68 }
  69
  70 /*
  71  * Common checks before entering the guest world.  Call with interrupts
  72  * disabled.
  73  *
  74  * returns:
  75  *
  76  * == 1 if we're ready to go into guest state
  77  * <= 0 if we need to go back to the host with return value
  78  */
  79 int kvmppc_prepare_to_enter(struct kvm_vcpu *vcpu)
  80 {
  81         int r;
  82
  83         WARN_ON(irqs_disabled());
  84         hard_irq_disable();
  85
  86         while (true) {
  87                 if (need_resched()) {
  88                         local_irq_enable();
  89                         cond_resched();
  90                         hard_irq_disable();
  91                         continue;
  92                 }
  93
  94                 if (signal_pending(current)) {
  95                         kvmppc_account_exit(vcpu, SIGNAL_EXITS);
  96                         vcpu->run->exit_reason = KVM_EXIT_INTR;
  97                         r = -EINTR;
  98                         break;
  99                 }
 100
 101                 vcpu->mode = IN_GUEST_MODE;
 102
 103                 /*
 104                  * Reading vcpu->requests must happen after setting vcpu->mode,
 105                  * so we don't miss a request because the requester sees
 106                  * OUTSIDE_GUEST_MODE and assumes we'll be checking requests
 107                  * before next entering the guest (and thus doesn't IPI).
 108                  * This also orders the write to mode from any reads
 109                  * to the page tables done while the VCPU is running.
 110                  * Please see the comment in kvm_flush_remote_tlbs.
 111                  */
 112                 smp_mb();
 113
 114                 if (kvm_request_pending(vcpu)) {
 115                         /* Make sure we process requests preemptable */
 116                         local_irq_enable();
 117                         trace_kvm_check_requests(vcpu);
 118                         r = kvmppc_core_check_requests(vcpu);
 119                         hard_irq_disable();
 120                         if (r > 0)
 121                                 continue;
 122                         break;
 123                 }
 124
 125                 if (kvmppc_core_prepare_to_enter(vcpu)) {
 126                         /* interrupts got enabled in between, so we
 127                            are back at square 1 */
 128                         continue;
 129                 }
 130
 131                 guest_enter_irqoff();
 132                 return 1;
 133         }
 134
 135         /* return to host */
 136         local_irq_enable();
 137         return r;
 138 }
 139 EXPORT_SYMBOL_GPL(kvmppc_prepare_to_enter);
 140
 141 #if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_KVM_BOOK3S_PR_POSSIBLE)
 142 static void kvmppc_swab_shared(struct kvm_vcpu *vcpu)
 143 {
 144         struct kvm_vcpu_arch_shared *shared = vcpu->arch.shared;
 145         int i;
 146
 147         shared->sprg0 = swab64(shared->sprg0);
 148         shared->sprg1 = swab64(shared->sprg1);
 149         shared->sprg2 = swab64(shared->sprg2);
 150         shared->sprg3 = swab64(shared->sprg3);
 151         shared->srr0 = swab64(shared->srr0);
 152         shared->srr1 = swab64(shared->srr1);
 153         shared->dar = swab64(shared->dar);
 154         shared->msr = swab64(shared->msr);
 155         shared->dsisr = swab32(shared->dsisr);
 156         shared->int_pending = swab32(shared->int_pending);
 157         for (i = 0; i < ARRAY_SIZE(shared->sr); i++)
 158                 shared->sr[i] = swab32(shared->sr[i]);
 159 }
 160 #endif
 161
 162 int kvmppc_kvm_pv(struct kvm_vcpu *vcpu)
 163 {
 164         int nr = kvmppc_get_gpr(vcpu, 11);
 165         int r;
 166         unsigned long __maybe_unused param1 = kvmppc_get_gpr(vcpu, 3);
 167         unsigned long __maybe_unused param2 = kvmppc_get_gpr(vcpu, 4);
 168         unsigned long __maybe_unused param3 = kvmppc_get_gpr(vcpu, 5);
 169         unsigned long __maybe_unused param4 = kvmppc_get_gpr(vcpu, 6);
 170         unsigned long r2 = 0;
 171
 172         if (!(kvmppc_get_msr(vcpu) & MSR_SF)) {
 173                 /* 32 bit mode */
 174                 param1 &= 0xffffffff;
 175                 param2 &= 0xffffffff;
 176                 param3 &= 0xffffffff;
 177                 param4 &= 0xffffffff;
 178         }
 179
 180         switch (nr) {
 181         case KVM_HCALL_TOKEN(KVM_HC_PPC_MAP_MAGIC_PAGE):
 182         {
 183 #if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_KVM_BOOK3S_PR_POSSIBLE)
 184                 /* Book3S can be little endian, find it out here */
 185                 int shared_big_endian = true;
 186                 if (vcpu->arch.intr_msr & MSR_LE)
 187                         shared_big_endian = false;
 188                 if (shared_big_endian != vcpu->arch.shared_big_endian)
 189                         kvmppc_swab_shared(vcpu);
 190                 vcpu->arch.shared_big_endian = shared_big_endian;
 191 #endif
 192
 193                 if (!(param2 & MAGIC_PAGE_FLAG_NOT_MAPPED_NX)) {
 194                         /*
 195                          * Older versions of the Linux magic page code had
 196                          * a bug where they would map their trampoline code
 197                          * NX. If that's the case, remove !PR NX capability.
 198                          */
 199                         vcpu->arch.disable_kernel_nx = true;
 200                         kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
 201                 }
 202
 203                 vcpu->arch.magic_page_pa = param1 & ~0xfffULL;
 204                 vcpu->arch.magic_page_ea = param2 & ~0xfffULL;
 205
 206 #ifdef CONFIG_PPC_64K_PAGES
 207                 /*
 208                  * Make sure our 4k magic page is in the same window of a 64k
 209                  * page within the guest and within the host's page.
 210                  */
 211                 if ((vcpu->arch.magic_page_pa & 0xf000) !=
 212                     ((ulong)vcpu->arch.shared & 0xf000)) {
 213                         void *old_shared = vcpu->arch.shared;
 214                         ulong shared = (ulong)vcpu->arch.shared;
 215                         void *new_shared;
 216
 217                         shared &= PAGE_MASK;
 218                         shared |= vcpu->arch.magic_page_pa & 0xf000;
 219                         new_shared = (void*)shared;
 220                         memcpy(new_shared, old_shared, 0x1000);
 221                         vcpu->arch.shared = new_shared;
 222                 }
 223 #endif
 224
 225                 r2 = KVM_MAGIC_FEAT_SR | KVM_MAGIC_FEAT_MAS0_TO_SPRG7;
 226
 227                 r = EV_SUCCESS;
 228                 break;
 229         }
 230         case KVM_HCALL_TOKEN(KVM_HC_FEATURES):
 231                 r = EV_SUCCESS;
 232 #if defined(CONFIG_PPC_BOOK3S) || defined(CONFIG_KVM_E500V2)
 233                 r2 |= (1 << KVM_FEATURE_MAGIC_PAGE);
 234 #endif
 235
 236                 /* Second return value is in r4 */
 237                 break;
 238         case EV_HCALL_TOKEN(EV_IDLE):
 239                 r = EV_SUCCESS;
 240                 kvm_vcpu_halt(vcpu);
 241                 break;
 242         default:
 243                 r = EV_UNIMPLEMENTED;
 244                 break;
 245         }
 246
 247         kvmppc_set_gpr(vcpu, 4, r2);
 248
 249         return r;
 250 }
 251 EXPORT_SYMBOL_GPL(kvmppc_kvm_pv);
 252
 253 int kvmppc_sanity_check(struct kvm_vcpu *vcpu)
 254 {
 255         int r = false;
 256
 257         /* We have to know what CPU to virtualize */
 258         if (!vcpu->arch.pvr)
 259                 goto out;
 260
 261         /* PAPR only works with book3s_64 */
 262         if ((vcpu->arch.cpu_type != KVM_CPU_3S_64) && vcpu->arch.papr_enabled)
 263                 goto out;
 264
 265         /* HV KVM can only do PAPR mode for now */
 266         if (!vcpu->arch.papr_enabled && is_kvmppc_hv_enabled(vcpu->kvm))
 267                 goto out;
 268
 269 #ifdef CONFIG_KVM_BOOKE_HV
 270         if (!cpu_has_feature(CPU_FTR_EMB_HV))
 271                 goto out;
 272 #endif
 273
 274         r = true;
 275
 276 out:
 277         vcpu->arch.sane = r;
 278         return r ? 0 : -EINVAL;
 279 }
 280 EXPORT_SYMBOL_GPL(kvmppc_sanity_check);
 281
 282 int kvmppc_emulate_mmio(struct kvm_vcpu *vcpu)
 283 {
 284         enum emulation_result er;
 285         int r;
 286
 287         er = kvmppc_emulate_loadstore(vcpu);
 288         switch (er) {
 289         case EMULATE_DONE:
 290                 /* Future optimization: only reload non-volatiles if they were
 291                  * actually modified. */
 292                 r = RESUME_GUEST_NV;
 293                 break;
 294         case EMULATE_AGAIN:
 295                 r = RESUME_GUEST;
 296                 break;
 297         case EMULATE_DO_MMIO:
 298                 vcpu->run->exit_reason = KVM_EXIT_MMIO;
 299                 /* We must reload nonvolatiles because "update" load/store
 300                  * instructions modify register state. */
 301                 /* Future optimization: only reload non-volatiles if they were
 302                  * actually modified. */
 303                 r = RESUME_HOST_NV;
 304                 break;
 305         case EMULATE_FAIL:
 306         {
 307                 ppc_inst_t last_inst;
 308
 309                 kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst);
 310                 kvm_debug_ratelimited("Guest access to device memory using unsupported instruction (opcode: %#08x)\n",
 311                                       ppc_inst_val(last_inst));
 312
 313                 /*
 314                  * Injecting a Data Storage here is a bit more
 315                  * accurate since the instruction that caused the
 316                  * access could still be a valid one.
 317                  */
 318                 if (!IS_ENABLED(CONFIG_BOOKE)) {
 319                         ulong dsisr = DSISR_BADACCESS;
 320
 321                         if (vcpu->mmio_is_write)
 322                                 dsisr |= DSISR_ISSTORE;
 323
 324                         kvmppc_core_queue_data_storage(vcpu,
 325                                         kvmppc_get_msr(vcpu) & SRR1_PREFIXED,
 326                                         vcpu->arch.vaddr_accessed, dsisr);
 327                 } else {
 328                         /*
 329                          * BookE does not send a SIGBUS on a bad
 330                          * fault, so use a Program interrupt instead
 331                          * to avoid a fault loop.
 332                          */
 333                         kvmppc_core_queue_program(vcpu, 0);
 334                 }
 335
 336                 r = RESUME_GUEST;
 337                 break;
 338         }
 339         default:
 340                 WARN_ON(1);
 341                 r = RESUME_GUEST;
 342         }
 343
 344         return r;
 345 }
 346 EXPORT_SYMBOL_GPL(kvmppc_emulate_mmio);
 347
 348 int kvmppc_st(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr,
 349               bool data)
 350 {
 351         ulong mp_pa = vcpu->arch.magic_page_pa & KVM_PAM & PAGE_MASK;
 352         struct kvmppc_pte pte;
 353         int r = -EINVAL;
 354
 355         vcpu->stat.st++;
 356
 357         if (vcpu->kvm->arch.kvm_ops && vcpu->kvm->arch.kvm_ops->store_to_eaddr)
 358                 r = vcpu->kvm->arch.kvm_ops->store_to_eaddr(vcpu, eaddr, ptr,
 359                                                             size);
 360
 361         if ((!r) || (r == -EAGAIN))
 362                 return r;
 363
 364         r = kvmppc_xlate(vcpu, *eaddr, data ? XLATE_DATA : XLATE_INST,
 365                          XLATE_WRITE, &pte);
 366         if (r < 0)
 367                 return r;
 368
 369         *eaddr = pte.raddr;
 370
 371         if (!pte.may_write)
 372                 return -EPERM;
 373
 374         /* Magic page override */
 375         if (kvmppc_supports_magic_page(vcpu) && mp_pa &&
 376             ((pte.raddr & KVM_PAM & PAGE_MASK) == mp_pa) &&
 377             !(kvmppc_get_msr(vcpu) & MSR_PR)) {
 378                 void *magic = vcpu->arch.shared;
 379                 magic += pte.eaddr & 0xfff;
 380                 memcpy(magic, ptr, size);
 381                 return EMULATE_DONE;
 382         }
 383
 384         if (kvm_write_guest(vcpu->kvm, pte.raddr, ptr, size))
 385                 return EMULATE_DO_MMIO;
 386
 387         return EMULATE_DONE;
 388 }
 389 EXPORT_SYMBOL_GPL(kvmppc_st);
 390
 391 int kvmppc_ld(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr,
 392                       bool data)
 393 {
 394         ulong mp_pa = vcpu->arch.magic_page_pa & KVM_PAM & PAGE_MASK;
 395         struct kvmppc_pte pte;
 396         int rc = -EINVAL;
 397
 398         vcpu->stat.ld++;
 399
 400         if (vcpu->kvm->arch.kvm_ops && vcpu->kvm->arch.kvm_ops->load_from_eaddr)
 401                 rc = vcpu->kvm->arch.kvm_ops->load_from_eaddr(vcpu, eaddr, ptr,
 402                                                               size);
 403
 404         if ((!rc) || (rc == -EAGAIN))
 405                 return rc;
 406
 407         rc = kvmppc_xlate(vcpu, *eaddr, data ? XLATE_DATA : XLATE_INST,
 408                           XLATE_READ, &pte);
 409         if (rc)
 410                 return rc;
 411
 412         *eaddr = pte.raddr;
 413
 414         if (!pte.may_read)
 415                 return -EPERM;
 416
 417         if (!data && !pte.may_execute)
 418                 return -ENOEXEC;
 419
 420         /* Magic page override */
 421         if (kvmppc_supports_magic_page(vcpu) && mp_pa &&
 422             ((pte.raddr & KVM_PAM & PAGE_MASK) == mp_pa) &&
 423             !(kvmppc_get_msr(vcpu) & MSR_PR)) {
 424                 void *magic = vcpu->arch.shared;
 425                 magic += pte.eaddr & 0xfff;
 426                 memcpy(ptr, magic, size);
 427                 return EMULATE_DONE;
 428         }
 429
 430         kvm_vcpu_srcu_read_lock(vcpu);
 431         rc = kvm_read_guest(vcpu->kvm, pte.raddr, ptr, size);
 432         kvm_vcpu_srcu_read_unlock(vcpu);
 433         if (rc)
 434                 return EMULATE_DO_MMIO;
 435
 436         return EMULATE_DONE;
 437 }
 438 EXPORT_SYMBOL_GPL(kvmppc_ld);
 439
 440 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
 441 {
 442         struct kvmppc_ops *kvm_ops = NULL;
 443         int r;
 444
 445         /*
 446          * if we have both HV and PR enabled, default is HV
 447          */
 448         if (type == 0) {
 449                 if (kvmppc_hv_ops)
 450                         kvm_ops = kvmppc_hv_ops;
 451                 else
 452                         kvm_ops = kvmppc_pr_ops;
 453                 if (!kvm_ops)
 454                         goto err_out;
 455         } else  if (type == KVM_VM_PPC_HV) {
 456                 if (!kvmppc_hv_ops)
 457                         goto err_out;
 458                 kvm_ops = kvmppc_hv_ops;
 459         } else if (type == KVM_VM_PPC_PR) {
 460                 if (!kvmppc_pr_ops)
 461                         goto err_out;
 462                 kvm_ops = kvmppc_pr_ops;
 463         } else
 464                 goto err_out;
 465
 466         if (!try_module_get(kvm_ops->owner))
 467                 return -ENOENT;
 468
 469         kvm->arch.kvm_ops = kvm_ops;
 470         r = kvmppc_core_init_vm(kvm);
 471         if (r)
 472                 module_put(kvm_ops->owner);
 473         return r;
 474 err_out:
 475         return -EINVAL;
 476 }
 477
 478 void kvm_arch_destroy_vm(struct kvm *kvm)
 479 {
 480 #ifdef CONFIG_KVM_XICS
 481         /*
 482          * We call kick_all_cpus_sync() to ensure that all
 483          * CPUs have executed any pending IPIs before we
 484          * continue and free VCPUs structures below.
 485          */
 486         if (is_kvmppc_hv_enabled(kvm))
 487                 kick_all_cpus_sync();
 488 #endif
 489
 490         kvm_destroy_vcpus(kvm);
 491
 492         mutex_lock(&kvm->lock);
 493
 494         kvmppc_core_destroy_vm(kvm);
 495
 496         mutex_unlock(&kvm->lock);
 497
 498         /* drop the module reference */
 499         module_put(kvm->arch.kvm_ops->owner);
 500 }
 501
 502 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
 503 {
 504         int r;
 505         /* Assume we're using HV mode when the HV module is loaded */
 506         int hv_enabled = kvmppc_hv_ops ? 1 : 0;
 507
 508         if (kvm) {
 509                 /*
 510                  * Hooray - we know which VM type we're running on. Depend on
 511                  * that rather than the guess above.
 512                  */
 513                 hv_enabled = is_kvmppc_hv_enabled(kvm);
 514         }
 515
 516         switch (ext) {
 517 #ifdef CONFIG_BOOKE
 518         case KVM_CAP_PPC_BOOKE_SREGS:
 519         case KVM_CAP_PPC_BOOKE_WATCHDOG:
 520         case KVM_CAP_PPC_EPR:
 521 #else
 522         case KVM_CAP_PPC_SEGSTATE:
 523         case KVM_CAP_PPC_HIOR:
 524         case KVM_CAP_PPC_PAPR:
 525 #endif
 526         case KVM_CAP_PPC_UNSET_IRQ:
 527         case KVM_CAP_PPC_IRQ_LEVEL:
 528         case KVM_CAP_ENABLE_CAP:
 529         case KVM_CAP_ONE_REG:
 530         case KVM_CAP_IOEVENTFD:
 531         case KVM_CAP_DEVICE_CTRL:
 532         case KVM_CAP_IMMEDIATE_EXIT:
 533         case KVM_CAP_SET_GUEST_DEBUG:
 534                 r = 1;
 535                 break;
 536         case KVM_CAP_PPC_GUEST_DEBUG_SSTEP:
 537         case KVM_CAP_PPC_PAIRED_SINGLES:
 538         case KVM_CAP_PPC_OSI:
 539         case KVM_CAP_PPC_GET_PVINFO:
 540 #if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC)
 541         case KVM_CAP_SW_TLB:
 542 #endif
 543                 /* We support this only for PR */
 544                 r = !hv_enabled;
 545                 break;
 546 #ifdef CONFIG_KVM_MPIC
 547         case KVM_CAP_IRQ_MPIC:
 548                 r = 1;
 549                 break;
 550 #endif
 551
 552 #ifdef CONFIG_PPC_BOOK3S_64
 553         case KVM_CAP_SPAPR_TCE:
 554         case KVM_CAP_SPAPR_TCE_64:
 555                 r = 1;
 556                 break;
 557         case KVM_CAP_SPAPR_TCE_VFIO:
 558                 r = !!cpu_has_feature(CPU_FTR_HVMODE);
 559                 break;
 560         case KVM_CAP_PPC_RTAS:
 561         case KVM_CAP_PPC_FIXUP_HCALL:
 562         case KVM_CAP_PPC_ENABLE_HCALL:
 563 #ifdef CONFIG_KVM_XICS
 564         case KVM_CAP_IRQ_XICS:
 565 #endif
 566         case KVM_CAP_PPC_GET_CPU_CHAR:
 567                 r = 1;
 568                 break;
 569 #ifdef CONFIG_KVM_XIVE
 570         case KVM_CAP_PPC_IRQ_XIVE:
 571                 /*
 572                  * We need XIVE to be enabled on the platform (implies
 573                  * a POWER9 processor) and the PowerNV platform, as
 574                  * nested is not yet supported.
 575                  */
 576                 r = xive_enabled() && !!cpu_has_feature(CPU_FTR_HVMODE) &&
 577                         kvmppc_xive_native_supported();
 578                 break;
 579 #endif
 580
 581 #ifdef CONFIG_HAVE_KVM_IRQFD
 582         case KVM_CAP_IRQFD_RESAMPLE:
 583                 r = !xive_enabled();
 584                 break;
 585 #endif
 586
 587         case KVM_CAP_PPC_ALLOC_HTAB:
 588                 r = hv_enabled;
 589                 break;
 590 #endif /* CONFIG_PPC_BOOK3S_64 */
 591 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
 592         case KVM_CAP_PPC_SMT:
 593                 r = 0;
 594                 if (kvm) {
 595                         if (kvm->arch.emul_smt_mode > 1)
 596                                 r = kvm->arch.emul_smt_mode;
 597                         else
 598                                 r = kvm->arch.smt_mode;
 599                 } else if (hv_enabled) {
 600                         if (cpu_has_feature(CPU_FTR_ARCH_300))
 601                                 r = 1;
 602                         else
 603                                 r = threads_per_subcore;
 604                 }
 605                 break;
 606         case KVM_CAP_PPC_SMT_POSSIBLE:
 607                 r = 1;
 608                 if (hv_enabled) {
 609                         if (!cpu_has_feature(CPU_FTR_ARCH_300))
 610                                 r = ((threads_per_subcore << 1) - 1);
 611                         else
 612                                 /* P9 can emulate dbells, so allow any mode */
 613                                 r = 8 | 4 | 2 | 1;
 614                 }
 615                 break;
 616         case KVM_CAP_PPC_RMA:
 617                 r = 0;
 618                 break;
 619         case KVM_CAP_PPC_HWRNG:
 620                 r = kvmppc_hwrng_present();
 621                 break;
 622         case KVM_CAP_PPC_MMU_RADIX:
 623                 r = !!(hv_enabled && radix_enabled());
 624                 break;
 625         case KVM_CAP_PPC_MMU_HASH_V3:
 626                 r = !!(hv_enabled && kvmppc_hv_ops->hash_v3_possible &&
 627                        kvmppc_hv_ops->hash_v3_possible());
 628                 break;
 629         case KVM_CAP_PPC_NESTED_HV:
 630                 r = !!(hv_enabled && kvmppc_hv_ops->enable_nested &&
 631                        !kvmppc_hv_ops->enable_nested(NULL));
 632                 break;
 633 #endif
 634         case KVM_CAP_SYNC_MMU:
 635 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
 636                 r = hv_enabled;
 637 #elif defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 638                 r = 1;
 639 #else
 640                 r = 0;
 641 #endif
 642                 break;
 643 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
 644         case KVM_CAP_PPC_HTAB_FD:
 645                 r = hv_enabled;
 646                 break;
 647 #endif
 648         case KVM_CAP_NR_VCPUS:
 649                 /*
 650                  * Recommending a number of CPUs is somewhat arbitrary; we
 651                  * return the number of present CPUs for -HV (since a host
 652                  * will have secondary threads "offline"), and for other KVM
 653                  * implementations just count online CPUs.
 654                  */
 655                 if (hv_enabled)
 656                         r = min_t(unsigned int, num_present_cpus(), KVM_MAX_VCPUS);
 657                 else
 658                         r = min_t(unsigned int, num_online_cpus(), KVM_MAX_VCPUS);
 659                 break;
 660         case KVM_CAP_MAX_VCPUS:
 661                 r = KVM_MAX_VCPUS;
 662                 break;
 663         case KVM_CAP_MAX_VCPU_ID:
 664                 r = KVM_MAX_VCPU_IDS;
 665                 break;
 666 #ifdef CONFIG_PPC_BOOK3S_64
 667         case KVM_CAP_PPC_GET_SMMU_INFO:
 668                 r = 1;
 669                 break;
 670         case KVM_CAP_SPAPR_MULTITCE:
 671                 r = 1;
 672                 break;
 673         case KVM_CAP_SPAPR_RESIZE_HPT:
 674                 r = !!hv_enabled;
 675                 break;
 676 #endif
 677 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
 678         case KVM_CAP_PPC_FWNMI:
 679                 r = hv_enabled;
 680                 break;
 681 #endif
 682 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
 683         case KVM_CAP_PPC_HTM:
 684                 r = !!(cur_cpu_spec->cpu_user_features2 & PPC_FEATURE2_HTM) ||
 685                      (hv_enabled && cpu_has_feature(CPU_FTR_P9_TM_HV_ASSIST));
 686                 break;
 687 #endif
 688 #if defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE)
 689         case KVM_CAP_PPC_SECURE_GUEST:
 690                 r = hv_enabled && kvmppc_hv_ops->enable_svm &&
 691                         !kvmppc_hv_ops->enable_svm(NULL);
 692                 break;
 693         case KVM_CAP_PPC_DAWR1:
 694                 r = !!(hv_enabled && kvmppc_hv_ops->enable_dawr1 &&
 695                        !kvmppc_hv_ops->enable_dawr1(NULL));
 696                 break;
 697         case KVM_CAP_PPC_RPT_INVALIDATE:
 698                 r = 1;
 699                 break;
 700 #endif
 701         case KVM_CAP_PPC_AIL_MODE_3:
 702                 r = 0;
 703                 /*
 704                  * KVM PR, POWER7, and some POWER9s don't support AIL=3 mode.
 705                  * The POWER9s can support it if the guest runs in hash mode,
 706                  * but QEMU doesn't necessarily query the capability in time.
 707                  */
 708                 if (hv_enabled) {
 709                         if (kvmhv_on_pseries()) {
 710                                 if (pseries_reloc_on_exception())
 711                                         r = 1;
 712                         } else if (cpu_has_feature(CPU_FTR_ARCH_207S) &&
 713                                   !cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) {
 714                                 r = 1;
 715                         }
 716                 }
 717                 break;
 718         default:
 719                 r = 0;
 720                 break;
 721         }
 722         return r;
 723
 724 }
 725
 726 long kvm_arch_dev_ioctl(struct file *filp,
 727                         unsigned int ioctl, unsigned long arg)
 728 {
 729         return -EINVAL;
 730 }
 731
 732 void kvm_arch_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
 733 {
 734         kvmppc_core_free_memslot(kvm, slot);
 735 }
 736
 737 int kvm_arch_prepare_memory_region(struct kvm *kvm,
 738                                    const struct kvm_memory_slot *old,
 739                                    struct kvm_memory_slot *new,
 740                                    enum kvm_mr_change change)
 741 {
 742         return kvmppc_core_prepare_memory_region(kvm, old, new, change);
 743 }
 744
 745 void kvm_arch_commit_memory_region(struct kvm *kvm,
 746                                    struct kvm_memory_slot *old,
 747                                    const struct kvm_memory_slot *new,
 748                                    enum kvm_mr_change change)
 749 {
 750         kvmppc_core_commit_memory_region(kvm, old, new, change);
 751 }
 752
 753 void kvm_arch_flush_shadow_memslot(struct kvm *kvm,
 754                                    struct kvm_memory_slot *slot)
 755 {
 756         kvmppc_core_flush_memslot(kvm, slot);
 757 }
 758
 759 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
 760 {
 761         return 0;
 762 }
 763
 764 static enum hrtimer_restart kvmppc_decrementer_wakeup(struct hrtimer *timer)
 765 {
 766         struct kvm_vcpu *vcpu;
 767
 768         vcpu = container_of(timer, struct kvm_vcpu, arch.dec_timer);
 769         kvmppc_decrementer_func(vcpu);
 770
 771         return HRTIMER_NORESTART;
 772 }
 773
 774 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
 775 {
 776         int err;
 777
 778         hrtimer_init(&vcpu->arch.dec_timer, CLOCK_REALTIME, HRTIMER_MODE_ABS);
 779         vcpu->arch.dec_timer.function = kvmppc_decrementer_wakeup;
 780
 781 #ifdef CONFIG_KVM_EXIT_TIMING
 782         mutex_init(&vcpu->arch.exit_timing_lock);
 783 #endif
 784         err = kvmppc_subarch_vcpu_init(vcpu);
 785         if (err)
 786                 return err;
 787
 788         err = kvmppc_core_vcpu_create(vcpu);
 789         if (err)
 790                 goto out_vcpu_uninit;
 791
 792         rcuwait_init(&vcpu->arch.wait);
 793         vcpu->arch.waitp = &vcpu->arch.wait;
 794         return 0;
 795
 796 out_vcpu_uninit:
 797         kvmppc_subarch_vcpu_uninit(vcpu);
 798         return err;
 799 }
 800
 801 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
 802 {
 803 }
 804
 805 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
 806 {
 807         /* Make sure we're not using the vcpu anymore */
 808         hrtimer_cancel(&vcpu->arch.dec_timer);
 809
 810         switch (vcpu->arch.irq_type) {
 811         case KVMPPC_IRQ_MPIC:
 812                 kvmppc_mpic_disconnect_vcpu(vcpu->arch.mpic, vcpu);
 813                 break;
 814         case KVMPPC_IRQ_XICS:
 815                 if (xics_on_xive())
 816                         kvmppc_xive_cleanup_vcpu(vcpu);
 817                 else
 818                         kvmppc_xics_free_icp(vcpu);
 819                 break;
 820         case KVMPPC_IRQ_XIVE:
 821                 kvmppc_xive_native_cleanup_vcpu(vcpu);
 822                 break;
 823         }
 824
 825         kvmppc_core_vcpu_free(vcpu);
 826
 827         kvmppc_subarch_vcpu_uninit(vcpu);
 828 }
 829
 830 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu)
 831 {
 832         return kvmppc_core_pending_dec(vcpu);
 833 }
 834
 835 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 836 {
 837 #ifdef CONFIG_BOOKE
 838         /*
 839          * vrsave (formerly usprg0) isn't used by Linux, but may
 840          * be used by the guest.
 841          *
 842          * On non-booke this is associated with Altivec and
 843          * is handled by code in book3s.c.
 844          */
 845         mtspr(SPRN_VRSAVE, vcpu->arch.vrsave);
 846 #endif
 847         kvmppc_core_vcpu_load(vcpu, cpu);
 848 }
 849
 850 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
 851 {
 852         kvmppc_core_vcpu_put(vcpu);
 853 #ifdef CONFIG_BOOKE
 854         vcpu->arch.vrsave = mfspr(SPRN_VRSAVE);
 855 #endif
 856 }
 857
 858 /*
 859  * irq_bypass_add_producer and irq_bypass_del_producer are only
 860  * useful if the architecture supports PCI passthrough.
 861  * irq_bypass_stop and irq_bypass_start are not needed and so
 862  * kvm_ops are not defined for them.
 863  */
 864 bool kvm_arch_has_irq_bypass(void)
 865 {
 866         return ((kvmppc_hv_ops && kvmppc_hv_ops->irq_bypass_add_producer) ||
 867                 (kvmppc_pr_ops && kvmppc_pr_ops->irq_bypass_add_producer));
 868 }
 869
 870 int kvm_arch_irq_bypass_add_producer(struct irq_bypass_consumer *cons,
 871                                      struct irq_bypass_producer *prod)
 872 {
 873         struct kvm_kernel_irqfd *irqfd =
 874                 container_of(cons, struct kvm_kernel_irqfd, consumer);
 875         struct kvm *kvm = irqfd->kvm;
 876
 877         if (kvm->arch.kvm_ops->irq_bypass_add_producer)
 878                 return kvm->arch.kvm_ops->irq_bypass_add_producer(cons, prod);
 879
 880         return 0;
 881 }
 882
 883 void kvm_arch_irq_bypass_del_producer(struct irq_bypass_consumer *cons,
 884                                       struct irq_bypass_producer *prod)
 885 {
 886         struct kvm_kernel_irqfd *irqfd =
 887                 container_of(cons, struct kvm_kernel_irqfd, consumer);
 888         struct kvm *kvm = irqfd->kvm;
 889
 890         if (kvm->arch.kvm_ops->irq_bypass_del_producer)
 891                 kvm->arch.kvm_ops->irq_bypass_del_producer(cons, prod);
 892 }
 893
 894 #ifdef CONFIG_VSX
 895 static inline int kvmppc_get_vsr_dword_offset(int index)
 896 {
 897         int offset;
 898
 899         if ((index != 0) && (index != 1))
 900                 return -1;
 901
 902 #ifdef __BIG_ENDIAN
 903         offset =  index;
 904 #else
 905         offset = 1 - index;
 906 #endif
 907
 908         return offset;
 909 }
 910
 911 static inline int kvmppc_get_vsr_word_offset(int index)
 912 {
 913         int offset;
 914
 915         if ((index > 3) || (index < 0))
 916                 return -1;
 917
 918 #ifdef __BIG_ENDIAN
 919         offset = index;
 920 #else
 921         offset = 3 - index;
 922 #endif
 923         return offset;
 924 }
 925
 926 static inline void kvmppc_set_vsr_dword(struct kvm_vcpu *vcpu,
 927         u64 gpr)
 928 {
 929         union kvmppc_one_reg val;
 930         int offset = kvmppc_get_vsr_dword_offset(vcpu->arch.mmio_vsx_offset);
 931         int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
 932
 933         if (offset == -1)
 934                 return;
 935
 936         if (index >= 32) {
 937                 val.vval = VCPU_VSX_VR(vcpu, index - 32);
 938                 val.vsxval[offset] = gpr;
 939                 VCPU_VSX_VR(vcpu, index - 32) = val.vval;
 940         } else {
 941                 VCPU_VSX_FPR(vcpu, index, offset) = gpr;
 942         }
 943 }
 944
 945 static inline void kvmppc_set_vsr_dword_dump(struct kvm_vcpu *vcpu,
 946         u64 gpr)
 947 {
 948         union kvmppc_one_reg val;
 949         int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
 950
 951         if (index >= 32) {
 952                 val.vval = VCPU_VSX_VR(vcpu, index - 32);
 953                 val.vsxval[0] = gpr;
 954                 val.vsxval[1] = gpr;
 955                 VCPU_VSX_VR(vcpu, index - 32) = val.vval;
 956         } else {
 957                 VCPU_VSX_FPR(vcpu, index, 0) = gpr;
 958                 VCPU_VSX_FPR(vcpu, index, 1) = gpr;
 959         }
 960 }
 961
 962 static inline void kvmppc_set_vsr_word_dump(struct kvm_vcpu *vcpu,
 963         u32 gpr)
 964 {
 965         union kvmppc_one_reg val;
 966         int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
 967
 968         if (index >= 32) {
 969                 val.vsx32val[0] = gpr;
 970                 val.vsx32val[1] = gpr;
 971                 val.vsx32val[2] = gpr;
 972                 val.vsx32val[3] = gpr;
 973                 VCPU_VSX_VR(vcpu, index - 32) = val.vval;
 974         } else {
 975                 val.vsx32val[0] = gpr;
 976                 val.vsx32val[1] = gpr;
 977                 VCPU_VSX_FPR(vcpu, index, 0) = val.vsxval[0];
 978                 VCPU_VSX_FPR(vcpu, index, 1) = val.vsxval[0];
 979         }
 980 }
 981
 982 static inline void kvmppc_set_vsr_word(struct kvm_vcpu *vcpu,
 983         u32 gpr32)
 984 {
 985         union kvmppc_one_reg val;
 986         int offset = kvmppc_get_vsr_word_offset(vcpu->arch.mmio_vsx_offset);
 987         int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
 988         int dword_offset, word_offset;
 989
 990         if (offset == -1)
 991                 return;
 992
 993         if (index >= 32) {
 994                 val.vval = VCPU_VSX_VR(vcpu, index - 32);
 995                 val.vsx32val[offset] = gpr32;
 996                 VCPU_VSX_VR(vcpu, index - 32) = val.vval;
 997         } else {
 998                 dword_offset = offset / 2;
 999                 word_offset = offset % 2;
1000                 val.vsxval[0] = VCPU_VSX_FPR(vcpu, index, dword_offset);
1001                 val.vsx32val[word_offset] = gpr32;
1002                 VCPU_VSX_FPR(vcpu, index, dword_offset) = val.vsxval[0];
1003         }
1004 }
1005 #endif /* CONFIG_VSX */
1006
1007 #ifdef CONFIG_ALTIVEC
1008 static inline int kvmppc_get_vmx_offset_generic(struct kvm_vcpu *vcpu,
1009                 int index, int element_size)
1010 {
1011         int offset;
1012         int elts = sizeof(vector128)/element_size;
1013
1014         if ((index < 0) || (index >= elts))
1015                 return -1;
1016
1017         if (kvmppc_need_byteswap(vcpu))
1018                 offset = elts - index - 1;
1019         else
1020                 offset = index;
1021
1022         return offset;
1023 }
1024
1025 static inline int kvmppc_get_vmx_dword_offset(struct kvm_vcpu *vcpu,
1026                 int index)
1027 {
1028         return kvmppc_get_vmx_offset_generic(vcpu, index, 8);
1029 }
1030
1031 static inline int kvmppc_get_vmx_word_offset(struct kvm_vcpu *vcpu,
1032                 int index)
1033 {
1034         return kvmppc_get_vmx_offset_generic(vcpu, index, 4);
1035 }
1036
1037 static inline int kvmppc_get_vmx_hword_offset(struct kvm_vcpu *vcpu,
1038                 int index)
1039 {
1040         return kvmppc_get_vmx_offset_generic(vcpu, index, 2);
1041 }
1042
1043 static inline int kvmppc_get_vmx_byte_offset(struct kvm_vcpu *vcpu,
1044                 int index)
1045 {
1046         return kvmppc_get_vmx_offset_generic(vcpu, index, 1);
1047 }
1048
1049
1050 static inline void kvmppc_set_vmx_dword(struct kvm_vcpu *vcpu,
1051         u64 gpr)
1052 {
1053         union kvmppc_one_reg val;
1054         int offset = kvmppc_get_vmx_dword_offset(vcpu,
1055                         vcpu->arch.mmio_vmx_offset);
1056         int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
1057
1058         if (offset == -1)
1059                 return;
1060
1061         val.vval = VCPU_VSX_VR(vcpu, index);
1062         val.vsxval[offset] = gpr;
1063         VCPU_VSX_VR(vcpu, index) = val.vval;
1064 }
1065
1066 static inline void kvmppc_set_vmx_word(struct kvm_vcpu *vcpu,
1067         u32 gpr32)
1068 {
1069         union kvmppc_one_reg val;
1070         int offset = kvmppc_get_vmx_word_offset(vcpu,
1071                         vcpu->arch.mmio_vmx_offset);
1072         int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
1073
1074         if (offset == -1)
1075                 return;
1076
1077         val.vval = VCPU_VSX_VR(vcpu, index);
1078         val.vsx32val[offset] = gpr32;
1079         VCPU_VSX_VR(vcpu, index) = val.vval;
1080 }
1081
1082 static inline void kvmppc_set_vmx_hword(struct kvm_vcpu *vcpu,
1083         u16 gpr16)
1084 {
1085         union kvmppc_one_reg val;
1086         int offset = kvmppc_get_vmx_hword_offset(vcpu,
1087                         vcpu->arch.mmio_vmx_offset);
1088         int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
1089
1090         if (offset == -1)
1091                 return;
1092
1093         val.vval = VCPU_VSX_VR(vcpu, index);
1094         val.vsx16val[offset] = gpr16;
1095         VCPU_VSX_VR(vcpu, index) = val.vval;
1096 }
1097
1098 static inline void kvmppc_set_vmx_byte(struct kvm_vcpu *vcpu,
1099         u8 gpr8)
1100 {
1101         union kvmppc_one_reg val;
1102         int offset = kvmppc_get_vmx_byte_offset(vcpu,
1103                         vcpu->arch.mmio_vmx_offset);
1104         int index = vcpu->arch.io_gpr & KVM_MMIO_REG_MASK;
1105
1106         if (offset == -1)
1107                 return;
1108
1109         val.vval = VCPU_VSX_VR(vcpu, index);
1110         val.vsx8val[offset] = gpr8;
1111         VCPU_VSX_VR(vcpu, index) = val.vval;
1112 }
1113 #endif /* CONFIG_ALTIVEC */
1114
1115 #ifdef CONFIG_PPC_FPU
1116 static inline u64 sp_to_dp(u32 fprs)
1117 {
1118         u64 fprd;
1119
1120         preempt_disable();
1121         enable_kernel_fp();
1122         asm ("lfs%U1%X1 0,%1; stfd%U0%X0 0,%0" : "=m<>" (fprd) : "m<>" (fprs)
1123              : "fr0");
1124         preempt_enable();
1125         return fprd;
1126 }
1127
1128 static inline u32 dp_to_sp(u64 fprd)
1129 {
1130         u32 fprs;
1131
1132         preempt_disable();
1133         enable_kernel_fp();
1134         asm ("lfd%U1%X1 0,%1; stfs%U0%X0 0,%0" : "=m<>" (fprs) : "m<>" (fprd)
1135              : "fr0");
1136         preempt_enable();
1137         return fprs;
1138 }
1139
1140 #else
1141 #define sp_to_dp(x)     (x)
1142 #define dp_to_sp(x)     (x)
1143 #endif /* CONFIG_PPC_FPU */
1144
1145 static void kvmppc_complete_mmio_load(struct kvm_vcpu *vcpu)
1146 {
1147         struct kvm_run *run = vcpu->run;
1148         u64 gpr;
1149
1150         if (run->mmio.len > sizeof(gpr))
1151                 return;
1152
1153         if (!vcpu->arch.mmio_host_swabbed) {
1154                 switch (run->mmio.len) {
1155                 case 8: gpr = *(u64 *)run->mmio.data; break;
1156                 case 4: gpr = *(u32 *)run->mmio.data; break;
1157                 case 2: gpr = *(u16 *)run->mmio.data; break;
1158                 case 1: gpr = *(u8 *)run->mmio.data; break;
1159                 }
1160         } else {
1161                 switch (run->mmio.len) {
1162                 case 8: gpr = swab64(*(u64 *)run->mmio.data); break;
1163                 case 4: gpr = swab32(*(u32 *)run->mmio.data); break;
1164                 case 2: gpr = swab16(*(u16 *)run->mmio.data); break;
1165                 case 1: gpr = *(u8 *)run->mmio.data; break;
1166                 }
1167         }
1168
1169         /* conversion between single and double precision */
1170         if ((vcpu->arch.mmio_sp64_extend) && (run->mmio.len == 4))
1171                 gpr = sp_to_dp(gpr);
1172
1173         if (vcpu->arch.mmio_sign_extend) {
1174                 switch (run->mmio.len) {
1175 #ifdef CONFIG_PPC64
1176                 case 4:
1177                         gpr = (s64)(s32)gpr;
1178                         break;
1179 #endif
1180                 case 2:
1181                         gpr = (s64)(s16)gpr;
1182                         break;
1183                 case 1:
1184                         gpr = (s64)(s8)gpr;
1185                         break;
1186                 }
1187         }
1188
1189         switch (vcpu->arch.io_gpr & KVM_MMIO_REG_EXT_MASK) {
1190         case KVM_MMIO_REG_GPR:
1191                 kvmppc_set_gpr(vcpu, vcpu->arch.io_gpr, gpr);
1192                 break;
1193         case KVM_MMIO_REG_FPR:
1194                 if (vcpu->kvm->arch.kvm_ops->giveup_ext)
1195                         vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_FP);
1196
1197                 VCPU_FPR(vcpu, vcpu->arch.io_gpr & KVM_MMIO_REG_MASK) = gpr;
1198                 break;
1199 #ifdef CONFIG_PPC_BOOK3S
1200         case KVM_MMIO_REG_QPR:
1201                 vcpu->arch.qpr[vcpu->arch.io_gpr & KVM_MMIO_REG_MASK] = gpr;
1202                 break;
1203         case KVM_MMIO_REG_FQPR:
1204                 VCPU_FPR(vcpu, vcpu->arch.io_gpr & KVM_MMIO_REG_MASK) = gpr;
1205                 vcpu->arch.qpr[vcpu->arch.io_gpr & KVM_MMIO_REG_MASK] = gpr;
1206                 break;
1207 #endif
1208 #ifdef CONFIG_VSX
1209         case KVM_MMIO_REG_VSX:
1210                 if (vcpu->kvm->arch.kvm_ops->giveup_ext)
1211                         vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_VSX);
1212
1213                 if (vcpu->arch.mmio_copy_type == KVMPPC_VSX_COPY_DWORD)
1214                         kvmppc_set_vsr_dword(vcpu, gpr);
1215                 else if (vcpu->arch.mmio_copy_type == KVMPPC_VSX_COPY_WORD)
1216                         kvmppc_set_vsr_word(vcpu, gpr);
1217                 else if (vcpu->arch.mmio_copy_type ==
1218                                 KVMPPC_VSX_COPY_DWORD_LOAD_DUMP)
1219                         kvmppc_set_vsr_dword_dump(vcpu, gpr);
1220                 else if (vcpu->arch.mmio_copy_type ==
1221                                 KVMPPC_VSX_COPY_WORD_LOAD_DUMP)
1222                         kvmppc_set_vsr_word_dump(vcpu, gpr);
1223                 break;
1224 #endif
1225 #ifdef CONFIG_ALTIVEC
1226         case KVM_MMIO_REG_VMX:
1227                 if (vcpu->kvm->arch.kvm_ops->giveup_ext)
1228                         vcpu->kvm->arch.kvm_ops->giveup_ext(vcpu, MSR_VEC);
1229
1230                 if (vcpu->arch.mmio_copy_type == KVMPPC_VMX_COPY_DWORD)
1231                         kvmppc_set_vmx_dword(vcpu, gpr);
1232                 else if (vcpu->arch.mmio_copy_type == KVMPPC_VMX_COPY_WORD)
1233                         kvmppc_set_vmx_word(vcpu, gpr);
1234                 else if (vcpu->arch.mmio_copy_type ==
1235                                 KVMPPC_VMX_COPY_HWORD)
1236                         kvmppc_set_vmx_hword(vcpu, gpr);
1237                 else if (vcpu->arch.mmio_copy_type ==
1238                                 KVMPPC_VMX_COPY_BYTE)
1239                         kvmppc_set_vmx_byte(vcpu, gpr);
1240                 break;
1241 #endif
1242 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
1243         case KVM_MMIO_REG_NESTED_GPR:
1244                 if (kvmppc_need_byteswap(vcpu))
1245                         gpr = swab64(gpr);
1246                 kvm_vcpu_write_guest(vcpu, vcpu->arch.nested_io_gpr, &gpr,
1247                                      sizeof(gpr));
1248                 break;
1249 #endif
1250         default:
1251                 BUG();
1252         }
1253 }
1254
1255 static int __kvmppc_handle_load(struct kvm_vcpu *vcpu,
1256                                 unsigned int rt, unsigned int bytes,
1257                                 int is_default_endian, int sign_extend)
1258 {
1259         struct kvm_run *run = vcpu->run;
1260         int idx, ret;
1261         bool host_swabbed;
1262
1263         /* Pity C doesn't have a logical XOR operator */
1264         if (kvmppc_need_byteswap(vcpu)) {
1265                 host_swabbed = is_default_endian;
1266         } else {
1267                 host_swabbed = !is_default_endian;
1268         }
1269
1270         if (bytes > sizeof(run->mmio.data))
1271                 return EMULATE_FAIL;
1272
1273         run->mmio.phys_addr = vcpu->arch.paddr_accessed;
1274         run->mmio.len = bytes;
1275         run->mmio.is_write = 0;
1276
1277         vcpu->arch.io_gpr = rt;
1278         vcpu->arch.mmio_host_swabbed = host_swabbed;
1279         vcpu->mmio_needed = 1;
1280         vcpu->mmio_is_write = 0;
1281         vcpu->arch.mmio_sign_extend = sign_extend;
1282
1283         idx = srcu_read_lock(&vcpu->kvm->srcu);
1284
1285         ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
1286                               bytes, &run->mmio.data);
1287
1288         srcu_read_unlock(&vcpu->kvm->srcu, idx);
1289
1290         if (!ret) {
1291                 kvmppc_complete_mmio_load(vcpu);
1292                 vcpu->mmio_needed = 0;
1293                 return EMULATE_DONE;
1294         }
1295
1296         return EMULATE_DO_MMIO;
1297 }
1298
1299 int kvmppc_handle_load(struct kvm_vcpu *vcpu,
1300                        unsigned int rt, unsigned int bytes,
1301                        int is_default_endian)
1302 {
1303         return __kvmppc_handle_load(vcpu, rt, bytes, is_default_endian, 0);
1304 }
1305 EXPORT_SYMBOL_GPL(kvmppc_handle_load);
1306
1307 /* Same as above, but sign extends */
1308 int kvmppc_handle_loads(struct kvm_vcpu *vcpu,
1309                         unsigned int rt, unsigned int bytes,
1310                         int is_default_endian)
1311 {
1312         return __kvmppc_handle_load(vcpu, rt, bytes, is_default_endian, 1);
1313 }
1314
1315 #ifdef CONFIG_VSX
1316 int kvmppc_handle_vsx_load(struct kvm_vcpu *vcpu,
1317                         unsigned int rt, unsigned int bytes,
1318                         int is_default_endian, int mmio_sign_extend)
1319 {
1320         enum emulation_result emulated = EMULATE_DONE;
1321
1322         /* Currently, mmio_vsx_copy_nums only allowed to be 4 or less */
1323         if (vcpu->arch.mmio_vsx_copy_nums > 4)
1324                 return EMULATE_FAIL;
1325
1326         while (vcpu->arch.mmio_vsx_copy_nums) {
1327                 emulated = __kvmppc_handle_load(vcpu, rt, bytes,
1328                         is_default_endian, mmio_sign_extend);
1329
1330                 if (emulated != EMULATE_DONE)
1331                         break;
1332
1333                 vcpu->arch.paddr_accessed += vcpu->run->mmio.len;
1334
1335                 vcpu->arch.mmio_vsx_copy_nums--;
1336                 vcpu->arch.mmio_vsx_offset++;
1337         }
1338         return emulated;
1339 }
1340 #endif /* CONFIG_VSX */
1341
1342 int kvmppc_handle_store(struct kvm_vcpu *vcpu,
1343                         u64 val, unsigned int bytes, int is_default_endian)
1344 {
1345         struct kvm_run *run = vcpu->run;
1346         void *data = run->mmio.data;
1347         int idx, ret;
1348         bool host_swabbed;
1349
1350         /* Pity C doesn't have a logical XOR operator */
1351         if (kvmppc_need_byteswap(vcpu)) {
1352                 host_swabbed = is_default_endian;
1353         } else {
1354                 host_swabbed = !is_default_endian;
1355         }
1356
1357         if (bytes > sizeof(run->mmio.data))
1358                 return EMULATE_FAIL;
1359
1360         run->mmio.phys_addr = vcpu->arch.paddr_accessed;
1361         run->mmio.len = bytes;
1362         run->mmio.is_write = 1;
1363         vcpu->mmio_needed = 1;
1364         vcpu->mmio_is_write = 1;
1365
1366         if ((vcpu->arch.mmio_sp64_extend) && (bytes == 4))
1367                 val = dp_to_sp(val);
1368
1369         /* Store the value at the lowest bytes in 'data'. */
1370         if (!host_swabbed) {
1371                 switch (bytes) {
1372                 case 8: *(u64 *)data = val; break;
1373                 case 4: *(u32 *)data = val; break;
1374                 case 2: *(u16 *)data = val; break;
1375                 case 1: *(u8  *)data = val; break;
1376                 }
1377         } else {
1378                 switch (bytes) {
1379                 case 8: *(u64 *)data = swab64(val); break;
1380                 case 4: *(u32 *)data = swab32(val); break;
1381                 case 2: *(u16 *)data = swab16(val); break;
1382                 case 1: *(u8  *)data = val; break;
1383                 }
1384         }
1385
1386         idx = srcu_read_lock(&vcpu->kvm->srcu);
1387
1388         ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, run->mmio.phys_addr,
1389                                bytes, &run->mmio.data);
1390
1391         srcu_read_unlock(&vcpu->kvm->srcu, idx);
1392
1393         if (!ret) {
1394                 vcpu->mmio_needed = 0;
1395                 return EMULATE_DONE;
1396         }
1397
1398         return EMULATE_DO_MMIO;
1399 }
1400 EXPORT_SYMBOL_GPL(kvmppc_handle_store);
1401
1402 #ifdef CONFIG_VSX
1403 static inline int kvmppc_get_vsr_data(struct kvm_vcpu *vcpu, int rs, u64 *val)
1404 {
1405         u32 dword_offset, word_offset;
1406         union kvmppc_one_reg reg;
1407         int vsx_offset = 0;
1408         int copy_type = vcpu->arch.mmio_copy_type;
1409         int result = 0;
1410
1411         switch (copy_type) {
1412         case KVMPPC_VSX_COPY_DWORD:
1413                 vsx_offset =
1414                         kvmppc_get_vsr_dword_offset(vcpu->arch.mmio_vsx_offset);
1415
1416                 if (vsx_offset == -1) {
1417                         result = -1;
1418                         break;
1419                 }
1420
1421                 if (rs < 32) {
1422                         *val = VCPU_VSX_FPR(vcpu, rs, vsx_offset);
1423                 } else {
1424                         reg.vval = VCPU_VSX_VR(vcpu, rs - 32);
1425                         *val = reg.vsxval[vsx_offset];
1426                 }
1427                 break;
1428
1429         case KVMPPC_VSX_COPY_WORD:
1430                 vsx_offset =
1431                         kvmppc_get_vsr_word_offset(vcpu->arch.mmio_vsx_offset);
1432
1433                 if (vsx_offset == -1) {
1434                         result = -1;
1435                         break;
1436                 }
1437
1438                 if (rs < 32) {
1439                         dword_offset = vsx_offset / 2;
1440                         word_offset = vsx_offset % 2;
1441                         reg.vsxval[0] = VCPU_VSX_FPR(vcpu, rs, dword_offset);
1442                         *val = reg.vsx32val[word_offset];
1443                 } else {
1444                         reg.vval = VCPU_VSX_VR(vcpu, rs - 32);
1445                         *val = reg.vsx32val[vsx_offset];
1446                 }
1447                 break;
1448
1449         default:
1450                 result = -1;
1451                 break;
1452         }
1453
1454         return result;
1455 }
1456
1457 int kvmppc_handle_vsx_store(struct kvm_vcpu *vcpu,
1458                         int rs, unsigned int bytes, int is_default_endian)
1459 {
1460         u64 val;
1461         enum emulation_result emulated = EMULATE_DONE;
1462
1463         vcpu->arch.io_gpr = rs;
1464
1465         /* Currently, mmio_vsx_copy_nums only allowed to be 4 or less */
1466         if (vcpu->arch.mmio_vsx_copy_nums > 4)
1467                 return EMULATE_FAIL;
1468
1469         while (vcpu->arch.mmio_vsx_copy_nums) {
1470                 if (kvmppc_get_vsr_data(vcpu, rs, &val) == -1)
1471                         return EMULATE_FAIL;
1472
1473                 emulated = kvmppc_handle_store(vcpu,
1474                          val, bytes, is_default_endian);
1475
1476                 if (emulated != EMULATE_DONE)
1477                         break;
1478
1479                 vcpu->arch.paddr_accessed += vcpu->run->mmio.len;
1480
1481                 vcpu->arch.mmio_vsx_copy_nums--;
1482                 vcpu->arch.mmio_vsx_offset++;
1483         }
1484
1485         return emulated;
1486 }
1487
1488 static int kvmppc_emulate_mmio_vsx_loadstore(struct kvm_vcpu *vcpu)
1489 {
1490         struct kvm_run *run = vcpu->run;
1491         enum emulation_result emulated = EMULATE_FAIL;
1492         int r;
1493
1494         vcpu->arch.paddr_accessed += run->mmio.len;
1495
1496         if (!vcpu->mmio_is_write) {
1497                 emulated = kvmppc_handle_vsx_load(vcpu, vcpu->arch.io_gpr,
1498                          run->mmio.len, 1, vcpu->arch.mmio_sign_extend);
1499         } else {
1500                 emulated = kvmppc_handle_vsx_store(vcpu,
1501                          vcpu->arch.io_gpr, run->mmio.len, 1);
1502         }
1503
1504         switch (emulated) {
1505         case EMULATE_DO_MMIO:
1506                 run->exit_reason = KVM_EXIT_MMIO;
1507                 r = RESUME_HOST;
1508                 break;
1509         case EMULATE_FAIL:
1510                 pr_info("KVM: MMIO emulation failed (VSX repeat)\n");
1511                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1512                 run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
1513                 r = RESUME_HOST;
1514                 break;
1515         default:
1516                 r = RESUME_GUEST;
1517                 break;
1518         }
1519         return r;
1520 }
1521 #endif /* CONFIG_VSX */
1522
1523 #ifdef CONFIG_ALTIVEC
1524 int kvmppc_handle_vmx_load(struct kvm_vcpu *vcpu,
1525                 unsigned int rt, unsigned int bytes, int is_default_endian)
1526 {
1527         enum emulation_result emulated = EMULATE_DONE;
1528
1529         if (vcpu->arch.mmio_vmx_copy_nums > 2)
1530                 return EMULATE_FAIL;
1531
1532         while (vcpu->arch.mmio_vmx_copy_nums) {
1533                 emulated = __kvmppc_handle_load(vcpu, rt, bytes,
1534                                 is_default_endian, 0);
1535
1536                 if (emulated != EMULATE_DONE)
1537                         break;
1538
1539                 vcpu->arch.paddr_accessed += vcpu->run->mmio.len;
1540                 vcpu->arch.mmio_vmx_copy_nums--;
1541                 vcpu->arch.mmio_vmx_offset++;
1542         }
1543
1544         return emulated;
1545 }
1546
1547 static int kvmppc_get_vmx_dword(struct kvm_vcpu *vcpu, int index, u64 *val)
1548 {
1549         union kvmppc_one_reg reg;
1550         int vmx_offset = 0;
1551         int result = 0;
1552
1553         vmx_offset =
1554                 kvmppc_get_vmx_dword_offset(vcpu, vcpu->arch.mmio_vmx_offset);
1555
1556         if (vmx_offset == -1)
1557                 return -1;
1558
1559         reg.vval = VCPU_VSX_VR(vcpu, index);
1560         *val = reg.vsxval[vmx_offset];
1561
1562         return result;
1563 }
1564
1565 static int kvmppc_get_vmx_word(struct kvm_vcpu *vcpu, int index, u64 *val)
1566 {
1567         union kvmppc_one_reg reg;
1568         int vmx_offset = 0;
1569         int result = 0;
1570
1571         vmx_offset =
1572                 kvmppc_get_vmx_word_offset(vcpu, vcpu->arch.mmio_vmx_offset);
1573
1574         if (vmx_offset == -1)
1575                 return -1;
1576
1577         reg.vval = VCPU_VSX_VR(vcpu, index);
1578         *val = reg.vsx32val[vmx_offset];
1579
1580         return result;
1581 }
1582
1583 static int kvmppc_get_vmx_hword(struct kvm_vcpu *vcpu, int index, u64 *val)
1584 {
1585         union kvmppc_one_reg reg;
1586         int vmx_offset = 0;
1587         int result = 0;
1588
1589         vmx_offset =
1590                 kvmppc_get_vmx_hword_offset(vcpu, vcpu->arch.mmio_vmx_offset);
1591
1592         if (vmx_offset == -1)
1593                 return -1;
1594
1595         reg.vval = VCPU_VSX_VR(vcpu, index);
1596         *val = reg.vsx16val[vmx_offset];
1597
1598         return result;
1599 }
1600
1601 static int kvmppc_get_vmx_byte(struct kvm_vcpu *vcpu, int index, u64 *val)
1602 {
1603         union kvmppc_one_reg reg;
1604         int vmx_offset = 0;
1605         int result = 0;
1606
1607         vmx_offset =
1608                 kvmppc_get_vmx_byte_offset(vcpu, vcpu->arch.mmio_vmx_offset);
1609
1610         if (vmx_offset == -1)
1611                 return -1;
1612
1613         reg.vval = VCPU_VSX_VR(vcpu, index);
1614         *val = reg.vsx8val[vmx_offset];
1615
1616         return result;
1617 }
1618
1619 int kvmppc_handle_vmx_store(struct kvm_vcpu *vcpu,
1620                 unsigned int rs, unsigned int bytes, int is_default_endian)
1621 {
1622         u64 val = 0;
1623         unsigned int index = rs & KVM_MMIO_REG_MASK;
1624         enum emulation_result emulated = EMULATE_DONE;
1625
1626         if (vcpu->arch.mmio_vmx_copy_nums > 2)
1627                 return EMULATE_FAIL;
1628
1629         vcpu->arch.io_gpr = rs;
1630
1631         while (vcpu->arch.mmio_vmx_copy_nums) {
1632                 switch (vcpu->arch.mmio_copy_type) {
1633                 case KVMPPC_VMX_COPY_DWORD:
1634                         if (kvmppc_get_vmx_dword(vcpu, index, &val) == -1)
1635                                 return EMULATE_FAIL;
1636
1637                         break;
1638                 case KVMPPC_VMX_COPY_WORD:
1639                         if (kvmppc_get_vmx_word(vcpu, index, &val) == -1)
1640                                 return EMULATE_FAIL;
1641                         break;
1642                 case KVMPPC_VMX_COPY_HWORD:
1643                         if (kvmppc_get_vmx_hword(vcpu, index, &val) == -1)
1644                                 return EMULATE_FAIL;
1645                         break;
1646                 case KVMPPC_VMX_COPY_BYTE:
1647                         if (kvmppc_get_vmx_byte(vcpu, index, &val) == -1)
1648                                 return EMULATE_FAIL;
1649                         break;
1650                 default:
1651                         return EMULATE_FAIL;
1652                 }
1653
1654                 emulated = kvmppc_handle_store(vcpu, val, bytes,
1655                                 is_default_endian);
1656                 if (emulated != EMULATE_DONE)
1657                         break;
1658
1659                 vcpu->arch.paddr_accessed += vcpu->run->mmio.len;
1660                 vcpu->arch.mmio_vmx_copy_nums--;
1661                 vcpu->arch.mmio_vmx_offset++;
1662         }
1663
1664         return emulated;
1665 }
1666
1667 static int kvmppc_emulate_mmio_vmx_loadstore(struct kvm_vcpu *vcpu)
1668 {
1669         struct kvm_run *run = vcpu->run;
1670         enum emulation_result emulated = EMULATE_FAIL;
1671         int r;
1672
1673         vcpu->arch.paddr_accessed += run->mmio.len;
1674
1675         if (!vcpu->mmio_is_write) {
1676                 emulated = kvmppc_handle_vmx_load(vcpu,
1677                                 vcpu->arch.io_gpr, run->mmio.len, 1);
1678         } else {
1679                 emulated = kvmppc_handle_vmx_store(vcpu,
1680                                 vcpu->arch.io_gpr, run->mmio.len, 1);
1681         }
1682
1683         switch (emulated) {
1684         case EMULATE_DO_MMIO:
1685                 run->exit_reason = KVM_EXIT_MMIO;
1686                 r = RESUME_HOST;
1687                 break;
1688         case EMULATE_FAIL:
1689                 pr_info("KVM: MMIO emulation failed (VMX repeat)\n");
1690                 run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
1691                 run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
1692                 r = RESUME_HOST;
1693                 break;
1694         default:
1695                 r = RESUME_GUEST;
1696                 break;
1697         }
1698         return r;
1699 }
1700 #endif /* CONFIG_ALTIVEC */
1701
1702 int kvm_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg)
1703 {
1704         int r = 0;
1705         union kvmppc_one_reg val;
1706         int size;
1707
1708         size = one_reg_size(reg->id);
1709         if (size > sizeof(val))
1710                 return -EINVAL;
1711
1712         r = kvmppc_get_one_reg(vcpu, reg->id, &val);
1713         if (r == -EINVAL) {
1714                 r = 0;
1715                 switch (reg->id) {
1716 #ifdef CONFIG_ALTIVEC
1717                 case KVM_REG_PPC_VR0 ... KVM_REG_PPC_VR31:
1718                         if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
1719                                 r = -ENXIO;
1720                                 break;
1721                         }
1722                         val.vval = vcpu->arch.vr.vr[reg->id - KVM_REG_PPC_VR0];
1723                         break;
1724                 case KVM_REG_PPC_VSCR:
1725                         if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
1726                                 r = -ENXIO;
1727                                 break;
1728                         }
1729                         val = get_reg_val(reg->id, vcpu->arch.vr.vscr.u[3]);
1730                         break;
1731                 case KVM_REG_PPC_VRSAVE:
1732                         val = get_reg_val(reg->id, vcpu->arch.vrsave);
1733                         break;
1734 #endif /* CONFIG_ALTIVEC */
1735                 default:
1736                         r = -EINVAL;
1737                         break;
1738                 }
1739         }
1740
1741         if (r)
1742                 return r;
1743
1744         if (copy_to_user((char __user *)(unsigned long)reg->addr, &val, size))
1745                 r = -EFAULT;
1746
1747         return r;
1748 }
1749
1750 int kvm_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu, struct kvm_one_reg *reg)
1751 {
1752         int r;
1753         union kvmppc_one_reg val;
1754         int size;
1755
1756         size = one_reg_size(reg->id);
1757         if (size > sizeof(val))
1758                 return -EINVAL;
1759
1760         if (copy_from_user(&val, (char __user *)(unsigned long)reg->addr, size))
1761                 return -EFAULT;
1762
1763         r = kvmppc_set_one_reg(vcpu, reg->id, &val);
1764         if (r == -EINVAL) {
1765                 r = 0;
1766                 switch (reg->id) {
1767 #ifdef CONFIG_ALTIVEC
1768                 case KVM_REG_PPC_VR0 ... KVM_REG_PPC_VR31:
1769                         if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
1770                                 r = -ENXIO;
1771                                 break;
1772                         }
1773                         vcpu->arch.vr.vr[reg->id - KVM_REG_PPC_VR0] = val.vval;
1774                         break;
1775                 case KVM_REG_PPC_VSCR:
1776                         if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
1777                                 r = -ENXIO;
1778                                 break;
1779                         }
1780                         vcpu->arch.vr.vscr.u[3] = set_reg_val(reg->id, val);
1781                         break;
1782                 case KVM_REG_PPC_VRSAVE:
1783                         if (!cpu_has_feature(CPU_FTR_ALTIVEC)) {
1784                                 r = -ENXIO;
1785                                 break;
1786                         }
1787                         vcpu->arch.vrsave = set_reg_val(reg->id, val);
1788                         break;
1789 #endif /* CONFIG_ALTIVEC */
1790                 default:
1791                         r = -EINVAL;
1792                         break;
1793                 }
1794         }
1795
1796         return r;
1797 }
1798
1799 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
1800 {
1801         struct kvm_run *run = vcpu->run;
1802         int r;
1803
1804         vcpu_load(vcpu);
1805
1806         if (vcpu->mmio_needed) {
1807                 vcpu->mmio_needed = 0;
1808                 if (!vcpu->mmio_is_write)
1809                         kvmppc_complete_mmio_load(vcpu);
1810 #ifdef CONFIG_VSX
1811                 if (vcpu->arch.mmio_vsx_copy_nums > 0) {
1812                         vcpu->arch.mmio_vsx_copy_nums--;
1813                         vcpu->arch.mmio_vsx_offset++;
1814                 }
1815
1816                 if (vcpu->arch.mmio_vsx_copy_nums > 0) {
1817                         r = kvmppc_emulate_mmio_vsx_loadstore(vcpu);
1818                         if (r == RESUME_HOST) {
1819                                 vcpu->mmio_needed = 1;
1820                                 goto out;
1821                         }
1822                 }
1823 #endif
1824 #ifdef CONFIG_ALTIVEC
1825                 if (vcpu->arch.mmio_vmx_copy_nums > 0) {
1826                         vcpu->arch.mmio_vmx_copy_nums--;
1827                         vcpu->arch.mmio_vmx_offset++;
1828                 }
1829
1830                 if (vcpu->arch.mmio_vmx_copy_nums > 0) {
1831                         r = kvmppc_emulate_mmio_vmx_loadstore(vcpu);
1832                         if (r == RESUME_HOST) {
1833                                 vcpu->mmio_needed = 1;
1834                                 goto out;
1835                         }
1836                 }
1837 #endif
1838         } else if (vcpu->arch.osi_needed) {
1839                 u64 *gprs = run->osi.gprs;
1840                 int i;
1841
1842                 for (i = 0; i < 32; i++)
1843                         kvmppc_set_gpr(vcpu, i, gprs[i]);
1844                 vcpu->arch.osi_needed = 0;
1845         } else if (vcpu->arch.hcall_needed) {
1846                 int i;
1847
1848                 kvmppc_set_gpr(vcpu, 3, run->papr_hcall.ret);
1849                 for (i = 0; i < 9; ++i)
1850                         kvmppc_set_gpr(vcpu, 4 + i, run->papr_hcall.args[i]);
1851                 vcpu->arch.hcall_needed = 0;
1852 #ifdef CONFIG_BOOKE
1853         } else if (vcpu->arch.epr_needed) {
1854                 kvmppc_set_epr(vcpu, run->epr.epr);
1855                 vcpu->arch.epr_needed = 0;
1856 #endif
1857         }
1858
1859         kvm_sigset_activate(vcpu);
1860
1861         if (run->immediate_exit)
1862                 r = -EINTR;
1863         else
1864                 r = kvmppc_vcpu_run(vcpu);
1865
1866         kvm_sigset_deactivate(vcpu);
1867
1868 #ifdef CONFIG_ALTIVEC
1869 out:
1870 #endif
1871
1872         /*
1873          * We're already returning to userspace, don't pass the
1874          * RESUME_HOST flags along.
1875          */
1876         if (r > 0)
1877                 r = 0;
1878
1879         vcpu_put(vcpu);
1880         return r;
1881 }
1882
1883 int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, struct kvm_interrupt *irq)
1884 {
1885         if (irq->irq == KVM_INTERRUPT_UNSET) {
1886                 kvmppc_core_dequeue_external(vcpu);
1887                 return 0;
1888         }
1889
1890         kvmppc_core_queue_external(vcpu, irq);
1891
1892         kvm_vcpu_kick(vcpu);
1893
1894         return 0;
1895 }
1896
1897 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
1898                                      struct kvm_enable_cap *cap)
1899 {
1900         int r;
1901
1902         if (cap->flags)
1903                 return -EINVAL;
1904
1905         switch (cap->cap) {
1906         case KVM_CAP_PPC_OSI:
1907                 r = 0;
1908                 vcpu->arch.osi_enabled = true;
1909                 break;
1910         case KVM_CAP_PPC_PAPR:
1911                 r = 0;
1912                 vcpu->arch.papr_enabled = true;
1913                 break;
1914         case KVM_CAP_PPC_EPR:
1915                 r = 0;
1916                 if (cap->args[0])
1917                         vcpu->arch.epr_flags |= KVMPPC_EPR_USER;
1918                 else
1919                         vcpu->arch.epr_flags &= ~KVMPPC_EPR_USER;
1920                 break;
1921 #ifdef CONFIG_BOOKE
1922         case KVM_CAP_PPC_BOOKE_WATCHDOG:
1923                 r = 0;
1924                 vcpu->arch.watchdog_enabled = true;
1925                 break;
1926 #endif
1927 #if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC)
1928         case KVM_CAP_SW_TLB: {
1929                 struct kvm_config_tlb cfg;
1930                 void __user *user_ptr = (void __user *)(uintptr_t)cap->args[0];
1931
1932                 r = -EFAULT;
1933                 if (copy_from_user(&cfg, user_ptr, sizeof(cfg)))
1934                         break;
1935
1936                 r = kvm_vcpu_ioctl_config_tlb(vcpu, &cfg);
1937                 break;
1938         }
1939 #endif
1940 #ifdef CONFIG_KVM_MPIC
1941         case KVM_CAP_IRQ_MPIC: {
1942                 struct fd f;
1943                 struct kvm_device *dev;
1944
1945                 r = -EBADF;
1946                 f = fdget(cap->args[0]);
1947                 if (!f.file)
1948                         break;
1949
1950                 r = -EPERM;
1951                 dev = kvm_device_from_filp(f.file);
1952                 if (dev)
1953                         r = kvmppc_mpic_connect_vcpu(dev, vcpu, cap->args[1]);
1954
1955                 fdput(f);
1956                 break;
1957         }
1958 #endif
1959 #ifdef CONFIG_KVM_XICS
1960         case KVM_CAP_IRQ_XICS: {
1961                 struct fd f;
1962                 struct kvm_device *dev;
1963
1964                 r = -EBADF;
1965                 f = fdget(cap->args[0]);
1966                 if (!f.file)
1967                         break;
1968
1969                 r = -EPERM;
1970                 dev = kvm_device_from_filp(f.file);
1971                 if (dev) {
1972                         if (xics_on_xive())
1973                                 r = kvmppc_xive_connect_vcpu(dev, vcpu, cap->args[1]);
1974                         else
1975                                 r = kvmppc_xics_connect_vcpu(dev, vcpu, cap->args[1]);
1976                 }
1977
1978                 fdput(f);
1979                 break;
1980         }
1981 #endif /* CONFIG_KVM_XICS */
1982 #ifdef CONFIG_KVM_XIVE
1983         case KVM_CAP_PPC_IRQ_XIVE: {
1984                 struct fd f;
1985                 struct kvm_device *dev;
1986
1987                 r = -EBADF;
1988                 f = fdget(cap->args[0]);
1989                 if (!f.file)
1990                         break;
1991
1992                 r = -ENXIO;
1993                 if (!xive_enabled())
1994                         break;
1995
1996                 r = -EPERM;
1997                 dev = kvm_device_from_filp(f.file);
1998                 if (dev)
1999                         r = kvmppc_xive_native_connect_vcpu(dev, vcpu,
2000                                                             cap->args[1]);
2001
2002                 fdput(f);
2003                 break;
2004         }
2005 #endif /* CONFIG_KVM_XIVE */
2006 #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
2007         case KVM_CAP_PPC_FWNMI:
2008                 r = -EINVAL;
2009                 if (!is_kvmppc_hv_enabled(vcpu->kvm))
2010                         break;
2011                 r = 0;
2012                 vcpu->kvm->arch.fwnmi_enabled = true;
2013                 break;
2014 #endif /* CONFIG_KVM_BOOK3S_HV_POSSIBLE */
2015         default:
2016                 r = -EINVAL;
2017                 break;
2018         }
2019
2020         if (!r)
2021                 r = kvmppc_sanity_check(vcpu);
2022
2023         return r;
2024 }
2025
2026 bool kvm_arch_intc_initialized(struct kvm *kvm)
2027 {
2028 #ifdef CONFIG_KVM_MPIC
2029         if (kvm->arch.mpic)
2030                 return true;
2031 #endif
2032 #ifdef CONFIG_KVM_XICS
2033         if (kvm->arch.xics || kvm->arch.xive)
2034                 return true;
2035 #endif
2036         return false;
2037 }
2038
2039 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
2040                                     struct kvm_mp_state *mp_state)
2041 {
2042         return -EINVAL;
2043 }
2044
2045 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
2046                                     struct kvm_mp_state *mp_state)
2047 {
2048         return -EINVAL;
2049 }
2050
2051 long kvm_arch_vcpu_async_ioctl(struct file *filp,
2052                                unsigned int ioctl, unsigned long arg)
2053 {
2054         struct kvm_vcpu *vcpu = filp->private_data;
2055         void __user *argp = (void __user *)arg;
2056
2057         if (ioctl == KVM_INTERRUPT) {
2058                 struct kvm_interrupt irq;
2059                 if (copy_from_user(&irq, argp, sizeof(irq)))
2060                         return -EFAULT;
2061                 return kvm_vcpu_ioctl_interrupt(vcpu, &irq);
2062         }
2063         return -ENOIOCTLCMD;
2064 }
2065
2066 long kvm_arch_vcpu_ioctl(struct file *filp,
2067                          unsigned int ioctl, unsigned long arg)
2068 {
2069         struct kvm_vcpu *vcpu = filp->private_data;
2070         void __user *argp = (void __user *)arg;
2071         long r;
2072
2073         switch (ioctl) {
2074         case KVM_ENABLE_CAP:
2075         {
2076                 struct kvm_enable_cap cap;
2077                 r = -EFAULT;
2078                 if (copy_from_user(&cap, argp, sizeof(cap)))
2079                         goto out;
2080                 vcpu_load(vcpu);
2081                 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
2082                 vcpu_put(vcpu);
2083                 break;
2084         }
2085
2086         case KVM_SET_ONE_REG:
2087         case KVM_GET_ONE_REG:
2088         {
2089                 struct kvm_one_reg reg;
2090                 r = -EFAULT;
2091                 if (copy_from_user(&reg, argp, sizeof(reg)))
2092                         goto out;
2093                 if (ioctl == KVM_SET_ONE_REG)
2094                         r = kvm_vcpu_ioctl_set_one_reg(vcpu, &reg);
2095                 else
2096                         r = kvm_vcpu_ioctl_get_one_reg(vcpu, &reg);
2097                 break;
2098         }
2099
2100 #if defined(CONFIG_KVM_E500V2) || defined(CONFIG_KVM_E500MC)
2101         case KVM_DIRTY_TLB: {
2102                 struct kvm_dirty_tlb dirty;
2103                 r = -EFAULT;
2104                 if (copy_from_user(&dirty, argp, sizeof(dirty)))
2105                         goto out;
2106                 vcpu_load(vcpu);
2107                 r = kvm_vcpu_ioctl_dirty_tlb(vcpu, &dirty);
2108                 vcpu_put(vcpu);
2109                 break;
2110         }
2111 #endif
2112         default:
2113                 r = -EINVAL;
2114         }
2115
2116 out:
2117         return r;
2118 }
2119
2120 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
2121 {
2122         return VM_FAULT_SIGBUS;
2123 }
2124
2125 static int kvm_vm_ioctl_get_pvinfo(struct kvm_ppc_pvinfo *pvinfo)
2126 {
2127         u32 inst_nop = 0x60000000;
2128 #ifdef CONFIG_KVM_BOOKE_HV
2129         u32 inst_sc1 = 0x44000022;
2130         pvinfo->hcall[0] = cpu_to_be32(inst_sc1);
2131         pvinfo->hcall[1] = cpu_to_be32(inst_nop);
2132         pvinfo->hcall[2] = cpu_to_be32(inst_nop);
2133         pvinfo->hcall[3] = cpu_to_be32(inst_nop);
2134 #else
2135         u32 inst_lis = 0x3c000000;
2136         u32 inst_ori = 0x60000000;
2137         u32 inst_sc = 0x44000002;
2138         u32 inst_imm_mask = 0xffff;
2139
2140         /*
2141          * The hypercall to get into KVM from within guest context is as
2142          * follows:
2143          *
2144          *    lis r0, r0, KVM_SC_MAGIC_R0@h
2145          *    ori r0, KVM_SC_MAGIC_R0@l
2146          *    sc
2147          *    nop
2148          */
2149         pvinfo->hcall[0] = cpu_to_be32(inst_lis | ((KVM_SC_MAGIC_R0 >> 16) & inst_imm_mask));
2150         pvinfo->hcall[1] = cpu_to_be32(inst_ori | (KVM_SC_MAGIC_R0 & inst_imm_mask));
2151         pvinfo->hcall[2] = cpu_to_be32(inst_sc);
2152         pvinfo->hcall[3] = cpu_to_be32(inst_nop);
2153 #endif
2154
2155         pvinfo->flags = KVM_PPC_PVINFO_FLAGS_EV_IDLE;
2156
2157         return 0;
2158 }
2159
2160 bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
2161 {
2162         int ret = 0;
2163
2164 #ifdef CONFIG_KVM_MPIC
2165         ret = ret || (kvm->arch.mpic != NULL);
2166 #endif
2167 #ifdef CONFIG_KVM_XICS
2168         ret = ret || (kvm->arch.xics != NULL);
2169         ret = ret || (kvm->arch.xive != NULL);
2170 #endif
2171         smp_rmb();
2172         return ret;
2173 }
2174
2175 int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_event,
2176                           bool line_status)
2177 {
2178         if (!kvm_arch_irqchip_in_kernel(kvm))
2179                 return -ENXIO;
2180
2181         irq_event->status = kvm_set_irq(kvm, KVM_USERSPACE_IRQ_SOURCE_ID,
2182                                         irq_event->irq, irq_event->level,
2183                                         line_status);
2184         return 0;
2185 }
2186
2187
2188 int kvm_vm_ioctl_enable_cap(struct kvm *kvm,
2189                             struct kvm_enable_cap *cap)
2190 {
2191         int r;
2192
2193         if (cap->flags)
2194                 return -EINVAL;
2195
2196         switch (cap->cap) {
2197 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
2198         case KVM_CAP_PPC_ENABLE_HCALL: {
2199                 unsigned long hcall = cap->args[0];
2200
2201                 r = -EINVAL;
2202                 if (hcall > MAX_HCALL_OPCODE || (hcall & 3) ||
2203                     cap->args[1] > 1)
2204                         break;
2205                 if (!kvmppc_book3s_hcall_implemented(kvm, hcall))
2206                         break;
2207                 if (cap->args[1])
2208                         set_bit(hcall / 4, kvm->arch.enabled_hcalls);
2209                 else
2210                         clear_bit(hcall / 4, kvm->arch.enabled_hcalls);
2211                 r = 0;
2212                 break;
2213         }
2214         case KVM_CAP_PPC_SMT: {
2215                 unsigned long mode = cap->args[0];
2216                 unsigned long flags = cap->args[1];
2217
2218                 r = -EINVAL;
2219                 if (kvm->arch.kvm_ops->set_smt_mode)
2220                         r = kvm->arch.kvm_ops->set_smt_mode(kvm, mode, flags);
2221                 break;
2222         }
2223
2224         case KVM_CAP_PPC_NESTED_HV:
2225                 r = -EINVAL;
2226                 if (!is_kvmppc_hv_enabled(kvm) ||
2227                     !kvm->arch.kvm_ops->enable_nested)
2228                         break;
2229                 r = kvm->arch.kvm_ops->enable_nested(kvm);
2230                 break;
2231 #endif
2232 #if defined(CONFIG_KVM_BOOK3S_HV_POSSIBLE)
2233         case KVM_CAP_PPC_SECURE_GUEST:
2234                 r = -EINVAL;
2235                 if (!is_kvmppc_hv_enabled(kvm) || !kvm->arch.kvm_ops->enable_svm)
2236                         break;
2237                 r = kvm->arch.kvm_ops->enable_svm(kvm);
2238                 break;
2239         case KVM_CAP_PPC_DAWR1:
2240                 r = -EINVAL;
2241                 if (!is_kvmppc_hv_enabled(kvm) || !kvm->arch.kvm_ops->enable_dawr1)
2242                         break;
2243                 r = kvm->arch.kvm_ops->enable_dawr1(kvm);
2244                 break;
2245 #endif
2246         default:
2247                 r = -EINVAL;
2248                 break;
2249         }
2250
2251         return r;
2252 }
2253
2254 #ifdef CONFIG_PPC_BOOK3S_64
2255 /*
2256  * These functions check whether the underlying hardware is safe
2257  * against attacks based on observing the effects of speculatively
2258  * executed instructions, and whether it supplies instructions for
2259  * use in workarounds.  The information comes from firmware, either
2260  * via the device tree on powernv platforms or from an hcall on
2261  * pseries platforms.
2262  */
2263 #ifdef CONFIG_PPC_PSERIES
2264 static int pseries_get_cpu_char(struct kvm_ppc_cpu_char *cp)
2265 {
2266         struct h_cpu_char_result c;
2267         unsigned long rc;
2268
2269         if (!machine_is(pseries))
2270                 return -ENOTTY;
2271
2272         rc = plpar_get_cpu_characteristics(&c);
2273         if (rc == H_SUCCESS) {
2274                 cp->character = c.character;
2275                 cp->behaviour = c.behaviour;
2276                 cp->character_mask = KVM_PPC_CPU_CHAR_SPEC_BAR_ORI31 |
2277                         KVM_PPC_CPU_CHAR_BCCTRL_SERIALISED |
2278                         KVM_PPC_CPU_CHAR_L1D_FLUSH_ORI30 |
2279                         KVM_PPC_CPU_CHAR_L1D_FLUSH_TRIG2 |
2280                         KVM_PPC_CPU_CHAR_L1D_THREAD_PRIV |
2281                         KVM_PPC_CPU_CHAR_BR_HINT_HONOURED |
2282                         KVM_PPC_CPU_CHAR_MTTRIG_THR_RECONF |
2283                         KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS |
2284                         KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST;
2285                 cp->behaviour_mask = KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY |
2286                         KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR |
2287                         KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR |
2288                         KVM_PPC_CPU_BEHAV_FLUSH_COUNT_CACHE;
2289         }
2290         return 0;
2291 }
2292 #else
2293 static int pseries_get_cpu_char(struct kvm_ppc_cpu_char *cp)
2294 {
2295         return -ENOTTY;
2296 }
2297 #endif
2298
2299 static inline bool have_fw_feat(struct device_node *fw_features,
2300                                 const char *state, const char *name)
2301 {
2302         struct device_node *np;
2303         bool r = false;
2304
2305         np = of_get_child_by_name(fw_features, name);
2306         if (np) {
2307                 r = of_property_read_bool(np, state);
2308                 of_node_put(np);
2309         }
2310         return r;
2311 }
2312
2313 static int kvmppc_get_cpu_char(struct kvm_ppc_cpu_char *cp)
2314 {
2315         struct device_node *np, *fw_features;
2316         int r;
2317
2318         memset(cp, 0, sizeof(*cp));
2319         r = pseries_get_cpu_char(cp);
2320         if (r != -ENOTTY)
2321                 return r;
2322
2323         np = of_find_node_by_name(NULL, "ibm,opal");
2324         if (np) {
2325                 fw_features = of_get_child_by_name(np, "fw-features");
2326                 of_node_put(np);
2327                 if (!fw_features)
2328                         return 0;
2329                 if (have_fw_feat(fw_features, "enabled",
2330                                  "inst-spec-barrier-ori31,31,0"))
2331                         cp->character |= KVM_PPC_CPU_CHAR_SPEC_BAR_ORI31;
2332                 if (have_fw_feat(fw_features, "enabled",
2333                                  "fw-bcctrl-serialized"))
2334                         cp->character |= KVM_PPC_CPU_CHAR_BCCTRL_SERIALISED;
2335                 if (have_fw_feat(fw_features, "enabled",
2336                                  "inst-l1d-flush-ori30,30,0"))
2337                         cp->character |= KVM_PPC_CPU_CHAR_L1D_FLUSH_ORI30;
2338                 if (have_fw_feat(fw_features, "enabled",
2339                                  "inst-l1d-flush-trig2"))
2340                         cp->character |= KVM_PPC_CPU_CHAR_L1D_FLUSH_TRIG2;
2341                 if (have_fw_feat(fw_features, "enabled",
2342                                  "fw-l1d-thread-split"))
2343                         cp->character |= KVM_PPC_CPU_CHAR_L1D_THREAD_PRIV;
2344                 if (have_fw_feat(fw_features, "enabled",
2345                                  "fw-count-cache-disabled"))
2346                         cp->character |= KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS;
2347                 if (have_fw_feat(fw_features, "enabled",
2348                                  "fw-count-cache-flush-bcctr2,0,0"))
2349                         cp->character |= KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST;
2350                 cp->character_mask = KVM_PPC_CPU_CHAR_SPEC_BAR_ORI31 |
2351                         KVM_PPC_CPU_CHAR_BCCTRL_SERIALISED |
2352                         KVM_PPC_CPU_CHAR_L1D_FLUSH_ORI30 |
2353                         KVM_PPC_CPU_CHAR_L1D_FLUSH_TRIG2 |
2354                         KVM_PPC_CPU_CHAR_L1D_THREAD_PRIV |
2355                         KVM_PPC_CPU_CHAR_COUNT_CACHE_DIS |
2356                         KVM_PPC_CPU_CHAR_BCCTR_FLUSH_ASSIST;
2357
2358                 if (have_fw_feat(fw_features, "enabled",
2359                                  "speculation-policy-favor-security"))
2360                         cp->behaviour |= KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY;
2361                 if (!have_fw_feat(fw_features, "disabled",
2362                                   "needs-l1d-flush-msr-pr-0-to-1"))
2363                         cp->behaviour |= KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR;
2364                 if (!have_fw_feat(fw_features, "disabled",
2365                                   "needs-spec-barrier-for-bound-checks"))
2366                         cp->behaviour |= KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR;
2367                 if (have_fw_feat(fw_features, "enabled",
2368                                  "needs-count-cache-flush-on-context-switch"))
2369                         cp->behaviour |= KVM_PPC_CPU_BEHAV_FLUSH_COUNT_CACHE;
2370                 cp->behaviour_mask = KVM_PPC_CPU_BEHAV_FAVOUR_SECURITY |
2371                         KVM_PPC_CPU_BEHAV_L1D_FLUSH_PR |
2372                         KVM_PPC_CPU_BEHAV_BNDS_CHK_SPEC_BAR |
2373                         KVM_PPC_CPU_BEHAV_FLUSH_COUNT_CACHE;
2374
2375                 of_node_put(fw_features);
2376         }
2377
2378         return 0;
2379 }
2380 #endif
2381
2382 int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
2383 {
2384         struct kvm *kvm __maybe_unused = filp->private_data;
2385         void __user *argp = (void __user *)arg;
2386         int r;
2387
2388         switch (ioctl) {
2389         case KVM_PPC_GET_PVINFO: {
2390                 struct kvm_ppc_pvinfo pvinfo;
2391                 memset(&pvinfo, 0, sizeof(pvinfo));
2392                 r = kvm_vm_ioctl_get_pvinfo(&pvinfo);
2393                 if (copy_to_user(argp, &pvinfo, sizeof(pvinfo))) {
2394                         r = -EFAULT;
2395                         goto out;
2396                 }
2397
2398                 break;
2399         }
2400 #ifdef CONFIG_SPAPR_TCE_IOMMU
2401         case KVM_CREATE_SPAPR_TCE_64: {
2402                 struct kvm_create_spapr_tce_64 create_tce_64;
2403
2404                 r = -EFAULT;
2405                 if (copy_from_user(&create_tce_64, argp, sizeof(create_tce_64)))
2406                         goto out;
2407                 if (create_tce_64.flags) {
2408                         r = -EINVAL;
2409                         goto out;
2410                 }
2411                 r = kvm_vm_ioctl_create_spapr_tce(kvm, &create_tce_64);
2412                 goto out;
2413         }
2414         case KVM_CREATE_SPAPR_TCE: {
2415                 struct kvm_create_spapr_tce create_tce;
2416                 struct kvm_create_spapr_tce_64 create_tce_64;
2417
2418                 r = -EFAULT;
2419                 if (copy_from_user(&create_tce, argp, sizeof(create_tce)))
2420                         goto out;
2421
2422                 create_tce_64.liobn = create_tce.liobn;
2423                 create_tce_64.page_shift = IOMMU_PAGE_SHIFT_4K;
2424                 create_tce_64.offset = 0;
2425                 create_tce_64.size = create_tce.window_size >>
2426                                 IOMMU_PAGE_SHIFT_4K;
2427                 create_tce_64.flags = 0;
2428                 r = kvm_vm_ioctl_create_spapr_tce(kvm, &create_tce_64);
2429                 goto out;
2430         }
2431 #endif
2432 #ifdef CONFIG_PPC_BOOK3S_64
2433         case KVM_PPC_GET_SMMU_INFO: {
2434                 struct kvm_ppc_smmu_info info;
2435                 struct kvm *kvm = filp->private_data;
2436
2437                 memset(&info, 0, sizeof(info));
2438                 r = kvm->arch.kvm_ops->get_smmu_info(kvm, &info);
2439                 if (r >= 0 && copy_to_user(argp, &info, sizeof(info)))
2440                         r = -EFAULT;
2441                 break;
2442         }
2443         case KVM_PPC_RTAS_DEFINE_TOKEN: {
2444                 struct kvm *kvm = filp->private_data;
2445
2446                 r = kvm_vm_ioctl_rtas_define_token(kvm, argp);
2447                 break;
2448         }
2449         case KVM_PPC_CONFIGURE_V3_MMU: {
2450                 struct kvm *kvm = filp->private_data;
2451                 struct kvm_ppc_mmuv3_cfg cfg;
2452
2453                 r = -EINVAL;
2454                 if (!kvm->arch.kvm_ops->configure_mmu)
2455                         goto out;
2456                 r = -EFAULT;
2457                 if (copy_from_user(&cfg, argp, sizeof(cfg)))
2458                         goto out;
2459                 r = kvm->arch.kvm_ops->configure_mmu(kvm, &cfg);
2460                 break;
2461         }
2462         case KVM_PPC_GET_RMMU_INFO: {
2463                 struct kvm *kvm = filp->private_data;
2464                 struct kvm_ppc_rmmu_info info;
2465
2466                 r = -EINVAL;
2467                 if (!kvm->arch.kvm_ops->get_rmmu_info)
2468                         goto out;
2469                 r = kvm->arch.kvm_ops->get_rmmu_info(kvm, &info);
2470                 if (r >= 0 && copy_to_user(argp, &info, sizeof(info)))
2471                         r = -EFAULT;
2472                 break;
2473         }
2474         case KVM_PPC_GET_CPU_CHAR: {
2475                 struct kvm_ppc_cpu_char cpuchar;
2476
2477                 r = kvmppc_get_cpu_char(&cpuchar);
2478                 if (r >= 0 && copy_to_user(argp, &cpuchar, sizeof(cpuchar)))
2479                         r = -EFAULT;
2480                 break;
2481         }
2482         case KVM_PPC_SVM_OFF: {
2483                 struct kvm *kvm = filp->private_data;
2484
2485                 r = 0;
2486                 if (!kvm->arch.kvm_ops->svm_off)
2487                         goto out;
2488
2489                 r = kvm->arch.kvm_ops->svm_off(kvm);
2490                 break;
2491         }
2492         default: {
2493                 struct kvm *kvm = filp->private_data;
2494                 r = kvm->arch.kvm_ops->arch_vm_ioctl(filp, ioctl, arg);
2495         }
2496 #else /* CONFIG_PPC_BOOK3S_64 */
2497         default:
2498                 r = -ENOTTY;
2499 #endif
2500         }
2501 out:
2502         return r;
2503 }
2504
2505 static DEFINE_IDA(lpid_inuse);
2506 static unsigned long nr_lpids;
2507
2508 long kvmppc_alloc_lpid(void)
2509 {
2510         int lpid;
2511
2512         /* The host LPID must always be 0 (allocation starts at 1) */
2513         lpid = ida_alloc_range(&lpid_inuse, 1, nr_lpids - 1, GFP_KERNEL);
2514         if (lpid < 0) {
2515                 if (lpid == -ENOMEM)
2516                         pr_err("%s: Out of memory\n", __func__);
2517                 else
2518                         pr_err("%s: No LPIDs free\n", __func__);
2519                 return -ENOMEM;
2520         }
2521
2522         return lpid;
2523 }
2524 EXPORT_SYMBOL_GPL(kvmppc_alloc_lpid);
2525
2526 void kvmppc_free_lpid(long lpid)
2527 {
2528         ida_free(&lpid_inuse, lpid);
2529 }
2530 EXPORT_SYMBOL_GPL(kvmppc_free_lpid);
2531
2532 /* nr_lpids_param includes the host LPID */
2533 void kvmppc_init_lpid(unsigned long nr_lpids_param)
2534 {
2535         nr_lpids = nr_lpids_param;
2536 }
2537 EXPORT_SYMBOL_GPL(kvmppc_init_lpid);
2538
2539 EXPORT_TRACEPOINT_SYMBOL_GPL(kvm_ppc_instr);
2540
2541 void kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu, struct dentry *debugfs_dentry)
2542 {
2543         if (vcpu->kvm->arch.kvm_ops->create_vcpu_debugfs)
2544                 vcpu->kvm->arch.kvm_ops->create_vcpu_debugfs(vcpu, debugfs_dentry);
2545 }
2546
2547 int kvm_arch_create_vm_debugfs(struct kvm *kvm)
2548 {
2549         if (kvm->arch.kvm_ops->create_vm_debugfs)
2550                 kvm->arch.kvm_ops->create_vm_debugfs(kvm);
2551         return 0;
2552 }