arch/powerpc/kvm/book3s_hv_nested.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright IBM Corporation, 2018
   4  * Authors Suraj Jitindar Singh <sjitindarsingh@gmail.com>
   5  *         Paul Mackerras <paulus@ozlabs.org>
   6  *
   7  * Description: KVM functions specific to running nested KVM-HV guests
   8  * on Book3S processors (specifically POWER9 and later).
   9  */
  10
  11 #include <linux/kernel.h>
  12 #include <linux/kvm_host.h>
  13 #include <linux/llist.h>
  14 #include <linux/pgtable.h>
  15
  16 #include <asm/kvm_ppc.h>
  17 #include <asm/kvm_book3s.h>
  18 #include <asm/mmu.h>
  19 #include <asm/pgalloc.h>
  20 #include <asm/pte-walk.h>
  21 #include <asm/reg.h>
  22 #include <asm/plpar_wrappers.h>
  23
  24 static struct patb_entry *pseries_partition_tb;
  25
  26 static void kvmhv_update_ptbl_cache(struct kvm_nested_guest *gp);
  27 static void kvmhv_free_memslot_nest_rmap(struct kvm_memory_slot *free);
  28
  29 void kvmhv_save_hv_regs(struct kvm_vcpu *vcpu, struct hv_guest_state *hr)
  30 {
  31         struct kvmppc_vcore *vc = vcpu->arch.vcore;
  32
  33         hr->pcr = vc->pcr | PCR_MASK;
  34         hr->dpdes = vc->dpdes;
  35         hr->hfscr = vcpu->arch.hfscr;
  36         hr->tb_offset = vc->tb_offset;
  37         hr->dawr0 = vcpu->arch.dawr0;
  38         hr->dawrx0 = vcpu->arch.dawrx0;
  39         hr->ciabr = vcpu->arch.ciabr;
  40         hr->purr = vcpu->arch.purr;
  41         hr->spurr = vcpu->arch.spurr;
  42         hr->ic = vcpu->arch.ic;
  43         hr->vtb = vc->vtb;
  44         hr->srr0 = vcpu->arch.shregs.srr0;
  45         hr->srr1 = vcpu->arch.shregs.srr1;
  46         hr->sprg[0] = vcpu->arch.shregs.sprg0;
  47         hr->sprg[1] = vcpu->arch.shregs.sprg1;
  48         hr->sprg[2] = vcpu->arch.shregs.sprg2;
  49         hr->sprg[3] = vcpu->arch.shregs.sprg3;
  50         hr->pidr = vcpu->arch.pid;
  51         hr->cfar = vcpu->arch.cfar;
  52         hr->ppr = vcpu->arch.ppr;
  53         hr->dawr1 = vcpu->arch.dawr1;
  54         hr->dawrx1 = vcpu->arch.dawrx1;
  55 }
  56
  57 /* Use noinline_for_stack due to https://bugs.llvm.org/show_bug.cgi?id=49610 */
  58 static noinline_for_stack void byteswap_pt_regs(struct pt_regs *regs)
  59 {
  60         unsigned long *addr = (unsigned long *) regs;
  61
  62         for (; addr < ((unsigned long *) (regs + 1)); addr++)
  63                 *addr = swab64(*addr);
  64 }
  65
  66 static void byteswap_hv_regs(struct hv_guest_state *hr)
  67 {
  68         hr->version = swab64(hr->version);
  69         hr->lpid = swab32(hr->lpid);
  70         hr->vcpu_token = swab32(hr->vcpu_token);
  71         hr->lpcr = swab64(hr->lpcr);
  72         hr->pcr = swab64(hr->pcr) | PCR_MASK;
  73         hr->amor = swab64(hr->amor);
  74         hr->dpdes = swab64(hr->dpdes);
  75         hr->hfscr = swab64(hr->hfscr);
  76         hr->tb_offset = swab64(hr->tb_offset);
  77         hr->dawr0 = swab64(hr->dawr0);
  78         hr->dawrx0 = swab64(hr->dawrx0);
  79         hr->ciabr = swab64(hr->ciabr);
  80         hr->hdec_expiry = swab64(hr->hdec_expiry);
  81         hr->purr = swab64(hr->purr);
  82         hr->spurr = swab64(hr->spurr);
  83         hr->ic = swab64(hr->ic);
  84         hr->vtb = swab64(hr->vtb);
  85         hr->hdar = swab64(hr->hdar);
  86         hr->hdsisr = swab64(hr->hdsisr);
  87         hr->heir = swab64(hr->heir);
  88         hr->asdr = swab64(hr->asdr);
  89         hr->srr0 = swab64(hr->srr0);
  90         hr->srr1 = swab64(hr->srr1);
  91         hr->sprg[0] = swab64(hr->sprg[0]);
  92         hr->sprg[1] = swab64(hr->sprg[1]);
  93         hr->sprg[2] = swab64(hr->sprg[2]);
  94         hr->sprg[3] = swab64(hr->sprg[3]);
  95         hr->pidr = swab64(hr->pidr);
  96         hr->cfar = swab64(hr->cfar);
  97         hr->ppr = swab64(hr->ppr);
  98         hr->dawr1 = swab64(hr->dawr1);
  99         hr->dawrx1 = swab64(hr->dawrx1);
 100 }
 101
 102 static void save_hv_return_state(struct kvm_vcpu *vcpu,
 103                                  struct hv_guest_state *hr)
 104 {
 105         struct kvmppc_vcore *vc = vcpu->arch.vcore;
 106
 107         hr->dpdes = vc->dpdes;
 108         hr->purr = vcpu->arch.purr;
 109         hr->spurr = vcpu->arch.spurr;
 110         hr->ic = vcpu->arch.ic;
 111         hr->vtb = vc->vtb;
 112         hr->srr0 = vcpu->arch.shregs.srr0;
 113         hr->srr1 = vcpu->arch.shregs.srr1;
 114         hr->sprg[0] = vcpu->arch.shregs.sprg0;
 115         hr->sprg[1] = vcpu->arch.shregs.sprg1;
 116         hr->sprg[2] = vcpu->arch.shregs.sprg2;
 117         hr->sprg[3] = vcpu->arch.shregs.sprg3;
 118         hr->pidr = vcpu->arch.pid;
 119         hr->cfar = vcpu->arch.cfar;
 120         hr->ppr = vcpu->arch.ppr;
 121         switch (vcpu->arch.trap) {
 122         case BOOK3S_INTERRUPT_H_DATA_STORAGE:
 123                 hr->hdar = vcpu->arch.fault_dar;
 124                 hr->hdsisr = vcpu->arch.fault_dsisr;
 125                 hr->asdr = vcpu->arch.fault_gpa;
 126                 break;
 127         case BOOK3S_INTERRUPT_H_INST_STORAGE:
 128                 hr->asdr = vcpu->arch.fault_gpa;
 129                 break;
 130         case BOOK3S_INTERRUPT_H_FAC_UNAVAIL:
 131                 hr->hfscr = ((~HFSCR_INTR_CAUSE & hr->hfscr) |
 132                              (HFSCR_INTR_CAUSE & vcpu->arch.hfscr));
 133                 break;
 134         case BOOK3S_INTERRUPT_H_EMUL_ASSIST:
 135                 hr->heir = vcpu->arch.emul_inst;
 136                 break;
 137         }
 138 }
 139
 140 static void restore_hv_regs(struct kvm_vcpu *vcpu, const struct hv_guest_state *hr)
 141 {
 142         struct kvmppc_vcore *vc = vcpu->arch.vcore;
 143
 144         vc->pcr = hr->pcr | PCR_MASK;
 145         vc->dpdes = hr->dpdes;
 146         vcpu->arch.hfscr = hr->hfscr;
 147         vcpu->arch.dawr0 = hr->dawr0;
 148         vcpu->arch.dawrx0 = hr->dawrx0;
 149         vcpu->arch.ciabr = hr->ciabr;
 150         vcpu->arch.purr = hr->purr;
 151         vcpu->arch.spurr = hr->spurr;
 152         vcpu->arch.ic = hr->ic;
 153         vc->vtb = hr->vtb;
 154         vcpu->arch.shregs.srr0 = hr->srr0;
 155         vcpu->arch.shregs.srr1 = hr->srr1;
 156         vcpu->arch.shregs.sprg0 = hr->sprg[0];
 157         vcpu->arch.shregs.sprg1 = hr->sprg[1];
 158         vcpu->arch.shregs.sprg2 = hr->sprg[2];
 159         vcpu->arch.shregs.sprg3 = hr->sprg[3];
 160         vcpu->arch.pid = hr->pidr;
 161         vcpu->arch.cfar = hr->cfar;
 162         vcpu->arch.ppr = hr->ppr;
 163         vcpu->arch.dawr1 = hr->dawr1;
 164         vcpu->arch.dawrx1 = hr->dawrx1;
 165 }
 166
 167 void kvmhv_restore_hv_return_state(struct kvm_vcpu *vcpu,
 168                                    struct hv_guest_state *hr)
 169 {
 170         struct kvmppc_vcore *vc = vcpu->arch.vcore;
 171
 172         vc->dpdes = hr->dpdes;
 173         vcpu->arch.hfscr = hr->hfscr;
 174         vcpu->arch.purr = hr->purr;
 175         vcpu->arch.spurr = hr->spurr;
 176         vcpu->arch.ic = hr->ic;
 177         vc->vtb = hr->vtb;
 178         vcpu->arch.fault_dar = hr->hdar;
 179         vcpu->arch.fault_dsisr = hr->hdsisr;
 180         vcpu->arch.fault_gpa = hr->asdr;
 181         vcpu->arch.emul_inst = hr->heir;
 182         vcpu->arch.shregs.srr0 = hr->srr0;
 183         vcpu->arch.shregs.srr1 = hr->srr1;
 184         vcpu->arch.shregs.sprg0 = hr->sprg[0];
 185         vcpu->arch.shregs.sprg1 = hr->sprg[1];
 186         vcpu->arch.shregs.sprg2 = hr->sprg[2];
 187         vcpu->arch.shregs.sprg3 = hr->sprg[3];
 188         vcpu->arch.pid = hr->pidr;
 189         vcpu->arch.cfar = hr->cfar;
 190         vcpu->arch.ppr = hr->ppr;
 191 }
 192
 193 static void kvmhv_nested_mmio_needed(struct kvm_vcpu *vcpu, u64 regs_ptr)
 194 {
 195         /* No need to reflect the page fault to L1, we've handled it */
 196         vcpu->arch.trap = 0;
 197
 198         /*
 199          * Since the L2 gprs have already been written back into L1 memory when
 200          * we complete the mmio, store the L1 memory location of the L2 gpr
 201          * being loaded into by the mmio so that the loaded value can be
 202          * written there in kvmppc_complete_mmio_load()
 203          */
 204         if (((vcpu->arch.io_gpr & KVM_MMIO_REG_EXT_MASK) == KVM_MMIO_REG_GPR)
 205             && (vcpu->mmio_is_write == 0)) {
 206                 vcpu->arch.nested_io_gpr = (gpa_t) regs_ptr +
 207                                            offsetof(struct pt_regs,
 208                                                     gpr[vcpu->arch.io_gpr]);
 209                 vcpu->arch.io_gpr = KVM_MMIO_REG_NESTED_GPR;
 210         }
 211 }
 212
 213 static int kvmhv_read_guest_state_and_regs(struct kvm_vcpu *vcpu,
 214                                            struct hv_guest_state *l2_hv,
 215                                            struct pt_regs *l2_regs,
 216                                            u64 hv_ptr, u64 regs_ptr)
 217 {
 218         int size;
 219
 220         if (kvm_vcpu_read_guest(vcpu, hv_ptr, &l2_hv->version,
 221                                 sizeof(l2_hv->version)))
 222                 return -1;
 223
 224         if (kvmppc_need_byteswap(vcpu))
 225                 l2_hv->version = swab64(l2_hv->version);
 226
 227         size = hv_guest_state_size(l2_hv->version);
 228         if (size < 0)
 229                 return -1;
 230
 231         return kvm_vcpu_read_guest(vcpu, hv_ptr, l2_hv, size) ||
 232                 kvm_vcpu_read_guest(vcpu, regs_ptr, l2_regs,
 233                                     sizeof(struct pt_regs));
 234 }
 235
 236 static int kvmhv_write_guest_state_and_regs(struct kvm_vcpu *vcpu,
 237                                             struct hv_guest_state *l2_hv,
 238                                             struct pt_regs *l2_regs,
 239                                             u64 hv_ptr, u64 regs_ptr)
 240 {
 241         int size;
 242
 243         size = hv_guest_state_size(l2_hv->version);
 244         if (size < 0)
 245                 return -1;
 246
 247         return kvm_vcpu_write_guest(vcpu, hv_ptr, l2_hv, size) ||
 248                 kvm_vcpu_write_guest(vcpu, regs_ptr, l2_regs,
 249                                      sizeof(struct pt_regs));
 250 }
 251
 252 static void load_l2_hv_regs(struct kvm_vcpu *vcpu,
 253                             const struct hv_guest_state *l2_hv,
 254                             const struct hv_guest_state *l1_hv, u64 *lpcr)
 255 {
 256         struct kvmppc_vcore *vc = vcpu->arch.vcore;
 257         u64 mask;
 258
 259         restore_hv_regs(vcpu, l2_hv);
 260
 261         /*
 262          * Don't let L1 change LPCR bits for the L2 except these:
 263          */
 264         mask = LPCR_DPFD | LPCR_ILE | LPCR_TC | LPCR_AIL | LPCR_LD | LPCR_MER;
 265
 266         /*
 267          * Additional filtering is required depending on hardware
 268          * and configuration.
 269          */
 270         *lpcr = kvmppc_filter_lpcr_hv(vcpu->kvm,
 271                                       (vc->lpcr & ~mask) | (*lpcr & mask));
 272
 273         /*
 274          * Don't let L1 enable features for L2 which we don't allow for L1,
 275          * but preserve the interrupt cause field.
 276          */
 277         vcpu->arch.hfscr = l2_hv->hfscr & (HFSCR_INTR_CAUSE | vcpu->arch.hfscr_permitted);
 278
 279         /* Don't let data address watchpoint match in hypervisor state */
 280         vcpu->arch.dawrx0 = l2_hv->dawrx0 & ~DAWRX_HYP;
 281         vcpu->arch.dawrx1 = l2_hv->dawrx1 & ~DAWRX_HYP;
 282
 283         /* Don't let completed instruction address breakpt match in HV state */
 284         if ((l2_hv->ciabr & CIABR_PRIV) == CIABR_PRIV_HYPER)
 285                 vcpu->arch.ciabr = l2_hv->ciabr & ~CIABR_PRIV;
 286 }
 287
 288 long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
 289 {
 290         long int err, r;
 291         struct kvm_nested_guest *l2;
 292         struct pt_regs l2_regs, saved_l1_regs;
 293         struct hv_guest_state l2_hv = {0}, saved_l1_hv;
 294         struct kvmppc_vcore *vc = vcpu->arch.vcore;
 295         u64 hv_ptr, regs_ptr;
 296         u64 hdec_exp, lpcr;
 297         s64 delta_purr, delta_spurr, delta_ic, delta_vtb;
 298
 299         if (vcpu->kvm->arch.l1_ptcr == 0)
 300                 return H_NOT_AVAILABLE;
 301
 302         if (MSR_TM_TRANSACTIONAL(vcpu->arch.shregs.msr))
 303                 return H_BAD_MODE;
 304
 305         /* copy parameters in */
 306         hv_ptr = kvmppc_get_gpr(vcpu, 4);
 307         regs_ptr = kvmppc_get_gpr(vcpu, 5);
 308         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
 309         err = kvmhv_read_guest_state_and_regs(vcpu, &l2_hv, &l2_regs,
 310                                               hv_ptr, regs_ptr);
 311         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
 312         if (err)
 313                 return H_PARAMETER;
 314
 315         if (kvmppc_need_byteswap(vcpu))
 316                 byteswap_hv_regs(&l2_hv);
 317         if (l2_hv.version > HV_GUEST_STATE_VERSION)
 318                 return H_P2;
 319
 320         if (kvmppc_need_byteswap(vcpu))
 321                 byteswap_pt_regs(&l2_regs);
 322         if (l2_hv.vcpu_token >= NR_CPUS)
 323                 return H_PARAMETER;
 324
 325         /*
 326          * L1 must have set up a suspended state to enter the L2 in a
 327          * transactional state, and only in that case. These have to be
 328          * filtered out here to prevent causing a TM Bad Thing in the
 329          * host HRFID. We could synthesize a TM Bad Thing back to the L1
 330          * here but there doesn't seem like much point.
 331          */
 332         if (MSR_TM_SUSPENDED(vcpu->arch.shregs.msr)) {
 333                 if (!MSR_TM_ACTIVE(l2_regs.msr))
 334                         return H_BAD_MODE;
 335         } else {
 336                 if (l2_regs.msr & MSR_TS_MASK)
 337                         return H_BAD_MODE;
 338                 if (WARN_ON_ONCE(vcpu->arch.shregs.msr & MSR_TS_MASK))
 339                         return H_BAD_MODE;
 340         }
 341
 342         /* translate lpid */
 343         l2 = kvmhv_get_nested(vcpu->kvm, l2_hv.lpid, true);
 344         if (!l2)
 345                 return H_PARAMETER;
 346         if (!l2->l1_gr_to_hr) {
 347                 mutex_lock(&l2->tlb_lock);
 348                 kvmhv_update_ptbl_cache(l2);
 349                 mutex_unlock(&l2->tlb_lock);
 350         }
 351
 352         /* save l1 values of things */
 353         vcpu->arch.regs.msr = vcpu->arch.shregs.msr;
 354         saved_l1_regs = vcpu->arch.regs;
 355         kvmhv_save_hv_regs(vcpu, &saved_l1_hv);
 356
 357         /* convert TB values/offsets to host (L0) values */
 358         hdec_exp = l2_hv.hdec_expiry - vc->tb_offset;
 359         vc->tb_offset += l2_hv.tb_offset;
 360
 361         /* set L1 state to L2 state */
 362         vcpu->arch.nested = l2;
 363         vcpu->arch.nested_vcpu_id = l2_hv.vcpu_token;
 364         vcpu->arch.nested_hfscr = l2_hv.hfscr;
 365         vcpu->arch.regs = l2_regs;
 366
 367         /* Guest must always run with ME enabled, HV disabled. */
 368         vcpu->arch.shregs.msr = (vcpu->arch.regs.msr | MSR_ME) & ~MSR_HV;
 369
 370         lpcr = l2_hv.lpcr;
 371         load_l2_hv_regs(vcpu, &l2_hv, &saved_l1_hv, &lpcr);
 372
 373         vcpu->arch.ret = RESUME_GUEST;
 374         vcpu->arch.trap = 0;
 375         do {
 376                 if (mftb() >= hdec_exp) {
 377                         vcpu->arch.trap = BOOK3S_INTERRUPT_HV_DECREMENTER;
 378                         r = RESUME_HOST;
 379                         break;
 380                 }
 381                 r = kvmhv_run_single_vcpu(vcpu, hdec_exp, lpcr);
 382         } while (is_kvmppc_resume_guest(r));
 383
 384         /* save L2 state for return */
 385         l2_regs = vcpu->arch.regs;
 386         l2_regs.msr = vcpu->arch.shregs.msr;
 387         delta_purr = vcpu->arch.purr - l2_hv.purr;
 388         delta_spurr = vcpu->arch.spurr - l2_hv.spurr;
 389         delta_ic = vcpu->arch.ic - l2_hv.ic;
 390         delta_vtb = vc->vtb - l2_hv.vtb;
 391         save_hv_return_state(vcpu, &l2_hv);
 392
 393         /* restore L1 state */
 394         vcpu->arch.nested = NULL;
 395         vcpu->arch.regs = saved_l1_regs;
 396         vcpu->arch.shregs.msr = saved_l1_regs.msr & ~MSR_TS_MASK;
 397         /* set L1 MSR TS field according to L2 transaction state */
 398         if (l2_regs.msr & MSR_TS_MASK)
 399                 vcpu->arch.shregs.msr |= MSR_TS_S;
 400         vc->tb_offset = saved_l1_hv.tb_offset;
 401         restore_hv_regs(vcpu, &saved_l1_hv);
 402         vcpu->arch.purr += delta_purr;
 403         vcpu->arch.spurr += delta_spurr;
 404         vcpu->arch.ic += delta_ic;
 405         vc->vtb += delta_vtb;
 406
 407         kvmhv_put_nested(l2);
 408
 409         /* copy l2_hv_state and regs back to guest */
 410         if (kvmppc_need_byteswap(vcpu)) {
 411                 byteswap_hv_regs(&l2_hv);
 412                 byteswap_pt_regs(&l2_regs);
 413         }
 414         vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
 415         err = kvmhv_write_guest_state_and_regs(vcpu, &l2_hv, &l2_regs,
 416                                                hv_ptr, regs_ptr);
 417         srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
 418         if (err)
 419                 return H_AUTHORITY;
 420
 421         if (r == -EINTR)
 422                 return H_INTERRUPT;
 423
 424         if (vcpu->mmio_needed) {
 425                 kvmhv_nested_mmio_needed(vcpu, regs_ptr);
 426                 return H_TOO_HARD;
 427         }
 428
 429         return vcpu->arch.trap;
 430 }
 431
 432 long kvmhv_nested_init(void)
 433 {
 434         long int ptb_order;
 435         unsigned long ptcr;
 436         long rc;
 437
 438         if (!kvmhv_on_pseries())
 439                 return 0;
 440         if (!radix_enabled())
 441                 return -ENODEV;
 442
 443         /* find log base 2 of KVMPPC_NR_LPIDS, rounding up */
 444         ptb_order = __ilog2(KVMPPC_NR_LPIDS - 1) + 1;
 445         if (ptb_order < 8)
 446                 ptb_order = 8;
 447         pseries_partition_tb = kmalloc(sizeof(struct patb_entry) << ptb_order,
 448                                        GFP_KERNEL);
 449         if (!pseries_partition_tb) {
 450                 pr_err("kvm-hv: failed to allocated nested partition table\n");
 451                 return -ENOMEM;
 452         }
 453
 454         ptcr = __pa(pseries_partition_tb) | (ptb_order - 8);
 455         rc = plpar_hcall_norets(H_SET_PARTITION_TABLE, ptcr);
 456         if (rc != H_SUCCESS) {
 457                 pr_err("kvm-hv: Parent hypervisor does not support nesting (rc=%ld)\n",
 458                        rc);
 459                 kfree(pseries_partition_tb);
 460                 pseries_partition_tb = NULL;
 461                 return -ENODEV;
 462         }
 463
 464         return 0;
 465 }
 466
 467 void kvmhv_nested_exit(void)
 468 {
 469         /*
 470          * N.B. the kvmhv_on_pseries() test is there because it enables
 471          * the compiler to remove the call to plpar_hcall_norets()
 472          * when CONFIG_PPC_PSERIES=n.
 473          */
 474         if (kvmhv_on_pseries() && pseries_partition_tb) {
 475                 plpar_hcall_norets(H_SET_PARTITION_TABLE, 0);
 476                 kfree(pseries_partition_tb);
 477                 pseries_partition_tb = NULL;
 478         }
 479 }
 480
 481 static void kvmhv_flush_lpid(unsigned int lpid)
 482 {
 483         long rc;
 484
 485         if (!kvmhv_on_pseries()) {
 486                 radix__flush_all_lpid(lpid);
 487                 return;
 488         }
 489
 490         if (!firmware_has_feature(FW_FEATURE_RPT_INVALIDATE))
 491                 rc = plpar_hcall_norets(H_TLB_INVALIDATE, H_TLBIE_P1_ENC(2, 0, 1),
 492                                         lpid, TLBIEL_INVAL_SET_LPID);
 493         else
 494                 rc = pseries_rpt_invalidate(lpid, H_RPTI_TARGET_CMMU,
 495                                             H_RPTI_TYPE_NESTED |
 496                                             H_RPTI_TYPE_TLB | H_RPTI_TYPE_PWC |
 497                                             H_RPTI_TYPE_PAT,
 498                                             H_RPTI_PAGE_ALL, 0, -1UL);
 499         if (rc)
 500                 pr_err("KVM: TLB LPID invalidation hcall failed, rc=%ld\n", rc);
 501 }
 502
 503 void kvmhv_set_ptbl_entry(unsigned int lpid, u64 dw0, u64 dw1)
 504 {
 505         if (!kvmhv_on_pseries()) {
 506                 mmu_partition_table_set_entry(lpid, dw0, dw1, true);
 507                 return;
 508         }
 509
 510         pseries_partition_tb[lpid].patb0 = cpu_to_be64(dw0);
 511         pseries_partition_tb[lpid].patb1 = cpu_to_be64(dw1);
 512         /* L0 will do the necessary barriers */
 513         kvmhv_flush_lpid(lpid);
 514 }
 515
 516 static void kvmhv_set_nested_ptbl(struct kvm_nested_guest *gp)
 517 {
 518         unsigned long dw0;
 519
 520         dw0 = PATB_HR | radix__get_tree_size() |
 521                 __pa(gp->shadow_pgtable) | RADIX_PGD_INDEX_SIZE;
 522         kvmhv_set_ptbl_entry(gp->shadow_lpid, dw0, gp->process_table);
 523 }
 524
 525 void kvmhv_vm_nested_init(struct kvm *kvm)
 526 {
 527         kvm->arch.max_nested_lpid = -1;
 528 }
 529
 530 /*
 531  * Handle the H_SET_PARTITION_TABLE hcall.
 532  * r4 = guest real address of partition table + log_2(size) - 12
 533  * (formatted as for the PTCR).
 534  */
 535 long kvmhv_set_partition_table(struct kvm_vcpu *vcpu)
 536 {
 537         struct kvm *kvm = vcpu->kvm;
 538         unsigned long ptcr = kvmppc_get_gpr(vcpu, 4);
 539         int srcu_idx;
 540         long ret = H_SUCCESS;
 541
 542         srcu_idx = srcu_read_lock(&kvm->srcu);
 543         /*
 544          * Limit the partition table to 4096 entries (because that's what
 545          * hardware supports), and check the base address.
 546          */
 547         if ((ptcr & PRTS_MASK) > 12 - 8 ||
 548             !kvm_is_visible_gfn(vcpu->kvm, (ptcr & PRTB_MASK) >> PAGE_SHIFT))
 549                 ret = H_PARAMETER;
 550         srcu_read_unlock(&kvm->srcu, srcu_idx);
 551         if (ret == H_SUCCESS)
 552                 kvm->arch.l1_ptcr = ptcr;
 553         return ret;
 554 }
 555
 556 /*
 557  * Handle the H_COPY_TOFROM_GUEST hcall.
 558  * r4 = L1 lpid of nested guest
 559  * r5 = pid
 560  * r6 = eaddr to access
 561  * r7 = to buffer (L1 gpa)
 562  * r8 = from buffer (L1 gpa)
 563  * r9 = n bytes to copy
 564  */
 565 long kvmhv_copy_tofrom_guest_nested(struct kvm_vcpu *vcpu)
 566 {
 567         struct kvm_nested_guest *gp;
 568         int l1_lpid = kvmppc_get_gpr(vcpu, 4);
 569         int pid = kvmppc_get_gpr(vcpu, 5);
 570         gva_t eaddr = kvmppc_get_gpr(vcpu, 6);
 571         gpa_t gp_to = (gpa_t) kvmppc_get_gpr(vcpu, 7);
 572         gpa_t gp_from = (gpa_t) kvmppc_get_gpr(vcpu, 8);
 573         void *buf;
 574         unsigned long n = kvmppc_get_gpr(vcpu, 9);
 575         bool is_load = !!gp_to;
 576         long rc;
 577
 578         if (gp_to && gp_from) /* One must be NULL to determine the direction */
 579                 return H_PARAMETER;
 580
 581         if (eaddr & (0xFFFUL << 52))
 582                 return H_PARAMETER;
 583
 584         buf = kzalloc(n, GFP_KERNEL | __GFP_NOWARN);
 585         if (!buf)
 586                 return H_NO_MEM;
 587
 588         gp = kvmhv_get_nested(vcpu->kvm, l1_lpid, false);
 589         if (!gp) {
 590                 rc = H_PARAMETER;
 591                 goto out_free;
 592         }
 593
 594         mutex_lock(&gp->tlb_lock);
 595
 596         if (is_load) {
 597                 /* Load from the nested guest into our buffer */
 598                 rc = __kvmhv_copy_tofrom_guest_radix(gp->shadow_lpid, pid,
 599                                                      eaddr, buf, NULL, n);
 600                 if (rc)
 601                         goto not_found;
 602
 603                 /* Write what was loaded into our buffer back to the L1 guest */
 604                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
 605                 rc = kvm_vcpu_write_guest(vcpu, gp_to, buf, n);
 606                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
 607                 if (rc)
 608                         goto not_found;
 609         } else {
 610                 /* Load the data to be stored from the L1 guest into our buf */
 611                 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
 612                 rc = kvm_vcpu_read_guest(vcpu, gp_from, buf, n);
 613                 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
 614                 if (rc)
 615                         goto not_found;
 616
 617                 /* Store from our buffer into the nested guest */
 618                 rc = __kvmhv_copy_tofrom_guest_radix(gp->shadow_lpid, pid,
 619                                                      eaddr, NULL, buf, n);
 620                 if (rc)
 621                         goto not_found;
 622         }
 623
 624 out_unlock:
 625         mutex_unlock(&gp->tlb_lock);
 626         kvmhv_put_nested(gp);
 627 out_free:
 628         kfree(buf);
 629         return rc;
 630 not_found:
 631         rc = H_NOT_FOUND;
 632         goto out_unlock;
 633 }
 634
 635 /*
 636  * Reload the partition table entry for a guest.
 637  * Caller must hold gp->tlb_lock.
 638  */
 639 static void kvmhv_update_ptbl_cache(struct kvm_nested_guest *gp)
 640 {
 641         int ret;
 642         struct patb_entry ptbl_entry;
 643         unsigned long ptbl_addr;
 644         struct kvm *kvm = gp->l1_host;
 645
 646         ret = -EFAULT;
 647         ptbl_addr = (kvm->arch.l1_ptcr & PRTB_MASK) + (gp->l1_lpid << 4);
 648         if (gp->l1_lpid < (1ul << ((kvm->arch.l1_ptcr & PRTS_MASK) + 8))) {
 649                 int srcu_idx = srcu_read_lock(&kvm->srcu);
 650                 ret = kvm_read_guest(kvm, ptbl_addr,
 651                                      &ptbl_entry, sizeof(ptbl_entry));
 652                 srcu_read_unlock(&kvm->srcu, srcu_idx);
 653         }
 654         if (ret) {
 655                 gp->l1_gr_to_hr = 0;
 656                 gp->process_table = 0;
 657         } else {
 658                 gp->l1_gr_to_hr = be64_to_cpu(ptbl_entry.patb0);
 659                 gp->process_table = be64_to_cpu(ptbl_entry.patb1);
 660         }
 661         kvmhv_set_nested_ptbl(gp);
 662 }
 663
 664 static struct kvm_nested_guest *kvmhv_alloc_nested(struct kvm *kvm, unsigned int lpid)
 665 {
 666         struct kvm_nested_guest *gp;
 667         long shadow_lpid;
 668
 669         gp = kzalloc(sizeof(*gp), GFP_KERNEL);
 670         if (!gp)
 671                 return NULL;
 672         gp->l1_host = kvm;
 673         gp->l1_lpid = lpid;
 674         mutex_init(&gp->tlb_lock);
 675         gp->shadow_pgtable = pgd_alloc(kvm->mm);
 676         if (!gp->shadow_pgtable)
 677                 goto out_free;
 678         shadow_lpid = kvmppc_alloc_lpid();
 679         if (shadow_lpid < 0)
 680                 goto out_free2;
 681         gp->shadow_lpid = shadow_lpid;
 682         gp->radix = 1;
 683
 684         memset(gp->prev_cpu, -1, sizeof(gp->prev_cpu));
 685
 686         return gp;
 687
 688  out_free2:
 689         pgd_free(kvm->mm, gp->shadow_pgtable);
 690  out_free:
 691         kfree(gp);
 692         return NULL;
 693 }
 694
 695 /*
 696  * Free up any resources allocated for a nested guest.
 697  */
 698 static void kvmhv_release_nested(struct kvm_nested_guest *gp)
 699 {
 700         struct kvm *kvm = gp->l1_host;
 701
 702         if (gp->shadow_pgtable) {
 703                 /*
 704                  * No vcpu is using this struct and no call to
 705                  * kvmhv_get_nested can find this struct,
 706                  * so we don't need to hold kvm->mmu_lock.
 707                  */
 708                 kvmppc_free_pgtable_radix(kvm, gp->shadow_pgtable,
 709                                           gp->shadow_lpid);
 710                 pgd_free(kvm->mm, gp->shadow_pgtable);
 711         }
 712         kvmhv_set_ptbl_entry(gp->shadow_lpid, 0, 0);
 713         kvmppc_free_lpid(gp->shadow_lpid);
 714         kfree(gp);
 715 }
 716
 717 static void kvmhv_remove_nested(struct kvm_nested_guest *gp)
 718 {
 719         struct kvm *kvm = gp->l1_host;
 720         int lpid = gp->l1_lpid;
 721         long ref;
 722
 723         spin_lock(&kvm->mmu_lock);
 724         if (gp == kvm->arch.nested_guests[lpid]) {
 725                 kvm->arch.nested_guests[lpid] = NULL;
 726                 if (lpid == kvm->arch.max_nested_lpid) {
 727                         while (--lpid >= 0 && !kvm->arch.nested_guests[lpid])
 728                                 ;
 729                         kvm->arch.max_nested_lpid = lpid;
 730                 }
 731                 --gp->refcnt;
 732         }
 733         ref = gp->refcnt;
 734         spin_unlock(&kvm->mmu_lock);
 735         if (ref == 0)
 736                 kvmhv_release_nested(gp);
 737 }
 738
 739 /*
 740  * Free up all nested resources allocated for this guest.
 741  * This is called with no vcpus of the guest running, when
 742  * switching the guest to HPT mode or when destroying the
 743  * guest.
 744  */
 745 void kvmhv_release_all_nested(struct kvm *kvm)
 746 {
 747         int i;
 748         struct kvm_nested_guest *gp;
 749         struct kvm_nested_guest *freelist = NULL;
 750         struct kvm_memory_slot *memslot;
 751         int srcu_idx;
 752
 753         spin_lock(&kvm->mmu_lock);
 754         for (i = 0; i <= kvm->arch.max_nested_lpid; i++) {
 755                 gp = kvm->arch.nested_guests[i];
 756                 if (!gp)
 757                         continue;
 758                 kvm->arch.nested_guests[i] = NULL;
 759                 if (--gp->refcnt == 0) {
 760                         gp->next = freelist;
 761                         freelist = gp;
 762                 }
 763         }
 764         kvm->arch.max_nested_lpid = -1;
 765         spin_unlock(&kvm->mmu_lock);
 766         while ((gp = freelist) != NULL) {
 767                 freelist = gp->next;
 768                 kvmhv_release_nested(gp);
 769         }
 770
 771         srcu_idx = srcu_read_lock(&kvm->srcu);
 772         kvm_for_each_memslot(memslot, kvm_memslots(kvm))
 773                 kvmhv_free_memslot_nest_rmap(memslot);
 774         srcu_read_unlock(&kvm->srcu, srcu_idx);
 775 }
 776
 777 /* caller must hold gp->tlb_lock */
 778 static void kvmhv_flush_nested(struct kvm_nested_guest *gp)
 779 {
 780         struct kvm *kvm = gp->l1_host;
 781
 782         spin_lock(&kvm->mmu_lock);
 783         kvmppc_free_pgtable_radix(kvm, gp->shadow_pgtable, gp->shadow_lpid);
 784         spin_unlock(&kvm->mmu_lock);
 785         kvmhv_flush_lpid(gp->shadow_lpid);
 786         kvmhv_update_ptbl_cache(gp);
 787         if (gp->l1_gr_to_hr == 0)
 788                 kvmhv_remove_nested(gp);
 789 }
 790
 791 struct kvm_nested_guest *kvmhv_get_nested(struct kvm *kvm, int l1_lpid,
 792                                           bool create)
 793 {
 794         struct kvm_nested_guest *gp, *newgp;
 795
 796         if (l1_lpid >= KVM_MAX_NESTED_GUESTS ||
 797             l1_lpid >= (1ul << ((kvm->arch.l1_ptcr & PRTS_MASK) + 12 - 4)))
 798                 return NULL;
 799
 800         spin_lock(&kvm->mmu_lock);
 801         gp = kvm->arch.nested_guests[l1_lpid];
 802         if (gp)
 803                 ++gp->refcnt;
 804         spin_unlock(&kvm->mmu_lock);
 805
 806         if (gp || !create)
 807                 return gp;
 808
 809         newgp = kvmhv_alloc_nested(kvm, l1_lpid);
 810         if (!newgp)
 811                 return NULL;
 812         spin_lock(&kvm->mmu_lock);
 813         if (kvm->arch.nested_guests[l1_lpid]) {
 814                 /* someone else beat us to it */
 815                 gp = kvm->arch.nested_guests[l1_lpid];
 816         } else {
 817                 kvm->arch.nested_guests[l1_lpid] = newgp;
 818                 ++newgp->refcnt;
 819                 gp = newgp;
 820                 newgp = NULL;
 821                 if (l1_lpid > kvm->arch.max_nested_lpid)
 822                         kvm->arch.max_nested_lpid = l1_lpid;
 823         }
 824         ++gp->refcnt;
 825         spin_unlock(&kvm->mmu_lock);
 826
 827         if (newgp)
 828                 kvmhv_release_nested(newgp);
 829
 830         return gp;
 831 }
 832
 833 void kvmhv_put_nested(struct kvm_nested_guest *gp)
 834 {
 835         struct kvm *kvm = gp->l1_host;
 836         long ref;
 837
 838         spin_lock(&kvm->mmu_lock);
 839         ref = --gp->refcnt;
 840         spin_unlock(&kvm->mmu_lock);
 841         if (ref == 0)
 842                 kvmhv_release_nested(gp);
 843 }
 844
 845 static struct kvm_nested_guest *kvmhv_find_nested(struct kvm *kvm, int lpid)
 846 {
 847         if (lpid > kvm->arch.max_nested_lpid)
 848                 return NULL;
 849         return kvm->arch.nested_guests[lpid];
 850 }
 851
 852 pte_t *find_kvm_nested_guest_pte(struct kvm *kvm, unsigned long lpid,
 853                                  unsigned long ea, unsigned *hshift)
 854 {
 855         struct kvm_nested_guest *gp;
 856         pte_t *pte;
 857
 858         gp = kvmhv_find_nested(kvm, lpid);
 859         if (!gp)
 860                 return NULL;
 861
 862         VM_WARN(!spin_is_locked(&kvm->mmu_lock),
 863                 "%s called with kvm mmu_lock not held \n", __func__);
 864         pte = __find_linux_pte(gp->shadow_pgtable, ea, NULL, hshift);
 865
 866         return pte;
 867 }
 868
 869 static inline bool kvmhv_n_rmap_is_equal(u64 rmap_1, u64 rmap_2)
 870 {
 871         return !((rmap_1 ^ rmap_2) & (RMAP_NESTED_LPID_MASK |
 872                                        RMAP_NESTED_GPA_MASK));
 873 }
 874
 875 void kvmhv_insert_nest_rmap(struct kvm *kvm, unsigned long *rmapp,
 876                             struct rmap_nested **n_rmap)
 877 {
 878         struct llist_node *entry = ((struct llist_head *) rmapp)->first;
 879         struct rmap_nested *cursor;
 880         u64 rmap, new_rmap = (*n_rmap)->rmap;
 881
 882         /* Are there any existing entries? */
 883         if (!(*rmapp)) {
 884                 /* No -> use the rmap as a single entry */
 885                 *rmapp = new_rmap | RMAP_NESTED_IS_SINGLE_ENTRY;
 886                 return;
 887         }
 888
 889         /* Do any entries match what we're trying to insert? */
 890         for_each_nest_rmap_safe(cursor, entry, &rmap) {
 891                 if (kvmhv_n_rmap_is_equal(rmap, new_rmap))
 892                         return;
 893         }
 894
 895         /* Do we need to create a list or just add the new entry? */
 896         rmap = *rmapp;
 897         if (rmap & RMAP_NESTED_IS_SINGLE_ENTRY) /* Not previously a list */
 898                 *rmapp = 0UL;
 899         llist_add(&((*n_rmap)->list), (struct llist_head *) rmapp);
 900         if (rmap & RMAP_NESTED_IS_SINGLE_ENTRY) /* Not previously a list */
 901                 (*n_rmap)->list.next = (struct llist_node *) rmap;
 902
 903         /* Set NULL so not freed by caller */
 904         *n_rmap = NULL;
 905 }
 906
 907 static void kvmhv_update_nest_rmap_rc(struct kvm *kvm, u64 n_rmap,
 908                                       unsigned long clr, unsigned long set,
 909                                       unsigned long hpa, unsigned long mask)
 910 {
 911         unsigned long gpa;
 912         unsigned int shift, lpid;
 913         pte_t *ptep;
 914
 915         gpa = n_rmap & RMAP_NESTED_GPA_MASK;
 916         lpid = (n_rmap & RMAP_NESTED_LPID_MASK) >> RMAP_NESTED_LPID_SHIFT;
 917
 918         /* Find the pte */
 919         ptep = find_kvm_nested_guest_pte(kvm, lpid, gpa, &shift);
 920         /*
 921          * If the pte is present and the pfn is still the same, update the pte.
 922          * If the pfn has changed then this is a stale rmap entry, the nested
 923          * gpa actually points somewhere else now, and there is nothing to do.
 924          * XXX A future optimisation would be to remove the rmap entry here.
 925          */
 926         if (ptep && pte_present(*ptep) && ((pte_val(*ptep) & mask) == hpa)) {
 927                 __radix_pte_update(ptep, clr, set);
 928                 kvmppc_radix_tlbie_page(kvm, gpa, shift, lpid);
 929         }
 930 }
 931
 932 /*
 933  * For a given list of rmap entries, update the rc bits in all ptes in shadow
 934  * page tables for nested guests which are referenced by the rmap list.
 935  */
 936 void kvmhv_update_nest_rmap_rc_list(struct kvm *kvm, unsigned long *rmapp,
 937                                     unsigned long clr, unsigned long set,
 938                                     unsigned long hpa, unsigned long nbytes)
 939 {
 940         struct llist_node *entry = ((struct llist_head *) rmapp)->first;
 941         struct rmap_nested *cursor;
 942         unsigned long rmap, mask;
 943
 944         if ((clr | set) & ~(_PAGE_DIRTY | _PAGE_ACCESSED))
 945                 return;
 946
 947         mask = PTE_RPN_MASK & ~(nbytes - 1);
 948         hpa &= mask;
 949
 950         for_each_nest_rmap_safe(cursor, entry, &rmap)
 951                 kvmhv_update_nest_rmap_rc(kvm, rmap, clr, set, hpa, mask);
 952 }
 953
 954 static void kvmhv_remove_nest_rmap(struct kvm *kvm, u64 n_rmap,
 955                                    unsigned long hpa, unsigned long mask)
 956 {
 957         struct kvm_nested_guest *gp;
 958         unsigned long gpa;
 959         unsigned int shift, lpid;
 960         pte_t *ptep;
 961
 962         gpa = n_rmap & RMAP_NESTED_GPA_MASK;
 963         lpid = (n_rmap & RMAP_NESTED_LPID_MASK) >> RMAP_NESTED_LPID_SHIFT;
 964         gp = kvmhv_find_nested(kvm, lpid);
 965         if (!gp)
 966                 return;
 967
 968         /* Find and invalidate the pte */
 969         ptep = find_kvm_nested_guest_pte(kvm, lpid, gpa, &shift);
 970         /* Don't spuriously invalidate ptes if the pfn has changed */
 971         if (ptep && pte_present(*ptep) && ((pte_val(*ptep) & mask) == hpa))
 972                 kvmppc_unmap_pte(kvm, ptep, gpa, shift, NULL, gp->shadow_lpid);
 973 }
 974
 975 static void kvmhv_remove_nest_rmap_list(struct kvm *kvm, unsigned long *rmapp,
 976                                         unsigned long hpa, unsigned long mask)
 977 {
 978         struct llist_node *entry = llist_del_all((struct llist_head *) rmapp);
 979         struct rmap_nested *cursor;
 980         unsigned long rmap;
 981
 982         for_each_nest_rmap_safe(cursor, entry, &rmap) {
 983                 kvmhv_remove_nest_rmap(kvm, rmap, hpa, mask);
 984                 kfree(cursor);
 985         }
 986 }
 987
 988 /* called with kvm->mmu_lock held */
 989 void kvmhv_remove_nest_rmap_range(struct kvm *kvm,
 990                                   const struct kvm_memory_slot *memslot,
 991                                   unsigned long gpa, unsigned long hpa,
 992                                   unsigned long nbytes)
 993 {
 994         unsigned long gfn, end_gfn;
 995         unsigned long addr_mask;
 996
 997         if (!memslot)
 998                 return;
 999         gfn = (gpa >> PAGE_SHIFT) - memslot->base_gfn;
1000         end_gfn = gfn + (nbytes >> PAGE_SHIFT);
1001
1002         addr_mask = PTE_RPN_MASK & ~(nbytes - 1);
1003         hpa &= addr_mask;
1004
1005         for (; gfn < end_gfn; gfn++) {
1006                 unsigned long *rmap = &memslot->arch.rmap[gfn];
1007                 kvmhv_remove_nest_rmap_list(kvm, rmap, hpa, addr_mask);
1008         }
1009 }
1010
1011 static void kvmhv_free_memslot_nest_rmap(struct kvm_memory_slot *free)
1012 {
1013         unsigned long page;
1014
1015         for (page = 0; page < free->npages; page++) {
1016                 unsigned long rmap, *rmapp = &free->arch.rmap[page];
1017                 struct rmap_nested *cursor;
1018                 struct llist_node *entry;
1019
1020                 entry = llist_del_all((struct llist_head *) rmapp);
1021                 for_each_nest_rmap_safe(cursor, entry, &rmap)
1022                         kfree(cursor);
1023         }
1024 }
1025
1026 static bool kvmhv_invalidate_shadow_pte(struct kvm_vcpu *vcpu,
1027                                         struct kvm_nested_guest *gp,
1028                                         long gpa, int *shift_ret)
1029 {
1030         struct kvm *kvm = vcpu->kvm;
1031         bool ret = false;
1032         pte_t *ptep;
1033         int shift;
1034
1035         spin_lock(&kvm->mmu_lock);
1036         ptep = find_kvm_nested_guest_pte(kvm, gp->l1_lpid, gpa, &shift);
1037         if (!shift)
1038                 shift = PAGE_SHIFT;
1039         if (ptep && pte_present(*ptep)) {
1040                 kvmppc_unmap_pte(kvm, ptep, gpa, shift, NULL, gp->shadow_lpid);
1041                 ret = true;
1042         }
1043         spin_unlock(&kvm->mmu_lock);
1044
1045         if (shift_ret)
1046                 *shift_ret = shift;
1047         return ret;
1048 }
1049
1050 static inline int get_ric(unsigned int instr)
1051 {
1052         return (instr >> 18) & 0x3;
1053 }
1054
1055 static inline int get_prs(unsigned int instr)
1056 {
1057         return (instr >> 17) & 0x1;
1058 }
1059
1060 static inline int get_r(unsigned int instr)
1061 {
1062         return (instr >> 16) & 0x1;
1063 }
1064
1065 static inline int get_lpid(unsigned long r_val)
1066 {
1067         return r_val & 0xffffffff;
1068 }
1069
1070 static inline int get_is(unsigned long r_val)
1071 {
1072         return (r_val >> 10) & 0x3;
1073 }
1074
1075 static inline int get_ap(unsigned long r_val)
1076 {
1077         return (r_val >> 5) & 0x7;
1078 }
1079
1080 static inline long get_epn(unsigned long r_val)
1081 {
1082         return r_val >> 12;
1083 }
1084
1085 static int kvmhv_emulate_tlbie_tlb_addr(struct kvm_vcpu *vcpu, int lpid,
1086                                         int ap, long epn)
1087 {
1088         struct kvm *kvm = vcpu->kvm;
1089         struct kvm_nested_guest *gp;
1090         long npages;
1091         int shift, shadow_shift;
1092         unsigned long addr;
1093
1094         shift = ap_to_shift(ap);
1095         addr = epn << 12;
1096         if (shift < 0)
1097                 /* Invalid ap encoding */
1098                 return -EINVAL;
1099
1100         addr &= ~((1UL << shift) - 1);
1101         npages = 1UL << (shift - PAGE_SHIFT);
1102
1103         gp = kvmhv_get_nested(kvm, lpid, false);
1104         if (!gp) /* No such guest -> nothing to do */
1105                 return 0;
1106         mutex_lock(&gp->tlb_lock);
1107
1108         /* There may be more than one host page backing this single guest pte */
1109         do {
1110                 kvmhv_invalidate_shadow_pte(vcpu, gp, addr, &shadow_shift);
1111
1112                 npages -= 1UL << (shadow_shift - PAGE_SHIFT);
1113                 addr += 1UL << shadow_shift;
1114         } while (npages > 0);
1115
1116         mutex_unlock(&gp->tlb_lock);
1117         kvmhv_put_nested(gp);
1118         return 0;
1119 }
1120
1121 static void kvmhv_emulate_tlbie_lpid(struct kvm_vcpu *vcpu,
1122                                      struct kvm_nested_guest *gp, int ric)
1123 {
1124         struct kvm *kvm = vcpu->kvm;
1125
1126         mutex_lock(&gp->tlb_lock);
1127         switch (ric) {
1128         case 0:
1129                 /* Invalidate TLB */
1130                 spin_lock(&kvm->mmu_lock);
1131                 kvmppc_free_pgtable_radix(kvm, gp->shadow_pgtable,
1132                                           gp->shadow_lpid);
1133                 kvmhv_flush_lpid(gp->shadow_lpid);
1134                 spin_unlock(&kvm->mmu_lock);
1135                 break;
1136         case 1:
1137                 /*
1138                  * Invalidate PWC
1139                  * We don't cache this -> nothing to do
1140                  */
1141                 break;
1142         case 2:
1143                 /* Invalidate TLB, PWC and caching of partition table entries */
1144                 kvmhv_flush_nested(gp);
1145                 break;
1146         default:
1147                 break;
1148         }
1149         mutex_unlock(&gp->tlb_lock);
1150 }
1151
1152 static void kvmhv_emulate_tlbie_all_lpid(struct kvm_vcpu *vcpu, int ric)
1153 {
1154         struct kvm *kvm = vcpu->kvm;
1155         struct kvm_nested_guest *gp;
1156         int i;
1157
1158         spin_lock(&kvm->mmu_lock);
1159         for (i = 0; i <= kvm->arch.max_nested_lpid; i++) {
1160                 gp = kvm->arch.nested_guests[i];
1161                 if (gp) {
1162                         spin_unlock(&kvm->mmu_lock);
1163                         kvmhv_emulate_tlbie_lpid(vcpu, gp, ric);
1164                         spin_lock(&kvm->mmu_lock);
1165                 }
1166         }
1167         spin_unlock(&kvm->mmu_lock);
1168 }
1169
1170 static int kvmhv_emulate_priv_tlbie(struct kvm_vcpu *vcpu, unsigned int instr,
1171                                     unsigned long rsval, unsigned long rbval)
1172 {
1173         struct kvm *kvm = vcpu->kvm;
1174         struct kvm_nested_guest *gp;
1175         int r, ric, prs, is, ap;
1176         int lpid;
1177         long epn;
1178         int ret = 0;
1179
1180         ric = get_ric(instr);
1181         prs = get_prs(instr);
1182         r = get_r(instr);
1183         lpid = get_lpid(rsval);
1184         is = get_is(rbval);
1185
1186         /*
1187          * These cases are invalid and are not handled:
1188          * r   != 1 -> Only radix supported
1189          * prs == 1 -> Not HV privileged
1190          * ric == 3 -> No cluster bombs for radix
1191          * is  == 1 -> Partition scoped translations not associated with pid
1192          * (!is) && (ric == 1 || ric == 2) -> Not supported by ISA
1193          */
1194         if ((!r) || (prs) || (ric == 3) || (is == 1) ||
1195             ((!is) && (ric == 1 || ric == 2)))
1196                 return -EINVAL;
1197
1198         switch (is) {
1199         case 0:
1200                 /*
1201                  * We know ric == 0
1202                  * Invalidate TLB for a given target address
1203                  */
1204                 epn = get_epn(rbval);
1205                 ap = get_ap(rbval);
1206                 ret = kvmhv_emulate_tlbie_tlb_addr(vcpu, lpid, ap, epn);
1207                 break;
1208         case 2:
1209                 /* Invalidate matching LPID */
1210                 gp = kvmhv_get_nested(kvm, lpid, false);
1211                 if (gp) {
1212                         kvmhv_emulate_tlbie_lpid(vcpu, gp, ric);
1213                         kvmhv_put_nested(gp);
1214                 }
1215                 break;
1216         case 3:
1217                 /* Invalidate ALL LPIDs */
1218                 kvmhv_emulate_tlbie_all_lpid(vcpu, ric);
1219                 break;
1220         default:
1221                 ret = -EINVAL;
1222                 break;
1223         }
1224
1225         return ret;
1226 }
1227
1228 /*
1229  * This handles the H_TLB_INVALIDATE hcall.
1230  * Parameters are (r4) tlbie instruction code, (r5) rS contents,
1231  * (r6) rB contents.
1232  */
1233 long kvmhv_do_nested_tlbie(struct kvm_vcpu *vcpu)
1234 {
1235         int ret;
1236
1237         ret = kvmhv_emulate_priv_tlbie(vcpu, kvmppc_get_gpr(vcpu, 4),
1238                         kvmppc_get_gpr(vcpu, 5), kvmppc_get_gpr(vcpu, 6));
1239         if (ret)
1240                 return H_PARAMETER;
1241         return H_SUCCESS;
1242 }
1243
1244 static long do_tlb_invalidate_nested_all(struct kvm_vcpu *vcpu,
1245                                          unsigned long lpid, unsigned long ric)
1246 {
1247         struct kvm *kvm = vcpu->kvm;
1248         struct kvm_nested_guest *gp;
1249
1250         gp = kvmhv_get_nested(kvm, lpid, false);
1251         if (gp) {
1252                 kvmhv_emulate_tlbie_lpid(vcpu, gp, ric);
1253                 kvmhv_put_nested(gp);
1254         }
1255         return H_SUCCESS;
1256 }
1257
1258 /*
1259  * Number of pages above which we invalidate the entire LPID rather than
1260  * flush individual pages.
1261  */
1262 static unsigned long tlb_range_flush_page_ceiling __read_mostly = 33;
1263
1264 static long do_tlb_invalidate_nested_tlb(struct kvm_vcpu *vcpu,
1265                                          unsigned long lpid,
1266                                          unsigned long pg_sizes,
1267                                          unsigned long start,
1268                                          unsigned long end)
1269 {
1270         int ret = H_P4;
1271         unsigned long addr, nr_pages;
1272         struct mmu_psize_def *def;
1273         unsigned long psize, ap, page_size;
1274         bool flush_lpid;
1275
1276         for (psize = 0; psize < MMU_PAGE_COUNT; psize++) {
1277                 def = &mmu_psize_defs[psize];
1278                 if (!(pg_sizes & def->h_rpt_pgsize))
1279                         continue;
1280
1281                 nr_pages = (end - start) >> def->shift;
1282                 flush_lpid = nr_pages > tlb_range_flush_page_ceiling;
1283                 if (flush_lpid)
1284                         return do_tlb_invalidate_nested_all(vcpu, lpid,
1285                                                         RIC_FLUSH_TLB);
1286                 addr = start;
1287                 ap = mmu_get_ap(psize);
1288                 page_size = 1UL << def->shift;
1289                 do {
1290                         ret = kvmhv_emulate_tlbie_tlb_addr(vcpu, lpid, ap,
1291                                                    get_epn(addr));
1292                         if (ret)
1293                                 return H_P4;
1294                         addr += page_size;
1295                 } while (addr < end);
1296         }
1297         return ret;
1298 }
1299
1300 /*
1301  * Performs partition-scoped invalidations for nested guests
1302  * as part of H_RPT_INVALIDATE hcall.
1303  */
1304 long do_h_rpt_invalidate_pat(struct kvm_vcpu *vcpu, unsigned long lpid,
1305                              unsigned long type, unsigned long pg_sizes,
1306                              unsigned long start, unsigned long end)
1307 {
1308         /*
1309          * If L2 lpid isn't valid, we need to return H_PARAMETER.
1310          *
1311          * However, nested KVM issues a L2 lpid flush call when creating
1312          * partition table entries for L2. This happens even before the
1313          * corresponding shadow lpid is created in HV which happens in
1314          * H_ENTER_NESTED call. Since we can't differentiate this case from
1315          * the invalid case, we ignore such flush requests and return success.
1316          */
1317         if (!kvmhv_find_nested(vcpu->kvm, lpid))
1318                 return H_SUCCESS;
1319
1320         /*
1321          * A flush all request can be handled by a full lpid flush only.
1322          */
1323         if ((type & H_RPTI_TYPE_NESTED_ALL) == H_RPTI_TYPE_NESTED_ALL)
1324                 return do_tlb_invalidate_nested_all(vcpu, lpid, RIC_FLUSH_ALL);
1325
1326         /*
1327          * We don't need to handle a PWC flush like process table here,
1328          * because intermediate partition scoped table in nested guest doesn't
1329          * really have PWC. Only level we have PWC is in L0 and for nested
1330          * invalidate at L0 we always do kvm_flush_lpid() which does
1331          * radix__flush_all_lpid(). For range invalidate at any level, we
1332          * are not removing the higher level page tables and hence there is
1333          * no PWC invalidate needed.
1334          *
1335          * if (type & H_RPTI_TYPE_PWC) {
1336          *      ret = do_tlb_invalidate_nested_all(vcpu, lpid, RIC_FLUSH_PWC);
1337          *      if (ret)
1338          *              return H_P4;
1339          * }
1340          */
1341
1342         if (start == 0 && end == -1)
1343                 return do_tlb_invalidate_nested_all(vcpu, lpid, RIC_FLUSH_TLB);
1344
1345         if (type & H_RPTI_TYPE_TLB)
1346                 return do_tlb_invalidate_nested_tlb(vcpu, lpid, pg_sizes,
1347                                                     start, end);
1348         return H_SUCCESS;
1349 }
1350
1351 /* Used to convert a nested guest real address to a L1 guest real address */
1352 static int kvmhv_translate_addr_nested(struct kvm_vcpu *vcpu,
1353                                        struct kvm_nested_guest *gp,
1354                                        unsigned long n_gpa, unsigned long dsisr,
1355                                        struct kvmppc_pte *gpte_p)
1356 {
1357         u64 fault_addr, flags = dsisr & DSISR_ISSTORE;
1358         int ret;
1359
1360         ret = kvmppc_mmu_walk_radix_tree(vcpu, n_gpa, gpte_p, gp->l1_gr_to_hr,
1361                                          &fault_addr);
1362
1363         if (ret) {
1364                 /* We didn't find a pte */
1365                 if (ret == -EINVAL) {
1366                         /* Unsupported mmu config */
1367                         flags |= DSISR_UNSUPP_MMU;
1368                 } else if (ret == -ENOENT) {
1369                         /* No translation found */
1370                         flags |= DSISR_NOHPTE;
1371                 } else if (ret == -EFAULT) {
1372                         /* Couldn't access L1 real address */
1373                         flags |= DSISR_PRTABLE_FAULT;
1374                         vcpu->arch.fault_gpa = fault_addr;
1375                 } else {
1376                         /* Unknown error */
1377                         return ret;
1378                 }
1379                 goto forward_to_l1;
1380         } else {
1381                 /* We found a pte -> check permissions */
1382                 if (dsisr & DSISR_ISSTORE) {
1383                         /* Can we write? */
1384                         if (!gpte_p->may_write) {
1385                                 flags |= DSISR_PROTFAULT;
1386                                 goto forward_to_l1;
1387                         }
1388                 } else if (vcpu->arch.trap == BOOK3S_INTERRUPT_H_INST_STORAGE) {
1389                         /* Can we execute? */
1390                         if (!gpte_p->may_execute) {
1391                                 flags |= SRR1_ISI_N_G_OR_CIP;
1392                                 goto forward_to_l1;
1393                         }
1394                 } else {
1395                         /* Can we read? */
1396                         if (!gpte_p->may_read && !gpte_p->may_write) {
1397                                 flags |= DSISR_PROTFAULT;
1398                                 goto forward_to_l1;
1399                         }
1400                 }
1401         }
1402
1403         return 0;
1404
1405 forward_to_l1:
1406         vcpu->arch.fault_dsisr = flags;
1407         if (vcpu->arch.trap == BOOK3S_INTERRUPT_H_INST_STORAGE) {
1408                 vcpu->arch.shregs.msr &= SRR1_MSR_BITS;
1409                 vcpu->arch.shregs.msr |= flags;
1410         }
1411         return RESUME_HOST;
1412 }
1413
1414 static long kvmhv_handle_nested_set_rc(struct kvm_vcpu *vcpu,
1415                                        struct kvm_nested_guest *gp,
1416                                        unsigned long n_gpa,
1417                                        struct kvmppc_pte gpte,
1418                                        unsigned long dsisr)
1419 {
1420         struct kvm *kvm = vcpu->kvm;
1421         bool writing = !!(dsisr & DSISR_ISSTORE);
1422         u64 pgflags;
1423         long ret;
1424
1425         /* Are the rc bits set in the L1 partition scoped pte? */
1426         pgflags = _PAGE_ACCESSED;
1427         if (writing)
1428                 pgflags |= _PAGE_DIRTY;
1429         if (pgflags & ~gpte.rc)
1430                 return RESUME_HOST;
1431
1432         spin_lock(&kvm->mmu_lock);
1433         /* Set the rc bit in the pte of our (L0) pgtable for the L1 guest */
1434         ret = kvmppc_hv_handle_set_rc(kvm, false, writing,
1435                                       gpte.raddr, kvm->arch.lpid);
1436         if (!ret) {
1437                 ret = -EINVAL;
1438                 goto out_unlock;
1439         }
1440
1441         /* Set the rc bit in the pte of the shadow_pgtable for the nest guest */
1442         ret = kvmppc_hv_handle_set_rc(kvm, true, writing,
1443                                       n_gpa, gp->l1_lpid);
1444         if (!ret)
1445                 ret = -EINVAL;
1446         else
1447                 ret = 0;
1448
1449 out_unlock:
1450         spin_unlock(&kvm->mmu_lock);
1451         return ret;
1452 }
1453
1454 static inline int kvmppc_radix_level_to_shift(int level)
1455 {
1456         switch (level) {
1457         case 2:
1458                 return PUD_SHIFT;
1459         case 1:
1460                 return PMD_SHIFT;
1461         default:
1462                 return PAGE_SHIFT;
1463         }
1464 }
1465
1466 static inline int kvmppc_radix_shift_to_level(int shift)
1467 {
1468         if (shift == PUD_SHIFT)
1469                 return 2;
1470         if (shift == PMD_SHIFT)
1471                 return 1;
1472         if (shift == PAGE_SHIFT)
1473                 return 0;
1474         WARN_ON_ONCE(1);
1475         return 0;
1476 }
1477
1478 /* called with gp->tlb_lock held */
1479 static long int __kvmhv_nested_page_fault(struct kvm_vcpu *vcpu,
1480                                           struct kvm_nested_guest *gp)
1481 {
1482         struct kvm *kvm = vcpu->kvm;
1483         struct kvm_memory_slot *memslot;
1484         struct rmap_nested *n_rmap;
1485         struct kvmppc_pte gpte;
1486         pte_t pte, *pte_p;
1487         unsigned long mmu_seq;
1488         unsigned long dsisr = vcpu->arch.fault_dsisr;
1489         unsigned long ea = vcpu->arch.fault_dar;
1490         unsigned long *rmapp;
1491         unsigned long n_gpa, gpa, gfn, perm = 0UL;
1492         unsigned int shift, l1_shift, level;
1493         bool writing = !!(dsisr & DSISR_ISSTORE);
1494         bool kvm_ro = false;
1495         long int ret;
1496
1497         if (!gp->l1_gr_to_hr) {
1498                 kvmhv_update_ptbl_cache(gp);
1499                 if (!gp->l1_gr_to_hr)
1500                         return RESUME_HOST;
1501         }
1502
1503         /* Convert the nested guest real address into a L1 guest real address */
1504
1505         n_gpa = vcpu->arch.fault_gpa & ~0xF000000000000FFFULL;
1506         if (!(dsisr & DSISR_PRTABLE_FAULT))
1507                 n_gpa |= ea & 0xFFF;
1508         ret = kvmhv_translate_addr_nested(vcpu, gp, n_gpa, dsisr, &gpte);
1509
1510         /*
1511          * If the hardware found a translation but we don't now have a usable
1512          * translation in the l1 partition-scoped tree, remove the shadow pte
1513          * and let the guest retry.
1514          */
1515         if (ret == RESUME_HOST &&
1516             (dsisr & (DSISR_PROTFAULT | DSISR_BADACCESS | DSISR_NOEXEC_OR_G |
1517                       DSISR_BAD_COPYPASTE)))
1518                 goto inval;
1519         if (ret)
1520                 return ret;
1521
1522         /* Failed to set the reference/change bits */
1523         if (dsisr & DSISR_SET_RC) {
1524                 ret = kvmhv_handle_nested_set_rc(vcpu, gp, n_gpa, gpte, dsisr);
1525                 if (ret == RESUME_HOST)
1526                         return ret;
1527                 if (ret)
1528                         goto inval;
1529                 dsisr &= ~DSISR_SET_RC;
1530                 if (!(dsisr & (DSISR_BAD_FAULT_64S | DSISR_NOHPTE |
1531                                DSISR_PROTFAULT)))
1532                         return RESUME_GUEST;
1533         }
1534
1535         /*
1536          * We took an HISI or HDSI while we were running a nested guest which
1537          * means we have no partition scoped translation for that. This means
1538          * we need to insert a pte for the mapping into our shadow_pgtable.
1539          */
1540
1541         l1_shift = gpte.page_shift;
1542         if (l1_shift < PAGE_SHIFT) {
1543                 /* We don't support l1 using a page size smaller than our own */
1544                 pr_err("KVM: L1 guest page shift (%d) less than our own (%d)\n",
1545                         l1_shift, PAGE_SHIFT);
1546                 return -EINVAL;
1547         }
1548         gpa = gpte.raddr;
1549         gfn = gpa >> PAGE_SHIFT;
1550
1551         /* 1. Get the corresponding host memslot */
1552
1553         memslot = gfn_to_memslot(kvm, gfn);
1554         if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID)) {
1555                 if (dsisr & (DSISR_PRTABLE_FAULT | DSISR_BADACCESS)) {
1556                         /* unusual error -> reflect to the guest as a DSI */
1557                         kvmppc_core_queue_data_storage(vcpu, ea, dsisr);
1558                         return RESUME_GUEST;
1559                 }
1560
1561                 /* passthrough of emulated MMIO case */
1562                 return kvmppc_hv_emulate_mmio(vcpu, gpa, ea, writing);
1563         }
1564         if (memslot->flags & KVM_MEM_READONLY) {
1565                 if (writing) {
1566                         /* Give the guest a DSI */
1567                         kvmppc_core_queue_data_storage(vcpu, ea,
1568                                         DSISR_ISSTORE | DSISR_PROTFAULT);
1569                         return RESUME_GUEST;
1570                 }
1571                 kvm_ro = true;
1572         }
1573
1574         /* 2. Find the host pte for this L1 guest real address */
1575
1576         /* Used to check for invalidations in progress */
1577         mmu_seq = kvm->mmu_notifier_seq;
1578         smp_rmb();
1579
1580         /* See if can find translation in our partition scoped tables for L1 */
1581         pte = __pte(0);
1582         spin_lock(&kvm->mmu_lock);
1583         pte_p = find_kvm_secondary_pte(kvm, gpa, &shift);
1584         if (!shift)
1585                 shift = PAGE_SHIFT;
1586         if (pte_p)
1587                 pte = *pte_p;
1588         spin_unlock(&kvm->mmu_lock);
1589
1590         if (!pte_present(pte) || (writing && !(pte_val(pte) & _PAGE_WRITE))) {
1591                 /* No suitable pte found -> try to insert a mapping */
1592                 ret = kvmppc_book3s_instantiate_page(vcpu, gpa, memslot,
1593                                         writing, kvm_ro, &pte, &level);
1594                 if (ret == -EAGAIN)
1595                         return RESUME_GUEST;
1596                 else if (ret)
1597                         return ret;
1598                 shift = kvmppc_radix_level_to_shift(level);
1599         }
1600         /* Align gfn to the start of the page */
1601         gfn = (gpa & ~((1UL << shift) - 1)) >> PAGE_SHIFT;
1602
1603         /* 3. Compute the pte we need to insert for nest_gpa -> host r_addr */
1604
1605         /* The permissions is the combination of the host and l1 guest ptes */
1606         perm |= gpte.may_read ? 0UL : _PAGE_READ;
1607         perm |= gpte.may_write ? 0UL : _PAGE_WRITE;
1608         perm |= gpte.may_execute ? 0UL : _PAGE_EXEC;
1609         /* Only set accessed/dirty (rc) bits if set in host and l1 guest ptes */
1610         perm |= (gpte.rc & _PAGE_ACCESSED) ? 0UL : _PAGE_ACCESSED;
1611         perm |= ((gpte.rc & _PAGE_DIRTY) && writing) ? 0UL : _PAGE_DIRTY;
1612         pte = __pte(pte_val(pte) & ~perm);
1613
1614         /* What size pte can we insert? */
1615         if (shift > l1_shift) {
1616                 u64 mask;
1617                 unsigned int actual_shift = PAGE_SHIFT;
1618                 if (PMD_SHIFT < l1_shift)
1619                         actual_shift = PMD_SHIFT;
1620                 mask = (1UL << shift) - (1UL << actual_shift);
1621                 pte = __pte(pte_val(pte) | (gpa & mask));
1622                 shift = actual_shift;
1623         }
1624         level = kvmppc_radix_shift_to_level(shift);
1625         n_gpa &= ~((1UL << shift) - 1);
1626
1627         /* 4. Insert the pte into our shadow_pgtable */
1628
1629         n_rmap = kzalloc(sizeof(*n_rmap), GFP_KERNEL);
1630         if (!n_rmap)
1631                 return RESUME_GUEST; /* Let the guest try again */
1632         n_rmap->rmap = (n_gpa & RMAP_NESTED_GPA_MASK) |
1633                 (((unsigned long) gp->l1_lpid) << RMAP_NESTED_LPID_SHIFT);
1634         rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn];
1635         ret = kvmppc_create_pte(kvm, gp->shadow_pgtable, pte, n_gpa, level,
1636                                 mmu_seq, gp->shadow_lpid, rmapp, &n_rmap);
1637         kfree(n_rmap);
1638         if (ret == -EAGAIN)
1639                 ret = RESUME_GUEST;     /* Let the guest try again */
1640
1641         return ret;
1642
1643  inval:
1644         kvmhv_invalidate_shadow_pte(vcpu, gp, n_gpa, NULL);
1645         return RESUME_GUEST;
1646 }
1647
1648 long int kvmhv_nested_page_fault(struct kvm_vcpu *vcpu)
1649 {
1650         struct kvm_nested_guest *gp = vcpu->arch.nested;
1651         long int ret;
1652
1653         mutex_lock(&gp->tlb_lock);
1654         ret = __kvmhv_nested_page_fault(vcpu, gp);
1655         mutex_unlock(&gp->tlb_lock);
1656         return ret;
1657 }
1658
1659 int kvmhv_nested_next_lpid(struct kvm *kvm, int lpid)
1660 {
1661         int ret = -1;
1662
1663         spin_lock(&kvm->mmu_lock);
1664         while (++lpid <= kvm->arch.max_nested_lpid) {
1665                 if (kvm->arch.nested_guests[lpid]) {
1666                         ret = lpid;
1667                         break;
1668                 }
1669         }
1670         spin_unlock(&kvm->mmu_lock);
1671         return ret;
1672 }