2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License, version 2, as
4 * published by the Free Software Foundation.
6 * This program is distributed in the hope that it will be useful,
7 * but WITHOUT ANY WARRANTY; without even the implied warranty of
8 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
9 * GNU General Public License for more details.
11 * Copyright 2011 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
13 * Derived from book3s_rmhandlers.S and other files, which are:
15 * Copyright SUSE Linux Products GmbH 2009
17 * Authors: Alexander Graf <agraf@suse.de>
20 #include <asm/ppc_asm.h>
21 #include <asm/code-patching-asm.h>
22 #include <asm/kvm_asm.h>
26 #include <asm/ptrace.h>
27 #include <asm/hvcall.h>
28 #include <asm/asm-offsets.h>
29 #include <asm/exception-64s.h>
30 #include <asm/kvm_book3s_asm.h>
31 #include <asm/mmu-hash64.h>
34 #define VCPU_GPRS_TM(reg) (((reg) * ULONG_SIZE) + VCPU_GPR_TM)
36 /* Values in HSTATE_NAPPING(r13) */
37 #define NAPPING_CEDE 1
38 #define NAPPING_NOVCPU 2
40 /* Stack frame offsets for kvmppc_hv_entry */
42 #define STACK_SLOT_TRAP (SFS-4)
43 #define STACK_SLOT_CIABR (SFS-16)
44 #define STACK_SLOT_DAWR (SFS-24)
45 #define STACK_SLOT_DAWRX (SFS-32)
48 * Call kvmppc_hv_entry in real mode.
49 * Must be called with interrupts hard-disabled.
53 * LR = return address to continue at after eventually re-enabling MMU
55 _GLOBAL_TOC(kvmppc_hv_entry_trampoline)
57 std r0, PPC_LR_STKOFF(r1)
60 LOAD_REG_ADDR(r5, kvmppc_call_hv_entry)
65 mtmsrd r0,1 /* clear RI in MSR */
71 ld r4, HSTATE_KVM_VCPU(r13)
74 /* Back from guest - restore host state and return to caller */
77 /* Restore host DABR and DABRX */
78 ld r5,HSTATE_DABR(r13)
82 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
85 ld r3,PACA_SPRG_VDSO(r13)
86 mtspr SPRN_SPRG_VDSO_WRITE,r3
88 /* Reload the host's PMU registers */
89 ld r3, PACALPPACAPTR(r13) /* is the host using the PMU? */
90 lbz r4, LPPACA_PMCINUSE(r3)
92 beq 23f /* skip if not */
94 ld r3, HSTATE_MMCR0(r13)
95 andi. r4, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
98 END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
99 lwz r3, HSTATE_PMC1(r13)
100 lwz r4, HSTATE_PMC2(r13)
101 lwz r5, HSTATE_PMC3(r13)
102 lwz r6, HSTATE_PMC4(r13)
103 lwz r8, HSTATE_PMC5(r13)
104 lwz r9, HSTATE_PMC6(r13)
111 ld r3, HSTATE_MMCR0(r13)
112 ld r4, HSTATE_MMCR1(r13)
113 ld r5, HSTATE_MMCRA(r13)
114 ld r6, HSTATE_SIAR(r13)
115 ld r7, HSTATE_SDAR(r13)
121 ld r8, HSTATE_MMCR2(r13)
122 ld r9, HSTATE_SIER(r13)
125 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
131 * Reload DEC. HDEC interrupts were disabled when
132 * we reloaded the host's LPCR value.
134 ld r3, HSTATE_DECEXP(r13)
139 /* hwthread_req may have got set by cede or no vcpu, so clear it */
141 stb r0, HSTATE_HWTHREAD_REQ(r13)
144 * For external and machine check interrupts, we need
145 * to call the Linux handler to process the interrupt.
146 * We do that by jumping to absolute address 0x500 for
147 * external interrupts, or the machine_check_fwnmi label
148 * for machine checks (since firmware might have patched
149 * the vector area at 0x200). The [h]rfid at the end of the
150 * handler will return to the book3s_hv_interrupts.S code.
151 * For other interrupts we do the rfid to get back
152 * to the book3s_hv_interrupts.S code here.
154 ld r8, 112+PPC_LR_STKOFF(r1)
156 ld r7, HSTATE_HOST_MSR(r13)
158 cmpwi cr1, r12, BOOK3S_INTERRUPT_MACHINE_CHECK
159 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
161 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL
162 beq 15f /* Invoke the H_DOORBELL handler */
163 cmpwi cr2, r12, BOOK3S_INTERRUPT_HMI
164 beq cr2, 14f /* HMI check */
166 /* RFI into the highmem handler, or branch to interrupt handler */
170 mtmsrd r6, 1 /* Clear RI in MSR */
173 beq cr1, 13f /* machine check */
176 /* On POWER7, we have external interrupts set to use HSRR0/1 */
177 11: mtspr SPRN_HSRR0, r8
181 13: b machine_check_fwnmi
183 14: mtspr SPRN_HSRR0, r8
185 b hmi_exception_after_realmode
187 15: mtspr SPRN_HSRR0, r8
191 kvmppc_primary_no_guest:
192 /* We handle this much like a ceded vcpu */
193 /* put the HDEC into the DEC, since HDEC interrupts don't wake us */
197 * Make sure the primary has finished the MMU switch.
198 * We should never get here on a secondary thread, but
199 * check it for robustness' sake.
201 ld r5, HSTATE_KVM_VCORE(r13)
202 65: lbz r0, VCORE_IN_GUEST(r5)
209 /* set our bit in napping_threads */
210 ld r5, HSTATE_KVM_VCORE(r13)
211 lbz r7, HSTATE_PTID(r13)
214 addi r6, r5, VCORE_NAPPING_THREADS
219 /* order napping_threads update vs testing entry_exit_map */
222 lwz r7, VCORE_ENTRY_EXIT(r5)
224 bge kvm_novcpu_exit /* another thread already exiting */
225 li r3, NAPPING_NOVCPU
226 stb r3, HSTATE_NAPPING(r13)
228 li r3, 0 /* Don't wake on privileged (OS) doorbell */
232 ld r1, HSTATE_HOST_R1(r13)
233 ld r5, HSTATE_KVM_VCORE(r13)
235 stb r0, HSTATE_NAPPING(r13)
237 /* check the wake reason */
238 bl kvmppc_check_wake_reason
240 /* see if any other thread is already exiting */
241 lwz r0, VCORE_ENTRY_EXIT(r5)
245 /* clear our bit in napping_threads */
246 lbz r7, HSTATE_PTID(r13)
249 addi r6, r5, VCORE_NAPPING_THREADS
255 /* See if the wake reason means we need to exit */
259 /* See if our timeslice has expired (HDEC is negative) */
261 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
265 /* Got an IPI but other vcpus aren't yet exiting, must be a latecomer */
266 ld r4, HSTATE_KVM_VCPU(r13)
268 beq kvmppc_primary_no_guest
270 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
271 addi r3, r4, VCPU_TB_RMENTRY
272 bl kvmhv_start_timing
277 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
278 ld r4, HSTATE_KVM_VCPU(r13)
281 addi r3, r4, VCPU_TB_RMEXIT
282 bl kvmhv_accumulate_time
285 stw r12, STACK_SLOT_TRAP(r1)
286 bl kvmhv_commence_exit
288 lwz r12, STACK_SLOT_TRAP(r1)
289 b kvmhv_switch_to_host
292 * We come in here when wakened from nap mode.
293 * Relocation is off and most register values are lost.
294 * r13 points to the PACA.
296 .globl kvm_start_guest
299 /* Set runlatch bit the minute you wake up from nap */
306 li r0,KVM_HWTHREAD_IN_KVM
307 stb r0,HSTATE_HWTHREAD_STATE(r13)
309 /* NV GPR values from power7_idle() will no longer be valid */
311 stb r0,PACA_NAPSTATELOST(r13)
313 /* were we napping due to cede? */
314 lbz r0,HSTATE_NAPPING(r13)
315 cmpwi r0,NAPPING_CEDE
317 cmpwi r0,NAPPING_NOVCPU
318 beq kvm_novcpu_wakeup
320 ld r1,PACAEMERGSP(r13)
321 subi r1,r1,STACK_FRAME_OVERHEAD
324 * We weren't napping due to cede, so this must be a secondary
325 * thread being woken up to run a guest, or being woken up due
326 * to a stray IPI. (Or due to some machine check or hypervisor
327 * maintenance interrupt while the core is in KVM.)
330 /* Check the wake reason in SRR1 to see why we got here */
331 bl kvmppc_check_wake_reason
335 /* get vcore pointer, NULL if we have nothing to run */
336 ld r5,HSTATE_KVM_VCORE(r13)
338 /* if we have no vcore to run, go back to sleep */
341 kvm_secondary_got_guest:
343 /* Set HSTATE_DSCR(r13) to something sensible */
344 ld r6, PACA_DSCR_DEFAULT(r13)
345 std r6, HSTATE_DSCR(r13)
347 /* On thread 0 of a subcore, set HDEC to max */
348 lbz r4, HSTATE_PTID(r13)
354 /* and set per-LPAR registers, if doing dynamic micro-threading */
355 ld r6, HSTATE_SPLIT_MODE(r13)
358 ld r0, KVM_SPLIT_RPR(r6)
360 ld r0, KVM_SPLIT_PMMAR(r6)
362 ld r0, KVM_SPLIT_LDBAR(r6)
366 /* Order load of vcpu after load of vcore */
368 ld r4, HSTATE_KVM_VCPU(r13)
371 /* Back from the guest, go back to nap */
372 /* Clear our vcpu and vcore pointers so we don't come back in early */
374 std r0, HSTATE_KVM_VCPU(r13)
376 * Once we clear HSTATE_KVM_VCORE(r13), the code in
377 * kvmppc_run_core() is going to assume that all our vcpu
378 * state is visible in memory. This lwsync makes sure
382 std r0, HSTATE_KVM_VCORE(r13)
385 * At this point we have finished executing in the guest.
386 * We need to wait for hwthread_req to become zero, since
387 * we may not turn on the MMU while hwthread_req is non-zero.
388 * While waiting we also need to check if we get given a vcpu to run.
391 lbz r3, HSTATE_HWTHREAD_REQ(r13)
395 li r0, KVM_HWTHREAD_IN_KERNEL
396 stb r0, HSTATE_HWTHREAD_STATE(r13)
397 /* need to recheck hwthread_req after a barrier, to avoid race */
399 lbz r3, HSTATE_HWTHREAD_REQ(r13)
403 * We jump to power7_wakeup_loss, which will return to the caller
404 * of power7_nap in the powernv cpu offline loop. The value we
405 * put in r3 becomes the return value for power7_nap.
409 rlwimi r4, r3, 0, LPCR_PECE0 | LPCR_PECE1
415 ld r5, HSTATE_KVM_VCORE(r13)
418 ld r3, HSTATE_SPLIT_MODE(r13)
421 lbz r0, KVM_SPLIT_DO_NAP(r3)
427 b kvm_secondary_got_guest
429 54: li r0, KVM_HWTHREAD_IN_KVM
430 stb r0, HSTATE_HWTHREAD_STATE(r13)
434 * Here the primary thread is trying to return the core to
435 * whole-core mode, so we need to nap.
439 * Ensure that secondary doesn't nap when it has
440 * its vcore pointer set.
442 sync /* matches smp_mb() before setting split_info.do_nap */
443 ld r0, HSTATE_KVM_VCORE(r13)
446 /* clear any pending message */
448 lis r6, (PPC_DBELL_SERVER << (63-36))@h
450 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
451 /* Set kvm_split_mode.napped[tid] = 1 */
452 ld r3, HSTATE_SPLIT_MODE(r13)
454 lhz r4, PACAPACAINDEX(r13)
455 clrldi r4, r4, 61 /* micro-threading => P8 => 8 threads/core */
456 addi r4, r4, KVM_SPLIT_NAPPED
458 /* Check the do_nap flag again after setting napped[] */
460 lbz r0, KVM_SPLIT_DO_NAP(r3)
463 li r3, (LPCR_PECEDH | LPCR_PECE0) >> 4
465 rlwimi r4, r3, 4, (LPCR_PECEDP | LPCR_PECEDH | LPCR_PECE0 | LPCR_PECE1)
468 std r0, HSTATE_SCRATCH0(r13)
470 ld r0, HSTATE_SCRATCH0(r13)
480 /******************************************************************************
484 *****************************************************************************/
486 .global kvmppc_hv_entry
491 * R4 = vcpu pointer (or NULL)
496 * all other volatile GPRS = free
499 std r0, PPC_LR_STKOFF(r1)
502 /* Save R1 in the PACA */
503 std r1, HSTATE_HOST_R1(r13)
505 li r6, KVM_GUEST_MODE_HOST_HV
506 stb r6, HSTATE_IN_GUEST(r13)
508 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
509 /* Store initial timestamp */
512 addi r3, r4, VCPU_TB_RMENTRY
513 bl kvmhv_start_timing
523 * POWER7/POWER8 host -> guest partition switch code.
524 * We don't have to lock against concurrent tlbies,
525 * but we do have to coordinate across hardware threads.
527 /* Set bit in entry map iff exit map is zero. */
528 ld r5, HSTATE_KVM_VCORE(r13)
530 lbz r6, HSTATE_PTID(r13)
532 addi r9, r5, VCORE_ENTRY_EXIT
534 cmpwi r3, 0x100 /* any threads starting to exit? */
535 bge secondary_too_late /* if so we're too late to the party */
540 /* Primary thread switches to guest partition. */
541 ld r9,VCORE_KVM(r5) /* pointer to struct kvm */
546 li r0,LPID_RSVD /* switch to reserved LPID */
549 mtspr SPRN_SDR1,r6 /* switch to partition page table */
553 /* See if we need to flush the TLB */
554 lhz r6,PACAPACAINDEX(r13) /* test_bit(cpu, need_tlb_flush) */
555 clrldi r7,r6,64-6 /* extract bit number (6 bits) */
556 srdi r6,r6,6 /* doubleword number */
557 sldi r6,r6,3 /* address offset */
559 addi r6,r6,KVM_NEED_FLUSH /* dword in kvm->arch.need_tlb_flush */
565 23: ldarx r7,0,r6 /* if set, clear the bit */
569 /* Flush the TLB of any entries for this LPID */
570 /* use arch 2.07S as a proxy for POWER8 */
572 li r6,512 /* POWER8 has 512 sets */
574 li r6,128 /* POWER7 has 128 sets */
575 ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
577 li r7,0x800 /* IS field = 0b10 */
584 /* Add timebase offset onto timebase */
585 22: ld r8,VCORE_TB_OFFSET(r5)
588 mftb r6 /* current host timebase */
590 mtspr SPRN_TBU40,r8 /* update upper 40 bits */
591 mftb r7 /* check if lower 24 bits overflowed */
596 addis r8,r8,0x100 /* if so, increment upper 40 bits */
599 /* Load guest PCR value to select appropriate compat mode */
600 37: ld r7, VCORE_PCR(r5)
607 /* DPDES is shared between threads */
608 ld r8, VCORE_DPDES(r5)
610 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
613 stb r0,VCORE_IN_GUEST(r5) /* signal secondaries to continue */
615 /* Do we have a guest vcpu to run? */
617 beq kvmppc_primary_no_guest
620 /* Load up guest SLB entries */
621 lwz r5,VCPU_SLB_MAX(r4)
626 1: ld r8,VCPU_SLB_E(r6)
629 addi r6,r6,VCPU_SLB_SIZE
632 /* Increment yield count if they have a VPA */
636 li r6, LPPACA_YIELDCOUNT
641 stb r6, VCPU_VPA_DIRTY(r4)
644 /* Save purr/spurr */
647 std r5,HSTATE_PURR(r13)
648 std r6,HSTATE_SPURR(r13)
654 /* Save host values of some registers */
659 std r5, STACK_SLOT_CIABR(r1)
660 std r6, STACK_SLOT_DAWR(r1)
661 std r7, STACK_SLOT_DAWRX(r1)
662 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
665 /* Set partition DABR */
666 /* Do this before re-enabling PMU to avoid P7 DABR corruption bug */
667 lwz r5,VCPU_DABRX(r4)
672 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
674 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
677 END_FTR_SECTION_IFSET(CPU_FTR_TM)
680 /* Load guest PMU registers */
681 /* R4 is live here (vcpu pointer) */
683 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
684 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
688 andi. r5, r3, MMCR0_PMAO_SYNC | MMCR0_PMAO
691 END_FTR_SECTION_IFSET(CPU_FTR_PMAO_BUG)
692 lwz r3, VCPU_PMC(r4) /* always load up guest PMU registers */
693 lwz r5, VCPU_PMC + 4(r4) /* to prevent information leak */
694 lwz r6, VCPU_PMC + 8(r4)
695 lwz r7, VCPU_PMC + 12(r4)
696 lwz r8, VCPU_PMC + 16(r4)
697 lwz r9, VCPU_PMC + 20(r4)
705 ld r5, VCPU_MMCR + 8(r4)
706 ld r6, VCPU_MMCR + 16(r4)
714 ld r5, VCPU_MMCR + 24(r4)
716 lwz r7, VCPU_PMC + 24(r4)
717 lwz r8, VCPU_PMC + 28(r4)
718 ld r9, VCPU_MMCR + 32(r4)
724 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
728 /* Load up FP, VMX and VSX registers */
731 ld r14, VCPU_GPR(R14)(r4)
732 ld r15, VCPU_GPR(R15)(r4)
733 ld r16, VCPU_GPR(R16)(r4)
734 ld r17, VCPU_GPR(R17)(r4)
735 ld r18, VCPU_GPR(R18)(r4)
736 ld r19, VCPU_GPR(R19)(r4)
737 ld r20, VCPU_GPR(R20)(r4)
738 ld r21, VCPU_GPR(R21)(r4)
739 ld r22, VCPU_GPR(R22)(r4)
740 ld r23, VCPU_GPR(R23)(r4)
741 ld r24, VCPU_GPR(R24)(r4)
742 ld r25, VCPU_GPR(R25)(r4)
743 ld r26, VCPU_GPR(R26)(r4)
744 ld r27, VCPU_GPR(R27)(r4)
745 ld r28, VCPU_GPR(R28)(r4)
746 ld r29, VCPU_GPR(R29)(r4)
747 ld r30, VCPU_GPR(R30)(r4)
748 ld r31, VCPU_GPR(R31)(r4)
750 /* Switch DSCR to guest value */
755 /* Skip next section on POWER7 */
757 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
758 /* Load up POWER8-specific registers */
760 lwz r6, VCPU_PSPB(r4)
766 ld r6, VCPU_DAWRX(r4)
767 ld r7, VCPU_CIABR(r4)
777 ld r8, VCPU_EBBHR(r4)
779 ld r5, VCPU_EBBRR(r4)
780 ld r6, VCPU_BESCR(r4)
781 ld r7, VCPU_CSIGR(r4)
787 ld r5, VCPU_TCSCR(r4)
789 lwz r7, VCPU_GUEST_PID(r4)
798 * Set the decrementer to the guest decrementer.
800 ld r8,VCPU_DEC_EXPIRES(r4)
801 /* r8 is a host timebase value here, convert to guest TB */
802 ld r5,HSTATE_KVM_VCORE(r13)
803 ld r6,VCORE_TB_OFFSET(r5)
810 ld r5, VCPU_SPRG0(r4)
811 ld r6, VCPU_SPRG1(r4)
812 ld r7, VCPU_SPRG2(r4)
813 ld r8, VCPU_SPRG3(r4)
819 /* Load up DAR and DSISR */
821 lwz r6, VCPU_DSISR(r4)
825 /* Restore AMR and UAMOR, set AMOR to all 1s */
833 /* Restore state of CTRL run bit; assume 1 on entry */
841 /* Secondary threads wait for primary to have done partition switch */
842 ld r5, HSTATE_KVM_VCORE(r13)
843 lbz r6, HSTATE_PTID(r13)
846 lbz r0, VCORE_IN_GUEST(r5)
850 20: lwz r3, VCORE_ENTRY_EXIT(r5)
853 lbz r0, VCORE_IN_GUEST(r5)
863 /* Check if HDEC expires soon */
865 cmpwi r3, 512 /* 1 microsecond */
874 kvmppc_cede_reentry: /* r4 = vcpu, r13 = paca */
882 deliver_guest_interrupt:
883 /* r11 = vcpu->arch.msr & ~MSR_HV */
884 rldicl r11, r11, 63 - MSR_HV_LG, 1
885 rotldi r11, r11, 1 + MSR_HV_LG
888 /* Check if we can deliver an external or decrementer interrupt now */
889 ld r0, VCPU_PENDING_EXC(r4)
890 rldicl r0, r0, 64 - BOOK3S_IRQPRIO_EXTERNAL_LEVEL, 63
892 andi. r8, r11, MSR_EE
894 /* Insert EXTERNAL_LEVEL bit into LPCR at the MER bit position */
895 rldimi r8, r0, LPCR_MER_SH, 63 - LPCR_MER_SH
899 li r0, BOOK3S_INTERRUPT_EXTERNAL
903 li r0, BOOK3S_INTERRUPT_DECREMENTER
906 12: mtspr SPRN_SRR0, r10
910 bl kvmppc_msr_interrupt
916 * R10: value for HSRR0
917 * R11: value for HSRR1
922 stb r0,VCPU_CEDED(r4) /* cancel cede */
926 /* Activate guest mode, so faults get handled by KVM */
927 li r9, KVM_GUEST_MODE_GUEST_HV
928 stb r9, HSTATE_IN_GUEST(r13)
930 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
931 /* Accumulate timing */
932 addi r3, r4, VCPU_TB_GUEST
933 bl kvmhv_accumulate_time
941 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
944 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
951 ld r1, VCPU_GPR(R1)(r4)
952 ld r2, VCPU_GPR(R2)(r4)
953 ld r3, VCPU_GPR(R3)(r4)
954 ld r5, VCPU_GPR(R5)(r4)
955 ld r6, VCPU_GPR(R6)(r4)
956 ld r7, VCPU_GPR(R7)(r4)
957 ld r8, VCPU_GPR(R8)(r4)
958 ld r9, VCPU_GPR(R9)(r4)
959 ld r10, VCPU_GPR(R10)(r4)
960 ld r11, VCPU_GPR(R11)(r4)
961 ld r12, VCPU_GPR(R12)(r4)
962 ld r13, VCPU_GPR(R13)(r4)
966 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
967 ld r0, VCPU_GPR(R0)(r4)
968 ld r4, VCPU_GPR(R4)(r4)
976 stw r12, VCPU_TRAP(r4)
977 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
978 addi r3, r4, VCPU_TB_RMEXIT
979 bl kvmhv_accumulate_time
981 11: b kvmhv_switch_to_host
988 li r12, BOOK3S_INTERRUPT_HV_DECREMENTER
989 12: stw r12, VCPU_TRAP(r4)
991 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
992 addi r3, r4, VCPU_TB_RMEXIT
993 bl kvmhv_accumulate_time
997 /******************************************************************************
1001 *****************************************************************************/
1004 * We come here from the first-level interrupt handlers.
1006 .globl kvmppc_interrupt_hv
1007 kvmppc_interrupt_hv:
1009 * Register contents:
1010 * R12 = interrupt vector
1012 * guest CR, R12 saved in shadow VCPU SCRATCH1/0
1013 * guest R13 saved in SPRN_SCRATCH0
1015 std r9, HSTATE_SCRATCH2(r13)
1017 lbz r9, HSTATE_IN_GUEST(r13)
1018 cmpwi r9, KVM_GUEST_MODE_HOST_HV
1019 beq kvmppc_bad_host_intr
1020 #ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
1021 cmpwi r9, KVM_GUEST_MODE_GUEST
1022 ld r9, HSTATE_SCRATCH2(r13)
1023 beq kvmppc_interrupt_pr
1025 /* We're now back in the host but in guest MMU context */
1026 li r9, KVM_GUEST_MODE_HOST_HV
1027 stb r9, HSTATE_IN_GUEST(r13)
1029 ld r9, HSTATE_KVM_VCPU(r13)
1031 /* Save registers */
1033 std r0, VCPU_GPR(R0)(r9)
1034 std r1, VCPU_GPR(R1)(r9)
1035 std r2, VCPU_GPR(R2)(r9)
1036 std r3, VCPU_GPR(R3)(r9)
1037 std r4, VCPU_GPR(R4)(r9)
1038 std r5, VCPU_GPR(R5)(r9)
1039 std r6, VCPU_GPR(R6)(r9)
1040 std r7, VCPU_GPR(R7)(r9)
1041 std r8, VCPU_GPR(R8)(r9)
1042 ld r0, HSTATE_SCRATCH2(r13)
1043 std r0, VCPU_GPR(R9)(r9)
1044 std r10, VCPU_GPR(R10)(r9)
1045 std r11, VCPU_GPR(R11)(r9)
1046 ld r3, HSTATE_SCRATCH0(r13)
1047 lwz r4, HSTATE_SCRATCH1(r13)
1048 std r3, VCPU_GPR(R12)(r9)
1051 ld r3, HSTATE_CFAR(r13)
1052 std r3, VCPU_CFAR(r9)
1053 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
1055 ld r4, HSTATE_PPR(r13)
1056 std r4, VCPU_PPR(r9)
1057 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
1059 /* Restore R1/R2 so we can handle faults */
1060 ld r1, HSTATE_HOST_R1(r13)
1063 mfspr r10, SPRN_SRR0
1064 mfspr r11, SPRN_SRR1
1065 std r10, VCPU_SRR0(r9)
1066 std r11, VCPU_SRR1(r9)
1067 andi. r0, r12, 2 /* need to read HSRR0/1? */
1069 mfspr r10, SPRN_HSRR0
1070 mfspr r11, SPRN_HSRR1
1072 1: std r10, VCPU_PC(r9)
1073 std r11, VCPU_MSR(r9)
1077 std r3, VCPU_GPR(R13)(r9)
1080 stw r12,VCPU_TRAP(r9)
1082 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1083 addi r3, r9, VCPU_TB_RMINTR
1085 bl kvmhv_accumulate_time
1086 ld r5, VCPU_GPR(R5)(r9)
1087 ld r6, VCPU_GPR(R6)(r9)
1088 ld r7, VCPU_GPR(R7)(r9)
1089 ld r8, VCPU_GPR(R8)(r9)
1092 /* Save HEIR (HV emulation assist reg) in emul_inst
1093 if this is an HEI (HV emulation interrupt, e40) */
1094 li r3,KVM_INST_FETCH_FAILED
1095 stw r3,VCPU_LAST_INST(r9)
1096 cmpwi r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
1099 11: stw r3,VCPU_HEIR(r9)
1101 /* these are volatile across C function calls */
1104 std r3, VCPU_CTR(r9)
1105 std r4, VCPU_XER(r9)
1107 /* If this is a page table miss then see if it's theirs or ours */
1108 cmpwi r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1110 cmpwi r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1113 /* See if this is a leftover HDEC interrupt */
1114 cmpwi r12,BOOK3S_INTERRUPT_HV_DECREMENTER
1119 bge fast_guest_return
1121 /* See if this is an hcall we can handle in real mode */
1122 cmpwi r12,BOOK3S_INTERRUPT_SYSCALL
1123 beq hcall_try_real_mode
1125 /* Hypervisor doorbell - exit only if host IPI flag set */
1126 cmpwi r12, BOOK3S_INTERRUPT_H_DOORBELL
1128 lbz r0, HSTATE_HOST_IPI(r13)
1133 /* External interrupt ? */
1134 cmpwi r12, BOOK3S_INTERRUPT_EXTERNAL
1135 bne+ guest_exit_cont
1137 /* External interrupt, first check for host_ipi. If this is
1138 * set, we know the host wants us out so let's do it now
1144 /* Check if any CPU is heading out to the host, if so head out too */
1145 4: ld r5, HSTATE_KVM_VCORE(r13)
1146 lwz r0, VCORE_ENTRY_EXIT(r5)
1149 blt deliver_guest_interrupt
1151 guest_exit_cont: /* r9 = vcpu, r12 = trap, r13 = paca */
1152 /* Save more register state */
1155 std r6, VCPU_DAR(r9)
1156 stw r7, VCPU_DSISR(r9)
1157 /* don't overwrite fault_dar/fault_dsisr if HDSI */
1158 cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
1160 std r6, VCPU_FAULT_DAR(r9)
1161 stw r7, VCPU_FAULT_DSISR(r9)
1163 /* See if it is a machine check */
1164 cmpwi r12, BOOK3S_INTERRUPT_MACHINE_CHECK
1165 beq machine_check_realmode
1167 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1168 addi r3, r9, VCPU_TB_RMEXIT
1170 bl kvmhv_accumulate_time
1173 /* Possibly flush the link stack here. */
1175 patch_site 1b patch__call_kvm_flush_link_stack
1178 /* Increment exit count, poke other threads to exit */
1179 bl kvmhv_commence_exit
1181 ld r9, HSTATE_KVM_VCPU(r13)
1182 lwz r12, VCPU_TRAP(r9)
1184 /* Stop others sending VCPU interrupts to this physical CPU */
1186 stw r0, VCPU_CPU(r9)
1187 stw r0, VCPU_THREAD_CPU(r9)
1189 /* Save guest CTRL register, set runlatch to 1 */
1191 stw r6,VCPU_CTRL(r9)
1197 /* Read the guest SLB and save it away */
1198 lwz r0,VCPU_SLB_NR(r9) /* number of entries in SLB */
1204 andis. r0,r8,SLB_ESID_V@h
1206 add r8,r8,r6 /* put index in */
1208 std r8,VCPU_SLB_E(r7)
1209 std r3,VCPU_SLB_V(r7)
1210 addi r7,r7,VCPU_SLB_SIZE
1214 stw r5,VCPU_SLB_MAX(r9)
1217 * Save the guest PURR/SPURR
1222 ld r8,VCPU_SPURR(r9)
1223 std r5,VCPU_PURR(r9)
1224 std r6,VCPU_SPURR(r9)
1229 * Restore host PURR/SPURR and add guest times
1230 * so that the time in the guest gets accounted.
1232 ld r3,HSTATE_PURR(r13)
1233 ld r4,HSTATE_SPURR(r13)
1244 /* r5 is a guest timebase value here, convert to host TB */
1245 ld r3,HSTATE_KVM_VCORE(r13)
1246 ld r4,VCORE_TB_OFFSET(r3)
1248 std r5,VCPU_DEC_EXPIRES(r9)
1252 END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_207S)
1253 /* Save POWER8-specific registers */
1257 std r5, VCPU_IAMR(r9)
1258 stw r6, VCPU_PSPB(r9)
1259 std r7, VCPU_FSCR(r9)
1264 std r6, VCPU_VTB(r9)
1265 std r7, VCPU_TAR(r9)
1266 mfspr r8, SPRN_EBBHR
1267 std r8, VCPU_EBBHR(r9)
1268 mfspr r5, SPRN_EBBRR
1269 mfspr r6, SPRN_BESCR
1270 mfspr r7, SPRN_CSIGR
1272 std r5, VCPU_EBBRR(r9)
1273 std r6, VCPU_BESCR(r9)
1274 std r7, VCPU_CSIGR(r9)
1275 std r8, VCPU_TACR(r9)
1276 mfspr r5, SPRN_TCSCR
1280 std r5, VCPU_TCSCR(r9)
1281 std r6, VCPU_ACOP(r9)
1282 stw r7, VCPU_GUEST_PID(r9)
1283 std r8, VCPU_WORT(r9)
1285 * Restore various registers to 0, where non-zero values
1286 * set by the guest could disrupt the host.
1291 mtspr SPRN_TCSCR, r0
1293 /* Set MMCRS to 1<<31 to freeze and disable the SPMC counters */
1296 mtspr SPRN_MMCRS, r0
1299 /* Save and reset AMR and UAMOR before turning on the MMU */
1303 std r6,VCPU_UAMOR(r9)
1306 mtspr SPRN_UAMOR, r6
1308 /* Switch DSCR back to host value */
1310 ld r7, HSTATE_DSCR(r13)
1311 std r8, VCPU_DSCR(r9)
1314 /* Save non-volatile GPRs */
1315 std r14, VCPU_GPR(R14)(r9)
1316 std r15, VCPU_GPR(R15)(r9)
1317 std r16, VCPU_GPR(R16)(r9)
1318 std r17, VCPU_GPR(R17)(r9)
1319 std r18, VCPU_GPR(R18)(r9)
1320 std r19, VCPU_GPR(R19)(r9)
1321 std r20, VCPU_GPR(R20)(r9)
1322 std r21, VCPU_GPR(R21)(r9)
1323 std r22, VCPU_GPR(R22)(r9)
1324 std r23, VCPU_GPR(R23)(r9)
1325 std r24, VCPU_GPR(R24)(r9)
1326 std r25, VCPU_GPR(R25)(r9)
1327 std r26, VCPU_GPR(R26)(r9)
1328 std r27, VCPU_GPR(R27)(r9)
1329 std r28, VCPU_GPR(R28)(r9)
1330 std r29, VCPU_GPR(R29)(r9)
1331 std r30, VCPU_GPR(R30)(r9)
1332 std r31, VCPU_GPR(R31)(r9)
1335 mfspr r3, SPRN_SPRG0
1336 mfspr r4, SPRN_SPRG1
1337 mfspr r5, SPRN_SPRG2
1338 mfspr r6, SPRN_SPRG3
1339 std r3, VCPU_SPRG0(r9)
1340 std r4, VCPU_SPRG1(r9)
1341 std r5, VCPU_SPRG2(r9)
1342 std r6, VCPU_SPRG3(r9)
1348 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
1351 END_FTR_SECTION_IFSET(CPU_FTR_TM)
1354 /* Increment yield count if they have a VPA */
1355 ld r8, VCPU_VPA(r9) /* do they have a VPA? */
1358 li r4, LPPACA_YIELDCOUNT
1363 stb r3, VCPU_VPA_DIRTY(r9)
1365 /* Save PMU registers if requested */
1366 /* r8 and cr0.eq are live here */
1369 * POWER8 seems to have a hardware bug where setting
1370 * MMCR0[PMAE] along with MMCR0[PMC1CE] and/or MMCR0[PMCjCE]
1371 * when some counters are already negative doesn't seem
1372 * to cause a performance monitor alert (and hence interrupt).
1373 * The effect of this is that when saving the PMU state,
1374 * if there is no PMU alert pending when we read MMCR0
1375 * before freezing the counters, but one becomes pending
1376 * before we read the counters, we lose it.
1377 * To work around this, we need a way to freeze the counters
1378 * before reading MMCR0. Normally, freezing the counters
1379 * is done by writing MMCR0 (to set MMCR0[FC]) which
1380 * unavoidably writes MMCR0[PMA0] as well. On POWER8,
1381 * we can also freeze the counters using MMCR2, by writing
1382 * 1s to all the counter freeze condition bits (there are
1383 * 9 bits each for 6 counters).
1385 li r3, -1 /* set all freeze bits */
1387 mfspr r10, SPRN_MMCR2
1388 mtspr SPRN_MMCR2, r3
1390 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1392 sldi r3, r3, 31 /* MMCR0_FC (freeze counters) bit */
1393 mfspr r4, SPRN_MMCR0 /* save MMCR0 */
1394 mtspr SPRN_MMCR0, r3 /* freeze all counters, disable ints */
1395 mfspr r6, SPRN_MMCRA
1396 /* Clear MMCRA in order to disable SDAR updates */
1398 mtspr SPRN_MMCRA, r7
1400 beq 21f /* if no VPA, save PMU stuff anyway */
1401 lbz r7, LPPACA_PMCINUSE(r8)
1402 cmpwi r7, 0 /* did they ask for PMU stuff to be saved? */
1404 std r3, VCPU_MMCR(r9) /* if not, set saved MMCR0 to FC */
1406 21: mfspr r5, SPRN_MMCR1
1409 std r4, VCPU_MMCR(r9)
1410 std r5, VCPU_MMCR + 8(r9)
1411 std r6, VCPU_MMCR + 16(r9)
1413 std r10, VCPU_MMCR + 24(r9)
1414 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1415 std r7, VCPU_SIAR(r9)
1416 std r8, VCPU_SDAR(r9)
1423 stw r3, VCPU_PMC(r9)
1424 stw r4, VCPU_PMC + 4(r9)
1425 stw r5, VCPU_PMC + 8(r9)
1426 stw r6, VCPU_PMC + 12(r9)
1427 stw r7, VCPU_PMC + 16(r9)
1428 stw r8, VCPU_PMC + 20(r9)
1431 mfspr r6, SPRN_SPMC1
1432 mfspr r7, SPRN_SPMC2
1433 mfspr r8, SPRN_MMCRS
1434 std r5, VCPU_SIER(r9)
1435 stw r6, VCPU_PMC + 24(r9)
1436 stw r7, VCPU_PMC + 28(r9)
1437 std r8, VCPU_MMCR + 32(r9)
1439 mtspr SPRN_MMCRS, r4
1440 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1448 /* Restore host values of some registers */
1450 ld r5, STACK_SLOT_CIABR(r1)
1451 ld r6, STACK_SLOT_DAWR(r1)
1452 ld r7, STACK_SLOT_DAWRX(r1)
1453 mtspr SPRN_CIABR, r5
1455 mtspr SPRN_DAWRX, r7
1456 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1459 * POWER7/POWER8 guest -> host partition switch code.
1460 * We don't have to lock against tlbies but we do
1461 * have to coordinate the hardware threads.
1463 kvmhv_switch_to_host:
1464 /* Secondary threads wait for primary to do partition switch */
1465 ld r5,HSTATE_KVM_VCORE(r13)
1466 ld r4,VCORE_KVM(r5) /* pointer to struct kvm */
1467 lbz r3,HSTATE_PTID(r13)
1471 13: lbz r3,VCORE_IN_GUEST(r5)
1477 /* Primary thread waits for all the secondaries to exit guest */
1478 15: lwz r3,VCORE_ENTRY_EXIT(r5)
1479 rlwinm r0,r3,32-8,0xff
1485 /* Did we actually switch to the guest at all? */
1486 lbz r6, VCORE_IN_GUEST(r5)
1490 /* Primary thread switches back to host partition */
1491 ld r6,KVM_HOST_SDR1(r4)
1492 lwz r7,KVM_HOST_LPID(r4)
1493 li r8,LPID_RSVD /* switch to reserved LPID */
1496 mtspr SPRN_SDR1,r6 /* switch to partition page table */
1501 /* DPDES is shared between threads */
1502 mfspr r7, SPRN_DPDES
1503 std r7, VCORE_DPDES(r5)
1504 /* clear DPDES so we don't get guest doorbells in the host */
1506 mtspr SPRN_DPDES, r8
1507 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1509 /* Subtract timebase offset from timebase */
1510 ld r8,VCORE_TB_OFFSET(r5)
1513 mftb r6 /* current guest timebase */
1515 mtspr SPRN_TBU40,r8 /* update upper 40 bits */
1516 mftb r7 /* check if lower 24 bits overflowed */
1521 addis r8,r8,0x100 /* if so, increment upper 40 bits */
1525 17: ld r0, VCORE_PCR(r5)
1531 /* Signal secondary CPUs to continue */
1532 stb r0,VCORE_IN_GUEST(r5)
1533 19: lis r8,0x7fff /* MAX_INT@h */
1536 16: ld r8,KVM_HOST_LPCR(r4)
1540 /* load host SLB entries */
1541 ld r8,PACA_SLBSHADOWPTR(r13)
1543 .rept SLB_NUM_BOLTED
1544 li r3, SLBSHADOW_SAVEAREA
1548 andis. r7,r5,SLB_ESID_V@h
1554 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
1555 /* Finish timing, if we have a vcpu */
1556 ld r4, HSTATE_KVM_VCPU(r13)
1560 bl kvmhv_accumulate_time
1563 /* Unset guest mode */
1564 li r0, KVM_GUEST_MODE_NONE
1565 stb r0, HSTATE_IN_GUEST(r13)
1567 ld r0, SFS+PPC_LR_STKOFF(r1)
1573 .global kvm_flush_link_stack
1574 kvm_flush_link_stack:
1575 /* Save LR into r0 */
1578 /* Flush the link stack. On Power8 it's up to 32 entries in size. */
1588 * Check whether an HDSI is an HPTE not found fault or something else.
1589 * If it is an HPTE not found fault that is due to the guest accessing
1590 * a page that they have mapped but which we have paged out, then
1591 * we continue on with the guest exit path. In all other cases,
1592 * reflect the HDSI to the guest as a DSI.
1596 mfspr r6, SPRN_HDSISR
1597 /* HPTE not found fault or protection fault? */
1598 andis. r0, r6, (DSISR_NOHPTE | DSISR_PROTFAULT)@h
1599 beq 1f /* if not, send it to the guest */
1600 andi. r0, r11, MSR_DR /* data relocation enabled? */
1603 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
1604 li r0, BOOK3S_INTERRUPT_DATA_SEGMENT
1605 bne 7f /* if no SLB entry found */
1606 4: std r4, VCPU_FAULT_DAR(r9)
1607 stw r6, VCPU_FAULT_DSISR(r9)
1609 /* Search the hash table. */
1610 mr r3, r9 /* vcpu pointer */
1611 li r7, 1 /* data fault */
1612 bl kvmppc_hpte_hv_fault
1613 ld r9, HSTATE_KVM_VCPU(r13)
1615 ld r11, VCPU_MSR(r9)
1616 li r12, BOOK3S_INTERRUPT_H_DATA_STORAGE
1617 cmpdi r3, 0 /* retry the instruction */
1619 cmpdi r3, -1 /* handle in kernel mode */
1621 cmpdi r3, -2 /* MMIO emulation; need instr word */
1624 /* Synthesize a DSI (or DSegI) for the guest */
1625 ld r4, VCPU_FAULT_DAR(r9)
1627 1: li r0, BOOK3S_INTERRUPT_DATA_STORAGE
1628 mtspr SPRN_DSISR, r6
1629 7: mtspr SPRN_DAR, r4
1630 mtspr SPRN_SRR0, r10
1631 mtspr SPRN_SRR1, r11
1633 bl kvmppc_msr_interrupt
1634 fast_interrupt_c_return:
1635 6: ld r7, VCPU_CTR(r9)
1642 3: ld r5, VCPU_KVM(r9) /* not relocated, use VRMA */
1643 ld r5, KVM_VRMA_SLB_V(r5)
1646 /* If this is for emulated MMIO, load the instruction word */
1647 2: li r8, KVM_INST_FETCH_FAILED /* In case lwz faults */
1649 /* Set guest mode to 'jump over instruction' so if lwz faults
1650 * we'll just continue at the next IP. */
1651 li r0, KVM_GUEST_MODE_SKIP
1652 stb r0, HSTATE_IN_GUEST(r13)
1654 /* Do the access with MSR:DR enabled */
1656 ori r4, r3, MSR_DR /* Enable paging for data */
1661 /* Store the result */
1662 stw r8, VCPU_LAST_INST(r9)
1664 /* Unset guest mode. */
1665 li r0, KVM_GUEST_MODE_HOST_HV
1666 stb r0, HSTATE_IN_GUEST(r13)
1670 * Similarly for an HISI, reflect it to the guest as an ISI unless
1671 * it is an HPTE not found fault for a page that we have paged out.
1674 andis. r0, r11, SRR1_ISI_NOPT@h
1676 andi. r0, r11, MSR_IR /* instruction relocation enabled? */
1679 PPC_SLBFEE_DOT(R5, R0) /* if so, look up SLB */
1680 li r0, BOOK3S_INTERRUPT_INST_SEGMENT
1681 bne 7f /* if no SLB entry found */
1683 /* Search the hash table. */
1684 mr r3, r9 /* vcpu pointer */
1687 li r7, 0 /* instruction fault */
1688 bl kvmppc_hpte_hv_fault
1689 ld r9, HSTATE_KVM_VCPU(r13)
1691 ld r11, VCPU_MSR(r9)
1692 li r12, BOOK3S_INTERRUPT_H_INST_STORAGE
1693 cmpdi r3, 0 /* retry the instruction */
1694 beq fast_interrupt_c_return
1695 cmpdi r3, -1 /* handle in kernel mode */
1698 /* Synthesize an ISI (or ISegI) for the guest */
1700 1: li r0, BOOK3S_INTERRUPT_INST_STORAGE
1701 7: mtspr SPRN_SRR0, r10
1702 mtspr SPRN_SRR1, r11
1704 bl kvmppc_msr_interrupt
1705 b fast_interrupt_c_return
1707 3: ld r6, VCPU_KVM(r9) /* not relocated, use VRMA */
1708 ld r5, KVM_VRMA_SLB_V(r6)
1712 * Try to handle an hcall in real mode.
1713 * Returns to the guest if we handle it, or continues on up to
1714 * the kernel if we can't (i.e. if we don't have a handler for
1715 * it, or if the handler returns H_TOO_HARD).
1717 * r5 - r8 contain hcall args,
1718 * r9 = vcpu, r10 = pc, r11 = msr, r12 = trap, r13 = paca
1720 hcall_try_real_mode:
1721 ld r3,VCPU_GPR(R3)(r9)
1723 /* sc 1 from userspace - reflect to guest syscall */
1724 bne sc_1_fast_return
1726 cmpldi r3,hcall_real_table_end - hcall_real_table
1728 /* See if this hcall is enabled for in-kernel handling */
1730 srdi r0, r3, 8 /* r0 = (r3 / 4) >> 6 */
1731 sldi r0, r0, 3 /* index into kvm->arch.enabled_hcalls[] */
1733 ld r0, KVM_ENABLED_HCALLS(r4)
1734 rlwinm r4, r3, 32-2, 0x3f /* r4 = (r3 / 4) & 0x3f */
1738 /* Get pointer to handler, if any, and call it */
1739 LOAD_REG_ADDR(r4, hcall_real_table)
1745 mr r3,r9 /* get vcpu pointer */
1746 ld r4,VCPU_GPR(R4)(r9)
1749 beq hcall_real_fallback
1750 ld r4,HSTATE_KVM_VCPU(r13)
1751 std r3,VCPU_GPR(R3)(r4)
1759 li r10, BOOK3S_INTERRUPT_SYSCALL
1760 bl kvmppc_msr_interrupt
1764 /* We've attempted a real mode hcall, but it's punted it back
1765 * to userspace. We need to restore some clobbered volatiles
1766 * before resuming the pass-it-to-qemu path */
1767 hcall_real_fallback:
1768 li r12,BOOK3S_INTERRUPT_SYSCALL
1769 ld r9, HSTATE_KVM_VCPU(r13)
1773 .globl hcall_real_table
1775 .long 0 /* 0 - unused */
1776 .long DOTSYM(kvmppc_h_remove) - hcall_real_table
1777 .long DOTSYM(kvmppc_h_enter) - hcall_real_table
1778 .long DOTSYM(kvmppc_h_read) - hcall_real_table
1779 .long DOTSYM(kvmppc_h_clear_mod) - hcall_real_table
1780 .long DOTSYM(kvmppc_h_clear_ref) - hcall_real_table
1781 .long DOTSYM(kvmppc_h_protect) - hcall_real_table
1782 .long DOTSYM(kvmppc_h_get_tce) - hcall_real_table
1783 .long DOTSYM(kvmppc_h_put_tce) - hcall_real_table
1784 .long 0 /* 0x24 - H_SET_SPRG0 */
1785 .long DOTSYM(kvmppc_h_set_dabr) - hcall_real_table
1800 #ifdef CONFIG_KVM_XICS
1801 .long DOTSYM(kvmppc_rm_h_eoi) - hcall_real_table
1802 .long DOTSYM(kvmppc_rm_h_cppr) - hcall_real_table
1803 .long DOTSYM(kvmppc_rm_h_ipi) - hcall_real_table
1804 .long 0 /* 0x70 - H_IPOLL */
1805 .long DOTSYM(kvmppc_rm_h_xirr) - hcall_real_table
1807 .long 0 /* 0x64 - H_EOI */
1808 .long 0 /* 0x68 - H_CPPR */
1809 .long 0 /* 0x6c - H_IPI */
1810 .long 0 /* 0x70 - H_IPOLL */
1811 .long 0 /* 0x74 - H_XIRR */
1839 .long DOTSYM(kvmppc_h_cede) - hcall_real_table
1840 .long DOTSYM(kvmppc_rm_h_confer) - hcall_real_table
1856 .long DOTSYM(kvmppc_h_bulk_remove) - hcall_real_table
1860 .long DOTSYM(kvmppc_h_set_xdabr) - hcall_real_table
1975 .long DOTSYM(kvmppc_h_random) - hcall_real_table
1976 .globl hcall_real_table_end
1977 hcall_real_table_end:
1979 _GLOBAL(kvmppc_h_set_xdabr)
1980 andi. r0, r5, DABRX_USER | DABRX_KERNEL
1982 li r0, DABRX_USER | DABRX_KERNEL | DABRX_BTI
1985 6: li r3, H_PARAMETER
1988 _GLOBAL(kvmppc_h_set_dabr)
1989 li r5, DABRX_USER | DABRX_KERNEL
1993 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
1994 std r4,VCPU_DABR(r3)
1995 stw r5, VCPU_DABRX(r3)
1996 mtspr SPRN_DABRX, r5
1997 /* Work around P7 bug where DABR can get corrupted on mtspr */
1998 1: mtspr SPRN_DABR,r4
2006 /* Emulate H_SET_DABR/X on P8 for the sake of compat mode guests */
2007 2: rlwimi r5, r4, 5, DAWRX_DR | DAWRX_DW
2008 rlwimi r5, r4, 2, DAWRX_WT
2010 std r4, VCPU_DAWR(r3)
2011 std r5, VCPU_DAWRX(r3)
2013 mtspr SPRN_DAWRX, r5
2017 _GLOBAL(kvmppc_h_cede) /* r3 = vcpu pointer, r11 = msr, r13 = paca */
2019 std r11,VCPU_MSR(r3)
2021 stb r0,VCPU_CEDED(r3)
2022 sync /* order setting ceded vs. testing prodded */
2023 lbz r5,VCPU_PRODDED(r3)
2025 bne kvm_cede_prodded
2026 li r12,0 /* set trap to 0 to say hcall is handled */
2027 stw r12,VCPU_TRAP(r3)
2029 std r0,VCPU_GPR(R3)(r3)
2032 * Set our bit in the bitmask of napping threads unless all the
2033 * other threads are already napping, in which case we send this
2036 ld r5,HSTATE_KVM_VCORE(r13)
2037 lbz r6,HSTATE_PTID(r13)
2038 lwz r8,VCORE_ENTRY_EXIT(r5)
2042 addi r6,r5,VCORE_NAPPING_THREADS
2049 /* order napping_threads update vs testing entry_exit_map */
2052 stb r0,HSTATE_NAPPING(r13)
2053 lwz r7,VCORE_ENTRY_EXIT(r5)
2055 bge 33f /* another thread already exiting */
2058 * Although not specifically required by the architecture, POWER7
2059 * preserves the following registers in nap mode, even if an SMT mode
2060 * switch occurs: SLB entries, PURR, SPURR, AMOR, UAMOR, AMR, SPRG0-3,
2061 * DAR, DSISR, DABR, DABRX, DSCR, PMCx, MMCRx, SIAR, SDAR.
2063 /* Save non-volatile GPRs */
2064 std r14, VCPU_GPR(R14)(r3)
2065 std r15, VCPU_GPR(R15)(r3)
2066 std r16, VCPU_GPR(R16)(r3)
2067 std r17, VCPU_GPR(R17)(r3)
2068 std r18, VCPU_GPR(R18)(r3)
2069 std r19, VCPU_GPR(R19)(r3)
2070 std r20, VCPU_GPR(R20)(r3)
2071 std r21, VCPU_GPR(R21)(r3)
2072 std r22, VCPU_GPR(R22)(r3)
2073 std r23, VCPU_GPR(R23)(r3)
2074 std r24, VCPU_GPR(R24)(r3)
2075 std r25, VCPU_GPR(R25)(r3)
2076 std r26, VCPU_GPR(R26)(r3)
2077 std r27, VCPU_GPR(R27)(r3)
2078 std r28, VCPU_GPR(R28)(r3)
2079 std r29, VCPU_GPR(R29)(r3)
2080 std r30, VCPU_GPR(R30)(r3)
2081 std r31, VCPU_GPR(R31)(r3)
2086 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2088 ld r9, HSTATE_KVM_VCPU(r13)
2090 END_FTR_SECTION_IFSET(CPU_FTR_TM)
2094 * Set DEC to the smaller of DEC and HDEC, so that we wake
2095 * no later than the end of our timeslice (HDEC interrupts
2096 * don't wake us from nap).
2105 /* save expiry time of guest decrementer */
2108 ld r4, HSTATE_KVM_VCPU(r13)
2109 ld r5, HSTATE_KVM_VCORE(r13)
2110 ld r6, VCORE_TB_OFFSET(r5)
2111 subf r3, r6, r3 /* convert to host TB value */
2112 std r3, VCPU_DEC_EXPIRES(r4)
2114 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2115 ld r4, HSTATE_KVM_VCPU(r13)
2116 addi r3, r4, VCPU_TB_CEDE
2117 bl kvmhv_accumulate_time
2120 lis r3, LPCR_PECEDP@h /* Do wake on privileged doorbell */
2123 * Take a nap until a decrementer or external or doobell interrupt
2124 * occurs, with PECE1 and PECE0 set in LPCR.
2125 * On POWER8, set PECEDH, and if we are ceding, also set PECEDP.
2126 * Also clear the runlatch bit before napping.
2129 mfspr r0, SPRN_CTRLF
2131 mtspr SPRN_CTRLT, r0
2134 stb r0,HSTATE_HWTHREAD_REQ(r13)
2136 ori r5,r5,LPCR_PECE0 | LPCR_PECE1
2138 ori r5, r5, LPCR_PECEDH
2139 rlwimi r5, r3, 0, LPCR_PECEDP
2140 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2144 std r0, HSTATE_SCRATCH0(r13)
2146 ld r0, HSTATE_SCRATCH0(r13)
2158 /* get vcpu pointer */
2159 ld r4, HSTATE_KVM_VCPU(r13)
2161 /* Woken by external or decrementer interrupt */
2162 ld r1, HSTATE_HOST_R1(r13)
2164 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2165 addi r3, r4, VCPU_TB_RMINTR
2166 bl kvmhv_accumulate_time
2169 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2171 bl kvmppc_restore_tm
2172 END_FTR_SECTION_IFSET(CPU_FTR_TM)
2175 /* load up FP state */
2178 /* Restore guest decrementer */
2179 ld r3, VCPU_DEC_EXPIRES(r4)
2180 ld r5, HSTATE_KVM_VCORE(r13)
2181 ld r6, VCORE_TB_OFFSET(r5)
2182 add r3, r3, r6 /* convert host TB to guest TB value */
2188 ld r14, VCPU_GPR(R14)(r4)
2189 ld r15, VCPU_GPR(R15)(r4)
2190 ld r16, VCPU_GPR(R16)(r4)
2191 ld r17, VCPU_GPR(R17)(r4)
2192 ld r18, VCPU_GPR(R18)(r4)
2193 ld r19, VCPU_GPR(R19)(r4)
2194 ld r20, VCPU_GPR(R20)(r4)
2195 ld r21, VCPU_GPR(R21)(r4)
2196 ld r22, VCPU_GPR(R22)(r4)
2197 ld r23, VCPU_GPR(R23)(r4)
2198 ld r24, VCPU_GPR(R24)(r4)
2199 ld r25, VCPU_GPR(R25)(r4)
2200 ld r26, VCPU_GPR(R26)(r4)
2201 ld r27, VCPU_GPR(R27)(r4)
2202 ld r28, VCPU_GPR(R28)(r4)
2203 ld r29, VCPU_GPR(R29)(r4)
2204 ld r30, VCPU_GPR(R30)(r4)
2205 ld r31, VCPU_GPR(R31)(r4)
2207 /* Check the wake reason in SRR1 to see why we got here */
2208 bl kvmppc_check_wake_reason
2210 /* clear our bit in vcore->napping_threads */
2211 34: ld r5,HSTATE_KVM_VCORE(r13)
2212 lbz r7,HSTATE_PTID(r13)
2215 addi r6,r5,VCORE_NAPPING_THREADS
2221 stb r0,HSTATE_NAPPING(r13)
2223 /* See if the wake reason means we need to exit */
2224 stw r12, VCPU_TRAP(r4)
2229 /* see if any other thread is already exiting */
2230 lwz r0,VCORE_ENTRY_EXIT(r5)
2234 b kvmppc_cede_reentry /* if not go back to guest */
2236 /* cede when already previously prodded case */
2239 stb r0,VCPU_PRODDED(r3)
2240 sync /* order testing prodded vs. clearing ceded */
2241 stb r0,VCPU_CEDED(r3)
2245 /* we've ceded but we want to give control to the host */
2247 ld r9, HSTATE_KVM_VCPU(r13)
2250 /* Try to handle a machine check in real mode */
2251 machine_check_realmode:
2252 mr r3, r9 /* get vcpu pointer */
2253 bl kvmppc_realmode_machine_check
2255 ld r9, HSTATE_KVM_VCPU(r13)
2256 li r12, BOOK3S_INTERRUPT_MACHINE_CHECK
2258 * Deliver unhandled/fatal (e.g. UE) MCE errors to guest through
2259 * machine check interrupt (set HSRR0 to 0x200). And for handled
2260 * errors (no-fatal), just go back to guest execution with current
2261 * HSRR0 instead of exiting guest. This new approach will inject
2262 * machine check to guest for fatal error causing guest to crash.
2264 * The old code used to return to host for unhandled errors which
2265 * was causing guest to hang with soft lockups inside guest and
2266 * makes it difficult to recover guest instance.
2268 * if we receive machine check with MSR(RI=0) then deliver it to
2269 * guest as machine check causing guest to crash.
2271 ld r11, VCPU_MSR(r9)
2272 andi. r10, r11, MSR_RI /* check for unrecoverable exception */
2273 beq 1f /* Deliver a machine check to guest */
2275 cmpdi r3, 0 /* Did we handle MCE ? */
2276 bne 2f /* Continue guest execution. */
2277 /* If not, deliver a machine check. SRR0/1 are already set */
2278 1: li r10, BOOK3S_INTERRUPT_MACHINE_CHECK
2279 bl kvmppc_msr_interrupt
2280 2: b fast_interrupt_c_return
2283 * Check the reason we woke from nap, and take appropriate action.
2285 * 0 if nothing needs to be done
2286 * 1 if something happened that needs to be handled by the host
2287 * -1 if there was a guest wakeup (IPI or msgsnd)
2289 * Also sets r12 to the interrupt vector for any interrupt that needs
2290 * to be handled now by the host (0x500 for external interrupt), or zero.
2291 * Modifies r0, r6, r7, r8.
2293 kvmppc_check_wake_reason:
2296 rlwinm r6, r6, 45-31, 0xf /* extract wake reason field (P8) */
2298 rlwinm r6, r6, 45-31, 0xe /* P7 wake reason field is 3 bits */
2299 ALT_FTR_SECTION_END_IFSET(CPU_FTR_ARCH_207S)
2300 cmpwi r6, 8 /* was it an external interrupt? */
2301 li r12, BOOK3S_INTERRUPT_EXTERNAL
2302 beq kvmppc_read_intr /* if so, see what it was */
2305 cmpwi r6, 6 /* was it the decrementer? */
2308 cmpwi r6, 5 /* privileged doorbell? */
2310 cmpwi r6, 3 /* hypervisor doorbell? */
2312 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S)
2313 li r3, 1 /* anything else, return 1 */
2316 /* hypervisor doorbell */
2317 3: li r12, BOOK3S_INTERRUPT_H_DOORBELL
2320 * Clear the doorbell as we will invoke the handler
2321 * explicitly in the guest exit path.
2323 lis r6, (PPC_DBELL_SERVER << (63-36))@h
2325 /* see if it's a host IPI */
2327 lbz r0, HSTATE_HOST_IPI(r13)
2330 /* if not, return -1 */
2335 * Determine what sort of external interrupt is pending (if any).
2337 * 0 if no interrupt is pending
2338 * 1 if an interrupt is pending that needs to be handled by the host
2339 * -1 if there was a guest wakeup IPI (which has now been cleared)
2340 * Modifies r0, r6, r7, r8, returns value in r3.
2343 /* see if a host IPI is pending */
2345 lbz r0, HSTATE_HOST_IPI(r13)
2349 /* Now read the interrupt from the ICP */
2350 ld r6, HSTATE_XICS_PHYS(r13)
2356 * Save XIRR for later. Since we get in in reverse endian on LE
2357 * systems, save it byte reversed and fetch it back in host endian.
2359 li r3, HSTATE_SAVED_XIRR
2361 #ifdef __LITTLE_ENDIAN__
2362 lwz r3, HSTATE_SAVED_XIRR(r13)
2366 rlwinm. r3, r3, 0, 0xffffff
2368 beq 1f /* if nothing pending in the ICP */
2370 /* We found something in the ICP...
2372 * If it's not an IPI, stash it in the PACA and return to
2373 * the host, we don't (yet) handle directing real external
2374 * interrupts directly to the guest
2376 cmpwi r3, XICS_IPI /* if there is, is it an IPI? */
2379 /* It's an IPI, clear the MFRR and EOI it */
2382 stbcix r3, r6, r8 /* clear the IPI */
2383 stwcix r0, r6, r7 /* EOI it */
2386 /* We need to re-check host IPI now in case it got set in the
2387 * meantime. If it's clear, we bounce the interrupt to the
2390 lbz r0, HSTATE_HOST_IPI(r13)
2394 /* OK, it's an IPI for us */
2399 42: /* It's not an IPI and it's for the host. We saved a copy of XIRR in
2400 * the PACA earlier, it will be picked up by the host ICP driver
2405 43: /* We raced with the host, we need to resend that IPI, bummer */
2407 stbcix r0, r6, r8 /* set the IPI */
2413 * Save away FP, VMX and VSX registers.
2415 * N.B. r30 and r31 are volatile across this function,
2416 * thus it is not callable from C.
2423 #ifdef CONFIG_ALTIVEC
2425 oris r8,r8,MSR_VEC@h
2426 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2430 oris r8,r8,MSR_VSX@h
2431 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2434 addi r3,r3,VCPU_FPRS
2436 #ifdef CONFIG_ALTIVEC
2438 addi r3,r31,VCPU_VRS
2440 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2442 mfspr r6,SPRN_VRSAVE
2443 stw r6,VCPU_VRSAVE(r31)
2448 * Load up FP, VMX and VSX registers
2450 * N.B. r30 and r31 are volatile across this function,
2451 * thus it is not callable from C.
2458 #ifdef CONFIG_ALTIVEC
2460 oris r8,r8,MSR_VEC@h
2461 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2465 oris r8,r8,MSR_VSX@h
2466 END_FTR_SECTION_IFSET(CPU_FTR_VSX)
2469 addi r3,r4,VCPU_FPRS
2471 #ifdef CONFIG_ALTIVEC
2473 addi r3,r31,VCPU_VRS
2475 END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
2477 lwz r7,VCPU_VRSAVE(r31)
2478 mtspr SPRN_VRSAVE,r7
2483 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
2485 * Save transactional state and TM-related registers.
2486 * Called with r9 pointing to the vcpu struct.
2487 * This can modify all checkpointed registers, but
2488 * restores r1, r2 and r9 (vcpu pointer) before exit.
2492 std r0, PPC_LR_STKOFF(r1)
2497 rldimi r8, r0, MSR_TM_LG, 63-MSR_TM_LG
2501 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
2502 beq 1f /* TM not active in guest. */
2504 std r1, HSTATE_HOST_R1(r13)
2505 li r3, TM_CAUSE_KVM_RESCHED
2507 /* Clear the MSR RI since r1, r13 are all going to be foobar. */
2511 /* All GPRs are volatile at this point. */
2514 /* Temporarily store r13 and r9 so we have some regs to play with */
2517 std r9, PACATMSCRATCH(r13)
2518 ld r9, HSTATE_KVM_VCPU(r13)
2520 /* Get a few more GPRs free. */
2521 std r29, VCPU_GPRS_TM(29)(r9)
2522 std r30, VCPU_GPRS_TM(30)(r9)
2523 std r31, VCPU_GPRS_TM(31)(r9)
2525 /* Save away PPR and DSCR soon so don't run with user values. */
2528 mfspr r30, SPRN_DSCR
2529 ld r29, HSTATE_DSCR(r13)
2530 mtspr SPRN_DSCR, r29
2532 /* Save all but r9, r13 & r29-r31 */
2535 .if (reg != 9) && (reg != 13)
2536 std reg, VCPU_GPRS_TM(reg)(r9)
2540 /* ... now save r13 */
2542 std r4, VCPU_GPRS_TM(13)(r9)
2543 /* ... and save r9 */
2544 ld r4, PACATMSCRATCH(r13)
2545 std r4, VCPU_GPRS_TM(9)(r9)
2547 /* Reload stack pointer and TOC. */
2548 ld r1, HSTATE_HOST_R1(r13)
2551 /* Set MSR RI now we have r1 and r13 back. */
2555 /* Save away checkpinted SPRs. */
2556 std r31, VCPU_PPR_TM(r9)
2557 std r30, VCPU_DSCR_TM(r9)
2564 std r5, VCPU_LR_TM(r9)
2565 stw r6, VCPU_CR_TM(r9)
2566 std r7, VCPU_CTR_TM(r9)
2567 std r8, VCPU_AMR_TM(r9)
2568 std r10, VCPU_TAR_TM(r9)
2569 std r11, VCPU_XER_TM(r9)
2571 /* Restore r12 as trap number. */
2572 lwz r12, VCPU_TRAP(r9)
2575 addi r3, r9, VCPU_FPRS_TM
2577 addi r3, r9, VCPU_VRS_TM
2579 mfspr r6, SPRN_VRSAVE
2580 stw r6, VCPU_VRSAVE_TM(r9)
2583 * We need to save these SPRs after the treclaim so that the software
2584 * error code is recorded correctly in the TEXASR. Also the user may
2585 * change these outside of a transaction, so they must always be
2588 mfspr r5, SPRN_TFHAR
2589 mfspr r6, SPRN_TFIAR
2590 mfspr r7, SPRN_TEXASR
2591 std r5, VCPU_TFHAR(r9)
2592 std r6, VCPU_TFIAR(r9)
2593 std r7, VCPU_TEXASR(r9)
2595 ld r0, PPC_LR_STKOFF(r1)
2600 * Restore transactional state and TM-related registers.
2601 * Called with r4 pointing to the vcpu struct.
2602 * This potentially modifies all checkpointed registers.
2603 * It restores r1, r2, r4 from the PACA.
2607 std r0, PPC_LR_STKOFF(r1)
2609 /* Turn on TM/FP/VSX/VMX so we can restore them. */
2615 oris r5, r5, (MSR_VEC | MSR_VSX)@h
2619 * The user may change these outside of a transaction, so they must
2620 * always be context switched.
2622 ld r5, VCPU_TFHAR(r4)
2623 ld r6, VCPU_TFIAR(r4)
2624 ld r7, VCPU_TEXASR(r4)
2625 mtspr SPRN_TFHAR, r5
2626 mtspr SPRN_TFIAR, r6
2627 mtspr SPRN_TEXASR, r7
2630 rldicl. r5, r5, 64 - MSR_TS_S_LG, 62
2631 beqlr /* TM not active in guest */
2632 std r1, HSTATE_HOST_R1(r13)
2634 /* Make sure the failure summary is set, otherwise we'll program check
2635 * when we trechkpt. It's possible that this might have been not set
2636 * on a kvmppc_set_one_reg() call but we shouldn't let this crash the
2639 oris r7, r7, (TEXASR_FS)@h
2640 mtspr SPRN_TEXASR, r7
2643 * We need to load up the checkpointed state for the guest.
2644 * We need to do this early as it will blow away any GPRs, VSRs and
2649 addi r3, r31, VCPU_FPRS_TM
2651 addi r3, r31, VCPU_VRS_TM
2654 lwz r7, VCPU_VRSAVE_TM(r4)
2655 mtspr SPRN_VRSAVE, r7
2657 ld r5, VCPU_LR_TM(r4)
2658 lwz r6, VCPU_CR_TM(r4)
2659 ld r7, VCPU_CTR_TM(r4)
2660 ld r8, VCPU_AMR_TM(r4)
2661 ld r9, VCPU_TAR_TM(r4)
2662 ld r10, VCPU_XER_TM(r4)
2671 * Load up PPR and DSCR values but don't put them in the actual SPRs
2672 * till the last moment to avoid running with userspace PPR and DSCR for
2675 ld r29, VCPU_DSCR_TM(r4)
2676 ld r30, VCPU_PPR_TM(r4)
2678 std r2, PACATMSCRATCH(r13) /* Save TOC */
2680 /* Clear the MSR RI since r1, r13 are all going to be foobar. */
2684 /* Load GPRs r0-r28 */
2687 ld reg, VCPU_GPRS_TM(reg)(r31)
2691 mtspr SPRN_DSCR, r29
2694 /* Load final GPRs */
2695 ld 29, VCPU_GPRS_TM(29)(r31)
2696 ld 30, VCPU_GPRS_TM(30)(r31)
2697 ld 31, VCPU_GPRS_TM(31)(r31)
2699 /* TM checkpointed state is now setup. All GPRs are now volatile. */
2702 /* Now let's get back the state we need. */
2705 ld r29, HSTATE_DSCR(r13)
2706 mtspr SPRN_DSCR, r29
2707 ld r4, HSTATE_KVM_VCPU(r13)
2708 ld r1, HSTATE_HOST_R1(r13)
2709 ld r2, PACATMSCRATCH(r13)
2711 /* Set the MSR RI since we have our registers back. */
2715 ld r0, PPC_LR_STKOFF(r1)
2721 * We come here if we get any exception or interrupt while we are
2722 * executing host real mode code while in guest MMU context.
2723 * For now just spin, but we should do something better.
2725 kvmppc_bad_host_intr:
2729 * This mimics the MSR transition on IRQ delivery. The new guest MSR is taken
2730 * from VCPU_INTR_MSR and is modified based on the required TM state changes.
2731 * r11 has the guest MSR value (in/out)
2732 * r9 has a vcpu pointer (in)
2733 * r0 is used as a scratch register
2735 kvmppc_msr_interrupt:
2736 rldicl r0, r11, 64 - MSR_TS_S_LG, 62
2737 cmpwi r0, 2 /* Check if we are in transactional state.. */
2738 ld r11, VCPU_INTR_MSR(r9)
2740 /* ... if transactional, change to suspended */
2742 1: rldimi r11, r0, MSR_TS_S_LG, 63 - MSR_TS_T_LG
2746 * This works around a hardware bug on POWER8E processors, where
2747 * writing a 1 to the MMCR0[PMAO] bit doesn't generate a
2748 * performance monitor interrupt. Instead, when we need to have
2749 * an interrupt pending, we have to arrange for a counter to overflow.
2753 mtspr SPRN_MMCR2, r3
2754 lis r3, (MMCR0_PMXE | MMCR0_FCECE)@h
2755 ori r3, r3, MMCR0_PMCjCE | MMCR0_C56RUN
2756 mtspr SPRN_MMCR0, r3
2763 #ifdef CONFIG_KVM_BOOK3S_HV_EXIT_TIMING
2765 * Start timing an activity
2766 * r3 = pointer to time accumulation struct, r4 = vcpu
2769 ld r5, HSTATE_KVM_VCORE(r13)
2770 lbz r6, VCORE_IN_GUEST(r5)
2772 beq 5f /* if in guest, need to */
2773 ld r6, VCORE_TB_OFFSET(r5) /* subtract timebase offset */
2776 std r3, VCPU_CUR_ACTIVITY(r4)
2777 std r5, VCPU_ACTIVITY_START(r4)
2781 * Accumulate time to one activity and start another.
2782 * r3 = pointer to new time accumulation struct, r4 = vcpu
2784 kvmhv_accumulate_time:
2785 ld r5, HSTATE_KVM_VCORE(r13)
2786 lbz r8, VCORE_IN_GUEST(r5)
2788 beq 4f /* if in guest, need to */
2789 ld r8, VCORE_TB_OFFSET(r5) /* subtract timebase offset */
2790 4: ld r5, VCPU_CUR_ACTIVITY(r4)
2791 ld r6, VCPU_ACTIVITY_START(r4)
2792 std r3, VCPU_CUR_ACTIVITY(r4)
2795 std r7, VCPU_ACTIVITY_START(r4)
2799 ld r8, TAS_SEQCOUNT(r5)
2802 std r8, TAS_SEQCOUNT(r5)
2804 ld r7, TAS_TOTAL(r5)
2806 std r7, TAS_TOTAL(r5)
2812 3: std r3, TAS_MIN(r5)
2818 std r8, TAS_SEQCOUNT(r5)