1 /* SPDX-License-Identifier: GPL-2.0-only */
3 * Copyright (C) 2012,2013 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
6 * Derived from arch/arm/include/kvm_emulate.h
7 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
8 * Author: Christoffer Dall <c.dall@virtualopensystems.com>
11 #ifndef __ARM64_KVM_EMULATE_H__
12 #define __ARM64_KVM_EMULATE_H__
14 #include <linux/kvm_host.h>
16 #include <asm/debug-monitors.h>
18 #include <asm/kvm_arm.h>
19 #include <asm/kvm_hyp.h>
20 #include <asm/kvm_mmio.h>
21 #include <asm/ptrace.h>
22 #include <asm/cputype.h>
25 unsigned long *vcpu_reg32(const struct kvm_vcpu *vcpu, u8 reg_num);
26 unsigned long vcpu_read_spsr32(const struct kvm_vcpu *vcpu);
27 void vcpu_write_spsr32(struct kvm_vcpu *vcpu, unsigned long v);
29 bool kvm_condition_valid32(const struct kvm_vcpu *vcpu);
30 void kvm_skip_instr32(struct kvm_vcpu *vcpu, bool is_wide_instr);
32 void kvm_inject_undefined(struct kvm_vcpu *vcpu);
33 void kvm_inject_vabt(struct kvm_vcpu *vcpu);
34 void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr);
35 void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr);
36 void kvm_inject_undef32(struct kvm_vcpu *vcpu);
37 void kvm_inject_dabt32(struct kvm_vcpu *vcpu, unsigned long addr);
38 void kvm_inject_pabt32(struct kvm_vcpu *vcpu, unsigned long addr);
40 static inline bool vcpu_el1_is_32bit(struct kvm_vcpu *vcpu)
42 return !(vcpu->arch.hcr_el2 & HCR_RW);
45 static inline void vcpu_reset_hcr(struct kvm_vcpu *vcpu)
47 vcpu->arch.hcr_el2 = HCR_GUEST_FLAGS;
48 if (is_kernel_in_hyp_mode())
49 vcpu->arch.hcr_el2 |= HCR_E2H;
50 if (cpus_have_const_cap(ARM64_HAS_RAS_EXTN)) {
51 /* route synchronous external abort exceptions to EL2 */
52 vcpu->arch.hcr_el2 |= HCR_TEA;
53 /* trap error record accesses */
54 vcpu->arch.hcr_el2 |= HCR_TERR;
56 if (cpus_have_const_cap(ARM64_HAS_STAGE2_FWB))
57 vcpu->arch.hcr_el2 |= HCR_FWB;
59 if (test_bit(KVM_ARM_VCPU_EL1_32BIT, vcpu->arch.features))
60 vcpu->arch.hcr_el2 &= ~HCR_RW;
63 * TID3: trap feature register accesses that we virtualise.
64 * For now this is conditional, since no AArch32 feature regs
65 * are currently virtualised.
67 if (!vcpu_el1_is_32bit(vcpu))
68 vcpu->arch.hcr_el2 |= HCR_TID3;
70 if (cpus_have_const_cap(ARM64_MISMATCHED_CACHE_TYPE) ||
71 vcpu_el1_is_32bit(vcpu))
72 vcpu->arch.hcr_el2 |= HCR_TID2;
75 static inline unsigned long *vcpu_hcr(struct kvm_vcpu *vcpu)
77 return (unsigned long *)&vcpu->arch.hcr_el2;
80 static inline void vcpu_clear_wfe_traps(struct kvm_vcpu *vcpu)
82 vcpu->arch.hcr_el2 &= ~HCR_TWE;
85 static inline void vcpu_set_wfe_traps(struct kvm_vcpu *vcpu)
87 vcpu->arch.hcr_el2 |= HCR_TWE;
90 static inline void vcpu_ptrauth_enable(struct kvm_vcpu *vcpu)
92 vcpu->arch.hcr_el2 |= (HCR_API | HCR_APK);
95 static inline void vcpu_ptrauth_disable(struct kvm_vcpu *vcpu)
97 vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK);
100 static inline unsigned long vcpu_get_vsesr(struct kvm_vcpu *vcpu)
102 return vcpu->arch.vsesr_el2;
105 static inline void vcpu_set_vsesr(struct kvm_vcpu *vcpu, u64 vsesr)
107 vcpu->arch.vsesr_el2 = vsesr;
110 static inline unsigned long *vcpu_pc(const struct kvm_vcpu *vcpu)
112 return (unsigned long *)&vcpu_gp_regs(vcpu)->regs.pc;
115 static inline unsigned long *__vcpu_elr_el1(const struct kvm_vcpu *vcpu)
117 return (unsigned long *)&vcpu_gp_regs(vcpu)->elr_el1;
120 static inline unsigned long vcpu_read_elr_el1(const struct kvm_vcpu *vcpu)
122 if (vcpu->arch.sysregs_loaded_on_cpu)
123 return read_sysreg_el1(SYS_ELR);
125 return *__vcpu_elr_el1(vcpu);
128 static inline void vcpu_write_elr_el1(const struct kvm_vcpu *vcpu, unsigned long v)
130 if (vcpu->arch.sysregs_loaded_on_cpu)
131 write_sysreg_el1(v, SYS_ELR);
133 *__vcpu_elr_el1(vcpu) = v;
136 static inline unsigned long *vcpu_cpsr(const struct kvm_vcpu *vcpu)
138 return (unsigned long *)&vcpu_gp_regs(vcpu)->regs.pstate;
141 static inline bool vcpu_mode_is_32bit(const struct kvm_vcpu *vcpu)
143 return !!(*vcpu_cpsr(vcpu) & PSR_MODE32_BIT);
146 static inline bool kvm_condition_valid(const struct kvm_vcpu *vcpu)
148 if (vcpu_mode_is_32bit(vcpu))
149 return kvm_condition_valid32(vcpu);
154 static inline void vcpu_set_thumb(struct kvm_vcpu *vcpu)
156 *vcpu_cpsr(vcpu) |= PSR_AA32_T_BIT;
160 * vcpu_get_reg and vcpu_set_reg should always be passed a register number
161 * coming from a read of ESR_EL2. Otherwise, it may give the wrong result on
162 * AArch32 with banked registers.
164 static inline unsigned long vcpu_get_reg(const struct kvm_vcpu *vcpu,
167 return (reg_num == 31) ? 0 : vcpu_gp_regs(vcpu)->regs.regs[reg_num];
170 static inline void vcpu_set_reg(struct kvm_vcpu *vcpu, u8 reg_num,
174 vcpu_gp_regs(vcpu)->regs.regs[reg_num] = val;
177 static inline unsigned long vcpu_read_spsr(const struct kvm_vcpu *vcpu)
179 if (vcpu_mode_is_32bit(vcpu))
180 return vcpu_read_spsr32(vcpu);
182 if (vcpu->arch.sysregs_loaded_on_cpu)
183 return read_sysreg_el1(SYS_SPSR);
185 return vcpu_gp_regs(vcpu)->spsr[KVM_SPSR_EL1];
188 static inline void vcpu_write_spsr(struct kvm_vcpu *vcpu, unsigned long v)
190 if (vcpu_mode_is_32bit(vcpu)) {
191 vcpu_write_spsr32(vcpu, v);
195 if (vcpu->arch.sysregs_loaded_on_cpu)
196 write_sysreg_el1(v, SYS_SPSR);
198 vcpu_gp_regs(vcpu)->spsr[KVM_SPSR_EL1] = v;
202 * The layout of SPSR for an AArch32 state is different when observed from an
203 * AArch64 SPSR_ELx or an AArch32 SPSR_*. This function generates the AArch32
204 * view given an AArch64 view.
206 * In ARM DDI 0487E.a see:
208 * - The AArch64 view (SPSR_EL2) in section C5.2.18, page C5-426
209 * - The AArch32 view (SPSR_abt) in section G8.2.126, page G8-6256
210 * - The AArch32 view (SPSR_und) in section G8.2.132, page G8-6280
212 * Which show the following differences:
214 * | Bit | AA64 | AA32 | Notes |
215 * +-----+------+------+-----------------------------|
216 * | 24 | DIT | J | J is RES0 in ARMv8 |
217 * | 21 | SS | DIT | SS doesn't exist in AArch32 |
219 * ... and all other bits are (currently) common.
221 static inline unsigned long host_spsr_to_spsr32(unsigned long spsr)
223 const unsigned long overlap = BIT(24) | BIT(21);
224 unsigned long dit = !!(spsr & PSR_AA32_DIT_BIT);
233 static inline bool vcpu_mode_priv(const struct kvm_vcpu *vcpu)
237 if (vcpu_mode_is_32bit(vcpu)) {
238 mode = *vcpu_cpsr(vcpu) & PSR_AA32_MODE_MASK;
239 return mode > PSR_AA32_MODE_USR;
242 mode = *vcpu_cpsr(vcpu) & PSR_MODE_MASK;
244 return mode != PSR_MODE_EL0t;
247 static inline u32 kvm_vcpu_get_hsr(const struct kvm_vcpu *vcpu)
249 return vcpu->arch.fault.esr_el2;
252 static inline int kvm_vcpu_get_condition(const struct kvm_vcpu *vcpu)
254 u32 esr = kvm_vcpu_get_hsr(vcpu);
256 if (esr & ESR_ELx_CV)
257 return (esr & ESR_ELx_COND_MASK) >> ESR_ELx_COND_SHIFT;
262 static inline unsigned long kvm_vcpu_get_hfar(const struct kvm_vcpu *vcpu)
264 return vcpu->arch.fault.far_el2;
267 static inline phys_addr_t kvm_vcpu_get_fault_ipa(const struct kvm_vcpu *vcpu)
269 return ((phys_addr_t)vcpu->arch.fault.hpfar_el2 & HPFAR_MASK) << 8;
272 static inline u64 kvm_vcpu_get_disr(const struct kvm_vcpu *vcpu)
274 return vcpu->arch.fault.disr_el1;
277 static inline u32 kvm_vcpu_hvc_get_imm(const struct kvm_vcpu *vcpu)
279 return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_xVC_IMM_MASK;
282 static inline bool kvm_vcpu_dabt_isvalid(const struct kvm_vcpu *vcpu)
284 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_ISV);
287 static inline bool kvm_vcpu_dabt_issext(const struct kvm_vcpu *vcpu)
289 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SSE);
292 static inline bool kvm_vcpu_dabt_issf(const struct kvm_vcpu *vcpu)
294 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SF);
297 static inline int kvm_vcpu_dabt_get_rd(const struct kvm_vcpu *vcpu)
299 return (kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SRT_MASK) >> ESR_ELx_SRT_SHIFT;
302 static __always_inline bool kvm_vcpu_abt_iss1tw(const struct kvm_vcpu *vcpu)
304 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_S1PTW);
307 static inline bool kvm_vcpu_dabt_iswrite(const struct kvm_vcpu *vcpu)
309 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_WNR) ||
310 kvm_vcpu_abt_iss1tw(vcpu); /* AF/DBM update */
313 static inline bool kvm_vcpu_dabt_is_cm(const struct kvm_vcpu *vcpu)
315 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_CM);
318 static inline int kvm_vcpu_dabt_get_as(const struct kvm_vcpu *vcpu)
320 return 1 << ((kvm_vcpu_get_hsr(vcpu) & ESR_ELx_SAS) >> ESR_ELx_SAS_SHIFT);
323 /* This one is not specific to Data Abort */
324 static inline bool kvm_vcpu_trap_il_is32bit(const struct kvm_vcpu *vcpu)
326 return !!(kvm_vcpu_get_hsr(vcpu) & ESR_ELx_IL);
329 static inline u8 kvm_vcpu_trap_get_class(const struct kvm_vcpu *vcpu)
331 return ESR_ELx_EC(kvm_vcpu_get_hsr(vcpu));
334 static inline bool kvm_vcpu_trap_is_iabt(const struct kvm_vcpu *vcpu)
336 return kvm_vcpu_trap_get_class(vcpu) == ESR_ELx_EC_IABT_LOW;
339 static inline bool kvm_vcpu_trap_is_exec_fault(const struct kvm_vcpu *vcpu)
341 return kvm_vcpu_trap_is_iabt(vcpu) && !kvm_vcpu_abt_iss1tw(vcpu);
344 static inline u8 kvm_vcpu_trap_get_fault(const struct kvm_vcpu *vcpu)
346 return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_FSC;
349 static inline u8 kvm_vcpu_trap_get_fault_type(const struct kvm_vcpu *vcpu)
351 return kvm_vcpu_get_hsr(vcpu) & ESR_ELx_FSC_TYPE;
354 static inline bool kvm_vcpu_dabt_isextabt(const struct kvm_vcpu *vcpu)
356 switch (kvm_vcpu_trap_get_fault(vcpu)) {
373 static inline int kvm_vcpu_sys_get_rt(struct kvm_vcpu *vcpu)
375 u32 esr = kvm_vcpu_get_hsr(vcpu);
376 return ESR_ELx_SYS64_ISS_RT(esr);
379 static inline bool kvm_is_write_fault(struct kvm_vcpu *vcpu)
381 if (kvm_vcpu_abt_iss1tw(vcpu)) {
383 * Only a permission fault on a S1PTW should be
384 * considered as a write. Otherwise, page tables baked
385 * in a read-only memslot will result in an exception
386 * being delivered in the guest.
388 * The drawback is that we end-up faulting twice if the
389 * guest is using any of HW AF/DB: a translation fault
390 * to map the page containing the PT (read only at
391 * first), then a permission fault to allow the flags
394 switch (kvm_vcpu_trap_get_fault_type(vcpu)) {
395 case ESR_ELx_FSC_PERM:
402 if (kvm_vcpu_trap_is_iabt(vcpu))
405 return kvm_vcpu_dabt_iswrite(vcpu);
408 static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu)
410 return vcpu_read_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK;
413 static inline bool kvm_arm_get_vcpu_workaround_2_flag(struct kvm_vcpu *vcpu)
415 return vcpu->arch.workaround_flags & VCPU_WORKAROUND_2_FLAG;
418 static inline void kvm_arm_set_vcpu_workaround_2_flag(struct kvm_vcpu *vcpu,
422 vcpu->arch.workaround_flags |= VCPU_WORKAROUND_2_FLAG;
424 vcpu->arch.workaround_flags &= ~VCPU_WORKAROUND_2_FLAG;
427 static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu)
429 if (vcpu_mode_is_32bit(vcpu)) {
430 *vcpu_cpsr(vcpu) |= PSR_AA32_E_BIT;
432 u64 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1);
434 vcpu_write_sys_reg(vcpu, sctlr, SCTLR_EL1);
438 static inline bool kvm_vcpu_is_be(struct kvm_vcpu *vcpu)
440 if (vcpu_mode_is_32bit(vcpu))
441 return !!(*vcpu_cpsr(vcpu) & PSR_AA32_E_BIT);
443 return !!(vcpu_read_sys_reg(vcpu, SCTLR_EL1) & (1 << 25));
446 static inline unsigned long vcpu_data_guest_to_host(struct kvm_vcpu *vcpu,
450 if (kvm_vcpu_is_be(vcpu)) {
455 return be16_to_cpu(data & 0xffff);
457 return be32_to_cpu(data & 0xffffffff);
459 return be64_to_cpu(data);
466 return le16_to_cpu(data & 0xffff);
468 return le32_to_cpu(data & 0xffffffff);
470 return le64_to_cpu(data);
474 return data; /* Leave LE untouched */
477 static inline unsigned long vcpu_data_host_to_guest(struct kvm_vcpu *vcpu,
481 if (kvm_vcpu_is_be(vcpu)) {
486 return cpu_to_be16(data & 0xffff);
488 return cpu_to_be32(data & 0xffffffff);
490 return cpu_to_be64(data);
497 return cpu_to_le16(data & 0xffff);
499 return cpu_to_le32(data & 0xffffffff);
501 return cpu_to_le64(data);
505 return data; /* Leave LE untouched */
508 static inline void kvm_skip_instr(struct kvm_vcpu *vcpu, bool is_wide_instr)
510 if (vcpu_mode_is_32bit(vcpu))
511 kvm_skip_instr32(vcpu, is_wide_instr);
515 /* advance the singlestep state machine */
516 *vcpu_cpsr(vcpu) &= ~DBG_SPSR_SS;
520 * Skip an instruction which has been emulated at hyp while most guest sysregs
523 static inline void __hyp_text __kvm_skip_instr(struct kvm_vcpu *vcpu)
525 *vcpu_pc(vcpu) = read_sysreg_el2(SYS_ELR);
526 vcpu->arch.ctxt.gp_regs.regs.pstate = read_sysreg_el2(SYS_SPSR);
528 kvm_skip_instr(vcpu, kvm_vcpu_trap_il_is32bit(vcpu));
530 write_sysreg_el2(vcpu->arch.ctxt.gp_regs.regs.pstate, SYS_SPSR);
531 write_sysreg_el2(*vcpu_pc(vcpu), SYS_ELR);
534 #endif /* __ARM64_KVM_EMULATE_H__ */