1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2012-2015 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
7 #include <hyp/adjust_pc.h>
9 #include <linux/compiler.h>
10 #include <linux/irqchip/arm-gic-v3.h>
11 #include <linux/kvm_host.h>
13 #include <asm/kvm_emulate.h>
14 #include <asm/kvm_hyp.h>
15 #include <asm/kvm_mmu.h>
17 #define vtr_to_max_lr_idx(v) ((v) & 0xf)
18 #define vtr_to_nr_pre_bits(v) ((((u32)(v) >> 26) & 7) + 1)
19 #define vtr_to_nr_apr_regs(v) (1 << (vtr_to_nr_pre_bits(v) - 5))
21 static u64 __gic_v3_get_lr(unsigned int lr)
25 return read_gicreg(ICH_LR0_EL2);
27 return read_gicreg(ICH_LR1_EL2);
29 return read_gicreg(ICH_LR2_EL2);
31 return read_gicreg(ICH_LR3_EL2);
33 return read_gicreg(ICH_LR4_EL2);
35 return read_gicreg(ICH_LR5_EL2);
37 return read_gicreg(ICH_LR6_EL2);
39 return read_gicreg(ICH_LR7_EL2);
41 return read_gicreg(ICH_LR8_EL2);
43 return read_gicreg(ICH_LR9_EL2);
45 return read_gicreg(ICH_LR10_EL2);
47 return read_gicreg(ICH_LR11_EL2);
49 return read_gicreg(ICH_LR12_EL2);
51 return read_gicreg(ICH_LR13_EL2);
53 return read_gicreg(ICH_LR14_EL2);
55 return read_gicreg(ICH_LR15_EL2);
61 static void __gic_v3_set_lr(u64 val, int lr)
65 write_gicreg(val, ICH_LR0_EL2);
68 write_gicreg(val, ICH_LR1_EL2);
71 write_gicreg(val, ICH_LR2_EL2);
74 write_gicreg(val, ICH_LR3_EL2);
77 write_gicreg(val, ICH_LR4_EL2);
80 write_gicreg(val, ICH_LR5_EL2);
83 write_gicreg(val, ICH_LR6_EL2);
86 write_gicreg(val, ICH_LR7_EL2);
89 write_gicreg(val, ICH_LR8_EL2);
92 write_gicreg(val, ICH_LR9_EL2);
95 write_gicreg(val, ICH_LR10_EL2);
98 write_gicreg(val, ICH_LR11_EL2);
101 write_gicreg(val, ICH_LR12_EL2);
104 write_gicreg(val, ICH_LR13_EL2);
107 write_gicreg(val, ICH_LR14_EL2);
110 write_gicreg(val, ICH_LR15_EL2);
115 static void __vgic_v3_write_ap0rn(u32 val, int n)
119 write_gicreg(val, ICH_AP0R0_EL2);
122 write_gicreg(val, ICH_AP0R1_EL2);
125 write_gicreg(val, ICH_AP0R2_EL2);
128 write_gicreg(val, ICH_AP0R3_EL2);
133 static void __vgic_v3_write_ap1rn(u32 val, int n)
137 write_gicreg(val, ICH_AP1R0_EL2);
140 write_gicreg(val, ICH_AP1R1_EL2);
143 write_gicreg(val, ICH_AP1R2_EL2);
146 write_gicreg(val, ICH_AP1R3_EL2);
151 static u32 __vgic_v3_read_ap0rn(int n)
157 val = read_gicreg(ICH_AP0R0_EL2);
160 val = read_gicreg(ICH_AP0R1_EL2);
163 val = read_gicreg(ICH_AP0R2_EL2);
166 val = read_gicreg(ICH_AP0R3_EL2);
175 static u32 __vgic_v3_read_ap1rn(int n)
181 val = read_gicreg(ICH_AP1R0_EL2);
184 val = read_gicreg(ICH_AP1R1_EL2);
187 val = read_gicreg(ICH_AP1R2_EL2);
190 val = read_gicreg(ICH_AP1R3_EL2);
199 void __vgic_v3_save_state(struct vgic_v3_cpu_if *cpu_if)
201 u64 used_lrs = cpu_if->used_lrs;
204 * Make sure stores to the GIC via the memory mapped interface
205 * are now visible to the system register interface when reading the
206 * LRs, and when reading back the VMCR on non-VHE systems.
208 if (used_lrs || !has_vhe()) {
209 if (!cpu_if->vgic_sre) {
215 if (used_lrs || cpu_if->its_vpe.its_vm) {
219 elrsr = read_gicreg(ICH_ELRSR_EL2);
221 write_gicreg(cpu_if->vgic_hcr & ~ICH_HCR_EN, ICH_HCR_EL2);
223 for (i = 0; i < used_lrs; i++) {
224 if (elrsr & (1 << i))
225 cpu_if->vgic_lr[i] &= ~ICH_LR_STATE;
227 cpu_if->vgic_lr[i] = __gic_v3_get_lr(i);
229 __gic_v3_set_lr(0, i);
234 void __vgic_v3_restore_state(struct vgic_v3_cpu_if *cpu_if)
236 u64 used_lrs = cpu_if->used_lrs;
239 if (used_lrs || cpu_if->its_vpe.its_vm) {
240 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
242 for (i = 0; i < used_lrs; i++)
243 __gic_v3_set_lr(cpu_if->vgic_lr[i], i);
247 * Ensure that writes to the LRs, and on non-VHE systems ensure that
248 * the write to the VMCR in __vgic_v3_activate_traps(), will have
249 * reached the (re)distributors. This ensure the guest will read the
250 * correct values from the memory-mapped interface.
252 if (used_lrs || !has_vhe()) {
253 if (!cpu_if->vgic_sre) {
260 void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if)
263 * VFIQEn is RES1 if ICC_SRE_EL1.SRE is 1. This causes a
264 * Group0 interrupt (as generated in GICv2 mode) to be
265 * delivered as a FIQ to the guest, with potentially fatal
266 * consequences. So we must make sure that ICC_SRE_EL1 has
267 * been actually programmed with the value we want before
268 * starting to mess with the rest of the GIC, and VMCR_EL2 in
269 * particular. This logic must be called before
270 * __vgic_v3_restore_state().
272 if (!cpu_if->vgic_sre) {
273 write_gicreg(0, ICC_SRE_EL1);
275 write_gicreg(cpu_if->vgic_vmcr, ICH_VMCR_EL2);
280 * Ensure that the write to the VMCR will have reached
281 * the (re)distributors. This ensure the guest will
282 * read the correct values from the memory-mapped
291 * Prevent the guest from touching the GIC system registers if
292 * SRE isn't enabled for GICv3 emulation.
294 write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
298 * If we need to trap system registers, we must write
299 * ICH_HCR_EL2 anyway, even if no interrupts are being
302 if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
303 cpu_if->its_vpe.its_vm)
304 write_gicreg(cpu_if->vgic_hcr, ICH_HCR_EL2);
307 void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if)
311 if (!cpu_if->vgic_sre) {
312 cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
315 val = read_gicreg(ICC_SRE_EL2);
316 write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
318 if (!cpu_if->vgic_sre) {
319 /* Make sure ENABLE is set at EL2 before setting SRE at EL1 */
321 write_gicreg(1, ICC_SRE_EL1);
325 * If we were trapping system registers, we enabled the VGIC even if
326 * no interrupts were being injected, and we disable it again here.
328 if (static_branch_unlikely(&vgic_v3_cpuif_trap) ||
329 cpu_if->its_vpe.its_vm)
330 write_gicreg(0, ICH_HCR_EL2);
333 void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if)
338 val = read_gicreg(ICH_VTR_EL2);
339 nr_pre_bits = vtr_to_nr_pre_bits(val);
341 switch (nr_pre_bits) {
343 cpu_if->vgic_ap0r[3] = __vgic_v3_read_ap0rn(3);
344 cpu_if->vgic_ap0r[2] = __vgic_v3_read_ap0rn(2);
347 cpu_if->vgic_ap0r[1] = __vgic_v3_read_ap0rn(1);
350 cpu_if->vgic_ap0r[0] = __vgic_v3_read_ap0rn(0);
353 switch (nr_pre_bits) {
355 cpu_if->vgic_ap1r[3] = __vgic_v3_read_ap1rn(3);
356 cpu_if->vgic_ap1r[2] = __vgic_v3_read_ap1rn(2);
359 cpu_if->vgic_ap1r[1] = __vgic_v3_read_ap1rn(1);
362 cpu_if->vgic_ap1r[0] = __vgic_v3_read_ap1rn(0);
366 void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if)
371 val = read_gicreg(ICH_VTR_EL2);
372 nr_pre_bits = vtr_to_nr_pre_bits(val);
374 switch (nr_pre_bits) {
376 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[3], 3);
377 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[2], 2);
380 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[1], 1);
383 __vgic_v3_write_ap0rn(cpu_if->vgic_ap0r[0], 0);
386 switch (nr_pre_bits) {
388 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[3], 3);
389 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[2], 2);
392 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[1], 1);
395 __vgic_v3_write_ap1rn(cpu_if->vgic_ap1r[0], 0);
399 void __vgic_v3_init_lrs(void)
401 int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2));
404 for (i = 0; i <= max_lr_idx; i++)
405 __gic_v3_set_lr(0, i);
409 * Return the GIC CPU configuration:
410 * - [31:0] ICH_VTR_EL2
412 * - [63] MMIO (GICv2) capable
414 u64 __vgic_v3_get_gic_config(void)
416 u64 val, sre = read_gicreg(ICC_SRE_EL1);
417 unsigned long flags = 0;
420 * To check whether we have a MMIO-based (GICv2 compatible)
421 * CPU interface, we need to disable the system register
422 * view. To do that safely, we have to prevent any interrupt
423 * from firing (which would be deadly).
425 * Note that this only makes sense on VHE, as interrupts are
426 * already masked for nVHE as part of the exception entry to
430 flags = local_daif_save();
433 * Table 11-2 "Permitted ICC_SRE_ELx.SRE settings" indicates
434 * that to be able to set ICC_SRE_EL1.SRE to 0, all the
435 * interrupt overrides must be set. You've got to love this.
437 sysreg_clear_set(hcr_el2, 0, HCR_AMO | HCR_FMO | HCR_IMO);
439 write_gicreg(0, ICC_SRE_EL1);
442 val = read_gicreg(ICC_SRE_EL1);
444 write_gicreg(sre, ICC_SRE_EL1);
446 sysreg_clear_set(hcr_el2, HCR_AMO | HCR_FMO | HCR_IMO, 0);
450 local_daif_restore(flags);
452 val = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63);
453 val |= read_gicreg(ICH_VTR_EL2);
458 u64 __vgic_v3_read_vmcr(void)
460 return read_gicreg(ICH_VMCR_EL2);
463 void __vgic_v3_write_vmcr(u32 vmcr)
465 write_gicreg(vmcr, ICH_VMCR_EL2);
468 static int __vgic_v3_bpr_min(void)
470 /* See Pseudocode for VPriorityGroup */
471 return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2));
474 static int __vgic_v3_get_group(struct kvm_vcpu *vcpu)
476 u64 esr = kvm_vcpu_get_esr(vcpu);
477 u8 crm = (esr & ESR_ELx_SYS64_ISS_CRM_MASK) >> ESR_ELx_SYS64_ISS_CRM_SHIFT;
482 #define GICv3_IDLE_PRIORITY 0xff
484 static int __vgic_v3_highest_priority_lr(struct kvm_vcpu *vcpu, u32 vmcr,
487 unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
488 u8 priority = GICv3_IDLE_PRIORITY;
491 for (i = 0; i < used_lrs; i++) {
492 u64 val = __gic_v3_get_lr(i);
493 u8 lr_prio = (val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
495 /* Not pending in the state? */
496 if ((val & ICH_LR_STATE) != ICH_LR_PENDING_BIT)
499 /* Group-0 interrupt, but Group-0 disabled? */
500 if (!(val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG0_MASK))
503 /* Group-1 interrupt, but Group-1 disabled? */
504 if ((val & ICH_LR_GROUP) && !(vmcr & ICH_VMCR_ENG1_MASK))
507 /* Not the highest priority? */
508 if (lr_prio >= priority)
511 /* This is a candidate */
518 *lr_val = ICC_IAR1_EL1_SPURIOUS;
523 static int __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, int intid,
526 unsigned int used_lrs = vcpu->arch.vgic_cpu.vgic_v3.used_lrs;
529 for (i = 0; i < used_lrs; i++) {
530 u64 val = __gic_v3_get_lr(i);
532 if ((val & ICH_LR_VIRTUAL_ID_MASK) == intid &&
533 (val & ICH_LR_ACTIVE_BIT)) {
539 *lr_val = ICC_IAR1_EL1_SPURIOUS;
543 static int __vgic_v3_get_highest_active_priority(void)
545 u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
549 for (i = 0; i < nr_apr_regs; i++) {
553 * The ICH_AP0Rn_EL2 and ICH_AP1Rn_EL2 registers
554 * contain the active priority levels for this VCPU
555 * for the maximum number of supported priority
556 * levels, and we return the full priority level only
557 * if the BPR is programmed to its minimum, otherwise
558 * we return a combination of the priority level and
559 * subpriority, as determined by the setting of the
560 * BPR, but without the full subpriority.
562 val = __vgic_v3_read_ap0rn(i);
563 val |= __vgic_v3_read_ap1rn(i);
569 return (hap + __ffs(val)) << __vgic_v3_bpr_min();
572 return GICv3_IDLE_PRIORITY;
575 static unsigned int __vgic_v3_get_bpr0(u32 vmcr)
577 return (vmcr & ICH_VMCR_BPR0_MASK) >> ICH_VMCR_BPR0_SHIFT;
580 static unsigned int __vgic_v3_get_bpr1(u32 vmcr)
584 if (vmcr & ICH_VMCR_CBPR_MASK) {
585 bpr = __vgic_v3_get_bpr0(vmcr);
589 bpr = (vmcr & ICH_VMCR_BPR1_MASK) >> ICH_VMCR_BPR1_SHIFT;
596 * Convert a priority to a preemption level, taking the relevant BPR
597 * into account by zeroing the sub-priority bits.
599 static u8 __vgic_v3_pri_to_pre(u8 pri, u32 vmcr, int grp)
604 bpr = __vgic_v3_get_bpr0(vmcr) + 1;
606 bpr = __vgic_v3_get_bpr1(vmcr);
608 return pri & (GENMASK(7, 0) << bpr);
612 * The priority value is independent of any of the BPR values, so we
613 * normalize it using the minimal BPR value. This guarantees that no
614 * matter what the guest does with its BPR, we can always set/get the
615 * same value of a priority.
617 static void __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp)
623 pre = __vgic_v3_pri_to_pre(pri, vmcr, grp);
624 ap = pre >> __vgic_v3_bpr_min();
628 val = __vgic_v3_read_ap0rn(apr);
629 __vgic_v3_write_ap0rn(val | BIT(ap % 32), apr);
631 val = __vgic_v3_read_ap1rn(apr);
632 __vgic_v3_write_ap1rn(val | BIT(ap % 32), apr);
636 static int __vgic_v3_clear_highest_active_priority(void)
638 u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2));
642 for (i = 0; i < nr_apr_regs; i++) {
646 ap0 = __vgic_v3_read_ap0rn(i);
647 ap1 = __vgic_v3_read_ap1rn(i);
653 c0 = ap0 ? __ffs(ap0) : 32;
654 c1 = ap1 ? __ffs(ap1) : 32;
656 /* Always clear the LSB, which is the highest priority */
659 __vgic_v3_write_ap0rn(ap0, i);
663 __vgic_v3_write_ap1rn(ap1, i);
667 /* Rescale to 8 bits of priority */
668 return hap << __vgic_v3_bpr_min();
671 return GICv3_IDLE_PRIORITY;
674 static void __vgic_v3_read_iar(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
680 grp = __vgic_v3_get_group(vcpu);
682 lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
686 if (grp != !!(lr_val & ICH_LR_GROUP))
689 pmr = (vmcr & ICH_VMCR_PMR_MASK) >> ICH_VMCR_PMR_SHIFT;
690 lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
694 if (__vgic_v3_get_highest_active_priority() <= __vgic_v3_pri_to_pre(lr_prio, vmcr, grp))
697 lr_val &= ~ICH_LR_STATE;
698 lr_val |= ICH_LR_ACTIVE_BIT;
699 __gic_v3_set_lr(lr_val, lr);
700 __vgic_v3_set_active_priority(lr_prio, vmcr, grp);
701 vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
705 vcpu_set_reg(vcpu, rt, ICC_IAR1_EL1_SPURIOUS);
708 static void __vgic_v3_clear_active_lr(int lr, u64 lr_val)
710 lr_val &= ~ICH_LR_ACTIVE_BIT;
711 if (lr_val & ICH_LR_HW) {
714 pid = (lr_val & ICH_LR_PHYS_ID_MASK) >> ICH_LR_PHYS_ID_SHIFT;
718 __gic_v3_set_lr(lr_val, lr);
721 static void __vgic_v3_bump_eoicount(void)
725 hcr = read_gicreg(ICH_HCR_EL2);
726 hcr += 1 << ICH_HCR_EOIcount_SHIFT;
727 write_gicreg(hcr, ICH_HCR_EL2);
730 static void __vgic_v3_write_dir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
732 u32 vid = vcpu_get_reg(vcpu, rt);
736 /* EOImode == 0, nothing to be done here */
737 if (!(vmcr & ICH_VMCR_EOIM_MASK))
740 /* No deactivate to be performed on an LPI */
741 if (vid >= VGIC_MIN_LPI)
744 lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
746 __vgic_v3_bump_eoicount();
750 __vgic_v3_clear_active_lr(lr, lr_val);
753 static void __vgic_v3_write_eoir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
755 u32 vid = vcpu_get_reg(vcpu, rt);
757 u8 lr_prio, act_prio;
760 grp = __vgic_v3_get_group(vcpu);
762 /* Drop priority in any case */
763 act_prio = __vgic_v3_clear_highest_active_priority();
765 lr = __vgic_v3_find_active_lr(vcpu, vid, &lr_val);
767 /* Do not bump EOIcount for LPIs that aren't in the LRs */
768 if (!(vid >= VGIC_MIN_LPI))
769 __vgic_v3_bump_eoicount();
773 /* EOImode == 1 and not an LPI, nothing to be done here */
774 if ((vmcr & ICH_VMCR_EOIM_MASK) && !(vid >= VGIC_MIN_LPI))
777 lr_prio = (lr_val & ICH_LR_PRIORITY_MASK) >> ICH_LR_PRIORITY_SHIFT;
779 /* If priorities or group do not match, the guest has fscked-up. */
780 if (grp != !!(lr_val & ICH_LR_GROUP) ||
781 __vgic_v3_pri_to_pre(lr_prio, vmcr, grp) != act_prio)
784 /* Let's now perform the deactivation */
785 __vgic_v3_clear_active_lr(lr, lr_val);
788 static void __vgic_v3_read_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
790 vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG0_MASK));
793 static void __vgic_v3_read_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
795 vcpu_set_reg(vcpu, rt, !!(vmcr & ICH_VMCR_ENG1_MASK));
798 static void __vgic_v3_write_igrpen0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
800 u64 val = vcpu_get_reg(vcpu, rt);
803 vmcr |= ICH_VMCR_ENG0_MASK;
805 vmcr &= ~ICH_VMCR_ENG0_MASK;
807 __vgic_v3_write_vmcr(vmcr);
810 static void __vgic_v3_write_igrpen1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
812 u64 val = vcpu_get_reg(vcpu, rt);
815 vmcr |= ICH_VMCR_ENG1_MASK;
817 vmcr &= ~ICH_VMCR_ENG1_MASK;
819 __vgic_v3_write_vmcr(vmcr);
822 static void __vgic_v3_read_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
824 vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr0(vmcr));
827 static void __vgic_v3_read_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
829 vcpu_set_reg(vcpu, rt, __vgic_v3_get_bpr1(vmcr));
832 static void __vgic_v3_write_bpr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
834 u64 val = vcpu_get_reg(vcpu, rt);
835 u8 bpr_min = __vgic_v3_bpr_min() - 1;
837 /* Enforce BPR limiting */
841 val <<= ICH_VMCR_BPR0_SHIFT;
842 val &= ICH_VMCR_BPR0_MASK;
843 vmcr &= ~ICH_VMCR_BPR0_MASK;
846 __vgic_v3_write_vmcr(vmcr);
849 static void __vgic_v3_write_bpr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
851 u64 val = vcpu_get_reg(vcpu, rt);
852 u8 bpr_min = __vgic_v3_bpr_min();
854 if (vmcr & ICH_VMCR_CBPR_MASK)
857 /* Enforce BPR limiting */
861 val <<= ICH_VMCR_BPR1_SHIFT;
862 val &= ICH_VMCR_BPR1_MASK;
863 vmcr &= ~ICH_VMCR_BPR1_MASK;
866 __vgic_v3_write_vmcr(vmcr);
869 static void __vgic_v3_read_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
873 if (!__vgic_v3_get_group(vcpu))
874 val = __vgic_v3_read_ap0rn(n);
876 val = __vgic_v3_read_ap1rn(n);
878 vcpu_set_reg(vcpu, rt, val);
881 static void __vgic_v3_write_apxrn(struct kvm_vcpu *vcpu, int rt, int n)
883 u32 val = vcpu_get_reg(vcpu, rt);
885 if (!__vgic_v3_get_group(vcpu))
886 __vgic_v3_write_ap0rn(val, n);
888 __vgic_v3_write_ap1rn(val, n);
891 static void __vgic_v3_read_apxr0(struct kvm_vcpu *vcpu,
894 __vgic_v3_read_apxrn(vcpu, rt, 0);
897 static void __vgic_v3_read_apxr1(struct kvm_vcpu *vcpu,
900 __vgic_v3_read_apxrn(vcpu, rt, 1);
903 static void __vgic_v3_read_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
905 __vgic_v3_read_apxrn(vcpu, rt, 2);
908 static void __vgic_v3_read_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
910 __vgic_v3_read_apxrn(vcpu, rt, 3);
913 static void __vgic_v3_write_apxr0(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
915 __vgic_v3_write_apxrn(vcpu, rt, 0);
918 static void __vgic_v3_write_apxr1(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
920 __vgic_v3_write_apxrn(vcpu, rt, 1);
923 static void __vgic_v3_write_apxr2(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
925 __vgic_v3_write_apxrn(vcpu, rt, 2);
928 static void __vgic_v3_write_apxr3(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
930 __vgic_v3_write_apxrn(vcpu, rt, 3);
933 static void __vgic_v3_read_hppir(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
938 grp = __vgic_v3_get_group(vcpu);
940 lr = __vgic_v3_highest_priority_lr(vcpu, vmcr, &lr_val);
944 lr_grp = !!(lr_val & ICH_LR_GROUP);
946 lr_val = ICC_IAR1_EL1_SPURIOUS;
949 vcpu_set_reg(vcpu, rt, lr_val & ICH_LR_VIRTUAL_ID_MASK);
952 static void __vgic_v3_read_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
954 vmcr &= ICH_VMCR_PMR_MASK;
955 vmcr >>= ICH_VMCR_PMR_SHIFT;
956 vcpu_set_reg(vcpu, rt, vmcr);
959 static void __vgic_v3_write_pmr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
961 u32 val = vcpu_get_reg(vcpu, rt);
963 val <<= ICH_VMCR_PMR_SHIFT;
964 val &= ICH_VMCR_PMR_MASK;
965 vmcr &= ~ICH_VMCR_PMR_MASK;
968 write_gicreg(vmcr, ICH_VMCR_EL2);
971 static void __vgic_v3_read_rpr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
973 u32 val = __vgic_v3_get_highest_active_priority();
974 vcpu_set_reg(vcpu, rt, val);
977 static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
981 vtr = read_gicreg(ICH_VTR_EL2);
983 val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT;
985 val |= ((vtr >> 23) & 7) << ICC_CTLR_EL1_ID_BITS_SHIFT;
987 if (kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_SEIS_MASK)
988 val |= BIT(ICC_CTLR_EL1_SEIS_SHIFT);
990 val |= ((vtr >> 21) & 1) << ICC_CTLR_EL1_A3V_SHIFT;
992 val |= ((vmcr & ICH_VMCR_EOIM_MASK) >> ICH_VMCR_EOIM_SHIFT) << ICC_CTLR_EL1_EOImode_SHIFT;
994 val |= (vmcr & ICH_VMCR_CBPR_MASK) >> ICH_VMCR_CBPR_SHIFT;
996 vcpu_set_reg(vcpu, rt, val);
999 static void __vgic_v3_write_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt)
1001 u32 val = vcpu_get_reg(vcpu, rt);
1003 if (val & ICC_CTLR_EL1_CBPR_MASK)
1004 vmcr |= ICH_VMCR_CBPR_MASK;
1006 vmcr &= ~ICH_VMCR_CBPR_MASK;
1008 if (val & ICC_CTLR_EL1_EOImode_MASK)
1009 vmcr |= ICH_VMCR_EOIM_MASK;
1011 vmcr &= ~ICH_VMCR_EOIM_MASK;
1013 write_gicreg(vmcr, ICH_VMCR_EL2);
1016 int __vgic_v3_perform_cpuif_access(struct kvm_vcpu *vcpu)
1021 void (*fn)(struct kvm_vcpu *, u32, int);
1025 esr = kvm_vcpu_get_esr(vcpu);
1026 if (vcpu_mode_is_32bit(vcpu)) {
1027 if (!kvm_condition_valid(vcpu)) {
1028 __kvm_skip_instr(vcpu);
1032 sysreg = esr_cp15_to_sysreg(esr);
1034 sysreg = esr_sys64_to_sysreg(esr);
1037 is_read = (esr & ESR_ELx_SYS64_ISS_DIR_MASK) == ESR_ELx_SYS64_ISS_DIR_READ;
1040 case SYS_ICC_IAR0_EL1:
1041 case SYS_ICC_IAR1_EL1:
1042 if (unlikely(!is_read))
1044 fn = __vgic_v3_read_iar;
1046 case SYS_ICC_EOIR0_EL1:
1047 case SYS_ICC_EOIR1_EL1:
1048 if (unlikely(is_read))
1050 fn = __vgic_v3_write_eoir;
1052 case SYS_ICC_IGRPEN1_EL1:
1054 fn = __vgic_v3_read_igrpen1;
1056 fn = __vgic_v3_write_igrpen1;
1058 case SYS_ICC_BPR1_EL1:
1060 fn = __vgic_v3_read_bpr1;
1062 fn = __vgic_v3_write_bpr1;
1064 case SYS_ICC_AP0Rn_EL1(0):
1065 case SYS_ICC_AP1Rn_EL1(0):
1067 fn = __vgic_v3_read_apxr0;
1069 fn = __vgic_v3_write_apxr0;
1071 case SYS_ICC_AP0Rn_EL1(1):
1072 case SYS_ICC_AP1Rn_EL1(1):
1074 fn = __vgic_v3_read_apxr1;
1076 fn = __vgic_v3_write_apxr1;
1078 case SYS_ICC_AP0Rn_EL1(2):
1079 case SYS_ICC_AP1Rn_EL1(2):
1081 fn = __vgic_v3_read_apxr2;
1083 fn = __vgic_v3_write_apxr2;
1085 case SYS_ICC_AP0Rn_EL1(3):
1086 case SYS_ICC_AP1Rn_EL1(3):
1088 fn = __vgic_v3_read_apxr3;
1090 fn = __vgic_v3_write_apxr3;
1092 case SYS_ICC_HPPIR0_EL1:
1093 case SYS_ICC_HPPIR1_EL1:
1094 if (unlikely(!is_read))
1096 fn = __vgic_v3_read_hppir;
1098 case SYS_ICC_IGRPEN0_EL1:
1100 fn = __vgic_v3_read_igrpen0;
1102 fn = __vgic_v3_write_igrpen0;
1104 case SYS_ICC_BPR0_EL1:
1106 fn = __vgic_v3_read_bpr0;
1108 fn = __vgic_v3_write_bpr0;
1110 case SYS_ICC_DIR_EL1:
1111 if (unlikely(is_read))
1113 fn = __vgic_v3_write_dir;
1115 case SYS_ICC_RPR_EL1:
1116 if (unlikely(!is_read))
1118 fn = __vgic_v3_read_rpr;
1120 case SYS_ICC_CTLR_EL1:
1122 fn = __vgic_v3_read_ctlr;
1124 fn = __vgic_v3_write_ctlr;
1126 case SYS_ICC_PMR_EL1:
1128 fn = __vgic_v3_read_pmr;
1130 fn = __vgic_v3_write_pmr;
1136 vmcr = __vgic_v3_read_vmcr();
1137 rt = kvm_vcpu_sys_get_rt(vcpu);
1140 __kvm_skip_instr(vcpu);