1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2012,2013 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
6 * Derived from arch/arm/kvm/coproc.c:
7 * Copyright (C) 2012 - Virtual Open Systems and Columbia University
8 * Authors: Rusty Russell <rusty@rustcorp.com.au>
9 * Christoffer Dall <c.dall@virtualopensystems.com>
12 #include <linux/bitfield.h>
13 #include <linux/bsearch.h>
14 #include <linux/cacheinfo.h>
15 #include <linux/kvm_host.h>
17 #include <linux/printk.h>
18 #include <linux/uaccess.h>
20 #include <asm/cacheflush.h>
21 #include <asm/cputype.h>
22 #include <asm/debug-monitors.h>
24 #include <asm/kvm_arm.h>
25 #include <asm/kvm_emulate.h>
26 #include <asm/kvm_hyp.h>
27 #include <asm/kvm_mmu.h>
28 #include <asm/kvm_nested.h>
29 #include <asm/perf_event.h>
30 #include <asm/sysreg.h>
32 #include <trace/events/kvm.h>
39 * For AArch32, we only take care of what is being trapped. Anything
40 * that has to do with init and userspace access has to go via the
44 static u64 sys_reg_to_index(const struct sys_reg_desc *reg);
45 static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
48 static bool bad_trap(struct kvm_vcpu *vcpu,
49 struct sys_reg_params *params,
50 const struct sys_reg_desc *r,
53 WARN_ONCE(1, "Unexpected %s\n", msg);
54 print_sys_reg_instr(params);
55 kvm_inject_undefined(vcpu);
59 static bool read_from_write_only(struct kvm_vcpu *vcpu,
60 struct sys_reg_params *params,
61 const struct sys_reg_desc *r)
63 return bad_trap(vcpu, params, r,
64 "sys_reg read to write-only register");
67 static bool write_to_read_only(struct kvm_vcpu *vcpu,
68 struct sys_reg_params *params,
69 const struct sys_reg_desc *r)
71 return bad_trap(vcpu, params, r,
72 "sys_reg write to read-only register");
75 #define PURE_EL2_SYSREG(el2) \
81 #define MAPPED_EL2_SYSREG(el2, el1, fn) \
88 static bool get_el2_to_el1_mapping(unsigned int reg,
89 unsigned int *el1r, u64 (**xlate)(u64))
92 PURE_EL2_SYSREG( VPIDR_EL2 );
93 PURE_EL2_SYSREG( VMPIDR_EL2 );
94 PURE_EL2_SYSREG( ACTLR_EL2 );
95 PURE_EL2_SYSREG( HCR_EL2 );
96 PURE_EL2_SYSREG( MDCR_EL2 );
97 PURE_EL2_SYSREG( HSTR_EL2 );
98 PURE_EL2_SYSREG( HACR_EL2 );
99 PURE_EL2_SYSREG( VTTBR_EL2 );
100 PURE_EL2_SYSREG( VTCR_EL2 );
101 PURE_EL2_SYSREG( RVBAR_EL2 );
102 PURE_EL2_SYSREG( TPIDR_EL2 );
103 PURE_EL2_SYSREG( HPFAR_EL2 );
104 PURE_EL2_SYSREG( CNTHCTL_EL2 );
105 MAPPED_EL2_SYSREG(SCTLR_EL2, SCTLR_EL1,
106 translate_sctlr_el2_to_sctlr_el1 );
107 MAPPED_EL2_SYSREG(CPTR_EL2, CPACR_EL1,
108 translate_cptr_el2_to_cpacr_el1 );
109 MAPPED_EL2_SYSREG(TTBR0_EL2, TTBR0_EL1,
110 translate_ttbr0_el2_to_ttbr0_el1 );
111 MAPPED_EL2_SYSREG(TTBR1_EL2, TTBR1_EL1, NULL );
112 MAPPED_EL2_SYSREG(TCR_EL2, TCR_EL1,
113 translate_tcr_el2_to_tcr_el1 );
114 MAPPED_EL2_SYSREG(VBAR_EL2, VBAR_EL1, NULL );
115 MAPPED_EL2_SYSREG(AFSR0_EL2, AFSR0_EL1, NULL );
116 MAPPED_EL2_SYSREG(AFSR1_EL2, AFSR1_EL1, NULL );
117 MAPPED_EL2_SYSREG(ESR_EL2, ESR_EL1, NULL );
118 MAPPED_EL2_SYSREG(FAR_EL2, FAR_EL1, NULL );
119 MAPPED_EL2_SYSREG(MAIR_EL2, MAIR_EL1, NULL );
120 MAPPED_EL2_SYSREG(AMAIR_EL2, AMAIR_EL1, NULL );
121 MAPPED_EL2_SYSREG(ELR_EL2, ELR_EL1, NULL );
122 MAPPED_EL2_SYSREG(SPSR_EL2, SPSR_EL1, NULL );
128 u64 vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
130 u64 val = 0x8badf00d8badf00d;
131 u64 (*xlate)(u64) = NULL;
134 if (!vcpu_get_flag(vcpu, SYSREGS_ON_CPU))
137 if (unlikely(get_el2_to_el1_mapping(reg, &el1r, &xlate))) {
138 if (!is_hyp_ctxt(vcpu))
142 * If this register does not have an EL1 counterpart,
143 * then read the stored EL2 version.
149 * If we have a non-VHE guest and that the sysreg
150 * requires translation to be used at EL1, use the
151 * in-memory copy instead.
153 if (!vcpu_el2_e2h_is_set(vcpu) && xlate)
156 /* Get the current version of the EL1 counterpart. */
157 WARN_ON(!__vcpu_read_sys_reg_from_cpu(el1r, &val));
161 /* EL1 register can't be on the CPU if the guest is in vEL2. */
162 if (unlikely(is_hyp_ctxt(vcpu)))
165 if (__vcpu_read_sys_reg_from_cpu(reg, &val))
169 return __vcpu_sys_reg(vcpu, reg);
172 void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
174 u64 (*xlate)(u64) = NULL;
177 if (!vcpu_get_flag(vcpu, SYSREGS_ON_CPU))
180 if (unlikely(get_el2_to_el1_mapping(reg, &el1r, &xlate))) {
181 if (!is_hyp_ctxt(vcpu))
185 * Always store a copy of the write to memory to avoid having
186 * to reverse-translate virtual EL2 system registers for a
187 * non-VHE guest hypervisor.
189 __vcpu_sys_reg(vcpu, reg) = val;
191 /* No EL1 counterpart? We're done here.? */
195 if (!vcpu_el2_e2h_is_set(vcpu) && xlate)
198 /* Redirect this to the EL1 version of the register. */
199 WARN_ON(!__vcpu_write_sys_reg_to_cpu(val, el1r));
203 /* EL1 register can't be on the CPU if the guest is in vEL2. */
204 if (unlikely(is_hyp_ctxt(vcpu)))
207 if (__vcpu_write_sys_reg_to_cpu(val, reg))
211 __vcpu_sys_reg(vcpu, reg) = val;
214 /* CSSELR values; used to index KVM_REG_ARM_DEMUX_ID_CCSIDR */
215 #define CSSELR_MAX 14
218 * Returns the minimum line size for the selected cache, expressed as
221 static u8 get_min_cache_line_size(bool icache)
223 u64 ctr = read_sanitised_ftr_reg(SYS_CTR_EL0);
227 field = SYS_FIELD_GET(CTR_EL0, IminLine, ctr);
229 field = SYS_FIELD_GET(CTR_EL0, DminLine, ctr);
232 * Cache line size is represented as Log2(words) in CTR_EL0.
233 * Log2(bytes) can be derived with the following:
235 * Log2(words) + 2 = Log2(bytes / 4) + 2
236 * = Log2(bytes) - 2 + 2
242 /* Which cache CCSIDR represents depends on CSSELR value. */
243 static u32 get_ccsidr(struct kvm_vcpu *vcpu, u32 csselr)
247 if (vcpu->arch.ccsidr)
248 return vcpu->arch.ccsidr[csselr];
250 line_size = get_min_cache_line_size(csselr & CSSELR_EL1_InD);
253 * Fabricate a CCSIDR value as the overriding value does not exist.
254 * The real CCSIDR value will not be used as it can vary by the
255 * physical CPU which the vcpu currently resides in.
257 * The line size is determined with get_min_cache_line_size(), which
258 * should be valid for all CPUs even if they have different cache
261 * The associativity bits are cleared, meaning the geometry of all data
262 * and unified caches (which are guaranteed to be PIPT and thus
263 * non-aliasing) are 1 set and 1 way.
264 * Guests should not be doing cache operations by set/way at all, and
265 * for this reason, we trap them and attempt to infer the intent, so
266 * that we can flush the entire guest's address space at the appropriate
267 * time. The exposed geometry minimizes the number of the traps.
268 * [If guests should attempt to infer aliasing properties from the
269 * geometry (which is not permitted by the architecture), they would
270 * only do so for virtually indexed caches.]
272 * We don't check if the cache level exists as it is allowed to return
273 * an UNKNOWN value if not.
275 return SYS_FIELD_PREP(CCSIDR_EL1, LineSize, line_size - 4);
278 static int set_ccsidr(struct kvm_vcpu *vcpu, u32 csselr, u32 val)
280 u8 line_size = FIELD_GET(CCSIDR_EL1_LineSize, val) + 4;
281 u32 *ccsidr = vcpu->arch.ccsidr;
284 if ((val & CCSIDR_EL1_RES0) ||
285 line_size < get_min_cache_line_size(csselr & CSSELR_EL1_InD))
289 if (val == get_ccsidr(vcpu, csselr))
292 ccsidr = kmalloc_array(CSSELR_MAX, sizeof(u32), GFP_KERNEL_ACCOUNT);
296 for (i = 0; i < CSSELR_MAX; i++)
297 ccsidr[i] = get_ccsidr(vcpu, i);
299 vcpu->arch.ccsidr = ccsidr;
302 ccsidr[csselr] = val;
307 static bool access_rw(struct kvm_vcpu *vcpu,
308 struct sys_reg_params *p,
309 const struct sys_reg_desc *r)
312 vcpu_write_sys_reg(vcpu, p->regval, r->reg);
314 p->regval = vcpu_read_sys_reg(vcpu, r->reg);
320 * See note at ARMv7 ARM B1.14.4 (TL;DR: S/W ops are not easily virtualized).
322 static bool access_dcsw(struct kvm_vcpu *vcpu,
323 struct sys_reg_params *p,
324 const struct sys_reg_desc *r)
327 return read_from_write_only(vcpu, p, r);
330 * Only track S/W ops if we don't have FWB. It still indicates
331 * that the guest is a bit broken (S/W operations should only
332 * be done by firmware, knowing that there is only a single
333 * CPU left in the system, and certainly not from non-secure
336 if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
337 kvm_set_way_flush(vcpu);
342 static bool access_dcgsw(struct kvm_vcpu *vcpu,
343 struct sys_reg_params *p,
344 const struct sys_reg_desc *r)
346 if (!kvm_has_mte(vcpu->kvm)) {
347 kvm_inject_undefined(vcpu);
351 /* Treat MTE S/W ops as we treat the classic ones: with contempt */
352 return access_dcsw(vcpu, p, r);
355 static void get_access_mask(const struct sys_reg_desc *r, u64 *mask, u64 *shift)
357 switch (r->aarch32_map) {
359 *mask = GENMASK_ULL(31, 0);
363 *mask = GENMASK_ULL(63, 32);
367 *mask = GENMASK_ULL(63, 0);
374 * Generic accessor for VM registers. Only called as long as HCR_TVM
375 * is set. If the guest enables the MMU, we stop trapping the VM
376 * sys_regs and leave it in complete control of the caches.
378 static bool access_vm_reg(struct kvm_vcpu *vcpu,
379 struct sys_reg_params *p,
380 const struct sys_reg_desc *r)
382 bool was_enabled = vcpu_has_cache_enabled(vcpu);
383 u64 val, mask, shift;
385 BUG_ON(!p->is_write);
387 get_access_mask(r, &mask, &shift);
390 val = vcpu_read_sys_reg(vcpu, r->reg);
396 val |= (p->regval & (mask >> shift)) << shift;
397 vcpu_write_sys_reg(vcpu, val, r->reg);
399 kvm_toggle_cache(vcpu, was_enabled);
403 static bool access_actlr(struct kvm_vcpu *vcpu,
404 struct sys_reg_params *p,
405 const struct sys_reg_desc *r)
410 return ignore_write(vcpu, p);
412 get_access_mask(r, &mask, &shift);
413 p->regval = (vcpu_read_sys_reg(vcpu, r->reg) & mask) >> shift;
419 * Trap handler for the GICv3 SGI generation system register.
420 * Forward the request to the VGIC emulation.
421 * The cp15_64 code makes sure this automatically works
422 * for both AArch64 and AArch32 accesses.
424 static bool access_gic_sgi(struct kvm_vcpu *vcpu,
425 struct sys_reg_params *p,
426 const struct sys_reg_desc *r)
431 return read_from_write_only(vcpu, p, r);
434 * In a system where GICD_CTLR.DS=1, a ICC_SGI0R_EL1 access generates
435 * Group0 SGIs only, while ICC_SGI1R_EL1 can generate either group,
436 * depending on the SGI configuration. ICC_ASGI1R_EL1 is effectively
437 * equivalent to ICC_SGI0R_EL1, as there is no "alternative" secure
440 if (p->Op0 == 0) { /* AArch32 */
442 default: /* Keep GCC quiet */
443 case 0: /* ICC_SGI1R */
446 case 1: /* ICC_ASGI1R */
447 case 2: /* ICC_SGI0R */
451 } else { /* AArch64 */
453 default: /* Keep GCC quiet */
454 case 5: /* ICC_SGI1R_EL1 */
457 case 6: /* ICC_ASGI1R_EL1 */
458 case 7: /* ICC_SGI0R_EL1 */
464 vgic_v3_dispatch_sgi(vcpu, p->regval, g1);
469 static bool access_gic_sre(struct kvm_vcpu *vcpu,
470 struct sys_reg_params *p,
471 const struct sys_reg_desc *r)
474 return ignore_write(vcpu, p);
476 p->regval = vcpu->arch.vgic_cpu.vgic_v3.vgic_sre;
480 static bool trap_raz_wi(struct kvm_vcpu *vcpu,
481 struct sys_reg_params *p,
482 const struct sys_reg_desc *r)
485 return ignore_write(vcpu, p);
487 return read_zero(vcpu, p);
490 static bool trap_undef(struct kvm_vcpu *vcpu,
491 struct sys_reg_params *p,
492 const struct sys_reg_desc *r)
494 kvm_inject_undefined(vcpu);
499 * ARMv8.1 mandates at least a trivial LORegion implementation, where all the
500 * RW registers are RES0 (which we can implement as RAZ/WI). On an ARMv8.0
501 * system, these registers should UNDEF. LORID_EL1 being a RO register, we
502 * treat it separately.
504 static bool trap_loregion(struct kvm_vcpu *vcpu,
505 struct sys_reg_params *p,
506 const struct sys_reg_desc *r)
508 u64 val = IDREG(vcpu->kvm, SYS_ID_AA64MMFR1_EL1);
509 u32 sr = reg_to_encoding(r);
511 if (!(val & (0xfUL << ID_AA64MMFR1_EL1_LO_SHIFT))) {
512 kvm_inject_undefined(vcpu);
516 if (p->is_write && sr == SYS_LORID_EL1)
517 return write_to_read_only(vcpu, p, r);
519 return trap_raz_wi(vcpu, p, r);
522 static bool trap_oslar_el1(struct kvm_vcpu *vcpu,
523 struct sys_reg_params *p,
524 const struct sys_reg_desc *r)
529 return read_from_write_only(vcpu, p, r);
531 /* Forward the OSLK bit to OSLSR */
532 oslsr = __vcpu_sys_reg(vcpu, OSLSR_EL1) & ~OSLSR_EL1_OSLK;
533 if (p->regval & OSLAR_EL1_OSLK)
534 oslsr |= OSLSR_EL1_OSLK;
536 __vcpu_sys_reg(vcpu, OSLSR_EL1) = oslsr;
540 static bool trap_oslsr_el1(struct kvm_vcpu *vcpu,
541 struct sys_reg_params *p,
542 const struct sys_reg_desc *r)
545 return write_to_read_only(vcpu, p, r);
547 p->regval = __vcpu_sys_reg(vcpu, r->reg);
551 static int set_oslsr_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
555 * The only modifiable bit is the OSLK bit. Refuse the write if
556 * userspace attempts to change any other bit in the register.
558 if ((val ^ rd->val) & ~OSLSR_EL1_OSLK)
561 __vcpu_sys_reg(vcpu, rd->reg) = val;
565 static bool trap_dbgauthstatus_el1(struct kvm_vcpu *vcpu,
566 struct sys_reg_params *p,
567 const struct sys_reg_desc *r)
570 return ignore_write(vcpu, p);
572 p->regval = read_sysreg(dbgauthstatus_el1);
578 * We want to avoid world-switching all the DBG registers all the
581 * - If we've touched any debug register, it is likely that we're
582 * going to touch more of them. It then makes sense to disable the
583 * traps and start doing the save/restore dance
584 * - If debug is active (DBG_MDSCR_KDE or DBG_MDSCR_MDE set), it is
585 * then mandatory to save/restore the registers, as the guest
588 * For this, we use a DIRTY bit, indicating the guest has modified the
589 * debug registers, used as follow:
592 * - If the dirty bit is set (because we're coming back from trapping),
593 * disable the traps, save host registers, restore guest registers.
594 * - If debug is actively in use (DBG_MDSCR_KDE or DBG_MDSCR_MDE set),
595 * set the dirty bit, disable the traps, save host registers,
596 * restore guest registers.
597 * - Otherwise, enable the traps
600 * - If the dirty bit is set, save guest registers, restore host
601 * registers and clear the dirty bit. This ensure that the host can
602 * now use the debug registers.
604 static bool trap_debug_regs(struct kvm_vcpu *vcpu,
605 struct sys_reg_params *p,
606 const struct sys_reg_desc *r)
608 access_rw(vcpu, p, r);
610 vcpu_set_flag(vcpu, DEBUG_DIRTY);
612 trace_trap_reg(__func__, r->reg, p->is_write, p->regval);
618 * reg_to_dbg/dbg_to_reg
620 * A 32 bit write to a debug register leave top bits alone
621 * A 32 bit read from a debug register only returns the bottom bits
623 * All writes will set the DEBUG_DIRTY flag to ensure the hyp code
624 * switches between host and guest values in future.
626 static void reg_to_dbg(struct kvm_vcpu *vcpu,
627 struct sys_reg_params *p,
628 const struct sys_reg_desc *rd,
631 u64 mask, shift, val;
633 get_access_mask(rd, &mask, &shift);
637 val |= (p->regval & (mask >> shift)) << shift;
640 vcpu_set_flag(vcpu, DEBUG_DIRTY);
643 static void dbg_to_reg(struct kvm_vcpu *vcpu,
644 struct sys_reg_params *p,
645 const struct sys_reg_desc *rd,
650 get_access_mask(rd, &mask, &shift);
651 p->regval = (*dbg_reg & mask) >> shift;
654 static bool trap_bvr(struct kvm_vcpu *vcpu,
655 struct sys_reg_params *p,
656 const struct sys_reg_desc *rd)
658 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm];
661 reg_to_dbg(vcpu, p, rd, dbg_reg);
663 dbg_to_reg(vcpu, p, rd, dbg_reg);
665 trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
670 static int set_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
673 vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm] = val;
677 static int get_bvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
680 *val = vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm];
684 static u64 reset_bvr(struct kvm_vcpu *vcpu,
685 const struct sys_reg_desc *rd)
687 vcpu->arch.vcpu_debug_state.dbg_bvr[rd->CRm] = rd->val;
691 static bool trap_bcr(struct kvm_vcpu *vcpu,
692 struct sys_reg_params *p,
693 const struct sys_reg_desc *rd)
695 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm];
698 reg_to_dbg(vcpu, p, rd, dbg_reg);
700 dbg_to_reg(vcpu, p, rd, dbg_reg);
702 trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
707 static int set_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
710 vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm] = val;
714 static int get_bcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
717 *val = vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm];
721 static u64 reset_bcr(struct kvm_vcpu *vcpu,
722 const struct sys_reg_desc *rd)
724 vcpu->arch.vcpu_debug_state.dbg_bcr[rd->CRm] = rd->val;
728 static bool trap_wvr(struct kvm_vcpu *vcpu,
729 struct sys_reg_params *p,
730 const struct sys_reg_desc *rd)
732 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm];
735 reg_to_dbg(vcpu, p, rd, dbg_reg);
737 dbg_to_reg(vcpu, p, rd, dbg_reg);
739 trace_trap_reg(__func__, rd->CRm, p->is_write,
740 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm]);
745 static int set_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
748 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm] = val;
752 static int get_wvr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
755 *val = vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm];
759 static u64 reset_wvr(struct kvm_vcpu *vcpu,
760 const struct sys_reg_desc *rd)
762 vcpu->arch.vcpu_debug_state.dbg_wvr[rd->CRm] = rd->val;
766 static bool trap_wcr(struct kvm_vcpu *vcpu,
767 struct sys_reg_params *p,
768 const struct sys_reg_desc *rd)
770 u64 *dbg_reg = &vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm];
773 reg_to_dbg(vcpu, p, rd, dbg_reg);
775 dbg_to_reg(vcpu, p, rd, dbg_reg);
777 trace_trap_reg(__func__, rd->CRm, p->is_write, *dbg_reg);
782 static int set_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
785 vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm] = val;
789 static int get_wcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
792 *val = vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm];
796 static u64 reset_wcr(struct kvm_vcpu *vcpu,
797 const struct sys_reg_desc *rd)
799 vcpu->arch.vcpu_debug_state.dbg_wcr[rd->CRm] = rd->val;
803 static u64 reset_amair_el1(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
805 u64 amair = read_sysreg(amair_el1);
806 vcpu_write_sys_reg(vcpu, amair, AMAIR_EL1);
810 static u64 reset_actlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
812 u64 actlr = read_sysreg(actlr_el1);
813 vcpu_write_sys_reg(vcpu, actlr, ACTLR_EL1);
817 static u64 reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
822 * Map the vcpu_id into the first three affinity level fields of
823 * the MPIDR. We limit the number of VCPUs in level 0 due to a
824 * limitation to 16 CPUs in that level in the ICC_SGIxR registers
825 * of the GICv3 to be able to address each CPU directly when
828 mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0);
829 mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1);
830 mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2);
831 mpidr |= (1ULL << 31);
832 vcpu_write_sys_reg(vcpu, mpidr, MPIDR_EL1);
837 static unsigned int pmu_visibility(const struct kvm_vcpu *vcpu,
838 const struct sys_reg_desc *r)
840 if (kvm_vcpu_has_pmu(vcpu))
846 static u64 reset_pmu_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
848 u64 mask = BIT(ARMV8_PMU_CYCLE_IDX);
849 u8 n = vcpu->kvm->arch.pmcr_n;
852 mask |= GENMASK(n - 1, 0);
854 reset_unknown(vcpu, r);
855 __vcpu_sys_reg(vcpu, r->reg) &= mask;
857 return __vcpu_sys_reg(vcpu, r->reg);
860 static u64 reset_pmevcntr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
862 reset_unknown(vcpu, r);
863 __vcpu_sys_reg(vcpu, r->reg) &= GENMASK(31, 0);
865 return __vcpu_sys_reg(vcpu, r->reg);
868 static u64 reset_pmevtyper(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
870 /* This thing will UNDEF, who cares about the reset value? */
871 if (!kvm_vcpu_has_pmu(vcpu))
874 reset_unknown(vcpu, r);
875 __vcpu_sys_reg(vcpu, r->reg) &= kvm_pmu_evtyper_mask(vcpu->kvm);
877 return __vcpu_sys_reg(vcpu, r->reg);
880 static u64 reset_pmselr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
882 reset_unknown(vcpu, r);
883 __vcpu_sys_reg(vcpu, r->reg) &= ARMV8_PMU_COUNTER_MASK;
885 return __vcpu_sys_reg(vcpu, r->reg);
888 static u64 reset_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
892 if (!kvm_supports_32bit_el0())
893 pmcr |= ARMV8_PMU_PMCR_LC;
896 * The value of PMCR.N field is included when the
897 * vCPU register is read via kvm_vcpu_read_pmcr().
899 __vcpu_sys_reg(vcpu, r->reg) = pmcr;
901 return __vcpu_sys_reg(vcpu, r->reg);
904 static bool check_pmu_access_disabled(struct kvm_vcpu *vcpu, u64 flags)
906 u64 reg = __vcpu_sys_reg(vcpu, PMUSERENR_EL0);
907 bool enabled = (reg & flags) || vcpu_mode_priv(vcpu);
910 kvm_inject_undefined(vcpu);
915 static bool pmu_access_el0_disabled(struct kvm_vcpu *vcpu)
917 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_EN);
920 static bool pmu_write_swinc_el0_disabled(struct kvm_vcpu *vcpu)
922 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_SW | ARMV8_PMU_USERENR_EN);
925 static bool pmu_access_cycle_counter_el0_disabled(struct kvm_vcpu *vcpu)
927 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_CR | ARMV8_PMU_USERENR_EN);
930 static bool pmu_access_event_counter_el0_disabled(struct kvm_vcpu *vcpu)
932 return check_pmu_access_disabled(vcpu, ARMV8_PMU_USERENR_ER | ARMV8_PMU_USERENR_EN);
935 static bool access_pmcr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
936 const struct sys_reg_desc *r)
940 if (pmu_access_el0_disabled(vcpu))
945 * Only update writeable bits of PMCR (continuing into
946 * kvm_pmu_handle_pmcr() as well)
948 val = kvm_vcpu_read_pmcr(vcpu);
949 val &= ~ARMV8_PMU_PMCR_MASK;
950 val |= p->regval & ARMV8_PMU_PMCR_MASK;
951 if (!kvm_supports_32bit_el0())
952 val |= ARMV8_PMU_PMCR_LC;
953 kvm_pmu_handle_pmcr(vcpu, val);
955 /* PMCR.P & PMCR.C are RAZ */
956 val = kvm_vcpu_read_pmcr(vcpu)
957 & ~(ARMV8_PMU_PMCR_P | ARMV8_PMU_PMCR_C);
964 static bool access_pmselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
965 const struct sys_reg_desc *r)
967 if (pmu_access_event_counter_el0_disabled(vcpu))
971 __vcpu_sys_reg(vcpu, PMSELR_EL0) = p->regval;
973 /* return PMSELR.SEL field */
974 p->regval = __vcpu_sys_reg(vcpu, PMSELR_EL0)
975 & ARMV8_PMU_COUNTER_MASK;
980 static bool access_pmceid(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
981 const struct sys_reg_desc *r)
983 u64 pmceid, mask, shift;
987 if (pmu_access_el0_disabled(vcpu))
990 get_access_mask(r, &mask, &shift);
992 pmceid = kvm_pmu_get_pmceid(vcpu, (p->Op2 & 1));
1001 static bool pmu_counter_idx_valid(struct kvm_vcpu *vcpu, u64 idx)
1005 pmcr = kvm_vcpu_read_pmcr(vcpu);
1006 val = FIELD_GET(ARMV8_PMU_PMCR_N, pmcr);
1007 if (idx >= val && idx != ARMV8_PMU_CYCLE_IDX) {
1008 kvm_inject_undefined(vcpu);
1015 static int get_pmu_evcntr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
1020 if (r->CRn == 9 && r->CRm == 13 && r->Op2 == 0)
1022 idx = ARMV8_PMU_CYCLE_IDX;
1025 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
1027 *val = kvm_pmu_get_counter_value(vcpu, idx);
1031 static bool access_pmu_evcntr(struct kvm_vcpu *vcpu,
1032 struct sys_reg_params *p,
1033 const struct sys_reg_desc *r)
1037 if (r->CRn == 9 && r->CRm == 13) {
1040 if (pmu_access_event_counter_el0_disabled(vcpu))
1043 idx = __vcpu_sys_reg(vcpu, PMSELR_EL0)
1044 & ARMV8_PMU_COUNTER_MASK;
1045 } else if (r->Op2 == 0) {
1047 if (pmu_access_cycle_counter_el0_disabled(vcpu))
1050 idx = ARMV8_PMU_CYCLE_IDX;
1052 } else if (r->CRn == 0 && r->CRm == 9) {
1054 if (pmu_access_event_counter_el0_disabled(vcpu))
1057 idx = ARMV8_PMU_CYCLE_IDX;
1058 } else if (r->CRn == 14 && (r->CRm & 12) == 8) {
1060 if (pmu_access_event_counter_el0_disabled(vcpu))
1063 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
1066 /* Catch any decoding mistake */
1067 WARN_ON(idx == ~0UL);
1069 if (!pmu_counter_idx_valid(vcpu, idx))
1073 if (pmu_access_el0_disabled(vcpu))
1076 kvm_pmu_set_counter_value(vcpu, idx, p->regval);
1078 p->regval = kvm_pmu_get_counter_value(vcpu, idx);
1084 static bool access_pmu_evtyper(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1085 const struct sys_reg_desc *r)
1089 if (pmu_access_el0_disabled(vcpu))
1092 if (r->CRn == 9 && r->CRm == 13 && r->Op2 == 1) {
1093 /* PMXEVTYPER_EL0 */
1094 idx = __vcpu_sys_reg(vcpu, PMSELR_EL0) & ARMV8_PMU_COUNTER_MASK;
1095 reg = PMEVTYPER0_EL0 + idx;
1096 } else if (r->CRn == 14 && (r->CRm & 12) == 12) {
1097 idx = ((r->CRm & 3) << 3) | (r->Op2 & 7);
1098 if (idx == ARMV8_PMU_CYCLE_IDX)
1099 reg = PMCCFILTR_EL0;
1101 /* PMEVTYPERn_EL0 */
1102 reg = PMEVTYPER0_EL0 + idx;
1107 if (!pmu_counter_idx_valid(vcpu, idx))
1111 kvm_pmu_set_counter_event_type(vcpu, p->regval, idx);
1112 kvm_vcpu_pmu_restore_guest(vcpu);
1114 p->regval = __vcpu_sys_reg(vcpu, reg);
1120 static int set_pmreg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, u64 val)
1124 val &= kvm_pmu_valid_counter_mask(vcpu);
1128 /* CRm[1] being set indicates a SET register, and CLR otherwise */
1132 /* Op2[0] being set indicates a SET register, and CLR otherwise */
1138 __vcpu_sys_reg(vcpu, r->reg) |= val;
1140 __vcpu_sys_reg(vcpu, r->reg) &= ~val;
1145 static int get_pmreg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, u64 *val)
1147 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
1149 *val = __vcpu_sys_reg(vcpu, r->reg) & mask;
1153 static bool access_pmcnten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1154 const struct sys_reg_desc *r)
1158 if (pmu_access_el0_disabled(vcpu))
1161 mask = kvm_pmu_valid_counter_mask(vcpu);
1163 val = p->regval & mask;
1165 /* accessing PMCNTENSET_EL0 */
1166 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) |= val;
1167 kvm_pmu_enable_counter_mask(vcpu, val);
1168 kvm_vcpu_pmu_restore_guest(vcpu);
1170 /* accessing PMCNTENCLR_EL0 */
1171 __vcpu_sys_reg(vcpu, PMCNTENSET_EL0) &= ~val;
1172 kvm_pmu_disable_counter_mask(vcpu, val);
1175 p->regval = __vcpu_sys_reg(vcpu, PMCNTENSET_EL0);
1181 static bool access_pminten(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1182 const struct sys_reg_desc *r)
1184 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
1186 if (check_pmu_access_disabled(vcpu, 0))
1190 u64 val = p->regval & mask;
1193 /* accessing PMINTENSET_EL1 */
1194 __vcpu_sys_reg(vcpu, PMINTENSET_EL1) |= val;
1196 /* accessing PMINTENCLR_EL1 */
1197 __vcpu_sys_reg(vcpu, PMINTENSET_EL1) &= ~val;
1199 p->regval = __vcpu_sys_reg(vcpu, PMINTENSET_EL1);
1205 static bool access_pmovs(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1206 const struct sys_reg_desc *r)
1208 u64 mask = kvm_pmu_valid_counter_mask(vcpu);
1210 if (pmu_access_el0_disabled(vcpu))
1215 /* accessing PMOVSSET_EL0 */
1216 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) |= (p->regval & mask);
1218 /* accessing PMOVSCLR_EL0 */
1219 __vcpu_sys_reg(vcpu, PMOVSSET_EL0) &= ~(p->regval & mask);
1221 p->regval = __vcpu_sys_reg(vcpu, PMOVSSET_EL0);
1227 static bool access_pmswinc(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1228 const struct sys_reg_desc *r)
1233 return read_from_write_only(vcpu, p, r);
1235 if (pmu_write_swinc_el0_disabled(vcpu))
1238 mask = kvm_pmu_valid_counter_mask(vcpu);
1239 kvm_pmu_software_increment(vcpu, p->regval & mask);
1243 static bool access_pmuserenr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1244 const struct sys_reg_desc *r)
1247 if (!vcpu_mode_priv(vcpu)) {
1248 kvm_inject_undefined(vcpu);
1252 __vcpu_sys_reg(vcpu, PMUSERENR_EL0) =
1253 p->regval & ARMV8_PMU_USERENR_MASK;
1255 p->regval = __vcpu_sys_reg(vcpu, PMUSERENR_EL0)
1256 & ARMV8_PMU_USERENR_MASK;
1262 static int get_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
1265 *val = kvm_vcpu_read_pmcr(vcpu);
1269 static int set_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
1272 u8 new_n = FIELD_GET(ARMV8_PMU_PMCR_N, val);
1273 struct kvm *kvm = vcpu->kvm;
1275 mutex_lock(&kvm->arch.config_lock);
1278 * The vCPU can't have more counters than the PMU hardware
1279 * implements. Ignore this error to maintain compatibility
1280 * with the existing KVM behavior.
1282 if (!kvm_vm_has_ran_once(kvm) &&
1283 new_n <= kvm_arm_pmu_get_max_counters(kvm))
1284 kvm->arch.pmcr_n = new_n;
1286 mutex_unlock(&kvm->arch.config_lock);
1289 * Ignore writes to RES0 bits, read only bits that are cleared on
1290 * vCPU reset, and writable bits that KVM doesn't support yet.
1291 * (i.e. only PMCR.N and bits [7:0] are mutable from userspace)
1292 * The LP bit is RES0 when FEAT_PMUv3p5 is not supported on the vCPU.
1293 * But, we leave the bit as it is here, as the vCPU's PMUver might
1294 * be changed later (NOTE: the bit will be cleared on first vCPU run
1297 val &= ARMV8_PMU_PMCR_MASK;
1299 /* The LC bit is RES1 when AArch32 is not supported */
1300 if (!kvm_supports_32bit_el0())
1301 val |= ARMV8_PMU_PMCR_LC;
1303 __vcpu_sys_reg(vcpu, r->reg) = val;
1307 /* Silly macro to expand the DBG{BCR,BVR,WVR,WCR}n_EL1 registers in one go */
1308 #define DBG_BCR_BVR_WCR_WVR_EL1(n) \
1309 { SYS_DESC(SYS_DBGBVRn_EL1(n)), \
1310 trap_bvr, reset_bvr, 0, 0, get_bvr, set_bvr }, \
1311 { SYS_DESC(SYS_DBGBCRn_EL1(n)), \
1312 trap_bcr, reset_bcr, 0, 0, get_bcr, set_bcr }, \
1313 { SYS_DESC(SYS_DBGWVRn_EL1(n)), \
1314 trap_wvr, reset_wvr, 0, 0, get_wvr, set_wvr }, \
1315 { SYS_DESC(SYS_DBGWCRn_EL1(n)), \
1316 trap_wcr, reset_wcr, 0, 0, get_wcr, set_wcr }
1318 #define PMU_SYS_REG(name) \
1319 SYS_DESC(SYS_##name), .reset = reset_pmu_reg, \
1320 .visibility = pmu_visibility
1322 /* Macro to expand the PMEVCNTRn_EL0 register */
1323 #define PMU_PMEVCNTR_EL0(n) \
1324 { PMU_SYS_REG(PMEVCNTRn_EL0(n)), \
1325 .reset = reset_pmevcntr, .get_user = get_pmu_evcntr, \
1326 .access = access_pmu_evcntr, .reg = (PMEVCNTR0_EL0 + n), }
1328 /* Macro to expand the PMEVTYPERn_EL0 register */
1329 #define PMU_PMEVTYPER_EL0(n) \
1330 { PMU_SYS_REG(PMEVTYPERn_EL0(n)), \
1331 .reset = reset_pmevtyper, \
1332 .access = access_pmu_evtyper, .reg = (PMEVTYPER0_EL0 + n), }
1334 static bool undef_access(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1335 const struct sys_reg_desc *r)
1337 kvm_inject_undefined(vcpu);
1342 /* Macro to expand the AMU counter and type registers*/
1343 #define AMU_AMEVCNTR0_EL0(n) { SYS_DESC(SYS_AMEVCNTR0_EL0(n)), undef_access }
1344 #define AMU_AMEVTYPER0_EL0(n) { SYS_DESC(SYS_AMEVTYPER0_EL0(n)), undef_access }
1345 #define AMU_AMEVCNTR1_EL0(n) { SYS_DESC(SYS_AMEVCNTR1_EL0(n)), undef_access }
1346 #define AMU_AMEVTYPER1_EL0(n) { SYS_DESC(SYS_AMEVTYPER1_EL0(n)), undef_access }
1348 static unsigned int ptrauth_visibility(const struct kvm_vcpu *vcpu,
1349 const struct sys_reg_desc *rd)
1351 return vcpu_has_ptrauth(vcpu) ? 0 : REG_HIDDEN;
1355 * If we land here on a PtrAuth access, that is because we didn't
1356 * fixup the access on exit by allowing the PtrAuth sysregs. The only
1357 * way this happens is when the guest does not have PtrAuth support
1360 #define __PTRAUTH_KEY(k) \
1361 { SYS_DESC(SYS_## k), undef_access, reset_unknown, k, \
1362 .visibility = ptrauth_visibility}
1364 #define PTRAUTH_KEY(k) \
1365 __PTRAUTH_KEY(k ## KEYLO_EL1), \
1366 __PTRAUTH_KEY(k ## KEYHI_EL1)
1368 static bool access_arch_timer(struct kvm_vcpu *vcpu,
1369 struct sys_reg_params *p,
1370 const struct sys_reg_desc *r)
1372 enum kvm_arch_timers tmr;
1373 enum kvm_arch_timer_regs treg;
1374 u64 reg = reg_to_encoding(r);
1377 case SYS_CNTP_TVAL_EL0:
1378 case SYS_AARCH32_CNTP_TVAL:
1380 treg = TIMER_REG_TVAL;
1382 case SYS_CNTP_CTL_EL0:
1383 case SYS_AARCH32_CNTP_CTL:
1385 treg = TIMER_REG_CTL;
1387 case SYS_CNTP_CVAL_EL0:
1388 case SYS_AARCH32_CNTP_CVAL:
1390 treg = TIMER_REG_CVAL;
1392 case SYS_CNTPCT_EL0:
1393 case SYS_CNTPCTSS_EL0:
1394 case SYS_AARCH32_CNTPCT:
1396 treg = TIMER_REG_CNT;
1399 print_sys_reg_msg(p, "%s", "Unhandled trapped timer register");
1400 kvm_inject_undefined(vcpu);
1405 kvm_arm_timer_write_sysreg(vcpu, tmr, treg, p->regval);
1407 p->regval = kvm_arm_timer_read_sysreg(vcpu, tmr, treg);
1412 static s64 kvm_arm64_ftr_safe_value(u32 id, const struct arm64_ftr_bits *ftrp,
1415 struct arm64_ftr_bits kvm_ftr = *ftrp;
1417 /* Some features have different safe value type in KVM than host features */
1419 case SYS_ID_AA64DFR0_EL1:
1420 switch (kvm_ftr.shift) {
1421 case ID_AA64DFR0_EL1_PMUVer_SHIFT:
1422 kvm_ftr.type = FTR_LOWER_SAFE;
1424 case ID_AA64DFR0_EL1_DebugVer_SHIFT:
1425 kvm_ftr.type = FTR_LOWER_SAFE;
1429 case SYS_ID_DFR0_EL1:
1430 if (kvm_ftr.shift == ID_DFR0_EL1_PerfMon_SHIFT)
1431 kvm_ftr.type = FTR_LOWER_SAFE;
1435 return arm64_ftr_safe_value(&kvm_ftr, new, cur);
1439 * arm64_check_features() - Check if a feature register value constitutes
1440 * a subset of features indicated by the idreg's KVM sanitised limit.
1442 * This function will check if each feature field of @val is the "safe" value
1443 * against idreg's KVM sanitised limit return from reset() callback.
1444 * If a field value in @val is the same as the one in limit, it is always
1445 * considered the safe value regardless For register fields that are not in
1446 * writable, only the value in limit is considered the safe value.
1448 * Return: 0 if all the fields are safe. Otherwise, return negative errno.
1450 static int arm64_check_features(struct kvm_vcpu *vcpu,
1451 const struct sys_reg_desc *rd,
1454 const struct arm64_ftr_reg *ftr_reg;
1455 const struct arm64_ftr_bits *ftrp = NULL;
1456 u32 id = reg_to_encoding(rd);
1457 u64 writable_mask = rd->val;
1458 u64 limit = rd->reset(vcpu, rd);
1462 * Hidden and unallocated ID registers may not have a corresponding
1463 * struct arm64_ftr_reg. Of course, if the register is RAZ we know the
1464 * only safe value is 0.
1466 if (sysreg_visible_as_raz(vcpu, rd))
1467 return val ? -E2BIG : 0;
1469 ftr_reg = get_arm64_ftr_reg(id);
1473 ftrp = ftr_reg->ftr_bits;
1475 for (; ftrp && ftrp->width; ftrp++) {
1476 s64 f_val, f_lim, safe_val;
1479 ftr_mask = arm64_ftr_mask(ftrp);
1480 if ((ftr_mask & writable_mask) != ftr_mask)
1483 f_val = arm64_ftr_value(ftrp, val);
1484 f_lim = arm64_ftr_value(ftrp, limit);
1490 safe_val = kvm_arm64_ftr_safe_value(id, ftrp, f_val, f_lim);
1492 if (safe_val != f_val)
1496 /* For fields that are not writable, values in limit are the safe values. */
1497 if ((val & ~mask) != (limit & ~mask))
1503 static u8 pmuver_to_perfmon(u8 pmuver)
1506 case ID_AA64DFR0_EL1_PMUVer_IMP:
1507 return ID_DFR0_EL1_PerfMon_PMUv3;
1508 case ID_AA64DFR0_EL1_PMUVer_IMP_DEF:
1509 return ID_DFR0_EL1_PerfMon_IMPDEF;
1511 /* Anything ARMv8.1+ and NI have the same value. For now. */
1516 /* Read a sanitised cpufeature ID register by sys_reg_desc */
1517 static u64 __kvm_read_sanitised_id_reg(const struct kvm_vcpu *vcpu,
1518 const struct sys_reg_desc *r)
1520 u32 id = reg_to_encoding(r);
1523 if (sysreg_visible_as_raz(vcpu, r))
1526 val = read_sanitised_ftr_reg(id);
1529 case SYS_ID_AA64PFR1_EL1:
1530 if (!kvm_has_mte(vcpu->kvm))
1531 val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTE);
1533 val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_SME);
1535 case SYS_ID_AA64ISAR1_EL1:
1536 if (!vcpu_has_ptrauth(vcpu))
1537 val &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_APA) |
1538 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_API) |
1539 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPA) |
1540 ARM64_FEATURE_MASK(ID_AA64ISAR1_EL1_GPI));
1542 case SYS_ID_AA64ISAR2_EL1:
1543 if (!vcpu_has_ptrauth(vcpu))
1544 val &= ~(ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_APA3) |
1545 ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_GPA3));
1546 if (!cpus_have_final_cap(ARM64_HAS_WFXT))
1547 val &= ~ARM64_FEATURE_MASK(ID_AA64ISAR2_EL1_WFxT);
1549 case SYS_ID_AA64MMFR2_EL1:
1550 val &= ~ID_AA64MMFR2_EL1_CCIDX_MASK;
1552 case SYS_ID_MMFR4_EL1:
1553 val &= ~ARM64_FEATURE_MASK(ID_MMFR4_EL1_CCIDX);
1560 static u64 kvm_read_sanitised_id_reg(struct kvm_vcpu *vcpu,
1561 const struct sys_reg_desc *r)
1563 return __kvm_read_sanitised_id_reg(vcpu, r);
1566 static u64 read_id_reg(const struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
1568 return IDREG(vcpu->kvm, reg_to_encoding(r));
1572 * Return true if the register's (Op0, Op1, CRn, CRm, Op2) is
1573 * (3, 0, 0, crm, op2), where 1<=crm<8, 0<=op2<8.
1575 static inline bool is_id_reg(u32 id)
1577 return (sys_reg_Op0(id) == 3 && sys_reg_Op1(id) == 0 &&
1578 sys_reg_CRn(id) == 0 && sys_reg_CRm(id) >= 1 &&
1579 sys_reg_CRm(id) < 8);
1582 static inline bool is_aa32_id_reg(u32 id)
1584 return (sys_reg_Op0(id) == 3 && sys_reg_Op1(id) == 0 &&
1585 sys_reg_CRn(id) == 0 && sys_reg_CRm(id) >= 1 &&
1586 sys_reg_CRm(id) <= 3);
1589 static unsigned int id_visibility(const struct kvm_vcpu *vcpu,
1590 const struct sys_reg_desc *r)
1592 u32 id = reg_to_encoding(r);
1595 case SYS_ID_AA64ZFR0_EL1:
1596 if (!vcpu_has_sve(vcpu))
1604 static unsigned int aa32_id_visibility(const struct kvm_vcpu *vcpu,
1605 const struct sys_reg_desc *r)
1608 * AArch32 ID registers are UNKNOWN if AArch32 isn't implemented at any
1609 * EL. Promote to RAZ/WI in order to guarantee consistency between
1612 if (!kvm_supports_32bit_el0())
1613 return REG_RAZ | REG_USER_WI;
1615 return id_visibility(vcpu, r);
1618 static unsigned int raz_visibility(const struct kvm_vcpu *vcpu,
1619 const struct sys_reg_desc *r)
1624 /* cpufeature ID register access trap handlers */
1626 static bool access_id_reg(struct kvm_vcpu *vcpu,
1627 struct sys_reg_params *p,
1628 const struct sys_reg_desc *r)
1631 return write_to_read_only(vcpu, p, r);
1633 p->regval = read_id_reg(vcpu, r);
1638 /* Visibility overrides for SVE-specific control registers */
1639 static unsigned int sve_visibility(const struct kvm_vcpu *vcpu,
1640 const struct sys_reg_desc *rd)
1642 if (vcpu_has_sve(vcpu))
1648 static u64 read_sanitised_id_aa64pfr0_el1(struct kvm_vcpu *vcpu,
1649 const struct sys_reg_desc *rd)
1651 u64 val = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
1653 if (!vcpu_has_sve(vcpu))
1654 val &= ~ID_AA64PFR0_EL1_SVE_MASK;
1657 * The default is to expose CSV2 == 1 if the HW isn't affected.
1658 * Although this is a per-CPU feature, we make it global because
1659 * asymmetric systems are just a nuisance.
1661 * Userspace can override this as long as it doesn't promise
1664 if (arm64_get_spectre_v2_state() == SPECTRE_UNAFFECTED) {
1665 val &= ~ID_AA64PFR0_EL1_CSV2_MASK;
1666 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, CSV2, IMP);
1668 if (arm64_get_meltdown_state() == SPECTRE_UNAFFECTED) {
1669 val &= ~ID_AA64PFR0_EL1_CSV3_MASK;
1670 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, CSV3, IMP);
1673 if (kvm_vgic_global_state.type == VGIC_V3) {
1674 val &= ~ID_AA64PFR0_EL1_GIC_MASK;
1675 val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, GIC, IMP);
1678 val &= ~ID_AA64PFR0_EL1_AMU_MASK;
1683 #define ID_REG_LIMIT_FIELD_ENUM(val, reg, field, limit) \
1685 u64 __f_val = FIELD_GET(reg##_##field##_MASK, val); \
1686 (val) &= ~reg##_##field##_MASK; \
1687 (val) |= FIELD_PREP(reg##_##field##_MASK, \
1688 min(__f_val, (u64)reg##_##field##_##limit)); \
1692 static u64 read_sanitised_id_aa64dfr0_el1(struct kvm_vcpu *vcpu,
1693 const struct sys_reg_desc *rd)
1695 u64 val = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
1697 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_AA64DFR0_EL1, DebugVer, V8P8);
1700 * Only initialize the PMU version if the vCPU was configured with one.
1702 val &= ~ID_AA64DFR0_EL1_PMUVer_MASK;
1703 if (kvm_vcpu_has_pmu(vcpu))
1704 val |= SYS_FIELD_PREP(ID_AA64DFR0_EL1, PMUVer,
1705 kvm_arm_pmu_get_pmuver_limit());
1707 /* Hide SPE from guests */
1708 val &= ~ID_AA64DFR0_EL1_PMSVer_MASK;
1713 static int set_id_aa64dfr0_el1(struct kvm_vcpu *vcpu,
1714 const struct sys_reg_desc *rd,
1717 u8 debugver = SYS_FIELD_GET(ID_AA64DFR0_EL1, DebugVer, val);
1718 u8 pmuver = SYS_FIELD_GET(ID_AA64DFR0_EL1, PMUVer, val);
1721 * Prior to commit 3d0dba5764b9 ("KVM: arm64: PMU: Move the
1722 * ID_AA64DFR0_EL1.PMUver limit to VM creation"), KVM erroneously
1723 * exposed an IMP_DEF PMU to userspace and the guest on systems w/
1724 * non-architectural PMUs. Of course, PMUv3 is the only game in town for
1725 * PMU virtualization, so the IMP_DEF value was rather user-hostile.
1727 * At minimum, we're on the hook to allow values that were given to
1728 * userspace by KVM. Cover our tracks here and replace the IMP_DEF value
1729 * with a more sensible NI. The value of an ID register changing under
1730 * the nose of the guest is unfortunate, but is certainly no more
1731 * surprising than an ill-guided PMU driver poking at impdef system
1732 * registers that end in an UNDEF...
1734 if (pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF)
1735 val &= ~ID_AA64DFR0_EL1_PMUVer_MASK;
1738 * ID_AA64DFR0_EL1.DebugVer is one of those awkward fields with a
1739 * nonzero minimum safe value.
1741 if (debugver < ID_AA64DFR0_EL1_DebugVer_IMP)
1744 return set_id_reg(vcpu, rd, val);
1747 static u64 read_sanitised_id_dfr0_el1(struct kvm_vcpu *vcpu,
1748 const struct sys_reg_desc *rd)
1750 u8 perfmon = pmuver_to_perfmon(kvm_arm_pmu_get_pmuver_limit());
1751 u64 val = read_sanitised_ftr_reg(SYS_ID_DFR0_EL1);
1753 val &= ~ID_DFR0_EL1_PerfMon_MASK;
1754 if (kvm_vcpu_has_pmu(vcpu))
1755 val |= SYS_FIELD_PREP(ID_DFR0_EL1, PerfMon, perfmon);
1757 val = ID_REG_LIMIT_FIELD_ENUM(val, ID_DFR0_EL1, CopDbg, Debugv8p8);
1762 static int set_id_dfr0_el1(struct kvm_vcpu *vcpu,
1763 const struct sys_reg_desc *rd,
1766 u8 perfmon = SYS_FIELD_GET(ID_DFR0_EL1, PerfMon, val);
1767 u8 copdbg = SYS_FIELD_GET(ID_DFR0_EL1, CopDbg, val);
1769 if (perfmon == ID_DFR0_EL1_PerfMon_IMPDEF) {
1770 val &= ~ID_DFR0_EL1_PerfMon_MASK;
1775 * Allow DFR0_EL1.PerfMon to be set from userspace as long as
1776 * it doesn't promise more than what the HW gives us on the
1777 * AArch64 side (as everything is emulated with that), and
1778 * that this is a PMUv3.
1780 if (perfmon != 0 && perfmon < ID_DFR0_EL1_PerfMon_PMUv3)
1783 if (copdbg < ID_DFR0_EL1_CopDbg_Armv8)
1786 return set_id_reg(vcpu, rd, val);
1790 * cpufeature ID register user accessors
1792 * For now, these registers are immutable for userspace, so no values
1793 * are stored, and for set_id_reg() we don't allow the effective value
1796 static int get_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1800 * Avoid locking if the VM has already started, as the ID registers are
1801 * guaranteed to be invariant at that point.
1803 if (kvm_vm_has_ran_once(vcpu->kvm)) {
1804 *val = read_id_reg(vcpu, rd);
1808 mutex_lock(&vcpu->kvm->arch.config_lock);
1809 *val = read_id_reg(vcpu, rd);
1810 mutex_unlock(&vcpu->kvm->arch.config_lock);
1815 static int set_id_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1818 u32 id = reg_to_encoding(rd);
1821 mutex_lock(&vcpu->kvm->arch.config_lock);
1824 * Once the VM has started the ID registers are immutable. Reject any
1825 * write that does not match the final register value.
1827 if (kvm_vm_has_ran_once(vcpu->kvm)) {
1828 if (val != read_id_reg(vcpu, rd))
1833 mutex_unlock(&vcpu->kvm->arch.config_lock);
1837 ret = arm64_check_features(vcpu, rd, val);
1839 IDREG(vcpu->kvm, id) = val;
1841 mutex_unlock(&vcpu->kvm->arch.config_lock);
1844 * arm64_check_features() returns -E2BIG to indicate the register's
1845 * feature set is a superset of the maximally-allowed register value.
1846 * While it would be nice to precisely describe this to userspace, the
1847 * existing UAPI for KVM_SET_ONE_REG has it that invalid register
1848 * writes return -EINVAL.
1855 static int get_raz_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1862 static int set_wi_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1868 static bool access_ctr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1869 const struct sys_reg_desc *r)
1872 return write_to_read_only(vcpu, p, r);
1874 p->regval = read_sanitised_ftr_reg(SYS_CTR_EL0);
1878 static bool access_clidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1879 const struct sys_reg_desc *r)
1882 return write_to_read_only(vcpu, p, r);
1884 p->regval = __vcpu_sys_reg(vcpu, r->reg);
1889 * Fabricate a CLIDR_EL1 value instead of using the real value, which can vary
1890 * by the physical CPU which the vcpu currently resides in.
1892 static u64 reset_clidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
1894 u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0);
1898 if ((ctr_el0 & CTR_EL0_IDC)) {
1900 * Data cache clean to the PoU is not required so LoUU and LoUIS
1901 * will not be set and a unified cache, which will be marked as
1902 * LoC, will be added.
1904 * If not DIC, let the unified cache L2 so that an instruction
1905 * cache can be added as L1 later.
1907 loc = (ctr_el0 & CTR_EL0_DIC) ? 1 : 2;
1908 clidr = CACHE_TYPE_UNIFIED << CLIDR_CTYPE_SHIFT(loc);
1911 * Data cache clean to the PoU is required so let L1 have a data
1912 * cache and mark it as LoUU and LoUIS. As L1 has a data cache,
1913 * it can be marked as LoC too.
1916 clidr = 1 << CLIDR_LOUU_SHIFT;
1917 clidr |= 1 << CLIDR_LOUIS_SHIFT;
1918 clidr |= CACHE_TYPE_DATA << CLIDR_CTYPE_SHIFT(1);
1922 * Instruction cache invalidation to the PoU is required so let L1 have
1923 * an instruction cache. If L1 already has a data cache, it will be
1924 * CACHE_TYPE_SEPARATE.
1926 if (!(ctr_el0 & CTR_EL0_DIC))
1927 clidr |= CACHE_TYPE_INST << CLIDR_CTYPE_SHIFT(1);
1929 clidr |= loc << CLIDR_LOC_SHIFT;
1932 * Add tag cache unified to data cache. Allocation tags and data are
1933 * unified in a cache line so that it looks valid even if there is only
1936 if (kvm_has_mte(vcpu->kvm))
1937 clidr |= 2 << CLIDR_TTYPE_SHIFT(loc);
1939 __vcpu_sys_reg(vcpu, r->reg) = clidr;
1941 return __vcpu_sys_reg(vcpu, r->reg);
1944 static int set_clidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *rd,
1947 u64 ctr_el0 = read_sanitised_ftr_reg(SYS_CTR_EL0);
1948 u64 idc = !CLIDR_LOC(val) || (!CLIDR_LOUIS(val) && !CLIDR_LOUU(val));
1950 if ((val & CLIDR_EL1_RES0) || (!(ctr_el0 & CTR_EL0_IDC) && idc))
1953 __vcpu_sys_reg(vcpu, rd->reg) = val;
1958 static bool access_csselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1959 const struct sys_reg_desc *r)
1964 vcpu_write_sys_reg(vcpu, p->regval, reg);
1966 p->regval = vcpu_read_sys_reg(vcpu, reg);
1970 static bool access_ccsidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p,
1971 const struct sys_reg_desc *r)
1976 return write_to_read_only(vcpu, p, r);
1978 csselr = vcpu_read_sys_reg(vcpu, CSSELR_EL1);
1979 csselr &= CSSELR_EL1_Level | CSSELR_EL1_InD;
1980 if (csselr < CSSELR_MAX)
1981 p->regval = get_ccsidr(vcpu, csselr);
1986 static unsigned int mte_visibility(const struct kvm_vcpu *vcpu,
1987 const struct sys_reg_desc *rd)
1989 if (kvm_has_mte(vcpu->kvm))
1995 #define MTE_REG(name) { \
1996 SYS_DESC(SYS_##name), \
1997 .access = undef_access, \
1998 .reset = reset_unknown, \
2000 .visibility = mte_visibility, \
2003 static unsigned int el2_visibility(const struct kvm_vcpu *vcpu,
2004 const struct sys_reg_desc *rd)
2006 if (vcpu_has_nv(vcpu))
2012 static bool bad_vncr_trap(struct kvm_vcpu *vcpu,
2013 struct sys_reg_params *p,
2014 const struct sys_reg_desc *r)
2017 * We really shouldn't be here, and this is likely the result
2018 * of a misconfigured trap, as this register should target the
2019 * VNCR page, and nothing else.
2021 return bad_trap(vcpu, p, r,
2022 "trap of VNCR-backed register");
2025 static bool bad_redir_trap(struct kvm_vcpu *vcpu,
2026 struct sys_reg_params *p,
2027 const struct sys_reg_desc *r)
2030 * We really shouldn't be here, and this is likely the result
2031 * of a misconfigured trap, as this register should target the
2032 * corresponding EL1, and nothing else.
2034 return bad_trap(vcpu, p, r,
2035 "trap of EL2 register redirected to EL1");
2038 #define EL2_REG(name, acc, rst, v) { \
2039 SYS_DESC(SYS_##name), \
2043 .visibility = el2_visibility, \
2047 #define EL2_REG_VNCR(name, rst, v) EL2_REG(name, bad_vncr_trap, rst, v)
2048 #define EL2_REG_REDIR(name, rst, v) EL2_REG(name, bad_redir_trap, rst, v)
2051 * EL{0,1}2 registers are the EL2 view on an EL0 or EL1 register when
2052 * HCR_EL2.E2H==1, and only in the sysreg table for convenience of
2053 * handling traps. Given that, they are always hidden from userspace.
2055 static unsigned int hidden_user_visibility(const struct kvm_vcpu *vcpu,
2056 const struct sys_reg_desc *rd)
2058 return REG_HIDDEN_USER;
2061 #define EL12_REG(name, acc, rst, v) { \
2062 SYS_DESC(SYS_##name##_EL12), \
2065 .reg = name##_EL1, \
2067 .visibility = hidden_user_visibility, \
2071 * Since reset() callback and field val are not used for idregs, they will be
2072 * used for specific purposes for idregs.
2073 * The reset() would return KVM sanitised register value. The value would be the
2074 * same as the host kernel sanitised value if there is no KVM sanitisation.
2075 * The val would be used as a mask indicating writable fields for the idreg.
2076 * Only bits with 1 are writable from userspace. This mask might not be
2077 * necessary in the future whenever all ID registers are enabled as writable
2081 #define ID_DESC(name) \
2082 SYS_DESC(SYS_##name), \
2083 .access = access_id_reg, \
2084 .get_user = get_id_reg \
2086 /* sys_reg_desc initialiser for known cpufeature ID registers */
2087 #define ID_SANITISED(name) { \
2089 .set_user = set_id_reg, \
2090 .visibility = id_visibility, \
2091 .reset = kvm_read_sanitised_id_reg, \
2095 /* sys_reg_desc initialiser for known cpufeature ID registers */
2096 #define AA32_ID_SANITISED(name) { \
2098 .set_user = set_id_reg, \
2099 .visibility = aa32_id_visibility, \
2100 .reset = kvm_read_sanitised_id_reg, \
2104 /* sys_reg_desc initialiser for writable ID registers */
2105 #define ID_WRITABLE(name, mask) { \
2107 .set_user = set_id_reg, \
2108 .visibility = id_visibility, \
2109 .reset = kvm_read_sanitised_id_reg, \
2114 * sys_reg_desc initialiser for architecturally unallocated cpufeature ID
2115 * register with encoding Op0=3, Op1=0, CRn=0, CRm=crm, Op2=op2
2116 * (1 <= crm < 8, 0 <= Op2 < 8).
2118 #define ID_UNALLOCATED(crm, op2) { \
2119 Op0(3), Op1(0), CRn(0), CRm(crm), Op2(op2), \
2120 .access = access_id_reg, \
2121 .get_user = get_id_reg, \
2122 .set_user = set_id_reg, \
2123 .visibility = raz_visibility, \
2124 .reset = kvm_read_sanitised_id_reg, \
2129 * sys_reg_desc initialiser for known ID registers that we hide from guests.
2130 * For now, these are exposed just like unallocated ID regs: they appear
2131 * RAZ for the guest.
2133 #define ID_HIDDEN(name) { \
2135 .set_user = set_id_reg, \
2136 .visibility = raz_visibility, \
2137 .reset = kvm_read_sanitised_id_reg, \
2141 static bool access_sp_el1(struct kvm_vcpu *vcpu,
2142 struct sys_reg_params *p,
2143 const struct sys_reg_desc *r)
2146 __vcpu_sys_reg(vcpu, SP_EL1) = p->regval;
2148 p->regval = __vcpu_sys_reg(vcpu, SP_EL1);
2153 static bool access_elr(struct kvm_vcpu *vcpu,
2154 struct sys_reg_params *p,
2155 const struct sys_reg_desc *r)
2158 vcpu_write_sys_reg(vcpu, p->regval, ELR_EL1);
2160 p->regval = vcpu_read_sys_reg(vcpu, ELR_EL1);
2165 static bool access_spsr(struct kvm_vcpu *vcpu,
2166 struct sys_reg_params *p,
2167 const struct sys_reg_desc *r)
2170 __vcpu_sys_reg(vcpu, SPSR_EL1) = p->regval;
2172 p->regval = __vcpu_sys_reg(vcpu, SPSR_EL1);
2178 * Architected system registers.
2179 * Important: Must be sorted ascending by Op0, Op1, CRn, CRm, Op2
2181 * Debug handling: We do trap most, if not all debug related system
2182 * registers. The implementation is good enough to ensure that a guest
2183 * can use these with minimal performance degradation. The drawback is
2184 * that we don't implement any of the external debug architecture.
2185 * This should be revisited if we ever encounter a more demanding
2188 static const struct sys_reg_desc sys_reg_descs[] = {
2189 { SYS_DESC(SYS_DC_ISW), access_dcsw },
2190 { SYS_DESC(SYS_DC_IGSW), access_dcgsw },
2191 { SYS_DESC(SYS_DC_IGDSW), access_dcgsw },
2192 { SYS_DESC(SYS_DC_CSW), access_dcsw },
2193 { SYS_DESC(SYS_DC_CGSW), access_dcgsw },
2194 { SYS_DESC(SYS_DC_CGDSW), access_dcgsw },
2195 { SYS_DESC(SYS_DC_CISW), access_dcsw },
2196 { SYS_DESC(SYS_DC_CIGSW), access_dcgsw },
2197 { SYS_DESC(SYS_DC_CIGDSW), access_dcgsw },
2199 DBG_BCR_BVR_WCR_WVR_EL1(0),
2200 DBG_BCR_BVR_WCR_WVR_EL1(1),
2201 { SYS_DESC(SYS_MDCCINT_EL1), trap_debug_regs, reset_val, MDCCINT_EL1, 0 },
2202 { SYS_DESC(SYS_MDSCR_EL1), trap_debug_regs, reset_val, MDSCR_EL1, 0 },
2203 DBG_BCR_BVR_WCR_WVR_EL1(2),
2204 DBG_BCR_BVR_WCR_WVR_EL1(3),
2205 DBG_BCR_BVR_WCR_WVR_EL1(4),
2206 DBG_BCR_BVR_WCR_WVR_EL1(5),
2207 DBG_BCR_BVR_WCR_WVR_EL1(6),
2208 DBG_BCR_BVR_WCR_WVR_EL1(7),
2209 DBG_BCR_BVR_WCR_WVR_EL1(8),
2210 DBG_BCR_BVR_WCR_WVR_EL1(9),
2211 DBG_BCR_BVR_WCR_WVR_EL1(10),
2212 DBG_BCR_BVR_WCR_WVR_EL1(11),
2213 DBG_BCR_BVR_WCR_WVR_EL1(12),
2214 DBG_BCR_BVR_WCR_WVR_EL1(13),
2215 DBG_BCR_BVR_WCR_WVR_EL1(14),
2216 DBG_BCR_BVR_WCR_WVR_EL1(15),
2218 { SYS_DESC(SYS_MDRAR_EL1), trap_raz_wi },
2219 { SYS_DESC(SYS_OSLAR_EL1), trap_oslar_el1 },
2220 { SYS_DESC(SYS_OSLSR_EL1), trap_oslsr_el1, reset_val, OSLSR_EL1,
2221 OSLSR_EL1_OSLM_IMPLEMENTED, .set_user = set_oslsr_el1, },
2222 { SYS_DESC(SYS_OSDLR_EL1), trap_raz_wi },
2223 { SYS_DESC(SYS_DBGPRCR_EL1), trap_raz_wi },
2224 { SYS_DESC(SYS_DBGCLAIMSET_EL1), trap_raz_wi },
2225 { SYS_DESC(SYS_DBGCLAIMCLR_EL1), trap_raz_wi },
2226 { SYS_DESC(SYS_DBGAUTHSTATUS_EL1), trap_dbgauthstatus_el1 },
2228 { SYS_DESC(SYS_MDCCSR_EL0), trap_raz_wi },
2229 { SYS_DESC(SYS_DBGDTR_EL0), trap_raz_wi },
2230 // DBGDTR[TR]X_EL0 share the same encoding
2231 { SYS_DESC(SYS_DBGDTRTX_EL0), trap_raz_wi },
2233 { SYS_DESC(SYS_DBGVCR32_EL2), trap_undef, reset_val, DBGVCR32_EL2, 0 },
2235 { SYS_DESC(SYS_MPIDR_EL1), NULL, reset_mpidr, MPIDR_EL1 },
2238 * ID regs: all ID_SANITISED() entries here must have corresponding
2239 * entries in arm64_ftr_regs[].
2242 /* AArch64 mappings of the AArch32 ID registers */
2244 AA32_ID_SANITISED(ID_PFR0_EL1),
2245 AA32_ID_SANITISED(ID_PFR1_EL1),
2246 { SYS_DESC(SYS_ID_DFR0_EL1),
2247 .access = access_id_reg,
2248 .get_user = get_id_reg,
2249 .set_user = set_id_dfr0_el1,
2250 .visibility = aa32_id_visibility,
2251 .reset = read_sanitised_id_dfr0_el1,
2252 .val = ID_DFR0_EL1_PerfMon_MASK |
2253 ID_DFR0_EL1_CopDbg_MASK, },
2254 ID_HIDDEN(ID_AFR0_EL1),
2255 AA32_ID_SANITISED(ID_MMFR0_EL1),
2256 AA32_ID_SANITISED(ID_MMFR1_EL1),
2257 AA32_ID_SANITISED(ID_MMFR2_EL1),
2258 AA32_ID_SANITISED(ID_MMFR3_EL1),
2261 AA32_ID_SANITISED(ID_ISAR0_EL1),
2262 AA32_ID_SANITISED(ID_ISAR1_EL1),
2263 AA32_ID_SANITISED(ID_ISAR2_EL1),
2264 AA32_ID_SANITISED(ID_ISAR3_EL1),
2265 AA32_ID_SANITISED(ID_ISAR4_EL1),
2266 AA32_ID_SANITISED(ID_ISAR5_EL1),
2267 AA32_ID_SANITISED(ID_MMFR4_EL1),
2268 AA32_ID_SANITISED(ID_ISAR6_EL1),
2271 AA32_ID_SANITISED(MVFR0_EL1),
2272 AA32_ID_SANITISED(MVFR1_EL1),
2273 AA32_ID_SANITISED(MVFR2_EL1),
2274 ID_UNALLOCATED(3,3),
2275 AA32_ID_SANITISED(ID_PFR2_EL1),
2276 ID_HIDDEN(ID_DFR1_EL1),
2277 AA32_ID_SANITISED(ID_MMFR5_EL1),
2278 ID_UNALLOCATED(3,7),
2280 /* AArch64 ID registers */
2282 { SYS_DESC(SYS_ID_AA64PFR0_EL1),
2283 .access = access_id_reg,
2284 .get_user = get_id_reg,
2285 .set_user = set_id_reg,
2286 .reset = read_sanitised_id_aa64pfr0_el1,
2287 .val = ~(ID_AA64PFR0_EL1_AMU |
2288 ID_AA64PFR0_EL1_MPAM |
2289 ID_AA64PFR0_EL1_SVE |
2290 ID_AA64PFR0_EL1_RAS |
2291 ID_AA64PFR0_EL1_GIC |
2292 ID_AA64PFR0_EL1_AdvSIMD |
2293 ID_AA64PFR0_EL1_FP), },
2294 ID_SANITISED(ID_AA64PFR1_EL1),
2295 ID_UNALLOCATED(4,2),
2296 ID_UNALLOCATED(4,3),
2297 ID_WRITABLE(ID_AA64ZFR0_EL1, ~ID_AA64ZFR0_EL1_RES0),
2298 ID_HIDDEN(ID_AA64SMFR0_EL1),
2299 ID_UNALLOCATED(4,6),
2300 ID_UNALLOCATED(4,7),
2303 { SYS_DESC(SYS_ID_AA64DFR0_EL1),
2304 .access = access_id_reg,
2305 .get_user = get_id_reg,
2306 .set_user = set_id_aa64dfr0_el1,
2307 .reset = read_sanitised_id_aa64dfr0_el1,
2308 .val = ID_AA64DFR0_EL1_PMUVer_MASK |
2309 ID_AA64DFR0_EL1_DebugVer_MASK, },
2310 ID_SANITISED(ID_AA64DFR1_EL1),
2311 ID_UNALLOCATED(5,2),
2312 ID_UNALLOCATED(5,3),
2313 ID_HIDDEN(ID_AA64AFR0_EL1),
2314 ID_HIDDEN(ID_AA64AFR1_EL1),
2315 ID_UNALLOCATED(5,6),
2316 ID_UNALLOCATED(5,7),
2319 ID_WRITABLE(ID_AA64ISAR0_EL1, ~ID_AA64ISAR0_EL1_RES0),
2320 ID_WRITABLE(ID_AA64ISAR1_EL1, ~(ID_AA64ISAR1_EL1_GPI |
2321 ID_AA64ISAR1_EL1_GPA |
2322 ID_AA64ISAR1_EL1_API |
2323 ID_AA64ISAR1_EL1_APA)),
2324 ID_WRITABLE(ID_AA64ISAR2_EL1, ~(ID_AA64ISAR2_EL1_RES0 |
2325 ID_AA64ISAR2_EL1_APA3 |
2326 ID_AA64ISAR2_EL1_GPA3)),
2327 ID_UNALLOCATED(6,3),
2328 ID_UNALLOCATED(6,4),
2329 ID_UNALLOCATED(6,5),
2330 ID_UNALLOCATED(6,6),
2331 ID_UNALLOCATED(6,7),
2334 ID_WRITABLE(ID_AA64MMFR0_EL1, ~(ID_AA64MMFR0_EL1_RES0 |
2335 ID_AA64MMFR0_EL1_TGRAN4_2 |
2336 ID_AA64MMFR0_EL1_TGRAN64_2 |
2337 ID_AA64MMFR0_EL1_TGRAN16_2)),
2338 ID_WRITABLE(ID_AA64MMFR1_EL1, ~(ID_AA64MMFR1_EL1_RES0 |
2339 ID_AA64MMFR1_EL1_HCX |
2340 ID_AA64MMFR1_EL1_XNX |
2341 ID_AA64MMFR1_EL1_TWED |
2342 ID_AA64MMFR1_EL1_XNX |
2343 ID_AA64MMFR1_EL1_VH |
2344 ID_AA64MMFR1_EL1_VMIDBits)),
2345 ID_WRITABLE(ID_AA64MMFR2_EL1, ~(ID_AA64MMFR2_EL1_RES0 |
2346 ID_AA64MMFR2_EL1_EVT |
2347 ID_AA64MMFR2_EL1_FWB |
2348 ID_AA64MMFR2_EL1_IDS |
2349 ID_AA64MMFR2_EL1_NV |
2350 ID_AA64MMFR2_EL1_CCIDX)),
2351 ID_SANITISED(ID_AA64MMFR3_EL1),
2352 ID_UNALLOCATED(7,4),
2353 ID_UNALLOCATED(7,5),
2354 ID_UNALLOCATED(7,6),
2355 ID_UNALLOCATED(7,7),
2357 { SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
2358 { SYS_DESC(SYS_ACTLR_EL1), access_actlr, reset_actlr, ACTLR_EL1 },
2359 { SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },
2364 { SYS_DESC(SYS_ZCR_EL1), NULL, reset_val, ZCR_EL1, 0, .visibility = sve_visibility },
2365 { SYS_DESC(SYS_TRFCR_EL1), undef_access },
2366 { SYS_DESC(SYS_SMPRI_EL1), undef_access },
2367 { SYS_DESC(SYS_SMCR_EL1), undef_access },
2368 { SYS_DESC(SYS_TTBR0_EL1), access_vm_reg, reset_unknown, TTBR0_EL1 },
2369 { SYS_DESC(SYS_TTBR1_EL1), access_vm_reg, reset_unknown, TTBR1_EL1 },
2370 { SYS_DESC(SYS_TCR_EL1), access_vm_reg, reset_val, TCR_EL1, 0 },
2371 { SYS_DESC(SYS_TCR2_EL1), access_vm_reg, reset_val, TCR2_EL1, 0 },
2379 { SYS_DESC(SYS_SPSR_EL1), access_spsr},
2380 { SYS_DESC(SYS_ELR_EL1), access_elr},
2382 { SYS_DESC(SYS_AFSR0_EL1), access_vm_reg, reset_unknown, AFSR0_EL1 },
2383 { SYS_DESC(SYS_AFSR1_EL1), access_vm_reg, reset_unknown, AFSR1_EL1 },
2384 { SYS_DESC(SYS_ESR_EL1), access_vm_reg, reset_unknown, ESR_EL1 },
2386 { SYS_DESC(SYS_ERRIDR_EL1), trap_raz_wi },
2387 { SYS_DESC(SYS_ERRSELR_EL1), trap_raz_wi },
2388 { SYS_DESC(SYS_ERXFR_EL1), trap_raz_wi },
2389 { SYS_DESC(SYS_ERXCTLR_EL1), trap_raz_wi },
2390 { SYS_DESC(SYS_ERXSTATUS_EL1), trap_raz_wi },
2391 { SYS_DESC(SYS_ERXADDR_EL1), trap_raz_wi },
2392 { SYS_DESC(SYS_ERXMISC0_EL1), trap_raz_wi },
2393 { SYS_DESC(SYS_ERXMISC1_EL1), trap_raz_wi },
2396 MTE_REG(TFSRE0_EL1),
2398 { SYS_DESC(SYS_FAR_EL1), access_vm_reg, reset_unknown, FAR_EL1 },
2399 { SYS_DESC(SYS_PAR_EL1), NULL, reset_unknown, PAR_EL1 },
2401 { SYS_DESC(SYS_PMSCR_EL1), undef_access },
2402 { SYS_DESC(SYS_PMSNEVFR_EL1), undef_access },
2403 { SYS_DESC(SYS_PMSICR_EL1), undef_access },
2404 { SYS_DESC(SYS_PMSIRR_EL1), undef_access },
2405 { SYS_DESC(SYS_PMSFCR_EL1), undef_access },
2406 { SYS_DESC(SYS_PMSEVFR_EL1), undef_access },
2407 { SYS_DESC(SYS_PMSLATFR_EL1), undef_access },
2408 { SYS_DESC(SYS_PMSIDR_EL1), undef_access },
2409 { SYS_DESC(SYS_PMBLIMITR_EL1), undef_access },
2410 { SYS_DESC(SYS_PMBPTR_EL1), undef_access },
2411 { SYS_DESC(SYS_PMBSR_EL1), undef_access },
2412 /* PMBIDR_EL1 is not trapped */
2414 { PMU_SYS_REG(PMINTENSET_EL1),
2415 .access = access_pminten, .reg = PMINTENSET_EL1,
2416 .get_user = get_pmreg, .set_user = set_pmreg },
2417 { PMU_SYS_REG(PMINTENCLR_EL1),
2418 .access = access_pminten, .reg = PMINTENSET_EL1,
2419 .get_user = get_pmreg, .set_user = set_pmreg },
2420 { SYS_DESC(SYS_PMMIR_EL1), trap_raz_wi },
2422 { SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 },
2423 { SYS_DESC(SYS_PIRE0_EL1), NULL, reset_unknown, PIRE0_EL1 },
2424 { SYS_DESC(SYS_PIR_EL1), NULL, reset_unknown, PIR_EL1 },
2425 { SYS_DESC(SYS_AMAIR_EL1), access_vm_reg, reset_amair_el1, AMAIR_EL1 },
2427 { SYS_DESC(SYS_LORSA_EL1), trap_loregion },
2428 { SYS_DESC(SYS_LOREA_EL1), trap_loregion },
2429 { SYS_DESC(SYS_LORN_EL1), trap_loregion },
2430 { SYS_DESC(SYS_LORC_EL1), trap_loregion },
2431 { SYS_DESC(SYS_LORID_EL1), trap_loregion },
2433 { SYS_DESC(SYS_VBAR_EL1), access_rw, reset_val, VBAR_EL1, 0 },
2434 { SYS_DESC(SYS_DISR_EL1), NULL, reset_val, DISR_EL1, 0 },
2436 { SYS_DESC(SYS_ICC_IAR0_EL1), write_to_read_only },
2437 { SYS_DESC(SYS_ICC_EOIR0_EL1), read_from_write_only },
2438 { SYS_DESC(SYS_ICC_HPPIR0_EL1), write_to_read_only },
2439 { SYS_DESC(SYS_ICC_DIR_EL1), read_from_write_only },
2440 { SYS_DESC(SYS_ICC_RPR_EL1), write_to_read_only },
2441 { SYS_DESC(SYS_ICC_SGI1R_EL1), access_gic_sgi },
2442 { SYS_DESC(SYS_ICC_ASGI1R_EL1), access_gic_sgi },
2443 { SYS_DESC(SYS_ICC_SGI0R_EL1), access_gic_sgi },
2444 { SYS_DESC(SYS_ICC_IAR1_EL1), write_to_read_only },
2445 { SYS_DESC(SYS_ICC_EOIR1_EL1), read_from_write_only },
2446 { SYS_DESC(SYS_ICC_HPPIR1_EL1), write_to_read_only },
2447 { SYS_DESC(SYS_ICC_SRE_EL1), access_gic_sre },
2449 { SYS_DESC(SYS_CONTEXTIDR_EL1), access_vm_reg, reset_val, CONTEXTIDR_EL1, 0 },
2450 { SYS_DESC(SYS_TPIDR_EL1), NULL, reset_unknown, TPIDR_EL1 },
2452 { SYS_DESC(SYS_ACCDATA_EL1), undef_access },
2454 { SYS_DESC(SYS_SCXTNUM_EL1), undef_access },
2456 { SYS_DESC(SYS_CNTKCTL_EL1), NULL, reset_val, CNTKCTL_EL1, 0},
2458 { SYS_DESC(SYS_CCSIDR_EL1), access_ccsidr },
2459 { SYS_DESC(SYS_CLIDR_EL1), access_clidr, reset_clidr, CLIDR_EL1,
2460 .set_user = set_clidr },
2461 { SYS_DESC(SYS_CCSIDR2_EL1), undef_access },
2462 { SYS_DESC(SYS_SMIDR_EL1), undef_access },
2463 { SYS_DESC(SYS_CSSELR_EL1), access_csselr, reset_unknown, CSSELR_EL1 },
2464 { SYS_DESC(SYS_CTR_EL0), access_ctr },
2465 { SYS_DESC(SYS_SVCR), undef_access },
2467 { PMU_SYS_REG(PMCR_EL0), .access = access_pmcr, .reset = reset_pmcr,
2468 .reg = PMCR_EL0, .get_user = get_pmcr, .set_user = set_pmcr },
2469 { PMU_SYS_REG(PMCNTENSET_EL0),
2470 .access = access_pmcnten, .reg = PMCNTENSET_EL0,
2471 .get_user = get_pmreg, .set_user = set_pmreg },
2472 { PMU_SYS_REG(PMCNTENCLR_EL0),
2473 .access = access_pmcnten, .reg = PMCNTENSET_EL0,
2474 .get_user = get_pmreg, .set_user = set_pmreg },
2475 { PMU_SYS_REG(PMOVSCLR_EL0),
2476 .access = access_pmovs, .reg = PMOVSSET_EL0,
2477 .get_user = get_pmreg, .set_user = set_pmreg },
2479 * PM_SWINC_EL0 is exposed to userspace as RAZ/WI, as it was
2480 * previously (and pointlessly) advertised in the past...
2482 { PMU_SYS_REG(PMSWINC_EL0),
2483 .get_user = get_raz_reg, .set_user = set_wi_reg,
2484 .access = access_pmswinc, .reset = NULL },
2485 { PMU_SYS_REG(PMSELR_EL0),
2486 .access = access_pmselr, .reset = reset_pmselr, .reg = PMSELR_EL0 },
2487 { PMU_SYS_REG(PMCEID0_EL0),
2488 .access = access_pmceid, .reset = NULL },
2489 { PMU_SYS_REG(PMCEID1_EL0),
2490 .access = access_pmceid, .reset = NULL },
2491 { PMU_SYS_REG(PMCCNTR_EL0),
2492 .access = access_pmu_evcntr, .reset = reset_unknown,
2493 .reg = PMCCNTR_EL0, .get_user = get_pmu_evcntr},
2494 { PMU_SYS_REG(PMXEVTYPER_EL0),
2495 .access = access_pmu_evtyper, .reset = NULL },
2496 { PMU_SYS_REG(PMXEVCNTR_EL0),
2497 .access = access_pmu_evcntr, .reset = NULL },
2499 * PMUSERENR_EL0 resets as unknown in 64bit mode while it resets as zero
2500 * in 32bit mode. Here we choose to reset it as zero for consistency.
2502 { PMU_SYS_REG(PMUSERENR_EL0), .access = access_pmuserenr,
2503 .reset = reset_val, .reg = PMUSERENR_EL0, .val = 0 },
2504 { PMU_SYS_REG(PMOVSSET_EL0),
2505 .access = access_pmovs, .reg = PMOVSSET_EL0,
2506 .get_user = get_pmreg, .set_user = set_pmreg },
2508 { SYS_DESC(SYS_TPIDR_EL0), NULL, reset_unknown, TPIDR_EL0 },
2509 { SYS_DESC(SYS_TPIDRRO_EL0), NULL, reset_unknown, TPIDRRO_EL0 },
2510 { SYS_DESC(SYS_TPIDR2_EL0), undef_access },
2512 { SYS_DESC(SYS_SCXTNUM_EL0), undef_access },
2514 { SYS_DESC(SYS_AMCR_EL0), undef_access },
2515 { SYS_DESC(SYS_AMCFGR_EL0), undef_access },
2516 { SYS_DESC(SYS_AMCGCR_EL0), undef_access },
2517 { SYS_DESC(SYS_AMUSERENR_EL0), undef_access },
2518 { SYS_DESC(SYS_AMCNTENCLR0_EL0), undef_access },
2519 { SYS_DESC(SYS_AMCNTENSET0_EL0), undef_access },
2520 { SYS_DESC(SYS_AMCNTENCLR1_EL0), undef_access },
2521 { SYS_DESC(SYS_AMCNTENSET1_EL0), undef_access },
2522 AMU_AMEVCNTR0_EL0(0),
2523 AMU_AMEVCNTR0_EL0(1),
2524 AMU_AMEVCNTR0_EL0(2),
2525 AMU_AMEVCNTR0_EL0(3),
2526 AMU_AMEVCNTR0_EL0(4),
2527 AMU_AMEVCNTR0_EL0(5),
2528 AMU_AMEVCNTR0_EL0(6),
2529 AMU_AMEVCNTR0_EL0(7),
2530 AMU_AMEVCNTR0_EL0(8),
2531 AMU_AMEVCNTR0_EL0(9),
2532 AMU_AMEVCNTR0_EL0(10),
2533 AMU_AMEVCNTR0_EL0(11),
2534 AMU_AMEVCNTR0_EL0(12),
2535 AMU_AMEVCNTR0_EL0(13),
2536 AMU_AMEVCNTR0_EL0(14),
2537 AMU_AMEVCNTR0_EL0(15),
2538 AMU_AMEVTYPER0_EL0(0),
2539 AMU_AMEVTYPER0_EL0(1),
2540 AMU_AMEVTYPER0_EL0(2),
2541 AMU_AMEVTYPER0_EL0(3),
2542 AMU_AMEVTYPER0_EL0(4),
2543 AMU_AMEVTYPER0_EL0(5),
2544 AMU_AMEVTYPER0_EL0(6),
2545 AMU_AMEVTYPER0_EL0(7),
2546 AMU_AMEVTYPER0_EL0(8),
2547 AMU_AMEVTYPER0_EL0(9),
2548 AMU_AMEVTYPER0_EL0(10),
2549 AMU_AMEVTYPER0_EL0(11),
2550 AMU_AMEVTYPER0_EL0(12),
2551 AMU_AMEVTYPER0_EL0(13),
2552 AMU_AMEVTYPER0_EL0(14),
2553 AMU_AMEVTYPER0_EL0(15),
2554 AMU_AMEVCNTR1_EL0(0),
2555 AMU_AMEVCNTR1_EL0(1),
2556 AMU_AMEVCNTR1_EL0(2),
2557 AMU_AMEVCNTR1_EL0(3),
2558 AMU_AMEVCNTR1_EL0(4),
2559 AMU_AMEVCNTR1_EL0(5),
2560 AMU_AMEVCNTR1_EL0(6),
2561 AMU_AMEVCNTR1_EL0(7),
2562 AMU_AMEVCNTR1_EL0(8),
2563 AMU_AMEVCNTR1_EL0(9),
2564 AMU_AMEVCNTR1_EL0(10),
2565 AMU_AMEVCNTR1_EL0(11),
2566 AMU_AMEVCNTR1_EL0(12),
2567 AMU_AMEVCNTR1_EL0(13),
2568 AMU_AMEVCNTR1_EL0(14),
2569 AMU_AMEVCNTR1_EL0(15),
2570 AMU_AMEVTYPER1_EL0(0),
2571 AMU_AMEVTYPER1_EL0(1),
2572 AMU_AMEVTYPER1_EL0(2),
2573 AMU_AMEVTYPER1_EL0(3),
2574 AMU_AMEVTYPER1_EL0(4),
2575 AMU_AMEVTYPER1_EL0(5),
2576 AMU_AMEVTYPER1_EL0(6),
2577 AMU_AMEVTYPER1_EL0(7),
2578 AMU_AMEVTYPER1_EL0(8),
2579 AMU_AMEVTYPER1_EL0(9),
2580 AMU_AMEVTYPER1_EL0(10),
2581 AMU_AMEVTYPER1_EL0(11),
2582 AMU_AMEVTYPER1_EL0(12),
2583 AMU_AMEVTYPER1_EL0(13),
2584 AMU_AMEVTYPER1_EL0(14),
2585 AMU_AMEVTYPER1_EL0(15),
2587 { SYS_DESC(SYS_CNTPCT_EL0), access_arch_timer },
2588 { SYS_DESC(SYS_CNTPCTSS_EL0), access_arch_timer },
2589 { SYS_DESC(SYS_CNTP_TVAL_EL0), access_arch_timer },
2590 { SYS_DESC(SYS_CNTP_CTL_EL0), access_arch_timer },
2591 { SYS_DESC(SYS_CNTP_CVAL_EL0), access_arch_timer },
2594 PMU_PMEVCNTR_EL0(0),
2595 PMU_PMEVCNTR_EL0(1),
2596 PMU_PMEVCNTR_EL0(2),
2597 PMU_PMEVCNTR_EL0(3),
2598 PMU_PMEVCNTR_EL0(4),
2599 PMU_PMEVCNTR_EL0(5),
2600 PMU_PMEVCNTR_EL0(6),
2601 PMU_PMEVCNTR_EL0(7),
2602 PMU_PMEVCNTR_EL0(8),
2603 PMU_PMEVCNTR_EL0(9),
2604 PMU_PMEVCNTR_EL0(10),
2605 PMU_PMEVCNTR_EL0(11),
2606 PMU_PMEVCNTR_EL0(12),
2607 PMU_PMEVCNTR_EL0(13),
2608 PMU_PMEVCNTR_EL0(14),
2609 PMU_PMEVCNTR_EL0(15),
2610 PMU_PMEVCNTR_EL0(16),
2611 PMU_PMEVCNTR_EL0(17),
2612 PMU_PMEVCNTR_EL0(18),
2613 PMU_PMEVCNTR_EL0(19),
2614 PMU_PMEVCNTR_EL0(20),
2615 PMU_PMEVCNTR_EL0(21),
2616 PMU_PMEVCNTR_EL0(22),
2617 PMU_PMEVCNTR_EL0(23),
2618 PMU_PMEVCNTR_EL0(24),
2619 PMU_PMEVCNTR_EL0(25),
2620 PMU_PMEVCNTR_EL0(26),
2621 PMU_PMEVCNTR_EL0(27),
2622 PMU_PMEVCNTR_EL0(28),
2623 PMU_PMEVCNTR_EL0(29),
2624 PMU_PMEVCNTR_EL0(30),
2625 /* PMEVTYPERn_EL0 */
2626 PMU_PMEVTYPER_EL0(0),
2627 PMU_PMEVTYPER_EL0(1),
2628 PMU_PMEVTYPER_EL0(2),
2629 PMU_PMEVTYPER_EL0(3),
2630 PMU_PMEVTYPER_EL0(4),
2631 PMU_PMEVTYPER_EL0(5),
2632 PMU_PMEVTYPER_EL0(6),
2633 PMU_PMEVTYPER_EL0(7),
2634 PMU_PMEVTYPER_EL0(8),
2635 PMU_PMEVTYPER_EL0(9),
2636 PMU_PMEVTYPER_EL0(10),
2637 PMU_PMEVTYPER_EL0(11),
2638 PMU_PMEVTYPER_EL0(12),
2639 PMU_PMEVTYPER_EL0(13),
2640 PMU_PMEVTYPER_EL0(14),
2641 PMU_PMEVTYPER_EL0(15),
2642 PMU_PMEVTYPER_EL0(16),
2643 PMU_PMEVTYPER_EL0(17),
2644 PMU_PMEVTYPER_EL0(18),
2645 PMU_PMEVTYPER_EL0(19),
2646 PMU_PMEVTYPER_EL0(20),
2647 PMU_PMEVTYPER_EL0(21),
2648 PMU_PMEVTYPER_EL0(22),
2649 PMU_PMEVTYPER_EL0(23),
2650 PMU_PMEVTYPER_EL0(24),
2651 PMU_PMEVTYPER_EL0(25),
2652 PMU_PMEVTYPER_EL0(26),
2653 PMU_PMEVTYPER_EL0(27),
2654 PMU_PMEVTYPER_EL0(28),
2655 PMU_PMEVTYPER_EL0(29),
2656 PMU_PMEVTYPER_EL0(30),
2658 * PMCCFILTR_EL0 resets as unknown in 64bit mode while it resets as zero
2659 * in 32bit mode. Here we choose to reset it as zero for consistency.
2661 { PMU_SYS_REG(PMCCFILTR_EL0), .access = access_pmu_evtyper,
2662 .reset = reset_val, .reg = PMCCFILTR_EL0, .val = 0 },
2664 EL2_REG_VNCR(VPIDR_EL2, reset_unknown, 0),
2665 EL2_REG_VNCR(VMPIDR_EL2, reset_unknown, 0),
2666 EL2_REG(SCTLR_EL2, access_rw, reset_val, SCTLR_EL2_RES1),
2667 EL2_REG(ACTLR_EL2, access_rw, reset_val, 0),
2668 EL2_REG_VNCR(HCR_EL2, reset_val, 0),
2669 EL2_REG(MDCR_EL2, access_rw, reset_val, 0),
2670 EL2_REG(CPTR_EL2, access_rw, reset_val, CPTR_NVHE_EL2_RES1),
2671 EL2_REG_VNCR(HSTR_EL2, reset_val, 0),
2672 EL2_REG_VNCR(HFGRTR_EL2, reset_val, 0),
2673 EL2_REG_VNCR(HFGWTR_EL2, reset_val, 0),
2674 EL2_REG_VNCR(HFGITR_EL2, reset_val, 0),
2675 EL2_REG_VNCR(HACR_EL2, reset_val, 0),
2677 EL2_REG_VNCR(HCRX_EL2, reset_val, 0),
2679 EL2_REG(TTBR0_EL2, access_rw, reset_val, 0),
2680 EL2_REG(TTBR1_EL2, access_rw, reset_val, 0),
2681 EL2_REG(TCR_EL2, access_rw, reset_val, TCR_EL2_RES1),
2682 EL2_REG_VNCR(VTTBR_EL2, reset_val, 0),
2683 EL2_REG_VNCR(VTCR_EL2, reset_val, 0),
2685 { SYS_DESC(SYS_DACR32_EL2), trap_undef, reset_unknown, DACR32_EL2 },
2686 EL2_REG_VNCR(HDFGRTR_EL2, reset_val, 0),
2687 EL2_REG_VNCR(HDFGWTR_EL2, reset_val, 0),
2688 EL2_REG_VNCR(HAFGRTR_EL2, reset_val, 0),
2689 EL2_REG_REDIR(SPSR_EL2, reset_val, 0),
2690 EL2_REG_REDIR(ELR_EL2, reset_val, 0),
2691 { SYS_DESC(SYS_SP_EL1), access_sp_el1},
2693 /* AArch32 SPSR_* are RES0 if trapped from a NV guest */
2694 { SYS_DESC(SYS_SPSR_irq), .access = trap_raz_wi,
2695 .visibility = hidden_user_visibility },
2696 { SYS_DESC(SYS_SPSR_abt), .access = trap_raz_wi,
2697 .visibility = hidden_user_visibility },
2698 { SYS_DESC(SYS_SPSR_und), .access = trap_raz_wi,
2699 .visibility = hidden_user_visibility },
2700 { SYS_DESC(SYS_SPSR_fiq), .access = trap_raz_wi,
2701 .visibility = hidden_user_visibility },
2703 { SYS_DESC(SYS_IFSR32_EL2), trap_undef, reset_unknown, IFSR32_EL2 },
2704 EL2_REG(AFSR0_EL2, access_rw, reset_val, 0),
2705 EL2_REG(AFSR1_EL2, access_rw, reset_val, 0),
2706 EL2_REG_REDIR(ESR_EL2, reset_val, 0),
2707 { SYS_DESC(SYS_FPEXC32_EL2), trap_undef, reset_val, FPEXC32_EL2, 0x700 },
2709 EL2_REG_REDIR(FAR_EL2, reset_val, 0),
2710 EL2_REG(HPFAR_EL2, access_rw, reset_val, 0),
2712 EL2_REG(MAIR_EL2, access_rw, reset_val, 0),
2713 EL2_REG(AMAIR_EL2, access_rw, reset_val, 0),
2715 EL2_REG(VBAR_EL2, access_rw, reset_val, 0),
2716 EL2_REG(RVBAR_EL2, access_rw, reset_val, 0),
2717 { SYS_DESC(SYS_RMR_EL2), trap_undef },
2719 EL2_REG(CONTEXTIDR_EL2, access_rw, reset_val, 0),
2720 EL2_REG(TPIDR_EL2, access_rw, reset_val, 0),
2722 EL2_REG_VNCR(CNTVOFF_EL2, reset_val, 0),
2723 EL2_REG(CNTHCTL_EL2, access_rw, reset_val, 0),
2725 EL12_REG(CNTKCTL, access_rw, reset_val, 0),
2727 EL2_REG(SP_EL2, NULL, reset_unknown, 0),
2730 static const struct sys_reg_desc *first_idreg;
2732 static bool trap_dbgdidr(struct kvm_vcpu *vcpu,
2733 struct sys_reg_params *p,
2734 const struct sys_reg_desc *r)
2737 return ignore_write(vcpu, p);
2739 u64 dfr = IDREG(vcpu->kvm, SYS_ID_AA64DFR0_EL1);
2740 u64 pfr = IDREG(vcpu->kvm, SYS_ID_AA64PFR0_EL1);
2741 u32 el3 = !!SYS_FIELD_GET(ID_AA64PFR0_EL1, EL3, pfr);
2743 p->regval = ((SYS_FIELD_GET(ID_AA64DFR0_EL1, WRPs, dfr) << 28) |
2744 (SYS_FIELD_GET(ID_AA64DFR0_EL1, BRPs, dfr) << 24) |
2745 (SYS_FIELD_GET(ID_AA64DFR0_EL1, CTX_CMPs, dfr) << 20) |
2746 (SYS_FIELD_GET(ID_AA64DFR0_EL1, DebugVer, dfr) << 16) |
2747 (1 << 15) | (el3 << 14) | (el3 << 12));
2753 * AArch32 debug register mappings
2755 * AArch32 DBGBVRn is mapped to DBGBVRn_EL1[31:0]
2756 * AArch32 DBGBXVRn is mapped to DBGBVRn_EL1[63:32]
2758 * None of the other registers share their location, so treat them as
2759 * if they were 64bit.
2761 #define DBG_BCR_BVR_WCR_WVR(n) \
2763 { AA32(LO), Op1( 0), CRn( 0), CRm((n)), Op2( 4), trap_bvr, NULL, n }, \
2765 { Op1( 0), CRn( 0), CRm((n)), Op2( 5), trap_bcr, NULL, n }, \
2767 { Op1( 0), CRn( 0), CRm((n)), Op2( 6), trap_wvr, NULL, n }, \
2769 { Op1( 0), CRn( 0), CRm((n)), Op2( 7), trap_wcr, NULL, n }
2771 #define DBGBXVR(n) \
2772 { AA32(HI), Op1( 0), CRn( 1), CRm((n)), Op2( 1), trap_bvr, NULL, n }
2775 * Trapped cp14 registers. We generally ignore most of the external
2776 * debug, on the principle that they don't really make sense to a
2777 * guest. Revisit this one day, would this principle change.
2779 static const struct sys_reg_desc cp14_regs[] = {
2781 { Op1( 0), CRn( 0), CRm( 0), Op2( 0), trap_dbgdidr },
2783 { Op1( 0), CRn( 0), CRm( 0), Op2( 2), trap_raz_wi },
2785 DBG_BCR_BVR_WCR_WVR(0),
2787 { Op1( 0), CRn( 0), CRm( 1), Op2( 0), trap_raz_wi },
2788 DBG_BCR_BVR_WCR_WVR(1),
2790 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), trap_debug_regs, NULL, MDCCINT_EL1 },
2792 { Op1( 0), CRn( 0), CRm( 2), Op2( 2), trap_debug_regs, NULL, MDSCR_EL1 },
2793 DBG_BCR_BVR_WCR_WVR(2),
2794 /* DBGDTR[RT]Xint */
2795 { Op1( 0), CRn( 0), CRm( 3), Op2( 0), trap_raz_wi },
2796 /* DBGDTR[RT]Xext */
2797 { Op1( 0), CRn( 0), CRm( 3), Op2( 2), trap_raz_wi },
2798 DBG_BCR_BVR_WCR_WVR(3),
2799 DBG_BCR_BVR_WCR_WVR(4),
2800 DBG_BCR_BVR_WCR_WVR(5),
2802 { Op1( 0), CRn( 0), CRm( 6), Op2( 0), trap_raz_wi },
2804 { Op1( 0), CRn( 0), CRm( 6), Op2( 2), trap_raz_wi },
2805 DBG_BCR_BVR_WCR_WVR(6),
2807 { Op1( 0), CRn( 0), CRm( 7), Op2( 0), trap_debug_regs, NULL, DBGVCR32_EL2 },
2808 DBG_BCR_BVR_WCR_WVR(7),
2809 DBG_BCR_BVR_WCR_WVR(8),
2810 DBG_BCR_BVR_WCR_WVR(9),
2811 DBG_BCR_BVR_WCR_WVR(10),
2812 DBG_BCR_BVR_WCR_WVR(11),
2813 DBG_BCR_BVR_WCR_WVR(12),
2814 DBG_BCR_BVR_WCR_WVR(13),
2815 DBG_BCR_BVR_WCR_WVR(14),
2816 DBG_BCR_BVR_WCR_WVR(15),
2818 /* DBGDRAR (32bit) */
2819 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), trap_raz_wi },
2823 { Op1( 0), CRn( 1), CRm( 0), Op2( 4), trap_oslar_el1 },
2826 { Op1( 0), CRn( 1), CRm( 1), Op2( 4), trap_oslsr_el1, NULL, OSLSR_EL1 },
2830 { Op1( 0), CRn( 1), CRm( 3), Op2( 4), trap_raz_wi },
2833 { Op1( 0), CRn( 1), CRm( 4), Op2( 4), trap_raz_wi },
2846 /* DBGDSAR (32bit) */
2847 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), trap_raz_wi },
2850 { Op1( 0), CRn( 7), CRm( 0), Op2( 7), trap_raz_wi },
2852 { Op1( 0), CRn( 7), CRm( 1), Op2( 7), trap_raz_wi },
2854 { Op1( 0), CRn( 7), CRm( 2), Op2( 7), trap_raz_wi },
2856 { Op1( 0), CRn( 7), CRm( 8), Op2( 6), trap_raz_wi },
2858 { Op1( 0), CRn( 7), CRm( 9), Op2( 6), trap_raz_wi },
2860 { Op1( 0), CRn( 7), CRm(14), Op2( 6), trap_dbgauthstatus_el1 },
2863 /* Trapped cp14 64bit registers */
2864 static const struct sys_reg_desc cp14_64_regs[] = {
2865 /* DBGDRAR (64bit) */
2866 { Op1( 0), CRm( 1), .access = trap_raz_wi },
2868 /* DBGDSAR (64bit) */
2869 { Op1( 0), CRm( 2), .access = trap_raz_wi },
2872 #define CP15_PMU_SYS_REG(_map, _Op1, _CRn, _CRm, _Op2) \
2874 Op1(_Op1), CRn(_CRn), CRm(_CRm), Op2(_Op2), \
2875 .visibility = pmu_visibility
2877 /* Macro to expand the PMEVCNTRn register */
2878 #define PMU_PMEVCNTR(n) \
2879 { CP15_PMU_SYS_REG(DIRECT, 0, 0b1110, \
2880 (0b1000 | (((n) >> 3) & 0x3)), ((n) & 0x7)), \
2881 .access = access_pmu_evcntr }
2883 /* Macro to expand the PMEVTYPERn register */
2884 #define PMU_PMEVTYPER(n) \
2885 { CP15_PMU_SYS_REG(DIRECT, 0, 0b1110, \
2886 (0b1100 | (((n) >> 3) & 0x3)), ((n) & 0x7)), \
2887 .access = access_pmu_evtyper }
2889 * Trapped cp15 registers. TTBR0/TTBR1 get a double encoding,
2890 * depending on the way they are accessed (as a 32bit or a 64bit
2893 static const struct sys_reg_desc cp15_regs[] = {
2894 { Op1( 0), CRn( 0), CRm( 0), Op2( 1), access_ctr },
2895 { Op1( 0), CRn( 1), CRm( 0), Op2( 0), access_vm_reg, NULL, SCTLR_EL1 },
2897 { AA32(LO), Op1( 0), CRn( 1), CRm( 0), Op2( 1), access_actlr, NULL, ACTLR_EL1 },
2899 { AA32(HI), Op1( 0), CRn( 1), CRm( 0), Op2( 3), access_actlr, NULL, ACTLR_EL1 },
2900 { Op1( 0), CRn( 2), CRm( 0), Op2( 0), access_vm_reg, NULL, TTBR0_EL1 },
2901 { Op1( 0), CRn( 2), CRm( 0), Op2( 1), access_vm_reg, NULL, TTBR1_EL1 },
2903 { AA32(LO), Op1( 0), CRn( 2), CRm( 0), Op2( 2), access_vm_reg, NULL, TCR_EL1 },
2905 { AA32(HI), Op1( 0), CRn( 2), CRm( 0), Op2( 3), access_vm_reg, NULL, TCR_EL1 },
2906 { Op1( 0), CRn( 3), CRm( 0), Op2( 0), access_vm_reg, NULL, DACR32_EL2 },
2908 { Op1( 0), CRn( 5), CRm( 0), Op2( 0), access_vm_reg, NULL, ESR_EL1 },
2909 { Op1( 0), CRn( 5), CRm( 0), Op2( 1), access_vm_reg, NULL, IFSR32_EL2 },
2911 { Op1( 0), CRn( 5), CRm( 1), Op2( 0), access_vm_reg, NULL, AFSR0_EL1 },
2913 { Op1( 0), CRn( 5), CRm( 1), Op2( 1), access_vm_reg, NULL, AFSR1_EL1 },
2915 { AA32(LO), Op1( 0), CRn( 6), CRm( 0), Op2( 0), access_vm_reg, NULL, FAR_EL1 },
2917 { AA32(HI), Op1( 0), CRn( 6), CRm( 0), Op2( 2), access_vm_reg, NULL, FAR_EL1 },
2920 * DC{C,I,CI}SW operations:
2922 { Op1( 0), CRn( 7), CRm( 6), Op2( 2), access_dcsw },
2923 { Op1( 0), CRn( 7), CRm(10), Op2( 2), access_dcsw },
2924 { Op1( 0), CRn( 7), CRm(14), Op2( 2), access_dcsw },
2927 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 0), .access = access_pmcr },
2928 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 1), .access = access_pmcnten },
2929 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 2), .access = access_pmcnten },
2930 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 3), .access = access_pmovs },
2931 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 4), .access = access_pmswinc },
2932 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 12, 5), .access = access_pmselr },
2933 { CP15_PMU_SYS_REG(LO, 0, 9, 12, 6), .access = access_pmceid },
2934 { CP15_PMU_SYS_REG(LO, 0, 9, 12, 7), .access = access_pmceid },
2935 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 13, 0), .access = access_pmu_evcntr },
2936 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 13, 1), .access = access_pmu_evtyper },
2937 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 13, 2), .access = access_pmu_evcntr },
2938 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 0), .access = access_pmuserenr },
2939 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 1), .access = access_pminten },
2940 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 2), .access = access_pminten },
2941 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 3), .access = access_pmovs },
2942 { CP15_PMU_SYS_REG(HI, 0, 9, 14, 4), .access = access_pmceid },
2943 { CP15_PMU_SYS_REG(HI, 0, 9, 14, 5), .access = access_pmceid },
2945 { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 6), .access = trap_raz_wi },
2948 { AA32(LO), Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, MAIR_EL1 },
2950 { AA32(HI), Op1( 0), CRn(10), CRm( 2), Op2( 1), access_vm_reg, NULL, MAIR_EL1 },
2952 { AA32(LO), Op1( 0), CRn(10), CRm( 3), Op2( 0), access_vm_reg, NULL, AMAIR_EL1 },
2954 { AA32(HI), Op1( 0), CRn(10), CRm( 3), Op2( 1), access_vm_reg, NULL, AMAIR_EL1 },
2957 { Op1( 0), CRn(12), CRm(12), Op2( 5), access_gic_sre },
2959 { Op1( 0), CRn(13), CRm( 0), Op2( 1), access_vm_reg, NULL, CONTEXTIDR_EL1 },
2962 { SYS_DESC(SYS_AARCH32_CNTP_TVAL), access_arch_timer },
2963 { SYS_DESC(SYS_AARCH32_CNTP_CTL), access_arch_timer },
3030 { CP15_PMU_SYS_REG(DIRECT, 0, 14, 15, 7), .access = access_pmu_evtyper },
3032 { Op1(1), CRn( 0), CRm( 0), Op2(0), access_ccsidr },
3033 { Op1(1), CRn( 0), CRm( 0), Op2(1), access_clidr },
3036 { Op1(1), CRn( 0), CRm( 0), Op2(2), undef_access },
3038 { Op1(2), CRn( 0), CRm( 0), Op2(0), access_csselr, NULL, CSSELR_EL1 },
3041 static const struct sys_reg_desc cp15_64_regs[] = {
3042 { Op1( 0), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR0_EL1 },
3043 { CP15_PMU_SYS_REG(DIRECT, 0, 0, 9, 0), .access = access_pmu_evcntr },
3044 { Op1( 0), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI1R */
3045 { SYS_DESC(SYS_AARCH32_CNTPCT), access_arch_timer },
3046 { Op1( 1), CRn( 0), CRm( 2), Op2( 0), access_vm_reg, NULL, TTBR1_EL1 },
3047 { Op1( 1), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_ASGI1R */
3048 { Op1( 2), CRn( 0), CRm(12), Op2( 0), access_gic_sgi }, /* ICC_SGI0R */
3049 { SYS_DESC(SYS_AARCH32_CNTP_CVAL), access_arch_timer },
3050 { SYS_DESC(SYS_AARCH32_CNTPCTSS), access_arch_timer },
3053 static bool check_sysreg_table(const struct sys_reg_desc *table, unsigned int n,
3058 for (i = 0; i < n; i++) {
3059 if (!is_32 && table[i].reg && !table[i].reset) {
3060 kvm_err("sys_reg table %pS entry %d lacks reset\n", &table[i], i);
3064 if (i && cmp_sys_reg(&table[i-1], &table[i]) >= 0) {
3065 kvm_err("sys_reg table %pS entry %d out of order\n", &table[i - 1], i - 1);
3073 int kvm_handle_cp14_load_store(struct kvm_vcpu *vcpu)
3075 kvm_inject_undefined(vcpu);
3079 static void perform_access(struct kvm_vcpu *vcpu,
3080 struct sys_reg_params *params,
3081 const struct sys_reg_desc *r)
3083 trace_kvm_sys_access(*vcpu_pc(vcpu), params, r);
3085 /* Check for regs disabled by runtime config */
3086 if (sysreg_hidden(vcpu, r)) {
3087 kvm_inject_undefined(vcpu);
3092 * Not having an accessor means that we have configured a trap
3093 * that we don't know how to handle. This certainly qualifies
3094 * as a gross bug that should be fixed right away.
3098 /* Skip instruction if instructed so */
3099 if (likely(r->access(vcpu, params, r)))
3104 * emulate_cp -- tries to match a sys_reg access in a handling table, and
3105 * call the corresponding trap handler.
3107 * @params: pointer to the descriptor of the access
3108 * @table: array of trap descriptors
3109 * @num: size of the trap descriptor array
3111 * Return true if the access has been handled, false if not.
3113 static bool emulate_cp(struct kvm_vcpu *vcpu,
3114 struct sys_reg_params *params,
3115 const struct sys_reg_desc *table,
3118 const struct sys_reg_desc *r;
3121 return false; /* Not handled */
3123 r = find_reg(params, table, num);
3126 perform_access(vcpu, params, r);
3134 static void unhandled_cp_access(struct kvm_vcpu *vcpu,
3135 struct sys_reg_params *params)
3137 u8 esr_ec = kvm_vcpu_trap_get_class(vcpu);
3141 case ESR_ELx_EC_CP15_32:
3142 case ESR_ELx_EC_CP15_64:
3145 case ESR_ELx_EC_CP14_MR:
3146 case ESR_ELx_EC_CP14_64:
3153 print_sys_reg_msg(params,
3154 "Unsupported guest CP%d access at: %08lx [%08lx]\n",
3155 cp, *vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
3156 kvm_inject_undefined(vcpu);
3160 * kvm_handle_cp_64 -- handles a mrrc/mcrr trap on a guest CP14/CP15 access
3161 * @vcpu: The VCPU pointer
3162 * @run: The kvm_run struct
3164 static int kvm_handle_cp_64(struct kvm_vcpu *vcpu,
3165 const struct sys_reg_desc *global,
3168 struct sys_reg_params params;
3169 u64 esr = kvm_vcpu_get_esr(vcpu);
3170 int Rt = kvm_vcpu_sys_get_rt(vcpu);
3171 int Rt2 = (esr >> 10) & 0x1f;
3173 params.CRm = (esr >> 1) & 0xf;
3174 params.is_write = ((esr & 1) == 0);
3177 params.Op1 = (esr >> 16) & 0xf;
3182 * Make a 64-bit value out of Rt and Rt2. As we use the same trap
3183 * backends between AArch32 and AArch64, we get away with it.
3185 if (params.is_write) {
3186 params.regval = vcpu_get_reg(vcpu, Rt) & 0xffffffff;
3187 params.regval |= vcpu_get_reg(vcpu, Rt2) << 32;
3191 * If the table contains a handler, handle the
3192 * potential register operation in the case of a read and return
3195 if (emulate_cp(vcpu, ¶ms, global, nr_global)) {
3196 /* Split up the value between registers for the read side */
3197 if (!params.is_write) {
3198 vcpu_set_reg(vcpu, Rt, lower_32_bits(params.regval));
3199 vcpu_set_reg(vcpu, Rt2, upper_32_bits(params.regval));
3205 unhandled_cp_access(vcpu, ¶ms);
3209 static bool emulate_sys_reg(struct kvm_vcpu *vcpu, struct sys_reg_params *params);
3212 * The CP10 ID registers are architecturally mapped to AArch64 feature
3213 * registers. Abuse that fact so we can rely on the AArch64 handler for accesses
3216 static bool kvm_esr_cp10_id_to_sys64(u64 esr, struct sys_reg_params *params)
3218 u8 reg_id = (esr >> 10) & 0xf;
3221 params->is_write = ((esr & 1) == 0);
3227 /* CP10 ID registers are read-only */
3228 valid = !params->is_write;
3250 kvm_pr_unimpl("Unhandled cp10 register %s: %u\n",
3251 params->is_write ? "write" : "read", reg_id);
3256 * kvm_handle_cp10_id() - Handles a VMRS trap on guest access to a 'Media and
3257 * VFP Register' from AArch32.
3258 * @vcpu: The vCPU pointer
3260 * MVFR{0-2} are architecturally mapped to the AArch64 MVFR{0-2}_EL1 registers.
3261 * Work out the correct AArch64 system register encoding and reroute to the
3262 * AArch64 system register emulation.
3264 int kvm_handle_cp10_id(struct kvm_vcpu *vcpu)
3266 int Rt = kvm_vcpu_sys_get_rt(vcpu);
3267 u64 esr = kvm_vcpu_get_esr(vcpu);
3268 struct sys_reg_params params;
3270 /* UNDEF on any unhandled register access */
3271 if (!kvm_esr_cp10_id_to_sys64(esr, ¶ms)) {
3272 kvm_inject_undefined(vcpu);
3276 if (emulate_sys_reg(vcpu, ¶ms))
3277 vcpu_set_reg(vcpu, Rt, params.regval);
3283 * kvm_emulate_cp15_id_reg() - Handles an MRC trap on a guest CP15 access where
3284 * CRn=0, which corresponds to the AArch32 feature
3286 * @vcpu: the vCPU pointer
3287 * @params: the system register access parameters.
3289 * Our cp15 system register tables do not enumerate the AArch32 feature
3290 * registers. Conveniently, our AArch64 table does, and the AArch32 system
3291 * register encoding can be trivially remapped into the AArch64 for the feature
3292 * registers: Append op0=3, leaving op1, CRn, CRm, and op2 the same.
3294 * According to DDI0487G.b G7.3.1, paragraph "Behavior of VMSAv8-32 32-bit
3295 * System registers with (coproc=0b1111, CRn==c0)", read accesses from this
3296 * range are either UNKNOWN or RES0. Rerouting remains architectural as we
3297 * treat undefined registers in this range as RAZ.
3299 static int kvm_emulate_cp15_id_reg(struct kvm_vcpu *vcpu,
3300 struct sys_reg_params *params)
3302 int Rt = kvm_vcpu_sys_get_rt(vcpu);
3304 /* Treat impossible writes to RO registers as UNDEFINED */
3305 if (params->is_write) {
3306 unhandled_cp_access(vcpu, params);
3313 * All registers where CRm > 3 are known to be UNKNOWN/RAZ from AArch32.
3314 * Avoid conflicting with future expansion of AArch64 feature registers
3315 * and simply treat them as RAZ here.
3317 if (params->CRm > 3)
3319 else if (!emulate_sys_reg(vcpu, params))
3322 vcpu_set_reg(vcpu, Rt, params->regval);
3327 * kvm_handle_cp_32 -- handles a mrc/mcr trap on a guest CP14/CP15 access
3328 * @vcpu: The VCPU pointer
3329 * @run: The kvm_run struct
3331 static int kvm_handle_cp_32(struct kvm_vcpu *vcpu,
3332 struct sys_reg_params *params,
3333 const struct sys_reg_desc *global,
3336 int Rt = kvm_vcpu_sys_get_rt(vcpu);
3338 params->regval = vcpu_get_reg(vcpu, Rt);
3340 if (emulate_cp(vcpu, params, global, nr_global)) {
3341 if (!params->is_write)
3342 vcpu_set_reg(vcpu, Rt, params->regval);
3346 unhandled_cp_access(vcpu, params);
3350 int kvm_handle_cp15_64(struct kvm_vcpu *vcpu)
3352 return kvm_handle_cp_64(vcpu, cp15_64_regs, ARRAY_SIZE(cp15_64_regs));
3355 int kvm_handle_cp15_32(struct kvm_vcpu *vcpu)
3357 struct sys_reg_params params;
3359 params = esr_cp1x_32_to_params(kvm_vcpu_get_esr(vcpu));
3362 * Certain AArch32 ID registers are handled by rerouting to the AArch64
3363 * system register table. Registers in the ID range where CRm=0 are
3364 * excluded from this scheme as they do not trivially map into AArch64
3365 * system register encodings.
3367 if (params.Op1 == 0 && params.CRn == 0 && params.CRm)
3368 return kvm_emulate_cp15_id_reg(vcpu, ¶ms);
3370 return kvm_handle_cp_32(vcpu, ¶ms, cp15_regs, ARRAY_SIZE(cp15_regs));
3373 int kvm_handle_cp14_64(struct kvm_vcpu *vcpu)
3375 return kvm_handle_cp_64(vcpu, cp14_64_regs, ARRAY_SIZE(cp14_64_regs));
3378 int kvm_handle_cp14_32(struct kvm_vcpu *vcpu)
3380 struct sys_reg_params params;
3382 params = esr_cp1x_32_to_params(kvm_vcpu_get_esr(vcpu));
3384 return kvm_handle_cp_32(vcpu, ¶ms, cp14_regs, ARRAY_SIZE(cp14_regs));
3387 static bool is_imp_def_sys_reg(struct sys_reg_params *params)
3389 // See ARM DDI 0487E.a, section D12.3.2
3390 return params->Op0 == 3 && (params->CRn & 0b1011) == 0b1011;
3394 * emulate_sys_reg - Emulate a guest access to an AArch64 system register
3395 * @vcpu: The VCPU pointer
3396 * @params: Decoded system register parameters
3398 * Return: true if the system register access was successful, false otherwise.
3400 static bool emulate_sys_reg(struct kvm_vcpu *vcpu,
3401 struct sys_reg_params *params)
3403 const struct sys_reg_desc *r;
3405 r = find_reg(params, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
3408 perform_access(vcpu, params, r);
3412 if (is_imp_def_sys_reg(params)) {
3413 kvm_inject_undefined(vcpu);
3415 print_sys_reg_msg(params,
3416 "Unsupported guest sys_reg access at: %lx [%08lx]\n",
3417 *vcpu_pc(vcpu), *vcpu_cpsr(vcpu));
3418 kvm_inject_undefined(vcpu);
3423 static void kvm_reset_id_regs(struct kvm_vcpu *vcpu)
3425 const struct sys_reg_desc *idreg = first_idreg;
3426 u32 id = reg_to_encoding(idreg);
3427 struct kvm *kvm = vcpu->kvm;
3429 if (test_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags))
3432 lockdep_assert_held(&kvm->arch.config_lock);
3434 /* Initialize all idregs */
3435 while (is_id_reg(id)) {
3436 IDREG(kvm, id) = idreg->reset(vcpu, idreg);
3439 id = reg_to_encoding(idreg);
3442 set_bit(KVM_ARCH_FLAG_ID_REGS_INITIALIZED, &kvm->arch.flags);
3446 * kvm_reset_sys_regs - sets system registers to reset value
3447 * @vcpu: The VCPU pointer
3449 * This function finds the right table above and sets the registers on the
3450 * virtual CPU struct to their architecturally defined reset values.
3452 void kvm_reset_sys_regs(struct kvm_vcpu *vcpu)
3456 kvm_reset_id_regs(vcpu);
3458 for (i = 0; i < ARRAY_SIZE(sys_reg_descs); i++) {
3459 const struct sys_reg_desc *r = &sys_reg_descs[i];
3461 if (is_id_reg(reg_to_encoding(r)))
3470 * kvm_handle_sys_reg -- handles a mrs/msr trap on a guest sys_reg access
3471 * @vcpu: The VCPU pointer
3473 int kvm_handle_sys_reg(struct kvm_vcpu *vcpu)
3475 struct sys_reg_params params;
3476 unsigned long esr = kvm_vcpu_get_esr(vcpu);
3477 int Rt = kvm_vcpu_sys_get_rt(vcpu);
3479 trace_kvm_handle_sys_reg(esr);
3481 if (__check_nv_sr_forward(vcpu))
3484 params = esr_sys64_to_params(esr);
3485 params.regval = vcpu_get_reg(vcpu, Rt);
3487 if (!emulate_sys_reg(vcpu, ¶ms))
3490 if (!params.is_write)
3491 vcpu_set_reg(vcpu, Rt, params.regval);
3495 /******************************************************************************
3497 *****************************************************************************/
3499 static bool index_to_params(u64 id, struct sys_reg_params *params)
3501 switch (id & KVM_REG_SIZE_MASK) {
3502 case KVM_REG_SIZE_U64:
3503 /* Any unused index bits means it's not valid. */
3504 if (id & ~(KVM_REG_ARCH_MASK | KVM_REG_SIZE_MASK
3505 | KVM_REG_ARM_COPROC_MASK
3506 | KVM_REG_ARM64_SYSREG_OP0_MASK
3507 | KVM_REG_ARM64_SYSREG_OP1_MASK
3508 | KVM_REG_ARM64_SYSREG_CRN_MASK
3509 | KVM_REG_ARM64_SYSREG_CRM_MASK
3510 | KVM_REG_ARM64_SYSREG_OP2_MASK))
3512 params->Op0 = ((id & KVM_REG_ARM64_SYSREG_OP0_MASK)
3513 >> KVM_REG_ARM64_SYSREG_OP0_SHIFT);
3514 params->Op1 = ((id & KVM_REG_ARM64_SYSREG_OP1_MASK)
3515 >> KVM_REG_ARM64_SYSREG_OP1_SHIFT);
3516 params->CRn = ((id & KVM_REG_ARM64_SYSREG_CRN_MASK)
3517 >> KVM_REG_ARM64_SYSREG_CRN_SHIFT);
3518 params->CRm = ((id & KVM_REG_ARM64_SYSREG_CRM_MASK)
3519 >> KVM_REG_ARM64_SYSREG_CRM_SHIFT);
3520 params->Op2 = ((id & KVM_REG_ARM64_SYSREG_OP2_MASK)
3521 >> KVM_REG_ARM64_SYSREG_OP2_SHIFT);
3528 const struct sys_reg_desc *get_reg_by_id(u64 id,
3529 const struct sys_reg_desc table[],
3532 struct sys_reg_params params;
3534 if (!index_to_params(id, ¶ms))
3537 return find_reg(¶ms, table, num);
3540 /* Decode an index value, and find the sys_reg_desc entry. */
3541 static const struct sys_reg_desc *
3542 id_to_sys_reg_desc(struct kvm_vcpu *vcpu, u64 id,
3543 const struct sys_reg_desc table[], unsigned int num)
3546 const struct sys_reg_desc *r;
3548 /* We only do sys_reg for now. */
3549 if ((id & KVM_REG_ARM_COPROC_MASK) != KVM_REG_ARM64_SYSREG)
3552 r = get_reg_by_id(id, table, num);
3554 /* Not saved in the sys_reg array and not otherwise accessible? */
3555 if (r && (!(r->reg || r->get_user) || sysreg_hidden(vcpu, r)))
3562 * These are the invariant sys_reg registers: we let the guest see the
3563 * host versions of these, so they're part of the guest state.
3565 * A future CPU may provide a mechanism to present different values to
3566 * the guest, or a future kvm may trap them.
3569 #define FUNCTION_INVARIANT(reg) \
3570 static u64 get_##reg(struct kvm_vcpu *v, \
3571 const struct sys_reg_desc *r) \
3573 ((struct sys_reg_desc *)r)->val = read_sysreg(reg); \
3574 return ((struct sys_reg_desc *)r)->val; \
3577 FUNCTION_INVARIANT(midr_el1)
3578 FUNCTION_INVARIANT(revidr_el1)
3579 FUNCTION_INVARIANT(aidr_el1)
3581 static u64 get_ctr_el0(struct kvm_vcpu *v, const struct sys_reg_desc *r)
3583 ((struct sys_reg_desc *)r)->val = read_sanitised_ftr_reg(SYS_CTR_EL0);
3584 return ((struct sys_reg_desc *)r)->val;
3587 /* ->val is filled in by kvm_sys_reg_table_init() */
3588 static struct sys_reg_desc invariant_sys_regs[] __ro_after_init = {
3589 { SYS_DESC(SYS_MIDR_EL1), NULL, get_midr_el1 },
3590 { SYS_DESC(SYS_REVIDR_EL1), NULL, get_revidr_el1 },
3591 { SYS_DESC(SYS_AIDR_EL1), NULL, get_aidr_el1 },
3592 { SYS_DESC(SYS_CTR_EL0), NULL, get_ctr_el0 },
3595 static int get_invariant_sys_reg(u64 id, u64 __user *uaddr)
3597 const struct sys_reg_desc *r;
3599 r = get_reg_by_id(id, invariant_sys_regs,
3600 ARRAY_SIZE(invariant_sys_regs));
3604 return put_user(r->val, uaddr);
3607 static int set_invariant_sys_reg(u64 id, u64 __user *uaddr)
3609 const struct sys_reg_desc *r;
3612 r = get_reg_by_id(id, invariant_sys_regs,
3613 ARRAY_SIZE(invariant_sys_regs));
3617 if (get_user(val, uaddr))
3620 /* This is what we mean by invariant: you can't change it. */
3627 static int demux_c15_get(struct kvm_vcpu *vcpu, u64 id, void __user *uaddr)
3630 u32 __user *uval = uaddr;
3632 /* Fail if we have unknown bits set. */
3633 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
3634 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
3637 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
3638 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
3639 if (KVM_REG_SIZE(id) != 4)
3641 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
3642 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
3643 if (val >= CSSELR_MAX)
3646 return put_user(get_ccsidr(vcpu, val), uval);
3652 static int demux_c15_set(struct kvm_vcpu *vcpu, u64 id, void __user *uaddr)
3655 u32 __user *uval = uaddr;
3657 /* Fail if we have unknown bits set. */
3658 if (id & ~(KVM_REG_ARCH_MASK|KVM_REG_SIZE_MASK|KVM_REG_ARM_COPROC_MASK
3659 | ((1 << KVM_REG_ARM_COPROC_SHIFT)-1)))
3662 switch (id & KVM_REG_ARM_DEMUX_ID_MASK) {
3663 case KVM_REG_ARM_DEMUX_ID_CCSIDR:
3664 if (KVM_REG_SIZE(id) != 4)
3666 val = (id & KVM_REG_ARM_DEMUX_VAL_MASK)
3667 >> KVM_REG_ARM_DEMUX_VAL_SHIFT;
3668 if (val >= CSSELR_MAX)
3671 if (get_user(newval, uval))
3674 return set_ccsidr(vcpu, val, newval);
3680 int kvm_sys_reg_get_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg,
3681 const struct sys_reg_desc table[], unsigned int num)
3683 u64 __user *uaddr = (u64 __user *)(unsigned long)reg->addr;
3684 const struct sys_reg_desc *r;
3688 r = id_to_sys_reg_desc(vcpu, reg->id, table, num);
3689 if (!r || sysreg_hidden_user(vcpu, r))
3693 ret = (r->get_user)(vcpu, r, &val);
3695 val = __vcpu_sys_reg(vcpu, r->reg);
3700 ret = put_user(val, uaddr);
3705 int kvm_arm_sys_reg_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
3707 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
3710 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
3711 return demux_c15_get(vcpu, reg->id, uaddr);
3713 err = get_invariant_sys_reg(reg->id, uaddr);
3717 return kvm_sys_reg_get_user(vcpu, reg,
3718 sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
3721 int kvm_sys_reg_set_user(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg,
3722 const struct sys_reg_desc table[], unsigned int num)
3724 u64 __user *uaddr = (u64 __user *)(unsigned long)reg->addr;
3725 const struct sys_reg_desc *r;
3729 if (get_user(val, uaddr))
3732 r = id_to_sys_reg_desc(vcpu, reg->id, table, num);
3733 if (!r || sysreg_hidden_user(vcpu, r))
3736 if (sysreg_user_write_ignore(vcpu, r))
3740 ret = (r->set_user)(vcpu, r, val);
3742 __vcpu_sys_reg(vcpu, r->reg) = val;
3749 int kvm_arm_sys_reg_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
3751 void __user *uaddr = (void __user *)(unsigned long)reg->addr;
3754 if ((reg->id & KVM_REG_ARM_COPROC_MASK) == KVM_REG_ARM_DEMUX)
3755 return demux_c15_set(vcpu, reg->id, uaddr);
3757 err = set_invariant_sys_reg(reg->id, uaddr);
3761 return kvm_sys_reg_set_user(vcpu, reg,
3762 sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
3765 static unsigned int num_demux_regs(void)
3770 static int write_demux_regids(u64 __user *uindices)
3772 u64 val = KVM_REG_ARM64 | KVM_REG_SIZE_U32 | KVM_REG_ARM_DEMUX;
3775 val |= KVM_REG_ARM_DEMUX_ID_CCSIDR;
3776 for (i = 0; i < CSSELR_MAX; i++) {
3777 if (put_user(val | i, uindices))
3784 static u64 sys_reg_to_index(const struct sys_reg_desc *reg)
3786 return (KVM_REG_ARM64 | KVM_REG_SIZE_U64 |
3787 KVM_REG_ARM64_SYSREG |
3788 (reg->Op0 << KVM_REG_ARM64_SYSREG_OP0_SHIFT) |
3789 (reg->Op1 << KVM_REG_ARM64_SYSREG_OP1_SHIFT) |
3790 (reg->CRn << KVM_REG_ARM64_SYSREG_CRN_SHIFT) |
3791 (reg->CRm << KVM_REG_ARM64_SYSREG_CRM_SHIFT) |
3792 (reg->Op2 << KVM_REG_ARM64_SYSREG_OP2_SHIFT));
3795 static bool copy_reg_to_user(const struct sys_reg_desc *reg, u64 __user **uind)
3800 if (put_user(sys_reg_to_index(reg), *uind))
3807 static int walk_one_sys_reg(const struct kvm_vcpu *vcpu,
3808 const struct sys_reg_desc *rd,
3810 unsigned int *total)
3813 * Ignore registers we trap but don't save,
3814 * and for which no custom user accessor is provided.
3816 if (!(rd->reg || rd->get_user))
3819 if (sysreg_hidden_user(vcpu, rd))
3822 if (!copy_reg_to_user(rd, uind))
3829 /* Assumed ordered tables, see kvm_sys_reg_table_init. */
3830 static int walk_sys_regs(struct kvm_vcpu *vcpu, u64 __user *uind)
3832 const struct sys_reg_desc *i2, *end2;
3833 unsigned int total = 0;
3837 end2 = sys_reg_descs + ARRAY_SIZE(sys_reg_descs);
3839 while (i2 != end2) {
3840 err = walk_one_sys_reg(vcpu, i2++, &uind, &total);
3847 unsigned long kvm_arm_num_sys_reg_descs(struct kvm_vcpu *vcpu)
3849 return ARRAY_SIZE(invariant_sys_regs)
3851 + walk_sys_regs(vcpu, (u64 __user *)NULL);
3854 int kvm_arm_copy_sys_reg_indices(struct kvm_vcpu *vcpu, u64 __user *uindices)
3859 /* Then give them all the invariant registers' indices. */
3860 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++) {
3861 if (put_user(sys_reg_to_index(&invariant_sys_regs[i]), uindices))
3866 err = walk_sys_regs(vcpu, uindices);
3871 return write_demux_regids(uindices);
3874 #define KVM_ARM_FEATURE_ID_RANGE_INDEX(r) \
3875 KVM_ARM_FEATURE_ID_RANGE_IDX(sys_reg_Op0(r), \
3881 static bool is_feature_id_reg(u32 encoding)
3883 return (sys_reg_Op0(encoding) == 3 &&
3884 (sys_reg_Op1(encoding) < 2 || sys_reg_Op1(encoding) == 3) &&
3885 sys_reg_CRn(encoding) == 0 &&
3886 sys_reg_CRm(encoding) <= 7);
3889 int kvm_vm_ioctl_get_reg_writable_masks(struct kvm *kvm, struct reg_mask_range *range)
3891 const void *zero_page = page_to_virt(ZERO_PAGE(0));
3892 u64 __user *masks = (u64 __user *)range->addr;
3894 /* Only feature id range is supported, reserved[13] must be zero. */
3896 memcmp(range->reserved, zero_page, sizeof(range->reserved)))
3899 /* Wipe the whole thing first */
3900 if (clear_user(masks, KVM_ARM_FEATURE_ID_RANGE_SIZE * sizeof(__u64)))
3903 for (int i = 0; i < ARRAY_SIZE(sys_reg_descs); i++) {
3904 const struct sys_reg_desc *reg = &sys_reg_descs[i];
3905 u32 encoding = reg_to_encoding(reg);
3908 if (!is_feature_id_reg(encoding) || !reg->set_user)
3912 * For ID registers, we return the writable mask. Other feature
3913 * registers return a full 64bit mask. That's not necessary
3914 * compliant with a given revision of the architecture, but the
3915 * RES0/RES1 definitions allow us to do that.
3917 if (is_id_reg(encoding)) {
3919 (is_aa32_id_reg(encoding) && !kvm_supports_32bit_el0()))
3926 if (put_user(val, (masks + KVM_ARM_FEATURE_ID_RANGE_INDEX(encoding))))
3933 int __init kvm_sys_reg_table_init(void)
3935 struct sys_reg_params params;
3939 /* Make sure tables are unique and in order. */
3940 valid &= check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs), false);
3941 valid &= check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs), true);
3942 valid &= check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs), true);
3943 valid &= check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs), true);
3944 valid &= check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), true);
3945 valid &= check_sysreg_table(invariant_sys_regs, ARRAY_SIZE(invariant_sys_regs), false);
3950 /* We abuse the reset function to overwrite the table itself. */
3951 for (i = 0; i < ARRAY_SIZE(invariant_sys_regs); i++)
3952 invariant_sys_regs[i].reset(NULL, &invariant_sys_regs[i]);
3954 /* Find the first idreg (SYS_ID_PFR0_EL1) in sys_reg_descs. */
3955 params = encoding_to_params(SYS_ID_PFR0_EL1);
3956 first_idreg = find_reg(¶ms, sys_reg_descs, ARRAY_SIZE(sys_reg_descs));
3960 if (kvm_get_mode() == KVM_MODE_NV)
3961 return populate_nv_trap_config();