1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/types.h>
3 #include <linux/interrupt.h>
5 #include <asm/xen/hypercall.h>
8 #include <xen/interface/xen.h>
9 #include <xen/interface/vcpu.h>
10 #include <xen/interface/xenpmu.h>
15 /* x86_pmu.handle_irq definition */
16 #include "../events/perf_event.h"
18 #define XENPMU_IRQ_PROCESSING 1
20 /* Shared page between hypervisor and domain */
21 struct xen_pmu_data *xenpmu_data;
25 static DEFINE_PER_CPU(struct xenpmu, xenpmu_shared);
26 #define get_xenpmu_data() (this_cpu_ptr(&xenpmu_shared)->xenpmu_data)
27 #define get_xenpmu_flags() (this_cpu_ptr(&xenpmu_shared)->flags)
29 /* Macro for computing address of a PMU MSR bank */
30 #define field_offset(ctxt, field) ((void *)((uintptr_t)ctxt + \
31 (uintptr_t)ctxt->field))
34 #define F15H_NUM_COUNTERS 6
35 #define F10H_NUM_COUNTERS 4
37 static __read_mostly uint32_t amd_counters_base;
38 static __read_mostly uint32_t amd_ctrls_base;
39 static __read_mostly int amd_msr_step;
40 static __read_mostly int k7_counters_mirrored;
41 static __read_mostly int amd_num_counters;
44 #define MSR_TYPE_COUNTER 0
45 #define MSR_TYPE_CTRL 1
46 #define MSR_TYPE_GLOBAL 2
47 #define MSR_TYPE_ARCH_COUNTER 3
48 #define MSR_TYPE_ARCH_CTRL 4
50 /* Number of general pmu registers (CPUID.EAX[0xa].EAX[8..15]) */
51 #define PMU_GENERAL_NR_SHIFT 8
52 #define PMU_GENERAL_NR_BITS 8
53 #define PMU_GENERAL_NR_MASK (((1 << PMU_GENERAL_NR_BITS) - 1) \
54 << PMU_GENERAL_NR_SHIFT)
56 /* Number of fixed pmu registers (CPUID.EDX[0xa].EDX[0..4]) */
57 #define PMU_FIXED_NR_SHIFT 0
58 #define PMU_FIXED_NR_BITS 5
59 #define PMU_FIXED_NR_MASK (((1 << PMU_FIXED_NR_BITS) - 1) \
60 << PMU_FIXED_NR_SHIFT)
62 /* Alias registers (0x4c1) for full-width writes to PMCs */
63 #define MSR_PMC_ALIAS_MASK (~(MSR_IA32_PERFCTR0 ^ MSR_IA32_PMC0))
65 #define INTEL_PMC_TYPE_SHIFT 30
67 static __read_mostly int intel_num_arch_counters, intel_num_fixed_counters;
70 static void xen_pmu_arch_init(void)
72 if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD) {
74 switch (boot_cpu_data.x86) {
76 amd_num_counters = F15H_NUM_COUNTERS;
77 amd_counters_base = MSR_F15H_PERF_CTR;
78 amd_ctrls_base = MSR_F15H_PERF_CTL;
80 k7_counters_mirrored = 1;
87 amd_num_counters = F10H_NUM_COUNTERS;
88 amd_counters_base = MSR_K7_PERFCTR0;
89 amd_ctrls_base = MSR_K7_EVNTSEL0;
91 k7_counters_mirrored = 0;
94 } else if (boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
95 amd_num_counters = F10H_NUM_COUNTERS;
96 amd_counters_base = MSR_K7_PERFCTR0;
97 amd_ctrls_base = MSR_K7_EVNTSEL0;
99 k7_counters_mirrored = 0;
101 uint32_t eax, ebx, ecx, edx;
103 cpuid(0xa, &eax, &ebx, &ecx, &edx);
105 intel_num_arch_counters = (eax & PMU_GENERAL_NR_MASK) >>
106 PMU_GENERAL_NR_SHIFT;
107 intel_num_fixed_counters = (edx & PMU_FIXED_NR_MASK) >>
112 static inline uint32_t get_fam15h_addr(u32 addr)
115 case MSR_K7_PERFCTR0:
116 case MSR_K7_PERFCTR1:
117 case MSR_K7_PERFCTR2:
118 case MSR_K7_PERFCTR3:
119 return MSR_F15H_PERF_CTR + (addr - MSR_K7_PERFCTR0);
120 case MSR_K7_EVNTSEL0:
121 case MSR_K7_EVNTSEL1:
122 case MSR_K7_EVNTSEL2:
123 case MSR_K7_EVNTSEL3:
124 return MSR_F15H_PERF_CTL + (addr - MSR_K7_EVNTSEL0);
132 static inline bool is_amd_pmu_msr(unsigned int msr)
134 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD &&
135 boot_cpu_data.x86_vendor != X86_VENDOR_HYGON)
138 if ((msr >= MSR_F15H_PERF_CTL &&
139 msr < MSR_F15H_PERF_CTR + (amd_num_counters * 2)) ||
140 (msr >= MSR_K7_EVNTSEL0 &&
141 msr < MSR_K7_PERFCTR0 + amd_num_counters))
147 static bool is_intel_pmu_msr(u32 msr_index, int *type, int *index)
151 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL &&
152 boot_cpu_data.x86_vendor != X86_VENDOR_CENTAUR &&
153 boot_cpu_data.x86_vendor != X86_VENDOR_ZHAOXIN)
157 case MSR_CORE_PERF_FIXED_CTR_CTRL:
158 case MSR_IA32_DS_AREA:
159 case MSR_IA32_PEBS_ENABLE:
160 *type = MSR_TYPE_CTRL;
163 case MSR_CORE_PERF_GLOBAL_CTRL:
164 case MSR_CORE_PERF_GLOBAL_STATUS:
165 case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
166 *type = MSR_TYPE_GLOBAL;
171 if ((msr_index >= MSR_CORE_PERF_FIXED_CTR0) &&
172 (msr_index < MSR_CORE_PERF_FIXED_CTR0 +
173 intel_num_fixed_counters)) {
174 *index = msr_index - MSR_CORE_PERF_FIXED_CTR0;
175 *type = MSR_TYPE_COUNTER;
179 if ((msr_index >= MSR_P6_EVNTSEL0) &&
180 (msr_index < MSR_P6_EVNTSEL0 + intel_num_arch_counters)) {
181 *index = msr_index - MSR_P6_EVNTSEL0;
182 *type = MSR_TYPE_ARCH_CTRL;
186 msr_index_pmc = msr_index & MSR_PMC_ALIAS_MASK;
187 if ((msr_index_pmc >= MSR_IA32_PERFCTR0) &&
188 (msr_index_pmc < MSR_IA32_PERFCTR0 +
189 intel_num_arch_counters)) {
190 *type = MSR_TYPE_ARCH_COUNTER;
191 *index = msr_index_pmc - MSR_IA32_PERFCTR0;
198 static bool xen_intel_pmu_emulate(unsigned int msr, u64 *val, int type,
199 int index, bool is_read)
201 uint64_t *reg = NULL;
202 struct xen_pmu_intel_ctxt *ctxt;
203 uint64_t *fix_counters;
204 struct xen_pmu_cntr_pair *arch_cntr_pair;
205 struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
206 uint8_t xenpmu_flags = get_xenpmu_flags();
209 if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING))
212 ctxt = &xenpmu_data->pmu.c.intel;
215 case MSR_CORE_PERF_GLOBAL_OVF_CTRL:
216 reg = &ctxt->global_ovf_ctrl;
218 case MSR_CORE_PERF_GLOBAL_STATUS:
219 reg = &ctxt->global_status;
221 case MSR_CORE_PERF_GLOBAL_CTRL:
222 reg = &ctxt->global_ctrl;
224 case MSR_CORE_PERF_FIXED_CTR_CTRL:
225 reg = &ctxt->fixed_ctrl;
229 case MSR_TYPE_COUNTER:
230 fix_counters = field_offset(ctxt, fixed_counters);
231 reg = &fix_counters[index];
233 case MSR_TYPE_ARCH_COUNTER:
234 arch_cntr_pair = field_offset(ctxt, arch_counters);
235 reg = &arch_cntr_pair[index].counter;
237 case MSR_TYPE_ARCH_CTRL:
238 arch_cntr_pair = field_offset(ctxt, arch_counters);
239 reg = &arch_cntr_pair[index].control;
252 if (msr == MSR_CORE_PERF_GLOBAL_OVF_CTRL)
253 ctxt->global_status &= (~(*val));
261 static bool xen_amd_pmu_emulate(unsigned int msr, u64 *val, bool is_read)
263 uint64_t *reg = NULL;
265 struct xen_pmu_amd_ctxt *ctxt;
266 uint64_t *counter_regs, *ctrl_regs;
267 struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
268 uint8_t xenpmu_flags = get_xenpmu_flags();
270 if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING))
273 if (k7_counters_mirrored &&
274 ((msr >= MSR_K7_EVNTSEL0) && (msr <= MSR_K7_PERFCTR3)))
275 msr = get_fam15h_addr(msr);
277 ctxt = &xenpmu_data->pmu.c.amd;
278 for (i = 0; i < amd_num_counters; i++) {
279 if (msr == amd_ctrls_base + off) {
280 ctrl_regs = field_offset(ctxt, ctrls);
283 } else if (msr == amd_counters_base + off) {
284 counter_regs = field_offset(ctxt, counters);
285 reg = &counter_regs[i];
302 static bool pmu_msr_chk_emulated(unsigned int msr, uint64_t *val, bool is_read,
307 if (is_amd_pmu_msr(msr))
308 *emul = xen_amd_pmu_emulate(msr, val, is_read);
309 else if (is_intel_pmu_msr(msr, &type, &index))
310 *emul = xen_intel_pmu_emulate(msr, val, type, index, is_read);
317 bool pmu_msr_read(unsigned int msr, uint64_t *val, int *err)
321 if (!pmu_msr_chk_emulated(msr, val, true, &emulated))
325 *val = err ? native_read_msr_safe(msr, err)
326 : native_read_msr(msr);
332 bool pmu_msr_write(unsigned int msr, uint32_t low, uint32_t high, int *err)
334 uint64_t val = ((uint64_t)high << 32) | low;
337 if (!pmu_msr_chk_emulated(msr, &val, false, &emulated))
342 *err = native_write_msr_safe(msr, low, high);
344 native_write_msr(msr, low, high);
350 static unsigned long long xen_amd_read_pmc(int counter)
352 struct xen_pmu_amd_ctxt *ctxt;
353 uint64_t *counter_regs;
354 struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
355 uint8_t xenpmu_flags = get_xenpmu_flags();
357 if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) {
361 msr = amd_counters_base + (counter * amd_msr_step);
362 return native_read_msr_safe(msr, &err);
365 ctxt = &xenpmu_data->pmu.c.amd;
366 counter_regs = field_offset(ctxt, counters);
367 return counter_regs[counter];
370 static unsigned long long xen_intel_read_pmc(int counter)
372 struct xen_pmu_intel_ctxt *ctxt;
373 uint64_t *fixed_counters;
374 struct xen_pmu_cntr_pair *arch_cntr_pair;
375 struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
376 uint8_t xenpmu_flags = get_xenpmu_flags();
378 if (!xenpmu_data || !(xenpmu_flags & XENPMU_IRQ_PROCESSING)) {
382 if (counter & (1 << INTEL_PMC_TYPE_SHIFT))
383 msr = MSR_CORE_PERF_FIXED_CTR0 + (counter & 0xffff);
385 msr = MSR_IA32_PERFCTR0 + counter;
387 return native_read_msr_safe(msr, &err);
390 ctxt = &xenpmu_data->pmu.c.intel;
391 if (counter & (1 << INTEL_PMC_TYPE_SHIFT)) {
392 fixed_counters = field_offset(ctxt, fixed_counters);
393 return fixed_counters[counter & 0xffff];
396 arch_cntr_pair = field_offset(ctxt, arch_counters);
397 return arch_cntr_pair[counter].counter;
400 unsigned long long xen_read_pmc(int counter)
402 if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL)
403 return xen_amd_read_pmc(counter);
405 return xen_intel_read_pmc(counter);
408 int pmu_apic_update(uint32_t val)
411 struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
414 pr_warn_once("%s: pmudata not initialized\n", __func__);
418 xenpmu_data->pmu.l.lapic_lvtpc = val;
420 if (get_xenpmu_flags() & XENPMU_IRQ_PROCESSING)
423 ret = HYPERVISOR_xenpmu_op(XENPMU_lvtpc_set, NULL);
429 static unsigned int xen_guest_state(void)
431 const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
432 unsigned int state = 0;
435 pr_warn_once("%s: pmudata not initialized\n", __func__);
439 if (!xen_initial_domain() || (xenpmu_data->domain_id >= DOMID_SELF))
442 state |= PERF_GUEST_ACTIVE;
444 if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_PV) {
445 if (xenpmu_data->pmu.pmu_flags & PMU_SAMPLE_USER)
446 state |= PERF_GUEST_USER;
447 } else if (xenpmu_data->pmu.r.regs.cpl & 3) {
448 state |= PERF_GUEST_USER;
454 static unsigned long xen_get_guest_ip(void)
456 const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
459 pr_warn_once("%s: pmudata not initialized\n", __func__);
463 return xenpmu_data->pmu.r.regs.ip;
466 static struct perf_guest_info_callbacks xen_guest_cbs = {
467 .state = xen_guest_state,
468 .get_ip = xen_get_guest_ip,
471 /* Convert registers from Xen's format to Linux' */
472 static void xen_convert_regs(const struct xen_pmu_regs *xen_regs,
473 struct pt_regs *regs, uint64_t pmu_flags)
475 regs->ip = xen_regs->ip;
476 regs->cs = xen_regs->cs;
477 regs->sp = xen_regs->sp;
479 if (pmu_flags & PMU_SAMPLE_PV) {
480 if (pmu_flags & PMU_SAMPLE_USER)
492 irqreturn_t xen_pmu_irq_handler(int irq, void *dev_id)
494 int err, ret = IRQ_NONE;
495 struct pt_regs regs = {0};
496 const struct xen_pmu_data *xenpmu_data = get_xenpmu_data();
497 uint8_t xenpmu_flags = get_xenpmu_flags();
500 pr_warn_once("%s: pmudata not initialized\n", __func__);
504 this_cpu_ptr(&xenpmu_shared)->flags =
505 xenpmu_flags | XENPMU_IRQ_PROCESSING;
506 xen_convert_regs(&xenpmu_data->pmu.r.regs, ®s,
507 xenpmu_data->pmu.pmu_flags);
508 if (x86_pmu.handle_irq(®s))
511 /* Write out cached context to HW */
512 err = HYPERVISOR_xenpmu_op(XENPMU_flush, NULL);
513 this_cpu_ptr(&xenpmu_shared)->flags = xenpmu_flags;
515 pr_warn_once("%s: failed hypercall, err: %d\n", __func__, err);
524 void xen_pmu_init(int cpu)
527 struct xen_pmu_params xp;
529 struct xen_pmu_data *xenpmu_data;
531 BUILD_BUG_ON(sizeof(struct xen_pmu_data) > PAGE_SIZE);
533 if (xen_hvm_domain() || (cpu != 0 && !is_xen_pmu))
536 xenpmu_data = (struct xen_pmu_data *)get_zeroed_page(GFP_KERNEL);
538 pr_err("VPMU init: No memory\n");
541 pfn = virt_to_pfn(xenpmu_data);
543 xp.val = pfn_to_mfn(pfn);
545 xp.version.maj = XENPMU_VER_MAJ;
546 xp.version.min = XENPMU_VER_MIN;
547 err = HYPERVISOR_xenpmu_op(XENPMU_init, &xp);
551 per_cpu(xenpmu_shared, cpu).xenpmu_data = xenpmu_data;
552 per_cpu(xenpmu_shared, cpu).flags = 0;
556 perf_register_guest_info_callbacks(&xen_guest_cbs);
563 if (err == -EOPNOTSUPP || err == -ENOSYS)
564 pr_info_once("VPMU disabled by hypervisor.\n");
566 pr_info_once("Could not initialize VPMU for cpu %d, error %d\n",
568 free_pages((unsigned long)xenpmu_data, 0);
571 void xen_pmu_finish(int cpu)
573 struct xen_pmu_params xp;
575 if (xen_hvm_domain())
579 xp.version.maj = XENPMU_VER_MAJ;
580 xp.version.min = XENPMU_VER_MIN;
582 (void)HYPERVISOR_xenpmu_op(XENPMU_finish, &xp);
584 free_pages((unsigned long)per_cpu(xenpmu_shared, cpu).xenpmu_data, 0);
585 per_cpu(xenpmu_shared, cpu).xenpmu_data = NULL;