1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Hypervisor supplied "24x7" performance counter support
5 * Author: Cody P Schafer <cody@linux.vnet.ibm.com>
6 * Copyright 2014 IBM Corporation.
9 #define pr_fmt(fmt) "hv-24x7: " fmt
11 #include <linux/perf_event.h>
12 #include <linux/rbtree.h>
13 #include <linux/module.h>
14 #include <linux/slab.h>
15 #include <linux/vmalloc.h>
17 #include <asm/cputhreads.h>
18 #include <asm/firmware.h>
19 #include <asm/hvcall.h>
21 #include <linux/byteorder/generic.h>
25 #include "hv-24x7-catalog.h"
26 #include "hv-common.h"
28 /* Version of the 24x7 hypervisor API that we should use in this machine. */
29 static int interface_version;
31 /* Whether we have to aggregate result data for some domains. */
32 static bool aggregate_result_elements;
34 static cpumask_t hv_24x7_cpumask;
36 static bool domain_is_valid(unsigned int domain)
39 #define DOMAIN(n, v, x, c) \
40 case HV_PERF_DOMAIN_##n: \
42 #include "hv-24x7-domains.h"
50 static bool is_physical_domain(unsigned int domain)
53 #define DOMAIN(n, v, x, c) \
54 case HV_PERF_DOMAIN_##n: \
56 #include "hv-24x7-domains.h"
64 * The Processor Module Information system parameter allows transferring
65 * of certain processor module information from the platform to the OS.
66 * Refer PAPR+ document to get parameter token value as '43'.
69 #define PROCESSOR_MODULE_INFO 43
71 static u32 phys_sockets; /* Physical sockets */
72 static u32 phys_chipspersocket; /* Physical chips per socket*/
73 static u32 phys_coresperchip; /* Physical cores per chip */
76 * read_24x7_sys_info()
77 * Retrieve the number of sockets and chips per socket and cores per
78 * chip details through the get-system-parameter rtas call.
80 void read_24x7_sys_info(void)
82 int call_status, len, ntypes;
84 spin_lock(&rtas_data_buf_lock);
87 * Making system parameter: chips and sockets and cores per chip
91 phys_chipspersocket = 1;
92 phys_coresperchip = 1;
94 call_status = rtas_call(rtas_token("ibm,get-system-parameter"), 3, 1,
96 PROCESSOR_MODULE_INFO,
100 if (call_status != 0) {
101 pr_err("Error calling get-system-parameter %d\n",
104 len = be16_to_cpup((__be16 *)&rtas_data_buf[0]);
108 ntypes = be16_to_cpup((__be16 *)&rtas_data_buf[2]);
113 phys_sockets = be16_to_cpup((__be16 *)&rtas_data_buf[4]);
114 phys_chipspersocket = be16_to_cpup((__be16 *)&rtas_data_buf[6]);
115 phys_coresperchip = be16_to_cpup((__be16 *)&rtas_data_buf[8]);
119 spin_unlock(&rtas_data_buf_lock);
122 /* Domains for which more than one result element are returned for each event. */
123 static bool domain_needs_aggregation(unsigned int domain)
125 return aggregate_result_elements &&
126 (domain == HV_PERF_DOMAIN_PHYS_CORE ||
127 (domain >= HV_PERF_DOMAIN_VCPU_HOME_CORE &&
128 domain <= HV_PERF_DOMAIN_VCPU_REMOTE_NODE));
131 static const char *domain_name(unsigned int domain)
133 if (!domain_is_valid(domain))
137 case HV_PERF_DOMAIN_PHYS_CHIP: return "Physical Chip";
138 case HV_PERF_DOMAIN_PHYS_CORE: return "Physical Core";
139 case HV_PERF_DOMAIN_VCPU_HOME_CORE: return "VCPU Home Core";
140 case HV_PERF_DOMAIN_VCPU_HOME_CHIP: return "VCPU Home Chip";
141 case HV_PERF_DOMAIN_VCPU_HOME_NODE: return "VCPU Home Node";
142 case HV_PERF_DOMAIN_VCPU_REMOTE_NODE: return "VCPU Remote Node";
145 WARN_ON_ONCE(domain);
149 static bool catalog_entry_domain_is_valid(unsigned int domain)
151 /* POWER8 doesn't support virtual domains. */
152 if (interface_version == 1)
153 return is_physical_domain(domain);
155 return domain_is_valid(domain);
159 * TODO: Merging events:
160 * - Think of the hcall as an interface to a 4d array of counters:
162 * - y = indexes in the domain (core, chip, vcpu, node, etc)
163 * - z = offset into the counter space
164 * - w = lpars (guest vms, "logical partitions")
165 * - A single request is: x,y,y_last,z,z_last,w,w_last
166 * - this means we can retrieve a rectangle of counters in y,z for a single x.
168 * - Things to consider (ignoring w):
169 * - input cost_per_request = 16
170 * - output cost_per_result(ys,zs) = 8 + 8 * ys + ys * zs
171 * - limited number of requests per hcall (must fit into 4K bytes)
172 * - 4k = 16 [buffer header] - 16 [request size] * request_count
173 * - 255 requests per hcall
174 * - sometimes it will be more efficient to read extra data and discard
179 * perf stat -e 'hv_24x7/domain=2,offset=8,vcpu=0,lpar=0xffffffff/'
182 /* u3 0-6, one of HV_24X7_PERF_DOMAIN */
183 EVENT_DEFINE_RANGE_FORMAT(domain, config, 0, 3);
185 EVENT_DEFINE_RANGE_FORMAT(core, config, 16, 31);
186 EVENT_DEFINE_RANGE_FORMAT(chip, config, 16, 31);
187 EVENT_DEFINE_RANGE_FORMAT(vcpu, config, 16, 31);
188 /* u32, see "data_offset" */
189 EVENT_DEFINE_RANGE_FORMAT(offset, config, 32, 63);
191 EVENT_DEFINE_RANGE_FORMAT(lpar, config1, 0, 15);
193 EVENT_DEFINE_RANGE(reserved1, config, 4, 15);
194 EVENT_DEFINE_RANGE(reserved2, config1, 16, 63);
195 EVENT_DEFINE_RANGE(reserved3, config2, 0, 63);
197 static struct attribute *format_attrs[] = {
198 &format_attr_domain.attr,
199 &format_attr_offset.attr,
200 &format_attr_core.attr,
201 &format_attr_chip.attr,
202 &format_attr_vcpu.attr,
203 &format_attr_lpar.attr,
207 static const struct attribute_group format_group = {
209 .attrs = format_attrs,
212 static struct attribute_group event_group = {
214 /* .attrs is set in init */
217 static struct attribute_group event_desc_group = {
218 .name = "event_descs",
219 /* .attrs is set in init */
222 static struct attribute_group event_long_desc_group = {
223 .name = "event_long_descs",
224 /* .attrs is set in init */
227 static struct kmem_cache *hv_page_cache;
229 static DEFINE_PER_CPU(int, hv_24x7_txn_flags);
230 static DEFINE_PER_CPU(int, hv_24x7_txn_err);
233 struct perf_event *events[255];
236 static DEFINE_PER_CPU(struct hv_24x7_hw, hv_24x7_hw);
239 * request_buffer and result_buffer are not required to be 4k aligned,
240 * but are not allowed to cross any 4k boundary. Aligning them to 4k is
241 * the simplest way to ensure that.
243 #define H24x7_DATA_BUFFER_SIZE 4096
244 static DEFINE_PER_CPU(char, hv_24x7_reqb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096);
245 static DEFINE_PER_CPU(char, hv_24x7_resb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096);
247 static unsigned int max_num_requests(int interface_version)
249 return (H24x7_DATA_BUFFER_SIZE - sizeof(struct hv_24x7_request_buffer))
250 / H24x7_REQUEST_SIZE(interface_version);
253 static char *event_name(struct hv_24x7_event_data *ev, int *len)
255 *len = be16_to_cpu(ev->event_name_len) - 2;
256 return (char *)ev->remainder;
259 static char *event_desc(struct hv_24x7_event_data *ev, int *len)
261 unsigned int nl = be16_to_cpu(ev->event_name_len);
262 __be16 *desc_len = (__be16 *)(ev->remainder + nl - 2);
264 *len = be16_to_cpu(*desc_len) - 2;
265 return (char *)ev->remainder + nl;
268 static char *event_long_desc(struct hv_24x7_event_data *ev, int *len)
270 unsigned int nl = be16_to_cpu(ev->event_name_len);
271 __be16 *desc_len_ = (__be16 *)(ev->remainder + nl - 2);
272 unsigned int desc_len = be16_to_cpu(*desc_len_);
273 __be16 *long_desc_len = (__be16 *)(ev->remainder + nl + desc_len - 2);
275 *len = be16_to_cpu(*long_desc_len) - 2;
276 return (char *)ev->remainder + nl + desc_len;
279 static bool event_fixed_portion_is_within(struct hv_24x7_event_data *ev,
284 return (start + offsetof(struct hv_24x7_event_data, remainder)) < end;
288 * Things we don't check:
289 * - padding for desc, name, and long/detailed desc is required to be '\0'
292 * Return NULL if we pass end,
293 * Otherwise return the address of the byte just following the event.
295 static void *event_end(struct hv_24x7_event_data *ev, void *end)
299 unsigned int dl, ldl;
300 unsigned int nl = be16_to_cpu(ev->event_name_len);
303 pr_debug("%s: name length too short: %d", __func__, nl);
307 if (start + nl > end) {
308 pr_debug("%s: start=%p + nl=%u > end=%p",
309 __func__, start, nl, end);
313 dl_ = (__be16 *)(ev->remainder + nl - 2);
314 if (!IS_ALIGNED((uintptr_t)dl_, 2))
315 pr_warn("desc len not aligned %p", dl_);
316 dl = be16_to_cpu(*dl_);
318 pr_debug("%s: desc len too short: %d", __func__, dl);
322 if (start + nl + dl > end) {
323 pr_debug("%s: (start=%p + nl=%u + dl=%u)=%p > end=%p",
324 __func__, start, nl, dl, start + nl + dl, end);
328 ldl_ = (__be16 *)(ev->remainder + nl + dl - 2);
329 if (!IS_ALIGNED((uintptr_t)ldl_, 2))
330 pr_warn("long desc len not aligned %p", ldl_);
331 ldl = be16_to_cpu(*ldl_);
333 pr_debug("%s: long desc len too short (ldl=%u)",
338 if (start + nl + dl + ldl > end) {
339 pr_debug("%s: start=%p + nl=%u + dl=%u + ldl=%u > end=%p",
340 __func__, start, nl, dl, ldl, end);
344 return start + nl + dl + ldl;
347 static long h_get_24x7_catalog_page_(unsigned long phys_4096,
348 unsigned long version, unsigned long index)
350 pr_devel("h_get_24x7_catalog_page(0x%lx, %lu, %lu)",
351 phys_4096, version, index);
353 WARN_ON(!IS_ALIGNED(phys_4096, 4096));
355 return plpar_hcall_norets(H_GET_24X7_CATALOG_PAGE,
356 phys_4096, version, index);
359 static long h_get_24x7_catalog_page(char page[], u64 version, u32 index)
361 return h_get_24x7_catalog_page_(virt_to_phys(page),
366 * Each event we find in the catalog, will have a sysfs entry. Format the
367 * data for this sysfs entry based on the event's domain.
369 * Events belonging to the Chip domain can only be monitored in that domain.
370 * i.e the domain for these events is a fixed/knwon value.
372 * Events belonging to the Core domain can be monitored either in the physical
373 * core or in one of the virtual CPU domains. So the domain value for these
374 * events must be specified by the user (i.e is a required parameter). Format
375 * the Core events with 'domain=?' so the perf-tool can error check required
378 * NOTE: For the Core domain events, rather than making domain a required
379 * parameter we could default it to PHYS_CORE and allowe users to
380 * override the domain to one of the VCPU domains.
382 * However, this can make the interface a little inconsistent.
384 * If we set domain=2 (PHYS_CHIP) and allow user to override this field
385 * the user may be tempted to also modify the "offset=x" field in which
386 * can lead to confusing usage. Consider the HPM_PCYC (offset=0x18) and
387 * HPM_INST (offset=0x20) events. With:
389 * perf stat -e hv_24x7/HPM_PCYC,offset=0x20/
391 * we end up monitoring HPM_INST, while the command line has HPM_PCYC.
393 * By not assigning a default value to the domain for the Core events,
394 * we can have simple guidelines:
396 * - Specifying values for parameters with "=?" is required.
398 * - Specifying (i.e overriding) values for other parameters
401 static char *event_fmt(struct hv_24x7_event_data *event, unsigned int domain)
405 const char *domain_str;
409 case HV_PERF_DOMAIN_PHYS_CHIP:
410 snprintf(buf, sizeof(buf), "%d", domain);
415 case HV_PERF_DOMAIN_PHYS_CORE:
426 return kasprintf(GFP_KERNEL,
427 "domain=%s,offset=0x%x,%s=?,lpar=%s",
429 be16_to_cpu(event->event_counter_offs) +
430 be16_to_cpu(event->event_group_record_offs),
435 /* Avoid trusting fw to NUL terminate strings */
436 static char *memdup_to_str(char *maybe_str, int max_len, gfp_t gfp)
438 return kasprintf(gfp, "%.*s", max_len, maybe_str);
441 static ssize_t device_show_string(struct device *dev,
442 struct device_attribute *attr, char *buf)
444 struct dev_ext_attribute *d;
446 d = container_of(attr, struct dev_ext_attribute, attr);
448 return sprintf(buf, "%s\n", (char *)d->var);
451 static ssize_t cpumask_show(struct device *dev,
452 struct device_attribute *attr, char *buf)
454 return cpumap_print_to_pagebuf(true, buf, &hv_24x7_cpumask);
457 static ssize_t sockets_show(struct device *dev,
458 struct device_attribute *attr, char *buf)
460 return sprintf(buf, "%d\n", phys_sockets);
463 static ssize_t chipspersocket_show(struct device *dev,
464 struct device_attribute *attr, char *buf)
466 return sprintf(buf, "%d\n", phys_chipspersocket);
469 static ssize_t coresperchip_show(struct device *dev,
470 struct device_attribute *attr, char *buf)
472 return sprintf(buf, "%d\n", phys_coresperchip);
475 static struct attribute *device_str_attr_create_(char *name, char *str)
477 struct dev_ext_attribute *attr = kzalloc(sizeof(*attr), GFP_KERNEL);
482 sysfs_attr_init(&attr->attr.attr);
485 attr->attr.attr.name = name;
486 attr->attr.attr.mode = 0444;
487 attr->attr.show = device_show_string;
489 return &attr->attr.attr;
493 * Allocate and initialize strings representing event attributes.
495 * NOTE: The strings allocated here are never destroyed and continue to
496 * exist till shutdown. This is to allow us to create as many events
497 * from the catalog as possible, even if we encounter errors with some.
498 * In case of changes to error paths in future, these may need to be
499 * freed by the caller.
501 static struct attribute *device_str_attr_create(char *name, int name_max,
503 char *str, size_t str_max)
506 char *s = memdup_to_str(str, str_max, GFP_KERNEL);
513 n = kasprintf(GFP_KERNEL, "%.*s", name_max, name);
515 n = kasprintf(GFP_KERNEL, "%.*s__%d", name_max, name,
520 a = device_str_attr_create_(n, s);
532 static struct attribute *event_to_attr(unsigned int ix,
533 struct hv_24x7_event_data *event,
538 char *ev_name, *a_ev_name, *val;
539 struct attribute *attr;
541 if (!domain_is_valid(domain)) {
542 pr_warn("catalog event %u has invalid domain %u\n",
547 val = event_fmt(event, domain);
551 ev_name = event_name(event, &event_name_len);
553 a_ev_name = kasprintf(GFP_KERNEL, "%.*s",
554 (int)event_name_len, ev_name);
556 a_ev_name = kasprintf(GFP_KERNEL, "%.*s__%d",
557 (int)event_name_len, ev_name, nonce);
562 attr = device_str_attr_create_(a_ev_name, val);
574 static struct attribute *event_to_desc_attr(struct hv_24x7_event_data *event,
578 char *name = event_name(event, &nl);
579 char *desc = event_desc(event, &dl);
581 /* If there isn't a description, don't create the sysfs file */
585 return device_str_attr_create(name, nl, nonce, desc, dl);
588 static struct attribute *
589 event_to_long_desc_attr(struct hv_24x7_event_data *event, int nonce)
592 char *name = event_name(event, &nl);
593 char *desc = event_long_desc(event, &dl);
595 /* If there isn't a description, don't create the sysfs file */
599 return device_str_attr_create(name, nl, nonce, desc, dl);
602 static int event_data_to_attrs(unsigned int ix, struct attribute **attrs,
603 struct hv_24x7_event_data *event, int nonce)
605 *attrs = event_to_attr(ix, event, event->domain, nonce);
621 static int memord(const void *d1, size_t s1, const void *d2, size_t s2)
628 return memcmp(d1, d2, s1);
631 static int ev_uniq_ord(const void *v1, size_t s1, unsigned int d1,
632 const void *v2, size_t s2, unsigned int d2)
634 int r = memord(v1, s1, v2, s2);
645 static int event_uniq_add(struct rb_root *root, const char *name, int nl,
648 struct rb_node **new = &(root->rb_node), *parent = NULL;
649 struct event_uniq *data;
651 /* Figure out where to put new node */
653 struct event_uniq *it;
656 it = rb_entry(*new, struct event_uniq, node);
657 result = ev_uniq_ord(name, nl, domain, it->name, it->nl,
662 new = &((*new)->rb_left);
664 new = &((*new)->rb_right);
667 pr_info("found a duplicate event %.*s, ct=%u\n", nl,
673 data = kmalloc(sizeof(*data), GFP_KERNEL);
677 *data = (struct event_uniq) {
684 /* Add new node and rebalance tree. */
685 rb_link_node(&data->node, parent, new);
686 rb_insert_color(&data->node, root);
692 static void event_uniq_destroy(struct rb_root *root)
695 * the strings we point to are in the giant block of memory filled by
696 * the catalog, and are freed separately.
698 struct event_uniq *pos, *n;
700 rbtree_postorder_for_each_entry_safe(pos, n, root, node)
706 * ensure the event structure's sizes are self consistent and don't cause us to
707 * read outside of the event
709 * On success, return the event length in bytes.
710 * Otherwise, return -1 (and print as appropriate).
712 static ssize_t catalog_event_len_validate(struct hv_24x7_event_data *event,
714 size_t event_data_bytes,
715 size_t event_entry_count,
716 size_t offset, void *end)
719 void *ev_end, *calc_ev_end;
721 if (offset >= event_data_bytes)
724 if (event_idx >= event_entry_count) {
725 pr_devel("catalog event data has %zu bytes of padding after last event\n",
726 event_data_bytes - offset);
730 if (!event_fixed_portion_is_within(event, end)) {
731 pr_warn("event %zu fixed portion is not within range\n",
736 ev_len = be16_to_cpu(event->length);
739 pr_info("event %zu has length %zu not divisible by 16: event=%pK\n",
740 event_idx, ev_len, event);
742 ev_end = (__u8 *)event + ev_len;
744 pr_warn("event %zu has .length=%zu, ends after buffer end: ev_end=%pK > end=%pK, offset=%zu\n",
745 event_idx, ev_len, ev_end, end,
750 calc_ev_end = event_end(event, end);
752 pr_warn("event %zu has a calculated length which exceeds buffer length %zu: event=%pK end=%pK, offset=%zu\n",
753 event_idx, event_data_bytes, event, end,
758 if (calc_ev_end > ev_end) {
759 pr_warn("event %zu exceeds its own length: event=%pK, end=%pK, offset=%zu, calc_ev_end=%pK\n",
760 event_idx, event, ev_end, offset, calc_ev_end);
768 * Return true incase of invalid or dummy events with names like RESERVED*
770 static bool ignore_event(const char *name)
772 return strncmp(name, "RESERVED", 8) == 0;
775 #define MAX_4K (SIZE_MAX / 4096)
777 static int create_events_from_catalog(struct attribute ***events_,
778 struct attribute ***event_descs_,
779 struct attribute ***event_long_descs_)
782 size_t catalog_len, catalog_page_len, event_entry_count,
783 event_data_len, event_data_offs,
784 event_data_bytes, junk_events, event_idx, event_attr_ct, i,
785 attr_max, event_idx_last, desc_ct, long_desc_ct;
787 uint64_t catalog_version_num;
788 struct attribute **events, **event_descs, **event_long_descs;
789 struct hv_24x7_catalog_page_0 *page_0 =
790 kmem_cache_alloc(hv_page_cache, GFP_KERNEL);
792 void *event_data, *end;
793 struct hv_24x7_event_data *event;
794 struct rb_root ev_uniq = RB_ROOT;
802 hret = h_get_24x7_catalog_page(page, 0, 0);
808 catalog_version_num = be64_to_cpu(page_0->version);
809 catalog_page_len = be32_to_cpu(page_0->length);
811 if (MAX_4K < catalog_page_len) {
812 pr_err("invalid page count: %zu\n", catalog_page_len);
817 catalog_len = catalog_page_len * 4096;
819 event_entry_count = be16_to_cpu(page_0->event_entry_count);
820 event_data_offs = be16_to_cpu(page_0->event_data_offs);
821 event_data_len = be16_to_cpu(page_0->event_data_len);
823 pr_devel("cv %llu cl %zu eec %zu edo %zu edl %zu\n",
824 catalog_version_num, catalog_len,
825 event_entry_count, event_data_offs, event_data_len);
827 if ((MAX_4K < event_data_len)
828 || (MAX_4K < event_data_offs)
829 || (MAX_4K - event_data_offs < event_data_len)) {
830 pr_err("invalid event data offs %zu and/or len %zu\n",
831 event_data_offs, event_data_len);
836 if ((event_data_offs + event_data_len) > catalog_page_len) {
837 pr_err("event data %zu-%zu does not fit inside catalog 0-%zu\n",
839 event_data_offs + event_data_len,
845 if (SIZE_MAX - 1 < event_entry_count) {
846 pr_err("event_entry_count %zu is invalid\n", event_entry_count);
851 event_data_bytes = event_data_len * 4096;
854 * event data can span several pages, events can cross between these
855 * pages. Use vmalloc to make this easier.
857 event_data = vmalloc(event_data_bytes);
859 pr_err("could not allocate event data\n");
864 end = event_data + event_data_bytes;
867 * using vmalloc_to_phys() like this only works if PAGE_SIZE is
870 BUILD_BUG_ON(PAGE_SIZE % 4096);
872 for (i = 0; i < event_data_len; i++) {
873 hret = h_get_24x7_catalog_page_(
874 vmalloc_to_phys(event_data + i * 4096),
876 i + event_data_offs);
878 pr_err("Failed to get event data in page %zu: rc=%ld\n",
879 i + event_data_offs, hret);
886 * scan the catalog to determine the number of attributes we need, and
887 * verify it at the same time.
889 for (junk_events = 0, event = event_data, event_idx = 0, attr_max = 0;
891 event_idx++, event = (void *)event + ev_len) {
892 size_t offset = (void *)event - (void *)event_data;
896 ev_len = catalog_event_len_validate(event, event_idx,
903 name = event_name(event, &nl);
905 if (ignore_event(name)) {
909 if (event->event_group_record_len == 0) {
910 pr_devel("invalid event %zu (%.*s): group_record_len == 0, skipping\n",
911 event_idx, nl, name);
916 if (!catalog_entry_domain_is_valid(event->domain)) {
917 pr_info("event %zu (%.*s) has invalid domain %d\n",
918 event_idx, nl, name, event->domain);
926 event_idx_last = event_idx;
927 if (event_idx_last != event_entry_count)
928 pr_warn("event buffer ended before listed # of events were parsed (got %zu, wanted %zu, junk %zu)\n",
929 event_idx_last, event_entry_count, junk_events);
931 events = kmalloc_array(attr_max + 1, sizeof(*events), GFP_KERNEL);
937 event_descs = kmalloc_array(event_idx + 1, sizeof(*event_descs),
944 event_long_descs = kmalloc_array(event_idx + 1,
945 sizeof(*event_long_descs), GFP_KERNEL);
946 if (!event_long_descs) {
951 /* Iterate over the catalog filling in the attribute vector */
952 for (junk_events = 0, event_attr_ct = 0, desc_ct = 0, long_desc_ct = 0,
953 event = event_data, event_idx = 0;
954 event_idx < event_idx_last;
955 event_idx++, ev_len = be16_to_cpu(event->length),
956 event = (void *)event + ev_len) {
961 * these are the only "bad" events that are intermixed and that
962 * we can ignore without issue. make sure to skip them here
964 if (event->event_group_record_len == 0)
966 if (!catalog_entry_domain_is_valid(event->domain))
969 name = event_name(event, &nl);
970 if (ignore_event(name))
973 nonce = event_uniq_add(&ev_uniq, name, nl, event->domain);
974 ct = event_data_to_attrs(event_idx, events + event_attr_ct,
977 pr_warn("event %zu (%.*s) creation failure, skipping\n",
978 event_idx, nl, name);
982 event_descs[desc_ct] = event_to_desc_attr(event, nonce);
983 if (event_descs[desc_ct])
985 event_long_descs[long_desc_ct] =
986 event_to_long_desc_attr(event, nonce);
987 if (event_long_descs[long_desc_ct])
992 pr_info("read %zu catalog entries, created %zu event attrs (%zu failures), %zu descs\n",
993 event_idx, event_attr_ct, junk_events, desc_ct);
995 events[event_attr_ct] = NULL;
996 event_descs[desc_ct] = NULL;
997 event_long_descs[long_desc_ct] = NULL;
999 event_uniq_destroy(&ev_uniq);
1001 kmem_cache_free(hv_page_cache, page);
1004 *event_descs_ = event_descs;
1005 *event_long_descs_ = event_long_descs;
1015 kmem_cache_free(hv_page_cache, page);
1018 *event_descs_ = NULL;
1019 *event_long_descs_ = NULL;
1023 static ssize_t catalog_read(struct file *filp, struct kobject *kobj,
1024 struct bin_attribute *bin_attr, char *buf,
1025 loff_t offset, size_t count)
1029 size_t catalog_len = 0, catalog_page_len = 0;
1030 loff_t page_offset = 0;
1031 loff_t offset_in_page;
1033 uint64_t catalog_version_num = 0;
1034 void *page = kmem_cache_alloc(hv_page_cache, GFP_USER);
1035 struct hv_24x7_catalog_page_0 *page_0 = page;
1040 hret = h_get_24x7_catalog_page(page, 0, 0);
1046 catalog_version_num = be64_to_cpu(page_0->version);
1047 catalog_page_len = be32_to_cpu(page_0->length);
1048 catalog_len = catalog_page_len * 4096;
1050 page_offset = offset / 4096;
1051 offset_in_page = offset % 4096;
1053 if (page_offset >= catalog_page_len)
1056 if (page_offset != 0) {
1057 hret = h_get_24x7_catalog_page(page, catalog_version_num,
1065 copy_len = 4096 - offset_in_page;
1066 if (copy_len > count)
1069 memcpy(buf, page+offset_in_page, copy_len);
1074 pr_err("h_get_24x7_catalog_page(ver=%lld, page=%lld) failed:"
1076 catalog_version_num, page_offset, hret);
1077 kmem_cache_free(hv_page_cache, page);
1079 pr_devel("catalog_read: offset=%lld(%lld) count=%zu "
1080 "catalog_len=%zu(%zu) => %zd\n", offset, page_offset,
1081 count, catalog_len, catalog_page_len, ret);
1086 static ssize_t domains_show(struct device *dev, struct device_attribute *attr,
1089 int d, n, count = 0;
1092 for (d = 0; d < HV_PERF_DOMAIN_MAX; d++) {
1093 str = domain_name(d);
1097 n = sprintf(page, "%d: %s\n", d, str);
1107 #define PAGE_0_ATTR(_name, _fmt, _expr) \
1108 static ssize_t _name##_show(struct device *dev, \
1109 struct device_attribute *dev_attr, \
1114 void *page = kmem_cache_alloc(hv_page_cache, GFP_USER); \
1115 struct hv_24x7_catalog_page_0 *page_0 = page; \
1118 hret = h_get_24x7_catalog_page(page, 0, 0); \
1123 ret = sprintf(buf, _fmt, _expr); \
1125 kmem_cache_free(hv_page_cache, page); \
1128 static DEVICE_ATTR_RO(_name)
1130 PAGE_0_ATTR(catalog_version, "%lld\n",
1131 (unsigned long long)be64_to_cpu(page_0->version));
1132 PAGE_0_ATTR(catalog_len, "%lld\n",
1133 (unsigned long long)be32_to_cpu(page_0->length) * 4096);
1134 static BIN_ATTR_RO(catalog, 0/* real length varies */);
1135 static DEVICE_ATTR_RO(domains);
1136 static DEVICE_ATTR_RO(sockets);
1137 static DEVICE_ATTR_RO(chipspersocket);
1138 static DEVICE_ATTR_RO(coresperchip);
1139 static DEVICE_ATTR_RO(cpumask);
1141 static struct bin_attribute *if_bin_attrs[] = {
1146 static struct attribute *cpumask_attrs[] = {
1147 &dev_attr_cpumask.attr,
1151 static const struct attribute_group cpumask_attr_group = {
1152 .attrs = cpumask_attrs,
1155 static struct attribute *if_attrs[] = {
1156 &dev_attr_catalog_len.attr,
1157 &dev_attr_catalog_version.attr,
1158 &dev_attr_domains.attr,
1159 &dev_attr_sockets.attr,
1160 &dev_attr_chipspersocket.attr,
1161 &dev_attr_coresperchip.attr,
1165 static const struct attribute_group if_group = {
1166 .name = "interface",
1167 .bin_attrs = if_bin_attrs,
1171 static const struct attribute_group *attr_groups[] = {
1175 &event_long_desc_group,
1177 &cpumask_attr_group,
1182 * Start the process for a new H_GET_24x7_DATA hcall.
1184 static void init_24x7_request(struct hv_24x7_request_buffer *request_buffer,
1185 struct hv_24x7_data_result_buffer *result_buffer)
1188 memset(request_buffer, 0, H24x7_DATA_BUFFER_SIZE);
1189 memset(result_buffer, 0, H24x7_DATA_BUFFER_SIZE);
1191 request_buffer->interface_version = interface_version;
1192 /* memset above set request_buffer->num_requests to 0 */
1196 * Commit (i.e perform) the H_GET_24x7_DATA hcall using the data collected
1197 * by 'init_24x7_request()' and 'add_event_to_24x7_request()'.
1199 static int make_24x7_request(struct hv_24x7_request_buffer *request_buffer,
1200 struct hv_24x7_data_result_buffer *result_buffer)
1205 * NOTE: Due to variable number of array elements in request and
1206 * result buffer(s), sizeof() is not reliable. Use the actual
1207 * allocated buffer size, H24x7_DATA_BUFFER_SIZE.
1209 ret = plpar_hcall_norets(H_GET_24X7_DATA,
1210 virt_to_phys(request_buffer), H24x7_DATA_BUFFER_SIZE,
1211 virt_to_phys(result_buffer), H24x7_DATA_BUFFER_SIZE);
1214 struct hv_24x7_request *req;
1216 req = request_buffer->requests;
1217 pr_notice_ratelimited("hcall failed: [%d %#x %#x %d] => ret 0x%lx (%ld) detail=0x%x failing ix=%x\n",
1218 req->performance_domain, req->data_offset,
1219 req->starting_ix, req->starting_lpar_ix,
1220 ret, ret, result_buffer->detailed_rc,
1221 result_buffer->failing_request_ix);
1229 * Add the given @event to the next slot in the 24x7 request_buffer.
1231 * Note that H_GET_24X7_DATA hcall allows reading several counters'
1232 * values in a single HCALL. We expect the caller to add events to the
1233 * request buffer one by one, make the HCALL and process the results.
1235 static int add_event_to_24x7_request(struct perf_event *event,
1236 struct hv_24x7_request_buffer *request_buffer)
1241 struct hv_24x7_request *req;
1243 if (request_buffer->num_requests >=
1244 max_num_requests(request_buffer->interface_version)) {
1245 pr_devel("Too many requests for 24x7 HCALL %d\n",
1246 request_buffer->num_requests);
1250 switch (event_get_domain(event)) {
1251 case HV_PERF_DOMAIN_PHYS_CHIP:
1252 idx = event_get_chip(event);
1254 case HV_PERF_DOMAIN_PHYS_CORE:
1255 idx = event_get_core(event);
1258 idx = event_get_vcpu(event);
1261 req_size = H24x7_REQUEST_SIZE(request_buffer->interface_version);
1263 i = request_buffer->num_requests++;
1264 req = (void *) request_buffer->requests + i * req_size;
1266 req->performance_domain = event_get_domain(event);
1267 req->data_size = cpu_to_be16(8);
1268 req->data_offset = cpu_to_be32(event_get_offset(event));
1269 req->starting_lpar_ix = cpu_to_be16(event_get_lpar(event));
1270 req->max_num_lpars = cpu_to_be16(1);
1271 req->starting_ix = cpu_to_be16(idx);
1272 req->max_ix = cpu_to_be16(1);
1274 if (request_buffer->interface_version > 1) {
1275 if (domain_needs_aggregation(req->performance_domain))
1276 req->max_num_thread_groups = -1;
1277 else if (req->performance_domain != HV_PERF_DOMAIN_PHYS_CHIP) {
1278 req->starting_thread_group_ix = idx % 2;
1279 req->max_num_thread_groups = 1;
1287 * get_count_from_result - get event count from all result elements in result
1289 * If the event corresponding to this result needs aggregation of the result
1290 * element values, then this function does that.
1292 * @event: Event associated with @res.
1293 * @resb: Result buffer containing @res.
1294 * @res: Result to work on.
1295 * @countp: Output variable containing the event count.
1296 * @next: Optional output variable pointing to the next result in @resb.
1298 static int get_count_from_result(struct perf_event *event,
1299 struct hv_24x7_data_result_buffer *resb,
1300 struct hv_24x7_result *res, u64 *countp,
1301 struct hv_24x7_result **next)
1303 u16 num_elements = be16_to_cpu(res->num_elements_returned);
1304 u16 data_size = be16_to_cpu(res->result_element_data_size);
1305 unsigned int data_offset;
1311 * We can bail out early if the result is empty.
1313 if (!num_elements) {
1314 pr_debug("Result of request %hhu is empty, nothing to do\n",
1318 *next = (struct hv_24x7_result *) res->elements;
1324 * Since we always specify 1 as the maximum for the smallest resource
1325 * we're requesting, there should to be only one element per result.
1326 * Except when an event needs aggregation, in which case there are more.
1328 if (num_elements != 1 &&
1329 !domain_needs_aggregation(event_get_domain(event))) {
1330 pr_err("Error: result of request %hhu has %hu elements\n",
1331 res->result_ix, num_elements);
1336 if (data_size != sizeof(u64)) {
1337 pr_debug("Error: result of request %hhu has data of %hu bytes\n",
1338 res->result_ix, data_size);
1343 if (resb->interface_version == 1)
1344 data_offset = offsetof(struct hv_24x7_result_element_v1,
1347 data_offset = offsetof(struct hv_24x7_result_element_v2,
1350 /* Go through the result elements in the result. */
1351 for (i = count = 0, element_data = res->elements + data_offset;
1353 i++, element_data += data_size + data_offset)
1354 count += be64_to_cpu(*((u64 *) element_data));
1358 /* The next result is after the last result element. */
1360 *next = element_data - data_offset;
1365 static int single_24x7_request(struct perf_event *event, u64 *count)
1368 struct hv_24x7_request_buffer *request_buffer;
1369 struct hv_24x7_data_result_buffer *result_buffer;
1371 BUILD_BUG_ON(sizeof(*request_buffer) > 4096);
1372 BUILD_BUG_ON(sizeof(*result_buffer) > 4096);
1374 request_buffer = (void *)get_cpu_var(hv_24x7_reqb);
1375 result_buffer = (void *)get_cpu_var(hv_24x7_resb);
1377 init_24x7_request(request_buffer, result_buffer);
1379 ret = add_event_to_24x7_request(event, request_buffer);
1383 ret = make_24x7_request(request_buffer, result_buffer);
1387 /* process result from hcall */
1388 ret = get_count_from_result(event, result_buffer,
1389 result_buffer->results, count, NULL);
1392 put_cpu_var(hv_24x7_reqb);
1393 put_cpu_var(hv_24x7_resb);
1398 static int h_24x7_event_init(struct perf_event *event)
1400 struct hv_perf_caps caps;
1401 unsigned int domain;
1406 if (event->attr.type != event->pmu->type)
1409 /* Unused areas must be 0 */
1410 if (event_get_reserved1(event) ||
1411 event_get_reserved2(event) ||
1412 event_get_reserved3(event)) {
1413 pr_devel("reserved set when forbidden 0x%llx(0x%llx) 0x%llx(0x%llx) 0x%llx(0x%llx)\n",
1415 event_get_reserved1(event),
1416 event->attr.config1,
1417 event_get_reserved2(event),
1418 event->attr.config2,
1419 event_get_reserved3(event));
1423 /* no branch sampling */
1424 if (has_branch_stack(event))
1427 /* offset must be 8 byte aligned */
1428 if (event_get_offset(event) % 8) {
1429 pr_devel("bad alignment\n");
1433 domain = event_get_domain(event);
1434 if (domain >= HV_PERF_DOMAIN_MAX) {
1435 pr_devel("invalid domain %d\n", domain);
1439 hret = hv_perf_caps_get(&caps);
1441 pr_devel("could not get capabilities: rc=%ld\n", hret);
1445 /* Physical domains & other lpars require extra capabilities */
1446 if (!caps.collect_privileged && (is_physical_domain(domain) ||
1447 (event_get_lpar(event) != event_get_lpar_max()))) {
1448 pr_devel("hv permissions disallow: is_physical_domain:%d, lpar=0x%llx\n",
1449 is_physical_domain(domain),
1450 event_get_lpar(event));
1454 /* Get the initial value of the counter for this event */
1455 if (single_24x7_request(event, &ct)) {
1456 pr_devel("test hcall failed\n");
1459 (void)local64_xchg(&event->hw.prev_count, ct);
1464 static u64 h_24x7_get_value(struct perf_event *event)
1468 if (single_24x7_request(event, &ct))
1469 /* We checked this in event init, shouldn't fail here... */
1475 static void update_event_count(struct perf_event *event, u64 now)
1479 prev = local64_xchg(&event->hw.prev_count, now);
1480 local64_add(now - prev, &event->count);
1483 static void h_24x7_event_read(struct perf_event *event)
1486 struct hv_24x7_request_buffer *request_buffer;
1487 struct hv_24x7_hw *h24x7hw;
1490 txn_flags = __this_cpu_read(hv_24x7_txn_flags);
1493 * If in a READ transaction, add this counter to the list of
1494 * counters to read during the next HCALL (i.e commit_txn()).
1495 * If not in a READ transaction, go ahead and make the HCALL
1496 * to read this counter by itself.
1499 if (txn_flags & PERF_PMU_TXN_READ) {
1503 if (__this_cpu_read(hv_24x7_txn_err))
1506 request_buffer = (void *)get_cpu_var(hv_24x7_reqb);
1508 ret = add_event_to_24x7_request(event, request_buffer);
1510 __this_cpu_write(hv_24x7_txn_err, ret);
1513 * Associate the event with the HCALL request index,
1514 * so ->commit_txn() can quickly find/update count.
1516 i = request_buffer->num_requests - 1;
1518 h24x7hw = &get_cpu_var(hv_24x7_hw);
1519 h24x7hw->events[i] = event;
1520 put_cpu_var(h24x7hw);
1523 put_cpu_var(hv_24x7_reqb);
1525 now = h_24x7_get_value(event);
1526 update_event_count(event, now);
1530 static void h_24x7_event_start(struct perf_event *event, int flags)
1532 if (flags & PERF_EF_RELOAD)
1533 local64_set(&event->hw.prev_count, h_24x7_get_value(event));
1536 static void h_24x7_event_stop(struct perf_event *event, int flags)
1538 h_24x7_event_read(event);
1541 static int h_24x7_event_add(struct perf_event *event, int flags)
1543 if (flags & PERF_EF_START)
1544 h_24x7_event_start(event, flags);
1550 * 24x7 counters only support READ transactions. They are
1551 * always counting and dont need/support ADD transactions.
1552 * Cache the flags, but otherwise ignore transactions that
1553 * are not PERF_PMU_TXN_READ.
1555 static void h_24x7_event_start_txn(struct pmu *pmu, unsigned int flags)
1557 struct hv_24x7_request_buffer *request_buffer;
1558 struct hv_24x7_data_result_buffer *result_buffer;
1560 /* We should not be called if we are already in a txn */
1561 WARN_ON_ONCE(__this_cpu_read(hv_24x7_txn_flags));
1563 __this_cpu_write(hv_24x7_txn_flags, flags);
1564 if (flags & ~PERF_PMU_TXN_READ)
1567 request_buffer = (void *)get_cpu_var(hv_24x7_reqb);
1568 result_buffer = (void *)get_cpu_var(hv_24x7_resb);
1570 init_24x7_request(request_buffer, result_buffer);
1572 put_cpu_var(hv_24x7_resb);
1573 put_cpu_var(hv_24x7_reqb);
1577 * Clean up transaction state.
1579 * NOTE: Ignore state of request and result buffers for now.
1580 * We will initialize them during the next read/txn.
1582 static void reset_txn(void)
1584 __this_cpu_write(hv_24x7_txn_flags, 0);
1585 __this_cpu_write(hv_24x7_txn_err, 0);
1589 * 24x7 counters only support READ transactions. They are always counting
1590 * and dont need/support ADD transactions. Clear ->txn_flags but otherwise
1591 * ignore transactions that are not of type PERF_PMU_TXN_READ.
1593 * For READ transactions, submit all pending 24x7 requests (i.e requests
1594 * that were queued by h_24x7_event_read()), to the hypervisor and update
1597 static int h_24x7_event_commit_txn(struct pmu *pmu)
1599 struct hv_24x7_request_buffer *request_buffer;
1600 struct hv_24x7_data_result_buffer *result_buffer;
1601 struct hv_24x7_result *res, *next_res;
1603 int i, ret, txn_flags;
1604 struct hv_24x7_hw *h24x7hw;
1606 txn_flags = __this_cpu_read(hv_24x7_txn_flags);
1607 WARN_ON_ONCE(!txn_flags);
1610 if (txn_flags & ~PERF_PMU_TXN_READ)
1613 ret = __this_cpu_read(hv_24x7_txn_err);
1617 request_buffer = (void *)get_cpu_var(hv_24x7_reqb);
1618 result_buffer = (void *)get_cpu_var(hv_24x7_resb);
1620 ret = make_24x7_request(request_buffer, result_buffer);
1624 h24x7hw = &get_cpu_var(hv_24x7_hw);
1626 /* Go through results in the result buffer to update event counts. */
1627 for (i = 0, res = result_buffer->results;
1628 i < result_buffer->num_results; i++, res = next_res) {
1629 struct perf_event *event = h24x7hw->events[res->result_ix];
1631 ret = get_count_from_result(event, result_buffer, res, &count,
1636 update_event_count(event, count);
1639 put_cpu_var(hv_24x7_hw);
1642 put_cpu_var(hv_24x7_resb);
1643 put_cpu_var(hv_24x7_reqb);
1650 * 24x7 counters only support READ transactions. They are always counting
1651 * and dont need/support ADD transactions. However, regardless of type
1652 * of transaction, all we need to do is cleanup, so we don't have to check
1653 * the type of transaction.
1655 static void h_24x7_event_cancel_txn(struct pmu *pmu)
1657 WARN_ON_ONCE(!__this_cpu_read(hv_24x7_txn_flags));
1661 static struct pmu h_24x7_pmu = {
1662 .task_ctx_nr = perf_invalid_context,
1665 .attr_groups = attr_groups,
1666 .event_init = h_24x7_event_init,
1667 .add = h_24x7_event_add,
1668 .del = h_24x7_event_stop,
1669 .start = h_24x7_event_start,
1670 .stop = h_24x7_event_stop,
1671 .read = h_24x7_event_read,
1672 .start_txn = h_24x7_event_start_txn,
1673 .commit_txn = h_24x7_event_commit_txn,
1674 .cancel_txn = h_24x7_event_cancel_txn,
1675 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
1678 static int ppc_hv_24x7_cpu_online(unsigned int cpu)
1680 if (cpumask_empty(&hv_24x7_cpumask))
1681 cpumask_set_cpu(cpu, &hv_24x7_cpumask);
1686 static int ppc_hv_24x7_cpu_offline(unsigned int cpu)
1690 /* Check if exiting cpu is used for collecting 24x7 events */
1691 if (!cpumask_test_and_clear_cpu(cpu, &hv_24x7_cpumask))
1694 /* Find a new cpu to collect 24x7 events */
1695 target = cpumask_last(cpu_active_mask);
1697 if (target < 0 || target >= nr_cpu_ids) {
1698 pr_err("hv_24x7: CPU hotplug init failed\n");
1702 /* Migrate 24x7 events to the new target */
1703 cpumask_set_cpu(target, &hv_24x7_cpumask);
1704 perf_pmu_migrate_context(&h_24x7_pmu, cpu, target);
1709 static int hv_24x7_cpu_hotplug_init(void)
1711 return cpuhp_setup_state(CPUHP_AP_PERF_POWERPC_HV_24x7_ONLINE,
1712 "perf/powerpc/hv_24x7:online",
1713 ppc_hv_24x7_cpu_online,
1714 ppc_hv_24x7_cpu_offline);
1717 static int hv_24x7_init(void)
1721 struct hv_perf_caps caps;
1723 if (!firmware_has_feature(FW_FEATURE_LPAR)) {
1724 pr_debug("not a virtualized system, not enabling\n");
1726 } else if (!cur_cpu_spec->oprofile_cpu_type)
1729 /* POWER8 only supports v1, while POWER9 only supports v2. */
1730 if (!strcmp(cur_cpu_spec->oprofile_cpu_type, "ppc64/power8"))
1731 interface_version = 1;
1733 interface_version = 2;
1735 /* SMT8 in POWER9 needs to aggregate result elements. */
1736 if (threads_per_core == 8)
1737 aggregate_result_elements = true;
1740 hret = hv_perf_caps_get(&caps);
1742 pr_debug("could not obtain capabilities, not enabling, rc=%ld\n",
1747 hv_page_cache = kmem_cache_create("hv-page-4096", 4096, 4096, 0, NULL);
1751 /* sampling not supported */
1752 h_24x7_pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
1754 r = create_events_from_catalog(&event_group.attrs,
1755 &event_desc_group.attrs,
1756 &event_long_desc_group.attrs);
1761 /* init cpuhotplug */
1762 r = hv_24x7_cpu_hotplug_init();
1766 r = perf_pmu_register(&h_24x7_pmu, h_24x7_pmu.name, -1);
1770 read_24x7_sys_info();
1775 device_initcall(hv_24x7_init);