1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Hypervisor supplied "24x7" performance counter support
5 * Author: Cody P Schafer <cody@linux.vnet.ibm.com>
6 * Copyright 2014 IBM Corporation.
9 #define pr_fmt(fmt) "hv-24x7: " fmt
11 #include <linux/perf_event.h>
12 #include <linux/rbtree.h>
13 #include <linux/module.h>
14 #include <linux/slab.h>
15 #include <linux/vmalloc.h>
17 #include <asm/cputhreads.h>
18 #include <asm/firmware.h>
19 #include <asm/hvcall.h>
21 #include <asm/papr-sysparm.h>
22 #include <linux/byteorder/generic.h>
26 #include "hv-24x7-catalog.h"
27 #include "hv-common.h"
29 /* Version of the 24x7 hypervisor API that we should use in this machine. */
30 static int interface_version;
32 /* Whether we have to aggregate result data for some domains. */
33 static bool aggregate_result_elements;
35 static cpumask_t hv_24x7_cpumask;
37 static bool domain_is_valid(unsigned int domain)
40 #define DOMAIN(n, v, x, c) \
41 case HV_PERF_DOMAIN_##n: \
43 #include "hv-24x7-domains.h"
51 static bool is_physical_domain(unsigned int domain)
54 #define DOMAIN(n, v, x, c) \
55 case HV_PERF_DOMAIN_##n: \
57 #include "hv-24x7-domains.h"
65 * The Processor Module Information system parameter allows transferring
66 * of certain processor module information from the platform to the OS.
67 * Refer PAPR+ document to get parameter token value as '43'.
70 static u32 phys_sockets; /* Physical sockets */
71 static u32 phys_chipspersocket; /* Physical chips per socket*/
72 static u32 phys_coresperchip; /* Physical cores per chip */
75 * read_24x7_sys_info()
76 * Retrieve the number of sockets and chips per socket and cores per
77 * chip details through the get-system-parameter rtas call.
79 void read_24x7_sys_info(void)
81 struct papr_sysparm_buf *buf;
84 * Making system parameter: chips and sockets and cores per chip
88 phys_chipspersocket = 1;
89 phys_coresperchip = 1;
91 buf = papr_sysparm_buf_alloc();
95 if (!papr_sysparm_get(PAPR_SYSPARM_PROC_MODULE_INFO, buf)) {
96 int ntypes = be16_to_cpup((__be16 *)&buf->val[0]);
97 int len = be16_to_cpu(buf->len);
99 if (len >= 8 && ntypes != 0) {
100 phys_sockets = be16_to_cpup((__be16 *)&buf->val[2]);
101 phys_chipspersocket = be16_to_cpup((__be16 *)&buf->val[4]);
102 phys_coresperchip = be16_to_cpup((__be16 *)&buf->val[6]);
106 papr_sysparm_buf_free(buf);
109 /* Domains for which more than one result element are returned for each event. */
110 static bool domain_needs_aggregation(unsigned int domain)
112 return aggregate_result_elements &&
113 (domain == HV_PERF_DOMAIN_PHYS_CORE ||
114 (domain >= HV_PERF_DOMAIN_VCPU_HOME_CORE &&
115 domain <= HV_PERF_DOMAIN_VCPU_REMOTE_NODE));
118 static const char *domain_name(unsigned int domain)
120 if (!domain_is_valid(domain))
124 case HV_PERF_DOMAIN_PHYS_CHIP: return "Physical Chip";
125 case HV_PERF_DOMAIN_PHYS_CORE: return "Physical Core";
126 case HV_PERF_DOMAIN_VCPU_HOME_CORE: return "VCPU Home Core";
127 case HV_PERF_DOMAIN_VCPU_HOME_CHIP: return "VCPU Home Chip";
128 case HV_PERF_DOMAIN_VCPU_HOME_NODE: return "VCPU Home Node";
129 case HV_PERF_DOMAIN_VCPU_REMOTE_NODE: return "VCPU Remote Node";
132 WARN_ON_ONCE(domain);
136 static bool catalog_entry_domain_is_valid(unsigned int domain)
138 /* POWER8 doesn't support virtual domains. */
139 if (interface_version == 1)
140 return is_physical_domain(domain);
142 return domain_is_valid(domain);
146 * TODO: Merging events:
147 * - Think of the hcall as an interface to a 4d array of counters:
149 * - y = indexes in the domain (core, chip, vcpu, node, etc)
150 * - z = offset into the counter space
151 * - w = lpars (guest vms, "logical partitions")
152 * - A single request is: x,y,y_last,z,z_last,w,w_last
153 * - this means we can retrieve a rectangle of counters in y,z for a single x.
155 * - Things to consider (ignoring w):
156 * - input cost_per_request = 16
157 * - output cost_per_result(ys,zs) = 8 + 8 * ys + ys * zs
158 * - limited number of requests per hcall (must fit into 4K bytes)
159 * - 4k = 16 [buffer header] - 16 [request size] * request_count
160 * - 255 requests per hcall
161 * - sometimes it will be more efficient to read extra data and discard
166 * perf stat -e 'hv_24x7/domain=2,offset=8,vcpu=0,lpar=0xffffffff/'
169 /* u3 0-6, one of HV_24X7_PERF_DOMAIN */
170 EVENT_DEFINE_RANGE_FORMAT(domain, config, 0, 3);
172 EVENT_DEFINE_RANGE_FORMAT(core, config, 16, 31);
173 EVENT_DEFINE_RANGE_FORMAT(chip, config, 16, 31);
174 EVENT_DEFINE_RANGE_FORMAT(vcpu, config, 16, 31);
175 /* u32, see "data_offset" */
176 EVENT_DEFINE_RANGE_FORMAT(offset, config, 32, 63);
178 EVENT_DEFINE_RANGE_FORMAT(lpar, config1, 0, 15);
180 EVENT_DEFINE_RANGE(reserved1, config, 4, 15);
181 EVENT_DEFINE_RANGE(reserved2, config1, 16, 63);
182 EVENT_DEFINE_RANGE(reserved3, config2, 0, 63);
184 static struct attribute *format_attrs[] = {
185 &format_attr_domain.attr,
186 &format_attr_offset.attr,
187 &format_attr_core.attr,
188 &format_attr_chip.attr,
189 &format_attr_vcpu.attr,
190 &format_attr_lpar.attr,
194 static const struct attribute_group format_group = {
196 .attrs = format_attrs,
199 static struct attribute_group event_group = {
201 /* .attrs is set in init */
204 static struct attribute_group event_desc_group = {
205 .name = "event_descs",
206 /* .attrs is set in init */
209 static struct attribute_group event_long_desc_group = {
210 .name = "event_long_descs",
211 /* .attrs is set in init */
214 static struct kmem_cache *hv_page_cache;
216 static DEFINE_PER_CPU(int, hv_24x7_txn_flags);
217 static DEFINE_PER_CPU(int, hv_24x7_txn_err);
220 struct perf_event *events[255];
223 static DEFINE_PER_CPU(struct hv_24x7_hw, hv_24x7_hw);
226 * request_buffer and result_buffer are not required to be 4k aligned,
227 * but are not allowed to cross any 4k boundary. Aligning them to 4k is
228 * the simplest way to ensure that.
230 #define H24x7_DATA_BUFFER_SIZE 4096
231 static DEFINE_PER_CPU(char, hv_24x7_reqb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096);
232 static DEFINE_PER_CPU(char, hv_24x7_resb[H24x7_DATA_BUFFER_SIZE]) __aligned(4096);
234 static unsigned int max_num_requests(int interface_version)
236 return (H24x7_DATA_BUFFER_SIZE - sizeof(struct hv_24x7_request_buffer))
237 / H24x7_REQUEST_SIZE(interface_version);
240 static char *event_name(struct hv_24x7_event_data *ev, int *len)
242 *len = be16_to_cpu(ev->event_name_len) - 2;
243 return (char *)ev->remainder;
246 static char *event_desc(struct hv_24x7_event_data *ev, int *len)
248 unsigned int nl = be16_to_cpu(ev->event_name_len);
249 __be16 *desc_len = (__be16 *)(ev->remainder + nl - 2);
251 *len = be16_to_cpu(*desc_len) - 2;
252 return (char *)ev->remainder + nl;
255 static char *event_long_desc(struct hv_24x7_event_data *ev, int *len)
257 unsigned int nl = be16_to_cpu(ev->event_name_len);
258 __be16 *desc_len_ = (__be16 *)(ev->remainder + nl - 2);
259 unsigned int desc_len = be16_to_cpu(*desc_len_);
260 __be16 *long_desc_len = (__be16 *)(ev->remainder + nl + desc_len - 2);
262 *len = be16_to_cpu(*long_desc_len) - 2;
263 return (char *)ev->remainder + nl + desc_len;
266 static bool event_fixed_portion_is_within(struct hv_24x7_event_data *ev,
271 return (start + offsetof(struct hv_24x7_event_data, remainder)) < end;
275 * Things we don't check:
276 * - padding for desc, name, and long/detailed desc is required to be '\0'
279 * Return NULL if we pass end,
280 * Otherwise return the address of the byte just following the event.
282 static void *event_end(struct hv_24x7_event_data *ev, void *end)
286 unsigned int dl, ldl;
287 unsigned int nl = be16_to_cpu(ev->event_name_len);
290 pr_debug("%s: name length too short: %d", __func__, nl);
294 if (start + nl > end) {
295 pr_debug("%s: start=%p + nl=%u > end=%p",
296 __func__, start, nl, end);
300 dl_ = (__be16 *)(ev->remainder + nl - 2);
301 if (!IS_ALIGNED((uintptr_t)dl_, 2))
302 pr_warn("desc len not aligned %p", dl_);
303 dl = be16_to_cpu(*dl_);
305 pr_debug("%s: desc len too short: %d", __func__, dl);
309 if (start + nl + dl > end) {
310 pr_debug("%s: (start=%p + nl=%u + dl=%u)=%p > end=%p",
311 __func__, start, nl, dl, start + nl + dl, end);
315 ldl_ = (__be16 *)(ev->remainder + nl + dl - 2);
316 if (!IS_ALIGNED((uintptr_t)ldl_, 2))
317 pr_warn("long desc len not aligned %p", ldl_);
318 ldl = be16_to_cpu(*ldl_);
320 pr_debug("%s: long desc len too short (ldl=%u)",
325 if (start + nl + dl + ldl > end) {
326 pr_debug("%s: start=%p + nl=%u + dl=%u + ldl=%u > end=%p",
327 __func__, start, nl, dl, ldl, end);
331 return start + nl + dl + ldl;
334 static long h_get_24x7_catalog_page_(unsigned long phys_4096,
335 unsigned long version, unsigned long index)
337 pr_devel("h_get_24x7_catalog_page(0x%lx, %lu, %lu)",
338 phys_4096, version, index);
340 WARN_ON(!IS_ALIGNED(phys_4096, 4096));
342 return plpar_hcall_norets(H_GET_24X7_CATALOG_PAGE,
343 phys_4096, version, index);
346 static long h_get_24x7_catalog_page(char page[], u64 version, u32 index)
348 return h_get_24x7_catalog_page_(virt_to_phys(page),
353 * Each event we find in the catalog, will have a sysfs entry. Format the
354 * data for this sysfs entry based on the event's domain.
356 * Events belonging to the Chip domain can only be monitored in that domain.
357 * i.e the domain for these events is a fixed/knwon value.
359 * Events belonging to the Core domain can be monitored either in the physical
360 * core or in one of the virtual CPU domains. So the domain value for these
361 * events must be specified by the user (i.e is a required parameter). Format
362 * the Core events with 'domain=?' so the perf-tool can error check required
365 * NOTE: For the Core domain events, rather than making domain a required
366 * parameter we could default it to PHYS_CORE and allowe users to
367 * override the domain to one of the VCPU domains.
369 * However, this can make the interface a little inconsistent.
371 * If we set domain=2 (PHYS_CHIP) and allow user to override this field
372 * the user may be tempted to also modify the "offset=x" field in which
373 * can lead to confusing usage. Consider the HPM_PCYC (offset=0x18) and
374 * HPM_INST (offset=0x20) events. With:
376 * perf stat -e hv_24x7/HPM_PCYC,offset=0x20/
378 * we end up monitoring HPM_INST, while the command line has HPM_PCYC.
380 * By not assigning a default value to the domain for the Core events,
381 * we can have simple guidelines:
383 * - Specifying values for parameters with "=?" is required.
385 * - Specifying (i.e overriding) values for other parameters
388 static char *event_fmt(struct hv_24x7_event_data *event, unsigned int domain)
392 const char *domain_str;
396 case HV_PERF_DOMAIN_PHYS_CHIP:
397 snprintf(buf, sizeof(buf), "%d", domain);
402 case HV_PERF_DOMAIN_PHYS_CORE:
413 return kasprintf(GFP_KERNEL,
414 "domain=%s,offset=0x%x,%s=?,lpar=%s",
416 be16_to_cpu(event->event_counter_offs) +
417 be16_to_cpu(event->event_group_record_offs),
422 /* Avoid trusting fw to NUL terminate strings */
423 static char *memdup_to_str(char *maybe_str, int max_len, gfp_t gfp)
425 return kasprintf(gfp, "%.*s", max_len, maybe_str);
428 static ssize_t device_show_string(struct device *dev,
429 struct device_attribute *attr, char *buf)
431 struct dev_ext_attribute *d;
433 d = container_of(attr, struct dev_ext_attribute, attr);
435 return sprintf(buf, "%s\n", (char *)d->var);
438 static ssize_t cpumask_show(struct device *dev,
439 struct device_attribute *attr, char *buf)
441 return cpumap_print_to_pagebuf(true, buf, &hv_24x7_cpumask);
444 static ssize_t sockets_show(struct device *dev,
445 struct device_attribute *attr, char *buf)
447 return sprintf(buf, "%d\n", phys_sockets);
450 static ssize_t chipspersocket_show(struct device *dev,
451 struct device_attribute *attr, char *buf)
453 return sprintf(buf, "%d\n", phys_chipspersocket);
456 static ssize_t coresperchip_show(struct device *dev,
457 struct device_attribute *attr, char *buf)
459 return sprintf(buf, "%d\n", phys_coresperchip);
462 static struct attribute *device_str_attr_create_(char *name, char *str)
464 struct dev_ext_attribute *attr = kzalloc(sizeof(*attr), GFP_KERNEL);
469 sysfs_attr_init(&attr->attr.attr);
472 attr->attr.attr.name = name;
473 attr->attr.attr.mode = 0444;
474 attr->attr.show = device_show_string;
476 return &attr->attr.attr;
480 * Allocate and initialize strings representing event attributes.
482 * NOTE: The strings allocated here are never destroyed and continue to
483 * exist till shutdown. This is to allow us to create as many events
484 * from the catalog as possible, even if we encounter errors with some.
485 * In case of changes to error paths in future, these may need to be
486 * freed by the caller.
488 static struct attribute *device_str_attr_create(char *name, int name_max,
490 char *str, size_t str_max)
493 char *s = memdup_to_str(str, str_max, GFP_KERNEL);
500 n = kasprintf(GFP_KERNEL, "%.*s", name_max, name);
502 n = kasprintf(GFP_KERNEL, "%.*s__%d", name_max, name,
507 a = device_str_attr_create_(n, s);
519 static struct attribute *event_to_attr(unsigned int ix,
520 struct hv_24x7_event_data *event,
525 char *ev_name, *a_ev_name, *val;
526 struct attribute *attr;
528 if (!domain_is_valid(domain)) {
529 pr_warn("catalog event %u has invalid domain %u\n",
534 val = event_fmt(event, domain);
538 ev_name = event_name(event, &event_name_len);
540 a_ev_name = kasprintf(GFP_KERNEL, "%.*s",
541 (int)event_name_len, ev_name);
543 a_ev_name = kasprintf(GFP_KERNEL, "%.*s__%d",
544 (int)event_name_len, ev_name, nonce);
549 attr = device_str_attr_create_(a_ev_name, val);
561 static struct attribute *event_to_desc_attr(struct hv_24x7_event_data *event,
565 char *name = event_name(event, &nl);
566 char *desc = event_desc(event, &dl);
568 /* If there isn't a description, don't create the sysfs file */
572 return device_str_attr_create(name, nl, nonce, desc, dl);
575 static struct attribute *
576 event_to_long_desc_attr(struct hv_24x7_event_data *event, int nonce)
579 char *name = event_name(event, &nl);
580 char *desc = event_long_desc(event, &dl);
582 /* If there isn't a description, don't create the sysfs file */
586 return device_str_attr_create(name, nl, nonce, desc, dl);
589 static int event_data_to_attrs(unsigned int ix, struct attribute **attrs,
590 struct hv_24x7_event_data *event, int nonce)
592 *attrs = event_to_attr(ix, event, event->domain, nonce);
608 static int memord(const void *d1, size_t s1, const void *d2, size_t s2)
615 return memcmp(d1, d2, s1);
618 static int ev_uniq_ord(const void *v1, size_t s1, unsigned int d1,
619 const void *v2, size_t s2, unsigned int d2)
621 int r = memord(v1, s1, v2, s2);
632 static int event_uniq_add(struct rb_root *root, const char *name, int nl,
635 struct rb_node **new = &(root->rb_node), *parent = NULL;
636 struct event_uniq *data;
638 /* Figure out where to put new node */
640 struct event_uniq *it;
643 it = rb_entry(*new, struct event_uniq, node);
644 result = ev_uniq_ord(name, nl, domain, it->name, it->nl,
649 new = &((*new)->rb_left);
651 new = &((*new)->rb_right);
654 pr_info("found a duplicate event %.*s, ct=%u\n", nl,
660 data = kmalloc(sizeof(*data), GFP_KERNEL);
664 *data = (struct event_uniq) {
671 /* Add new node and rebalance tree. */
672 rb_link_node(&data->node, parent, new);
673 rb_insert_color(&data->node, root);
679 static void event_uniq_destroy(struct rb_root *root)
682 * the strings we point to are in the giant block of memory filled by
683 * the catalog, and are freed separately.
685 struct event_uniq *pos, *n;
687 rbtree_postorder_for_each_entry_safe(pos, n, root, node)
693 * ensure the event structure's sizes are self consistent and don't cause us to
694 * read outside of the event
696 * On success, return the event length in bytes.
697 * Otherwise, return -1 (and print as appropriate).
699 static ssize_t catalog_event_len_validate(struct hv_24x7_event_data *event,
701 size_t event_data_bytes,
702 size_t event_entry_count,
703 size_t offset, void *end)
706 void *ev_end, *calc_ev_end;
708 if (offset >= event_data_bytes)
711 if (event_idx >= event_entry_count) {
712 pr_devel("catalog event data has %zu bytes of padding after last event\n",
713 event_data_bytes - offset);
717 if (!event_fixed_portion_is_within(event, end)) {
718 pr_warn("event %zu fixed portion is not within range\n",
723 ev_len = be16_to_cpu(event->length);
726 pr_info("event %zu has length %zu not divisible by 16: event=%pK\n",
727 event_idx, ev_len, event);
729 ev_end = (__u8 *)event + ev_len;
731 pr_warn("event %zu has .length=%zu, ends after buffer end: ev_end=%pK > end=%pK, offset=%zu\n",
732 event_idx, ev_len, ev_end, end,
737 calc_ev_end = event_end(event, end);
739 pr_warn("event %zu has a calculated length which exceeds buffer length %zu: event=%pK end=%pK, offset=%zu\n",
740 event_idx, event_data_bytes, event, end,
745 if (calc_ev_end > ev_end) {
746 pr_warn("event %zu exceeds its own length: event=%pK, end=%pK, offset=%zu, calc_ev_end=%pK\n",
747 event_idx, event, ev_end, offset, calc_ev_end);
755 * Return true incase of invalid or dummy events with names like RESERVED*
757 static bool ignore_event(const char *name)
759 return strncmp(name, "RESERVED", 8) == 0;
762 #define MAX_4K (SIZE_MAX / 4096)
764 static int create_events_from_catalog(struct attribute ***events_,
765 struct attribute ***event_descs_,
766 struct attribute ***event_long_descs_)
769 size_t catalog_len, catalog_page_len, event_entry_count,
770 event_data_len, event_data_offs,
771 event_data_bytes, junk_events, event_idx, event_attr_ct, i,
772 attr_max, event_idx_last, desc_ct, long_desc_ct;
774 uint64_t catalog_version_num;
775 struct attribute **events, **event_descs, **event_long_descs;
776 struct hv_24x7_catalog_page_0 *page_0 =
777 kmem_cache_alloc(hv_page_cache, GFP_KERNEL);
779 void *event_data, *end;
780 struct hv_24x7_event_data *event;
781 struct rb_root ev_uniq = RB_ROOT;
789 hret = h_get_24x7_catalog_page(page, 0, 0);
795 catalog_version_num = be64_to_cpu(page_0->version);
796 catalog_page_len = be32_to_cpu(page_0->length);
798 if (MAX_4K < catalog_page_len) {
799 pr_err("invalid page count: %zu\n", catalog_page_len);
804 catalog_len = catalog_page_len * 4096;
806 event_entry_count = be16_to_cpu(page_0->event_entry_count);
807 event_data_offs = be16_to_cpu(page_0->event_data_offs);
808 event_data_len = be16_to_cpu(page_0->event_data_len);
810 pr_devel("cv %llu cl %zu eec %zu edo %zu edl %zu\n",
811 catalog_version_num, catalog_len,
812 event_entry_count, event_data_offs, event_data_len);
814 if ((MAX_4K < event_data_len)
815 || (MAX_4K < event_data_offs)
816 || (MAX_4K - event_data_offs < event_data_len)) {
817 pr_err("invalid event data offs %zu and/or len %zu\n",
818 event_data_offs, event_data_len);
823 if ((event_data_offs + event_data_len) > catalog_page_len) {
824 pr_err("event data %zu-%zu does not fit inside catalog 0-%zu\n",
826 event_data_offs + event_data_len,
832 if (SIZE_MAX - 1 < event_entry_count) {
833 pr_err("event_entry_count %zu is invalid\n", event_entry_count);
838 event_data_bytes = event_data_len * 4096;
841 * event data can span several pages, events can cross between these
842 * pages. Use vmalloc to make this easier.
844 event_data = vmalloc(event_data_bytes);
846 pr_err("could not allocate event data\n");
851 end = event_data + event_data_bytes;
854 * using vmalloc_to_phys() like this only works if PAGE_SIZE is
857 BUILD_BUG_ON(PAGE_SIZE % 4096);
859 for (i = 0; i < event_data_len; i++) {
860 hret = h_get_24x7_catalog_page_(
861 vmalloc_to_phys(event_data + i * 4096),
863 i + event_data_offs);
865 pr_err("Failed to get event data in page %zu: rc=%ld\n",
866 i + event_data_offs, hret);
873 * scan the catalog to determine the number of attributes we need, and
874 * verify it at the same time.
876 for (junk_events = 0, event = event_data, event_idx = 0, attr_max = 0;
878 event_idx++, event = (void *)event + ev_len) {
879 size_t offset = (void *)event - (void *)event_data;
883 ev_len = catalog_event_len_validate(event, event_idx,
890 name = event_name(event, &nl);
892 if (ignore_event(name)) {
896 if (event->event_group_record_len == 0) {
897 pr_devel("invalid event %zu (%.*s): group_record_len == 0, skipping\n",
898 event_idx, nl, name);
903 if (!catalog_entry_domain_is_valid(event->domain)) {
904 pr_info("event %zu (%.*s) has invalid domain %d\n",
905 event_idx, nl, name, event->domain);
913 event_idx_last = event_idx;
914 if (event_idx_last != event_entry_count)
915 pr_warn("event buffer ended before listed # of events were parsed (got %zu, wanted %zu, junk %zu)\n",
916 event_idx_last, event_entry_count, junk_events);
918 events = kmalloc_array(attr_max + 1, sizeof(*events), GFP_KERNEL);
924 event_descs = kmalloc_array(event_idx + 1, sizeof(*event_descs),
931 event_long_descs = kmalloc_array(event_idx + 1,
932 sizeof(*event_long_descs), GFP_KERNEL);
933 if (!event_long_descs) {
938 /* Iterate over the catalog filling in the attribute vector */
939 for (junk_events = 0, event_attr_ct = 0, desc_ct = 0, long_desc_ct = 0,
940 event = event_data, event_idx = 0;
941 event_idx < event_idx_last;
942 event_idx++, ev_len = be16_to_cpu(event->length),
943 event = (void *)event + ev_len) {
948 * these are the only "bad" events that are intermixed and that
949 * we can ignore without issue. make sure to skip them here
951 if (event->event_group_record_len == 0)
953 if (!catalog_entry_domain_is_valid(event->domain))
956 name = event_name(event, &nl);
957 if (ignore_event(name))
960 nonce = event_uniq_add(&ev_uniq, name, nl, event->domain);
961 ct = event_data_to_attrs(event_idx, events + event_attr_ct,
964 pr_warn("event %zu (%.*s) creation failure, skipping\n",
965 event_idx, nl, name);
969 event_descs[desc_ct] = event_to_desc_attr(event, nonce);
970 if (event_descs[desc_ct])
972 event_long_descs[long_desc_ct] =
973 event_to_long_desc_attr(event, nonce);
974 if (event_long_descs[long_desc_ct])
979 pr_info("read %zu catalog entries, created %zu event attrs (%zu failures), %zu descs\n",
980 event_idx, event_attr_ct, junk_events, desc_ct);
982 events[event_attr_ct] = NULL;
983 event_descs[desc_ct] = NULL;
984 event_long_descs[long_desc_ct] = NULL;
986 event_uniq_destroy(&ev_uniq);
988 kmem_cache_free(hv_page_cache, page);
991 *event_descs_ = event_descs;
992 *event_long_descs_ = event_long_descs;
1002 kmem_cache_free(hv_page_cache, page);
1005 *event_descs_ = NULL;
1006 *event_long_descs_ = NULL;
1010 static ssize_t catalog_read(struct file *filp, struct kobject *kobj,
1011 struct bin_attribute *bin_attr, char *buf,
1012 loff_t offset, size_t count)
1016 size_t catalog_len = 0, catalog_page_len = 0;
1017 loff_t page_offset = 0;
1018 loff_t offset_in_page;
1020 uint64_t catalog_version_num = 0;
1021 void *page = kmem_cache_alloc(hv_page_cache, GFP_USER);
1022 struct hv_24x7_catalog_page_0 *page_0 = page;
1027 hret = h_get_24x7_catalog_page(page, 0, 0);
1033 catalog_version_num = be64_to_cpu(page_0->version);
1034 catalog_page_len = be32_to_cpu(page_0->length);
1035 catalog_len = catalog_page_len * 4096;
1037 page_offset = offset / 4096;
1038 offset_in_page = offset % 4096;
1040 if (page_offset >= catalog_page_len)
1043 if (page_offset != 0) {
1044 hret = h_get_24x7_catalog_page(page, catalog_version_num,
1052 copy_len = 4096 - offset_in_page;
1053 if (copy_len > count)
1056 memcpy(buf, page+offset_in_page, copy_len);
1061 pr_err("h_get_24x7_catalog_page(ver=%lld, page=%lld) failed:"
1063 catalog_version_num, page_offset, hret);
1064 kmem_cache_free(hv_page_cache, page);
1066 pr_devel("catalog_read: offset=%lld(%lld) count=%zu "
1067 "catalog_len=%zu(%zu) => %zd\n", offset, page_offset,
1068 count, catalog_len, catalog_page_len, ret);
1073 static ssize_t domains_show(struct device *dev, struct device_attribute *attr,
1076 int d, n, count = 0;
1079 for (d = 0; d < HV_PERF_DOMAIN_MAX; d++) {
1080 str = domain_name(d);
1084 n = sprintf(page, "%d: %s\n", d, str);
1094 #define PAGE_0_ATTR(_name, _fmt, _expr) \
1095 static ssize_t _name##_show(struct device *dev, \
1096 struct device_attribute *dev_attr, \
1101 void *page = kmem_cache_alloc(hv_page_cache, GFP_USER); \
1102 struct hv_24x7_catalog_page_0 *page_0 = page; \
1105 hret = h_get_24x7_catalog_page(page, 0, 0); \
1110 ret = sprintf(buf, _fmt, _expr); \
1112 kmem_cache_free(hv_page_cache, page); \
1115 static DEVICE_ATTR_RO(_name)
1117 PAGE_0_ATTR(catalog_version, "%lld\n",
1118 (unsigned long long)be64_to_cpu(page_0->version));
1119 PAGE_0_ATTR(catalog_len, "%lld\n",
1120 (unsigned long long)be32_to_cpu(page_0->length) * 4096);
1121 static BIN_ATTR_RO(catalog, 0/* real length varies */);
1122 static DEVICE_ATTR_RO(domains);
1123 static DEVICE_ATTR_RO(sockets);
1124 static DEVICE_ATTR_RO(chipspersocket);
1125 static DEVICE_ATTR_RO(coresperchip);
1126 static DEVICE_ATTR_RO(cpumask);
1128 static struct bin_attribute *if_bin_attrs[] = {
1133 static struct attribute *cpumask_attrs[] = {
1134 &dev_attr_cpumask.attr,
1138 static const struct attribute_group cpumask_attr_group = {
1139 .attrs = cpumask_attrs,
1142 static struct attribute *if_attrs[] = {
1143 &dev_attr_catalog_len.attr,
1144 &dev_attr_catalog_version.attr,
1145 &dev_attr_domains.attr,
1146 &dev_attr_sockets.attr,
1147 &dev_attr_chipspersocket.attr,
1148 &dev_attr_coresperchip.attr,
1152 static const struct attribute_group if_group = {
1153 .name = "interface",
1154 .bin_attrs = if_bin_attrs,
1158 static const struct attribute_group *attr_groups[] = {
1162 &event_long_desc_group,
1164 &cpumask_attr_group,
1169 * Start the process for a new H_GET_24x7_DATA hcall.
1171 static void init_24x7_request(struct hv_24x7_request_buffer *request_buffer,
1172 struct hv_24x7_data_result_buffer *result_buffer)
1175 memset(request_buffer, 0, H24x7_DATA_BUFFER_SIZE);
1176 memset(result_buffer, 0, H24x7_DATA_BUFFER_SIZE);
1178 request_buffer->interface_version = interface_version;
1179 /* memset above set request_buffer->num_requests to 0 */
1183 * Commit (i.e perform) the H_GET_24x7_DATA hcall using the data collected
1184 * by 'init_24x7_request()' and 'add_event_to_24x7_request()'.
1186 static int make_24x7_request(struct hv_24x7_request_buffer *request_buffer,
1187 struct hv_24x7_data_result_buffer *result_buffer)
1192 * NOTE: Due to variable number of array elements in request and
1193 * result buffer(s), sizeof() is not reliable. Use the actual
1194 * allocated buffer size, H24x7_DATA_BUFFER_SIZE.
1196 ret = plpar_hcall_norets(H_GET_24X7_DATA,
1197 virt_to_phys(request_buffer), H24x7_DATA_BUFFER_SIZE,
1198 virt_to_phys(result_buffer), H24x7_DATA_BUFFER_SIZE);
1201 struct hv_24x7_request *req;
1203 req = request_buffer->requests;
1204 pr_notice_ratelimited("hcall failed: [%d %#x %#x %d] => ret 0x%lx (%ld) detail=0x%x failing ix=%x\n",
1205 req->performance_domain, req->data_offset,
1206 req->starting_ix, req->starting_lpar_ix,
1207 ret, ret, result_buffer->detailed_rc,
1208 result_buffer->failing_request_ix);
1216 * Add the given @event to the next slot in the 24x7 request_buffer.
1218 * Note that H_GET_24X7_DATA hcall allows reading several counters'
1219 * values in a single HCALL. We expect the caller to add events to the
1220 * request buffer one by one, make the HCALL and process the results.
1222 static int add_event_to_24x7_request(struct perf_event *event,
1223 struct hv_24x7_request_buffer *request_buffer)
1228 struct hv_24x7_request *req;
1230 if (request_buffer->num_requests >=
1231 max_num_requests(request_buffer->interface_version)) {
1232 pr_devel("Too many requests for 24x7 HCALL %d\n",
1233 request_buffer->num_requests);
1237 switch (event_get_domain(event)) {
1238 case HV_PERF_DOMAIN_PHYS_CHIP:
1239 idx = event_get_chip(event);
1241 case HV_PERF_DOMAIN_PHYS_CORE:
1242 idx = event_get_core(event);
1245 idx = event_get_vcpu(event);
1248 req_size = H24x7_REQUEST_SIZE(request_buffer->interface_version);
1250 i = request_buffer->num_requests++;
1251 req = (void *) request_buffer->requests + i * req_size;
1253 req->performance_domain = event_get_domain(event);
1254 req->data_size = cpu_to_be16(8);
1255 req->data_offset = cpu_to_be32(event_get_offset(event));
1256 req->starting_lpar_ix = cpu_to_be16(event_get_lpar(event));
1257 req->max_num_lpars = cpu_to_be16(1);
1258 req->starting_ix = cpu_to_be16(idx);
1259 req->max_ix = cpu_to_be16(1);
1261 if (request_buffer->interface_version > 1) {
1262 if (domain_needs_aggregation(req->performance_domain))
1263 req->max_num_thread_groups = -1;
1264 else if (req->performance_domain != HV_PERF_DOMAIN_PHYS_CHIP) {
1265 req->starting_thread_group_ix = idx % 2;
1266 req->max_num_thread_groups = 1;
1274 * get_count_from_result - get event count from all result elements in result
1276 * If the event corresponding to this result needs aggregation of the result
1277 * element values, then this function does that.
1279 * @event: Event associated with @res.
1280 * @resb: Result buffer containing @res.
1281 * @res: Result to work on.
1282 * @countp: Output variable containing the event count.
1283 * @next: Optional output variable pointing to the next result in @resb.
1285 static int get_count_from_result(struct perf_event *event,
1286 struct hv_24x7_data_result_buffer *resb,
1287 struct hv_24x7_result *res, u64 *countp,
1288 struct hv_24x7_result **next)
1290 u16 num_elements = be16_to_cpu(res->num_elements_returned);
1291 u16 data_size = be16_to_cpu(res->result_element_data_size);
1292 unsigned int data_offset;
1298 * We can bail out early if the result is empty.
1300 if (!num_elements) {
1301 pr_debug("Result of request %hhu is empty, nothing to do\n",
1305 *next = (struct hv_24x7_result *) res->elements;
1311 * Since we always specify 1 as the maximum for the smallest resource
1312 * we're requesting, there should to be only one element per result.
1313 * Except when an event needs aggregation, in which case there are more.
1315 if (num_elements != 1 &&
1316 !domain_needs_aggregation(event_get_domain(event))) {
1317 pr_err("Error: result of request %hhu has %hu elements\n",
1318 res->result_ix, num_elements);
1323 if (data_size != sizeof(u64)) {
1324 pr_debug("Error: result of request %hhu has data of %hu bytes\n",
1325 res->result_ix, data_size);
1330 if (resb->interface_version == 1)
1331 data_offset = offsetof(struct hv_24x7_result_element_v1,
1334 data_offset = offsetof(struct hv_24x7_result_element_v2,
1337 /* Go through the result elements in the result. */
1338 for (i = count = 0, element_data = res->elements + data_offset;
1340 i++, element_data += data_size + data_offset)
1341 count += be64_to_cpu(*((__be64 *)element_data));
1345 /* The next result is after the last result element. */
1347 *next = element_data - data_offset;
1352 static int single_24x7_request(struct perf_event *event, u64 *count)
1355 struct hv_24x7_request_buffer *request_buffer;
1356 struct hv_24x7_data_result_buffer *result_buffer;
1358 BUILD_BUG_ON(sizeof(*request_buffer) > 4096);
1359 BUILD_BUG_ON(sizeof(*result_buffer) > 4096);
1361 request_buffer = (void *)get_cpu_var(hv_24x7_reqb);
1362 result_buffer = (void *)get_cpu_var(hv_24x7_resb);
1364 init_24x7_request(request_buffer, result_buffer);
1366 ret = add_event_to_24x7_request(event, request_buffer);
1370 ret = make_24x7_request(request_buffer, result_buffer);
1374 /* process result from hcall */
1375 ret = get_count_from_result(event, result_buffer,
1376 result_buffer->results, count, NULL);
1379 put_cpu_var(hv_24x7_reqb);
1380 put_cpu_var(hv_24x7_resb);
1385 static int h_24x7_event_init(struct perf_event *event)
1387 struct hv_perf_caps caps;
1388 unsigned int domain;
1393 if (event->attr.type != event->pmu->type)
1396 /* Unused areas must be 0 */
1397 if (event_get_reserved1(event) ||
1398 event_get_reserved2(event) ||
1399 event_get_reserved3(event)) {
1400 pr_devel("reserved set when forbidden 0x%llx(0x%llx) 0x%llx(0x%llx) 0x%llx(0x%llx)\n",
1402 event_get_reserved1(event),
1403 event->attr.config1,
1404 event_get_reserved2(event),
1405 event->attr.config2,
1406 event_get_reserved3(event));
1410 /* no branch sampling */
1411 if (has_branch_stack(event))
1414 /* offset must be 8 byte aligned */
1415 if (event_get_offset(event) % 8) {
1416 pr_devel("bad alignment\n");
1420 domain = event_get_domain(event);
1421 if (domain == 0 || domain >= HV_PERF_DOMAIN_MAX) {
1422 pr_devel("invalid domain %d\n", domain);
1426 hret = hv_perf_caps_get(&caps);
1428 pr_devel("could not get capabilities: rc=%ld\n", hret);
1432 /* Physical domains & other lpars require extra capabilities */
1433 if (!caps.collect_privileged && (is_physical_domain(domain) ||
1434 (event_get_lpar(event) != event_get_lpar_max()))) {
1435 pr_devel("hv permissions disallow: is_physical_domain:%d, lpar=0x%llx\n",
1436 is_physical_domain(domain),
1437 event_get_lpar(event));
1441 /* Get the initial value of the counter for this event */
1442 if (single_24x7_request(event, &ct)) {
1443 pr_devel("test hcall failed\n");
1446 (void)local64_xchg(&event->hw.prev_count, ct);
1451 static u64 h_24x7_get_value(struct perf_event *event)
1455 if (single_24x7_request(event, &ct))
1456 /* We checked this in event init, shouldn't fail here... */
1462 static void update_event_count(struct perf_event *event, u64 now)
1466 prev = local64_xchg(&event->hw.prev_count, now);
1467 local64_add(now - prev, &event->count);
1470 static void h_24x7_event_read(struct perf_event *event)
1473 struct hv_24x7_request_buffer *request_buffer;
1474 struct hv_24x7_hw *h24x7hw;
1477 txn_flags = __this_cpu_read(hv_24x7_txn_flags);
1480 * If in a READ transaction, add this counter to the list of
1481 * counters to read during the next HCALL (i.e commit_txn()).
1482 * If not in a READ transaction, go ahead and make the HCALL
1483 * to read this counter by itself.
1486 if (txn_flags & PERF_PMU_TXN_READ) {
1490 if (__this_cpu_read(hv_24x7_txn_err))
1493 request_buffer = (void *)get_cpu_var(hv_24x7_reqb);
1495 ret = add_event_to_24x7_request(event, request_buffer);
1497 __this_cpu_write(hv_24x7_txn_err, ret);
1500 * Associate the event with the HCALL request index,
1501 * so ->commit_txn() can quickly find/update count.
1503 i = request_buffer->num_requests - 1;
1505 h24x7hw = &get_cpu_var(hv_24x7_hw);
1506 h24x7hw->events[i] = event;
1507 put_cpu_var(h24x7hw);
1510 put_cpu_var(hv_24x7_reqb);
1512 now = h_24x7_get_value(event);
1513 update_event_count(event, now);
1517 static void h_24x7_event_start(struct perf_event *event, int flags)
1519 if (flags & PERF_EF_RELOAD)
1520 local64_set(&event->hw.prev_count, h_24x7_get_value(event));
1523 static void h_24x7_event_stop(struct perf_event *event, int flags)
1525 h_24x7_event_read(event);
1528 static int h_24x7_event_add(struct perf_event *event, int flags)
1530 if (flags & PERF_EF_START)
1531 h_24x7_event_start(event, flags);
1537 * 24x7 counters only support READ transactions. They are
1538 * always counting and dont need/support ADD transactions.
1539 * Cache the flags, but otherwise ignore transactions that
1540 * are not PERF_PMU_TXN_READ.
1542 static void h_24x7_event_start_txn(struct pmu *pmu, unsigned int flags)
1544 struct hv_24x7_request_buffer *request_buffer;
1545 struct hv_24x7_data_result_buffer *result_buffer;
1547 /* We should not be called if we are already in a txn */
1548 WARN_ON_ONCE(__this_cpu_read(hv_24x7_txn_flags));
1550 __this_cpu_write(hv_24x7_txn_flags, flags);
1551 if (flags & ~PERF_PMU_TXN_READ)
1554 request_buffer = (void *)get_cpu_var(hv_24x7_reqb);
1555 result_buffer = (void *)get_cpu_var(hv_24x7_resb);
1557 init_24x7_request(request_buffer, result_buffer);
1559 put_cpu_var(hv_24x7_resb);
1560 put_cpu_var(hv_24x7_reqb);
1564 * Clean up transaction state.
1566 * NOTE: Ignore state of request and result buffers for now.
1567 * We will initialize them during the next read/txn.
1569 static void reset_txn(void)
1571 __this_cpu_write(hv_24x7_txn_flags, 0);
1572 __this_cpu_write(hv_24x7_txn_err, 0);
1576 * 24x7 counters only support READ transactions. They are always counting
1577 * and dont need/support ADD transactions. Clear ->txn_flags but otherwise
1578 * ignore transactions that are not of type PERF_PMU_TXN_READ.
1580 * For READ transactions, submit all pending 24x7 requests (i.e requests
1581 * that were queued by h_24x7_event_read()), to the hypervisor and update
1584 static int h_24x7_event_commit_txn(struct pmu *pmu)
1586 struct hv_24x7_request_buffer *request_buffer;
1587 struct hv_24x7_data_result_buffer *result_buffer;
1588 struct hv_24x7_result *res, *next_res;
1590 int i, ret, txn_flags;
1591 struct hv_24x7_hw *h24x7hw;
1593 txn_flags = __this_cpu_read(hv_24x7_txn_flags);
1594 WARN_ON_ONCE(!txn_flags);
1597 if (txn_flags & ~PERF_PMU_TXN_READ)
1600 ret = __this_cpu_read(hv_24x7_txn_err);
1604 request_buffer = (void *)get_cpu_var(hv_24x7_reqb);
1605 result_buffer = (void *)get_cpu_var(hv_24x7_resb);
1607 ret = make_24x7_request(request_buffer, result_buffer);
1611 h24x7hw = &get_cpu_var(hv_24x7_hw);
1613 /* Go through results in the result buffer to update event counts. */
1614 for (i = 0, res = result_buffer->results;
1615 i < result_buffer->num_results; i++, res = next_res) {
1616 struct perf_event *event = h24x7hw->events[res->result_ix];
1618 ret = get_count_from_result(event, result_buffer, res, &count,
1623 update_event_count(event, count);
1626 put_cpu_var(hv_24x7_hw);
1629 put_cpu_var(hv_24x7_resb);
1630 put_cpu_var(hv_24x7_reqb);
1637 * 24x7 counters only support READ transactions. They are always counting
1638 * and dont need/support ADD transactions. However, regardless of type
1639 * of transaction, all we need to do is cleanup, so we don't have to check
1640 * the type of transaction.
1642 static void h_24x7_event_cancel_txn(struct pmu *pmu)
1644 WARN_ON_ONCE(!__this_cpu_read(hv_24x7_txn_flags));
1648 static struct pmu h_24x7_pmu = {
1649 .task_ctx_nr = perf_invalid_context,
1652 .attr_groups = attr_groups,
1653 .event_init = h_24x7_event_init,
1654 .add = h_24x7_event_add,
1655 .del = h_24x7_event_stop,
1656 .start = h_24x7_event_start,
1657 .stop = h_24x7_event_stop,
1658 .read = h_24x7_event_read,
1659 .start_txn = h_24x7_event_start_txn,
1660 .commit_txn = h_24x7_event_commit_txn,
1661 .cancel_txn = h_24x7_event_cancel_txn,
1662 .capabilities = PERF_PMU_CAP_NO_EXCLUDE,
1665 static int ppc_hv_24x7_cpu_online(unsigned int cpu)
1667 if (cpumask_empty(&hv_24x7_cpumask))
1668 cpumask_set_cpu(cpu, &hv_24x7_cpumask);
1673 static int ppc_hv_24x7_cpu_offline(unsigned int cpu)
1677 /* Check if exiting cpu is used for collecting 24x7 events */
1678 if (!cpumask_test_and_clear_cpu(cpu, &hv_24x7_cpumask))
1681 /* Find a new cpu to collect 24x7 events */
1682 target = cpumask_last(cpu_active_mask);
1684 if (target < 0 || target >= nr_cpu_ids) {
1685 pr_err("hv_24x7: CPU hotplug init failed\n");
1689 /* Migrate 24x7 events to the new target */
1690 cpumask_set_cpu(target, &hv_24x7_cpumask);
1691 perf_pmu_migrate_context(&h_24x7_pmu, cpu, target);
1696 static int hv_24x7_cpu_hotplug_init(void)
1698 return cpuhp_setup_state(CPUHP_AP_PERF_POWERPC_HV_24x7_ONLINE,
1699 "perf/powerpc/hv_24x7:online",
1700 ppc_hv_24x7_cpu_online,
1701 ppc_hv_24x7_cpu_offline);
1704 static int hv_24x7_init(void)
1708 unsigned int pvr = mfspr(SPRN_PVR);
1709 struct hv_perf_caps caps;
1711 if (!firmware_has_feature(FW_FEATURE_LPAR)) {
1712 pr_debug("not a virtualized system, not enabling\n");
1716 /* POWER8 only supports v1, while POWER9 only supports v2. */
1717 if (PVR_VER(pvr) == PVR_POWER8 || PVR_VER(pvr) == PVR_POWER8E ||
1718 PVR_VER(pvr) == PVR_POWER8NVL)
1719 interface_version = 1;
1721 interface_version = 2;
1723 /* SMT8 in POWER9 needs to aggregate result elements. */
1724 if (threads_per_core == 8)
1725 aggregate_result_elements = true;
1728 hret = hv_perf_caps_get(&caps);
1730 pr_debug("could not obtain capabilities, not enabling, rc=%ld\n",
1735 hv_page_cache = kmem_cache_create("hv-page-4096", 4096, 4096, 0, NULL);
1739 /* sampling not supported */
1740 h_24x7_pmu.capabilities |= PERF_PMU_CAP_NO_INTERRUPT;
1742 r = create_events_from_catalog(&event_group.attrs,
1743 &event_desc_group.attrs,
1744 &event_long_desc_group.attrs);
1749 /* init cpuhotplug */
1750 r = hv_24x7_cpu_hotplug_init();
1754 r = perf_pmu_register(&h_24x7_pmu, h_24x7_pmu.name, -1);
1758 read_24x7_sys_info();
1763 device_initcall(hv_24x7_init);