drivers/perf/marvell_cn10k_tad_pmu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Marvell CN10K LLC-TAD perf driver
   3  *
   4  * Copyright (C) 2021 Marvell
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #define pr_fmt(fmt) "tad_pmu: " fmt
  12
  13 #include <linux/module.h>
  14 #include <linux/of.h>
  15 #include <linux/of_address.h>
  16 #include <linux/of_device.h>
  17 #include <linux/cpuhotplug.h>
  18 #include <linux/perf_event.h>
  19 #include <linux/platform_device.h>
  20
  21 #define TAD_PFC_OFFSET          0x0
  22 #define TAD_PFC(counter)        (TAD_PFC_OFFSET | (counter << 3))
  23 #define TAD_PRF_OFFSET          0x100
  24 #define TAD_PRF(counter)        (TAD_PRF_OFFSET | (counter << 3))
  25 #define TAD_PRF_CNTSEL_MASK     0xFF
  26 #define TAD_MAX_COUNTERS        8
  27
  28 #define to_tad_pmu(p) (container_of(p, struct tad_pmu, pmu))
  29
  30 struct tad_region {
  31         void __iomem    *base;
  32 };
  33
  34 struct tad_pmu {
  35         struct pmu pmu;
  36         struct tad_region *regions;
  37         u32 region_cnt;
  38         unsigned int cpu;
  39         struct hlist_node node;
  40         struct perf_event *events[TAD_MAX_COUNTERS];
  41         DECLARE_BITMAP(counters_map, TAD_MAX_COUNTERS);
  42 };
  43
  44 static int tad_pmu_cpuhp_state;
  45
  46 static void tad_pmu_event_counter_read(struct perf_event *event)
  47 {
  48         struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu);
  49         struct hw_perf_event *hwc = &event->hw;
  50         u32 counter_idx = hwc->idx;
  51         u64 prev, new;
  52         int i;
  53
  54         do {
  55                 prev = local64_read(&hwc->prev_count);
  56                 for (i = 0, new = 0; i < tad_pmu->region_cnt; i++)
  57                         new += readq(tad_pmu->regions[i].base +
  58                                      TAD_PFC(counter_idx));
  59         } while (local64_cmpxchg(&hwc->prev_count, prev, new) != prev);
  60
  61         local64_add(new - prev, &event->count);
  62 }
  63
  64 static void tad_pmu_event_counter_stop(struct perf_event *event, int flags)
  65 {
  66         struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu);
  67         struct hw_perf_event *hwc = &event->hw;
  68         u32 counter_idx = hwc->idx;
  69         int i;
  70
  71         /* TAD()_PFC() stop counting on the write
  72          * which sets TAD()_PRF()[CNTSEL] == 0
  73          */
  74         for (i = 0; i < tad_pmu->region_cnt; i++) {
  75                 writeq_relaxed(0, tad_pmu->regions[i].base +
  76                                TAD_PRF(counter_idx));
  77         }
  78
  79         tad_pmu_event_counter_read(event);
  80         hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
  81 }
  82
  83 static void tad_pmu_event_counter_start(struct perf_event *event, int flags)
  84 {
  85         struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu);
  86         struct hw_perf_event *hwc = &event->hw;
  87         u32 event_idx = event->attr.config;
  88         u32 counter_idx = hwc->idx;
  89         u64 reg_val;
  90         int i;
  91
  92         hwc->state = 0;
  93
  94         /* Typically TAD_PFC() are zeroed to start counting */
  95         for (i = 0; i < tad_pmu->region_cnt; i++)
  96                 writeq_relaxed(0, tad_pmu->regions[i].base +
  97                                TAD_PFC(counter_idx));
  98
  99         /* TAD()_PFC() start counting on the write
 100          * which sets TAD()_PRF()[CNTSEL] != 0
 101          */
 102         for (i = 0; i < tad_pmu->region_cnt; i++) {
 103                 reg_val = readq_relaxed(tad_pmu->regions[i].base +
 104                                         TAD_PRF(counter_idx));
 105                 reg_val |= (event_idx & 0xFF);
 106                 writeq_relaxed(reg_val, tad_pmu->regions[i].base +
 107                                TAD_PRF(counter_idx));
 108         }
 109 }
 110
 111 static void tad_pmu_event_counter_del(struct perf_event *event, int flags)
 112 {
 113         struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu);
 114         struct hw_perf_event *hwc = &event->hw;
 115         int idx = hwc->idx;
 116
 117         tad_pmu_event_counter_stop(event, flags | PERF_EF_UPDATE);
 118         tad_pmu->events[idx] = NULL;
 119         clear_bit(idx, tad_pmu->counters_map);
 120 }
 121
 122 static int tad_pmu_event_counter_add(struct perf_event *event, int flags)
 123 {
 124         struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu);
 125         struct hw_perf_event *hwc = &event->hw;
 126         int idx;
 127
 128         /* Get a free counter for this event */
 129         idx = find_first_zero_bit(tad_pmu->counters_map, TAD_MAX_COUNTERS);
 130         if (idx == TAD_MAX_COUNTERS)
 131                 return -EAGAIN;
 132
 133         set_bit(idx, tad_pmu->counters_map);
 134
 135         hwc->idx = idx;
 136         hwc->state = PERF_HES_STOPPED;
 137         tad_pmu->events[idx] = event;
 138
 139         if (flags & PERF_EF_START)
 140                 tad_pmu_event_counter_start(event, flags);
 141
 142         return 0;
 143 }
 144
 145 static int tad_pmu_event_init(struct perf_event *event)
 146 {
 147         struct tad_pmu *tad_pmu = to_tad_pmu(event->pmu);
 148
 149         if (event->attr.type != event->pmu->type)
 150                 return -ENOENT;
 151
 152         if (!event->attr.disabled)
 153                 return -EINVAL;
 154
 155         if (event->state != PERF_EVENT_STATE_OFF)
 156                 return -EINVAL;
 157
 158         event->cpu = tad_pmu->cpu;
 159         event->hw.idx = -1;
 160         event->hw.config_base = event->attr.config;
 161
 162         return 0;
 163 }
 164
 165 static ssize_t tad_pmu_event_show(struct device *dev,
 166                                 struct device_attribute *attr, char *page)
 167 {
 168         struct perf_pmu_events_attr *pmu_attr;
 169
 170         pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr);
 171         return sysfs_emit(page, "event=0x%02llx\n", pmu_attr->id);
 172 }
 173
 174 #define TAD_PMU_EVENT_ATTR(name, config)                        \
 175         PMU_EVENT_ATTR_ID(name, tad_pmu_event_show, config)
 176
 177 static struct attribute *tad_pmu_event_attrs[] = {
 178         TAD_PMU_EVENT_ATTR(tad_none, 0x0),
 179         TAD_PMU_EVENT_ATTR(tad_req_msh_in_any, 0x1),
 180         TAD_PMU_EVENT_ATTR(tad_req_msh_in_mn, 0x2),
 181         TAD_PMU_EVENT_ATTR(tad_req_msh_in_exlmn, 0x3),
 182         TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_any, 0x4),
 183         TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_mn, 0x5),
 184         TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_exlmn, 0x6),
 185         TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_dss, 0x7),
 186         TAD_PMU_EVENT_ATTR(tad_rsp_msh_in_retry_dss, 0x8),
 187         TAD_PMU_EVENT_ATTR(tad_dat_msh_in_any, 0x9),
 188         TAD_PMU_EVENT_ATTR(tad_dat_msh_in_dss, 0xa),
 189         TAD_PMU_EVENT_ATTR(tad_req_msh_out_any, 0xb),
 190         TAD_PMU_EVENT_ATTR(tad_req_msh_out_dss_rd, 0xc),
 191         TAD_PMU_EVENT_ATTR(tad_req_msh_out_dss_wr, 0xd),
 192         TAD_PMU_EVENT_ATTR(tad_req_msh_out_evict, 0xe),
 193         TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_any, 0xf),
 194         TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_retry_exlmn, 0x10),
 195         TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_retry_mn, 0x11),
 196         TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_exlmn, 0x12),
 197         TAD_PMU_EVENT_ATTR(tad_rsp_msh_out_mn, 0x13),
 198         TAD_PMU_EVENT_ATTR(tad_snp_msh_out_any, 0x14),
 199         TAD_PMU_EVENT_ATTR(tad_snp_msh_out_mn, 0x15),
 200         TAD_PMU_EVENT_ATTR(tad_snp_msh_out_exlmn, 0x16),
 201         TAD_PMU_EVENT_ATTR(tad_dat_msh_out_any, 0x17),
 202         TAD_PMU_EVENT_ATTR(tad_dat_msh_out_fill, 0x18),
 203         TAD_PMU_EVENT_ATTR(tad_dat_msh_out_dss, 0x19),
 204         TAD_PMU_EVENT_ATTR(tad_alloc_dtg, 0x1a),
 205         TAD_PMU_EVENT_ATTR(tad_alloc_ltg, 0x1b),
 206         TAD_PMU_EVENT_ATTR(tad_alloc_any, 0x1c),
 207         TAD_PMU_EVENT_ATTR(tad_hit_dtg, 0x1d),
 208         TAD_PMU_EVENT_ATTR(tad_hit_ltg, 0x1e),
 209         TAD_PMU_EVENT_ATTR(tad_hit_any, 0x1f),
 210         TAD_PMU_EVENT_ATTR(tad_tag_rd, 0x20),
 211         TAD_PMU_EVENT_ATTR(tad_dat_rd, 0x21),
 212         TAD_PMU_EVENT_ATTR(tad_dat_rd_byp, 0x22),
 213         TAD_PMU_EVENT_ATTR(tad_ifb_occ, 0x23),
 214         TAD_PMU_EVENT_ATTR(tad_req_occ, 0x24),
 215         NULL
 216 };
 217
 218 static const struct attribute_group tad_pmu_events_attr_group = {
 219         .name = "events",
 220         .attrs = tad_pmu_event_attrs,
 221 };
 222
 223 PMU_FORMAT_ATTR(event, "config:0-7");
 224
 225 static struct attribute *tad_pmu_format_attrs[] = {
 226         &format_attr_event.attr,
 227         NULL
 228 };
 229
 230 static struct attribute_group tad_pmu_format_attr_group = {
 231         .name = "format",
 232         .attrs = tad_pmu_format_attrs,
 233 };
 234
 235 static ssize_t tad_pmu_cpumask_show(struct device *dev,
 236                                 struct device_attribute *attr, char *buf)
 237 {
 238         struct tad_pmu *tad_pmu = to_tad_pmu(dev_get_drvdata(dev));
 239
 240         return cpumap_print_to_pagebuf(true, buf, cpumask_of(tad_pmu->cpu));
 241 }
 242
 243 static DEVICE_ATTR(cpumask, 0444, tad_pmu_cpumask_show, NULL);
 244
 245 static struct attribute *tad_pmu_cpumask_attrs[] = {
 246         &dev_attr_cpumask.attr,
 247         NULL
 248 };
 249
 250 static struct attribute_group tad_pmu_cpumask_attr_group = {
 251         .attrs = tad_pmu_cpumask_attrs,
 252 };
 253
 254 static const struct attribute_group *tad_pmu_attr_groups[] = {
 255         &tad_pmu_events_attr_group,
 256         &tad_pmu_format_attr_group,
 257         &tad_pmu_cpumask_attr_group,
 258         NULL
 259 };
 260
 261 static int tad_pmu_probe(struct platform_device *pdev)
 262 {
 263         struct device_node *node = pdev->dev.of_node;
 264         struct tad_region *regions;
 265         struct tad_pmu *tad_pmu;
 266         struct resource *res;
 267         u32 tad_pmu_page_size;
 268         u32 tad_page_size;
 269         u32 tad_cnt;
 270         int i, ret;
 271         char *name;
 272
 273         tad_pmu = devm_kzalloc(&pdev->dev, sizeof(*tad_pmu), GFP_KERNEL);
 274         if (!tad_pmu)
 275                 return -ENOMEM;
 276
 277         platform_set_drvdata(pdev, tad_pmu);
 278
 279         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 280         if (!res) {
 281                 dev_err(&pdev->dev, "Mem resource not found\n");
 282                 return -ENODEV;
 283         }
 284
 285         ret = of_property_read_u32(node, "marvell,tad-page-size",
 286                                    &tad_page_size);
 287         if (ret) {
 288                 dev_err(&pdev->dev, "Can't find tad-page-size property\n");
 289                 return ret;
 290         }
 291
 292         ret = of_property_read_u32(node, "marvell,tad-pmu-page-size",
 293                                    &tad_pmu_page_size);
 294         if (ret) {
 295                 dev_err(&pdev->dev, "Can't find tad-pmu-page-size property\n");
 296                 return ret;
 297         }
 298
 299         ret = of_property_read_u32(node, "marvell,tad-cnt", &tad_cnt);
 300         if (ret) {
 301                 dev_err(&pdev->dev, "Can't find tad-cnt property\n");
 302                 return ret;
 303         }
 304
 305         regions = devm_kcalloc(&pdev->dev, tad_cnt,
 306                                sizeof(*regions), GFP_KERNEL);
 307         if (!regions)
 308                 return -ENOMEM;
 309
 310         /* ioremap the distributed TAD pmu regions */
 311         for (i = 0; i < tad_cnt && res->start < res->end; i++) {
 312                 regions[i].base = devm_ioremap(&pdev->dev,
 313                                                res->start,
 314                                                tad_pmu_page_size);
 315                 if (!regions[i].base) {
 316                         dev_err(&pdev->dev, "TAD%d ioremap fail\n", i);
 317                         return -ENOMEM;
 318                 }
 319                 res->start += tad_page_size;
 320         }
 321
 322         tad_pmu->regions = regions;
 323         tad_pmu->region_cnt = tad_cnt;
 324
 325         tad_pmu->pmu = (struct pmu) {
 326
 327                 .module         = THIS_MODULE,
 328                 .attr_groups    = tad_pmu_attr_groups,
 329                 .capabilities   = PERF_PMU_CAP_NO_EXCLUDE |
 330                                   PERF_PMU_CAP_NO_INTERRUPT,
 331                 .task_ctx_nr    = perf_invalid_context,
 332
 333                 .event_init     = tad_pmu_event_init,
 334                 .add            = tad_pmu_event_counter_add,
 335                 .del            = tad_pmu_event_counter_del,
 336                 .start          = tad_pmu_event_counter_start,
 337                 .stop           = tad_pmu_event_counter_stop,
 338                 .read           = tad_pmu_event_counter_read,
 339         };
 340
 341         tad_pmu->cpu = raw_smp_processor_id();
 342
 343         /* Register pmu instance for cpu hotplug */
 344         ret = cpuhp_state_add_instance_nocalls(tad_pmu_cpuhp_state,
 345                                                &tad_pmu->node);
 346         if (ret) {
 347                 dev_err(&pdev->dev, "Error %d registering hotplug\n", ret);
 348                 return ret;
 349         }
 350
 351         name = "tad";
 352         ret = perf_pmu_register(&tad_pmu->pmu, name, -1);
 353         if (ret)
 354                 cpuhp_state_remove_instance_nocalls(tad_pmu_cpuhp_state,
 355                                                     &tad_pmu->node);
 356
 357         return ret;
 358 }
 359
 360 static int tad_pmu_remove(struct platform_device *pdev)
 361 {
 362         struct tad_pmu *pmu = platform_get_drvdata(pdev);
 363
 364         cpuhp_state_remove_instance_nocalls(tad_pmu_cpuhp_state,
 365                                                 &pmu->node);
 366         perf_pmu_unregister(&pmu->pmu);
 367
 368         return 0;
 369 }
 370
 371 #ifdef CONFIG_OF
 372 static const struct of_device_id tad_pmu_of_match[] = {
 373         { .compatible = "marvell,cn10k-tad-pmu", },
 374         {},
 375 };
 376 #endif
 377
 378 static struct platform_driver tad_pmu_driver = {
 379         .driver         = {
 380                 .name   = "cn10k_tad_pmu",
 381                 .of_match_table = of_match_ptr(tad_pmu_of_match),
 382                 .suppress_bind_attrs = true,
 383         },
 384         .probe          = tad_pmu_probe,
 385         .remove         = tad_pmu_remove,
 386 };
 387
 388 static int tad_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
 389 {
 390         struct tad_pmu *pmu = hlist_entry_safe(node, struct tad_pmu, node);
 391         unsigned int target;
 392
 393         if (cpu != pmu->cpu)
 394                 return 0;
 395
 396         target = cpumask_any_but(cpu_online_mask, cpu);
 397         if (target >= nr_cpu_ids)
 398                 return 0;
 399
 400         perf_pmu_migrate_context(&pmu->pmu, cpu, target);
 401         pmu->cpu = target;
 402
 403         return 0;
 404 }
 405
 406 static int __init tad_pmu_init(void)
 407 {
 408         int ret;
 409
 410         ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN,
 411                                       "perf/cn10k/tadpmu:online",
 412                                       NULL,
 413                                       tad_pmu_offline_cpu);
 414         if (ret < 0)
 415                 return ret;
 416         tad_pmu_cpuhp_state = ret;
 417         return platform_driver_register(&tad_pmu_driver);
 418 }
 419
 420 static void __exit tad_pmu_exit(void)
 421 {
 422         platform_driver_unregister(&tad_pmu_driver);
 423         cpuhp_remove_multi_state(tad_pmu_cpuhp_state);
 424 }
 425
 426 module_init(tad_pmu_init);
 427 module_exit(tad_pmu_exit);
 428
 429 MODULE_DESCRIPTION("Marvell CN10K LLC-TAD Perf driver");
 430 MODULE_AUTHOR("Bhaskara Budiredla <bbudiredla@marvell.com>");
 431 MODULE_LICENSE("GPL v2");