drivers/perf/arm_cspmu/nvidia_cspmu.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
   4  *
   5  */
   6
   7 /* Support for NVIDIA specific attributes. */
   8
   9 #include <linux/module.h>
  10 #include <linux/topology.h>
  11
  12 #include "arm_cspmu.h"
  13
  14 #define NV_PCIE_PORT_COUNT           10ULL
  15 #define NV_PCIE_FILTER_ID_MASK       GENMASK_ULL(NV_PCIE_PORT_COUNT - 1, 0)
  16
  17 #define NV_NVL_C2C_PORT_COUNT        2ULL
  18 #define NV_NVL_C2C_FILTER_ID_MASK    GENMASK_ULL(NV_NVL_C2C_PORT_COUNT - 1, 0)
  19
  20 #define NV_CNVL_PORT_COUNT           4ULL
  21 #define NV_CNVL_FILTER_ID_MASK       GENMASK_ULL(NV_CNVL_PORT_COUNT - 1, 0)
  22
  23 #define NV_GENERIC_FILTER_ID_MASK    GENMASK_ULL(31, 0)
  24
  25 #define NV_PRODID_MASK               GENMASK(31, 0)
  26
  27 #define NV_FORMAT_NAME_GENERIC  0
  28
  29 #define to_nv_cspmu_ctx(cspmu)  ((struct nv_cspmu_ctx *)(cspmu->impl.ctx))
  30
  31 #define NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _num, _suff, _config)        \
  32         ARM_CSPMU_EVENT_ATTR(_pref##_num##_suff, _config)
  33
  34 #define NV_CSPMU_EVENT_ATTR_4(_pref, _suff, _config)                    \
  35         NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _0_, _suff, _config),        \
  36         NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _1_, _suff, _config + 1),    \
  37         NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _2_, _suff, _config + 2),    \
  38         NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _3_, _suff, _config + 3)
  39
  40 struct nv_cspmu_ctx {
  41         const char *name;
  42         u32 filter_mask;
  43         u32 filter_default_val;
  44         struct attribute **event_attr;
  45         struct attribute **format_attr;
  46 };
  47
  48 static struct attribute *scf_pmu_event_attrs[] = {
  49         ARM_CSPMU_EVENT_ATTR(bus_cycles,                        0x1d),
  50
  51         ARM_CSPMU_EVENT_ATTR(scf_cache_allocate,                0xF0),
  52         ARM_CSPMU_EVENT_ATTR(scf_cache_refill,                  0xF1),
  53         ARM_CSPMU_EVENT_ATTR(scf_cache,                         0xF2),
  54         ARM_CSPMU_EVENT_ATTR(scf_cache_wb,                      0xF3),
  55
  56         NV_CSPMU_EVENT_ATTR_4(socket, rd_data,                  0x101),
  57         NV_CSPMU_EVENT_ATTR_4(socket, dl_rsp,                   0x105),
  58         NV_CSPMU_EVENT_ATTR_4(socket, wb_data,                  0x109),
  59         NV_CSPMU_EVENT_ATTR_4(socket, ev_rsp,                   0x10d),
  60         NV_CSPMU_EVENT_ATTR_4(socket, prb_data,                 0x111),
  61
  62         NV_CSPMU_EVENT_ATTR_4(socket, rd_outstanding,           0x115),
  63         NV_CSPMU_EVENT_ATTR_4(socket, dl_outstanding,           0x119),
  64         NV_CSPMU_EVENT_ATTR_4(socket, wb_outstanding,           0x11d),
  65         NV_CSPMU_EVENT_ATTR_4(socket, wr_outstanding,           0x121),
  66         NV_CSPMU_EVENT_ATTR_4(socket, ev_outstanding,           0x125),
  67         NV_CSPMU_EVENT_ATTR_4(socket, prb_outstanding,          0x129),
  68
  69         NV_CSPMU_EVENT_ATTR_4(socket, rd_access,                0x12d),
  70         NV_CSPMU_EVENT_ATTR_4(socket, dl_access,                0x131),
  71         NV_CSPMU_EVENT_ATTR_4(socket, wb_access,                0x135),
  72         NV_CSPMU_EVENT_ATTR_4(socket, wr_access,                0x139),
  73         NV_CSPMU_EVENT_ATTR_4(socket, ev_access,                0x13d),
  74         NV_CSPMU_EVENT_ATTR_4(socket, prb_access,               0x141),
  75
  76         NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_data,                0x145),
  77         NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_access,              0x149),
  78         NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_access,              0x14d),
  79         NV_CSPMU_EVENT_ATTR_4(ocu, gmem_rd_outstanding,         0x151),
  80         NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_outstanding,         0x155),
  81
  82         NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_data,                 0x159),
  83         NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_access,               0x15d),
  84         NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_access,               0x161),
  85         NV_CSPMU_EVENT_ATTR_4(ocu, rem_rd_outstanding,          0x165),
  86         NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_outstanding,          0x169),
  87
  88         ARM_CSPMU_EVENT_ATTR(gmem_rd_data,                      0x16d),
  89         ARM_CSPMU_EVENT_ATTR(gmem_rd_access,                    0x16e),
  90         ARM_CSPMU_EVENT_ATTR(gmem_rd_outstanding,               0x16f),
  91         ARM_CSPMU_EVENT_ATTR(gmem_dl_rsp,                       0x170),
  92         ARM_CSPMU_EVENT_ATTR(gmem_dl_access,                    0x171),
  93         ARM_CSPMU_EVENT_ATTR(gmem_dl_outstanding,               0x172),
  94         ARM_CSPMU_EVENT_ATTR(gmem_wb_data,                      0x173),
  95         ARM_CSPMU_EVENT_ATTR(gmem_wb_access,                    0x174),
  96         ARM_CSPMU_EVENT_ATTR(gmem_wb_outstanding,               0x175),
  97         ARM_CSPMU_EVENT_ATTR(gmem_ev_rsp,                       0x176),
  98         ARM_CSPMU_EVENT_ATTR(gmem_ev_access,                    0x177),
  99         ARM_CSPMU_EVENT_ATTR(gmem_ev_outstanding,               0x178),
 100         ARM_CSPMU_EVENT_ATTR(gmem_wr_data,                      0x179),
 101         ARM_CSPMU_EVENT_ATTR(gmem_wr_outstanding,               0x17a),
 102         ARM_CSPMU_EVENT_ATTR(gmem_wr_access,                    0x17b),
 103
 104         NV_CSPMU_EVENT_ATTR_4(socket, wr_data,                  0x17c),
 105
 106         NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_data,                0x180),
 107         NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_data,                0x184),
 108         NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wr_access,              0x188),
 109         NV_CSPMU_EVENT_ATTR_4(ocu, gmem_wb_outstanding,         0x18c),
 110
 111         NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_data,                 0x190),
 112         NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_data,                 0x194),
 113         NV_CSPMU_EVENT_ATTR_4(ocu, rem_wr_access,               0x198),
 114         NV_CSPMU_EVENT_ATTR_4(ocu, rem_wb_outstanding,          0x19c),
 115
 116         ARM_CSPMU_EVENT_ATTR(gmem_wr_total_bytes,               0x1a0),
 117         ARM_CSPMU_EVENT_ATTR(remote_socket_wr_total_bytes,      0x1a1),
 118         ARM_CSPMU_EVENT_ATTR(remote_socket_rd_data,             0x1a2),
 119         ARM_CSPMU_EVENT_ATTR(remote_socket_rd_outstanding,      0x1a3),
 120         ARM_CSPMU_EVENT_ATTR(remote_socket_rd_access,           0x1a4),
 121
 122         ARM_CSPMU_EVENT_ATTR(cmem_rd_data,                      0x1a5),
 123         ARM_CSPMU_EVENT_ATTR(cmem_rd_access,                    0x1a6),
 124         ARM_CSPMU_EVENT_ATTR(cmem_rd_outstanding,               0x1a7),
 125         ARM_CSPMU_EVENT_ATTR(cmem_dl_rsp,                       0x1a8),
 126         ARM_CSPMU_EVENT_ATTR(cmem_dl_access,                    0x1a9),
 127         ARM_CSPMU_EVENT_ATTR(cmem_dl_outstanding,               0x1aa),
 128         ARM_CSPMU_EVENT_ATTR(cmem_wb_data,                      0x1ab),
 129         ARM_CSPMU_EVENT_ATTR(cmem_wb_access,                    0x1ac),
 130         ARM_CSPMU_EVENT_ATTR(cmem_wb_outstanding,               0x1ad),
 131         ARM_CSPMU_EVENT_ATTR(cmem_ev_rsp,                       0x1ae),
 132         ARM_CSPMU_EVENT_ATTR(cmem_ev_access,                    0x1af),
 133         ARM_CSPMU_EVENT_ATTR(cmem_ev_outstanding,               0x1b0),
 134         ARM_CSPMU_EVENT_ATTR(cmem_wr_data,                      0x1b1),
 135         ARM_CSPMU_EVENT_ATTR(cmem_wr_outstanding,               0x1b2),
 136
 137         NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_data,                0x1b3),
 138         NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_access,              0x1b7),
 139         NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_access,              0x1bb),
 140         NV_CSPMU_EVENT_ATTR_4(ocu, cmem_rd_outstanding,         0x1bf),
 141         NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_outstanding,         0x1c3),
 142
 143         ARM_CSPMU_EVENT_ATTR(ocu_prb_access,                    0x1c7),
 144         ARM_CSPMU_EVENT_ATTR(ocu_prb_data,                      0x1c8),
 145         ARM_CSPMU_EVENT_ATTR(ocu_prb_outstanding,               0x1c9),
 146
 147         ARM_CSPMU_EVENT_ATTR(cmem_wr_access,                    0x1ca),
 148
 149         NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_access,              0x1cb),
 150         NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_data,                0x1cf),
 151         NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wr_data,                0x1d3),
 152         NV_CSPMU_EVENT_ATTR_4(ocu, cmem_wb_outstanding,         0x1d7),
 153
 154         ARM_CSPMU_EVENT_ATTR(cmem_wr_total_bytes,               0x1db),
 155
 156         ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
 157         NULL,
 158 };
 159
 160 static struct attribute *mcf_pmu_event_attrs[] = {
 161         ARM_CSPMU_EVENT_ATTR(rd_bytes_loc,                      0x0),
 162         ARM_CSPMU_EVENT_ATTR(rd_bytes_rem,                      0x1),
 163         ARM_CSPMU_EVENT_ATTR(wr_bytes_loc,                      0x2),
 164         ARM_CSPMU_EVENT_ATTR(wr_bytes_rem,                      0x3),
 165         ARM_CSPMU_EVENT_ATTR(total_bytes_loc,                   0x4),
 166         ARM_CSPMU_EVENT_ATTR(total_bytes_rem,                   0x5),
 167         ARM_CSPMU_EVENT_ATTR(rd_req_loc,                        0x6),
 168         ARM_CSPMU_EVENT_ATTR(rd_req_rem,                        0x7),
 169         ARM_CSPMU_EVENT_ATTR(wr_req_loc,                        0x8),
 170         ARM_CSPMU_EVENT_ATTR(wr_req_rem,                        0x9),
 171         ARM_CSPMU_EVENT_ATTR(total_req_loc,                     0xa),
 172         ARM_CSPMU_EVENT_ATTR(total_req_rem,                     0xb),
 173         ARM_CSPMU_EVENT_ATTR(rd_cum_outs_loc,                   0xc),
 174         ARM_CSPMU_EVENT_ATTR(rd_cum_outs_rem,                   0xd),
 175         ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
 176         NULL,
 177 };
 178
 179 static struct attribute *generic_pmu_event_attrs[] = {
 180         ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
 181         NULL,
 182 };
 183
 184 static struct attribute *scf_pmu_format_attrs[] = {
 185         ARM_CSPMU_FORMAT_EVENT_ATTR,
 186         NULL,
 187 };
 188
 189 static struct attribute *pcie_pmu_format_attrs[] = {
 190         ARM_CSPMU_FORMAT_EVENT_ATTR,
 191         ARM_CSPMU_FORMAT_ATTR(root_port, "config1:0-9"),
 192         NULL,
 193 };
 194
 195 static struct attribute *nvlink_c2c_pmu_format_attrs[] = {
 196         ARM_CSPMU_FORMAT_EVENT_ATTR,
 197         NULL,
 198 };
 199
 200 static struct attribute *cnvlink_pmu_format_attrs[] = {
 201         ARM_CSPMU_FORMAT_EVENT_ATTR,
 202         ARM_CSPMU_FORMAT_ATTR(rem_socket, "config1:0-3"),
 203         NULL,
 204 };
 205
 206 static struct attribute *generic_pmu_format_attrs[] = {
 207         ARM_CSPMU_FORMAT_EVENT_ATTR,
 208         ARM_CSPMU_FORMAT_FILTER_ATTR,
 209         NULL,
 210 };
 211
 212 static struct attribute **
 213 nv_cspmu_get_event_attrs(const struct arm_cspmu *cspmu)
 214 {
 215         const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
 216
 217         return ctx->event_attr;
 218 }
 219
 220 static struct attribute **
 221 nv_cspmu_get_format_attrs(const struct arm_cspmu *cspmu)
 222 {
 223         const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
 224
 225         return ctx->format_attr;
 226 }
 227
 228 static const char *
 229 nv_cspmu_get_name(const struct arm_cspmu *cspmu)
 230 {
 231         const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
 232
 233         return ctx->name;
 234 }
 235
 236 static u32 nv_cspmu_event_filter(const struct perf_event *event)
 237 {
 238         const struct nv_cspmu_ctx *ctx =
 239                 to_nv_cspmu_ctx(to_arm_cspmu(event->pmu));
 240
 241         if (ctx->filter_mask == 0)
 242                 return ctx->filter_default_val;
 243
 244         return event->attr.config1 & ctx->filter_mask;
 245 }
 246
 247 enum nv_cspmu_name_fmt {
 248         NAME_FMT_GENERIC,
 249         NAME_FMT_SOCKET
 250 };
 251
 252 struct nv_cspmu_match {
 253         u32 prodid;
 254         u32 prodid_mask;
 255         u64 filter_mask;
 256         u32 filter_default_val;
 257         const char *name_pattern;
 258         enum nv_cspmu_name_fmt name_fmt;
 259         struct attribute **event_attr;
 260         struct attribute **format_attr;
 261 };
 262
 263 static const struct nv_cspmu_match nv_cspmu_match[] = {
 264         {
 265           .prodid = 0x103,
 266           .prodid_mask = NV_PRODID_MASK,
 267           .filter_mask = NV_PCIE_FILTER_ID_MASK,
 268           .filter_default_val = NV_PCIE_FILTER_ID_MASK,
 269           .name_pattern = "nvidia_pcie_pmu_%u",
 270           .name_fmt = NAME_FMT_SOCKET,
 271           .event_attr = mcf_pmu_event_attrs,
 272           .format_attr = pcie_pmu_format_attrs
 273         },
 274         {
 275           .prodid = 0x104,
 276           .prodid_mask = NV_PRODID_MASK,
 277           .filter_mask = 0x0,
 278           .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK,
 279           .name_pattern = "nvidia_nvlink_c2c1_pmu_%u",
 280           .name_fmt = NAME_FMT_SOCKET,
 281           .event_attr = mcf_pmu_event_attrs,
 282           .format_attr = nvlink_c2c_pmu_format_attrs
 283         },
 284         {
 285           .prodid = 0x105,
 286           .prodid_mask = NV_PRODID_MASK,
 287           .filter_mask = 0x0,
 288           .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK,
 289           .name_pattern = "nvidia_nvlink_c2c0_pmu_%u",
 290           .name_fmt = NAME_FMT_SOCKET,
 291           .event_attr = mcf_pmu_event_attrs,
 292           .format_attr = nvlink_c2c_pmu_format_attrs
 293         },
 294         {
 295           .prodid = 0x106,
 296           .prodid_mask = NV_PRODID_MASK,
 297           .filter_mask = NV_CNVL_FILTER_ID_MASK,
 298           .filter_default_val = NV_CNVL_FILTER_ID_MASK,
 299           .name_pattern = "nvidia_cnvlink_pmu_%u",
 300           .name_fmt = NAME_FMT_SOCKET,
 301           .event_attr = mcf_pmu_event_attrs,
 302           .format_attr = cnvlink_pmu_format_attrs
 303         },
 304         {
 305           .prodid = 0x2CF,
 306           .prodid_mask = NV_PRODID_MASK,
 307           .filter_mask = 0x0,
 308           .filter_default_val = 0x0,
 309           .name_pattern = "nvidia_scf_pmu_%u",
 310           .name_fmt = NAME_FMT_SOCKET,
 311           .event_attr = scf_pmu_event_attrs,
 312           .format_attr = scf_pmu_format_attrs
 313         },
 314         {
 315           .prodid = 0,
 316           .prodid_mask = 0,
 317           .filter_mask = NV_GENERIC_FILTER_ID_MASK,
 318           .filter_default_val = NV_GENERIC_FILTER_ID_MASK,
 319           .name_pattern = "nvidia_uncore_pmu_%u",
 320           .name_fmt = NAME_FMT_GENERIC,
 321           .event_attr = generic_pmu_event_attrs,
 322           .format_attr = generic_pmu_format_attrs
 323         },
 324 };
 325
 326 static char *nv_cspmu_format_name(const struct arm_cspmu *cspmu,
 327                                   const struct nv_cspmu_match *match)
 328 {
 329         char *name;
 330         struct device *dev = cspmu->dev;
 331
 332         static atomic_t pmu_generic_idx = {0};
 333
 334         switch (match->name_fmt) {
 335         case NAME_FMT_SOCKET: {
 336                 const int cpu = cpumask_first(&cspmu->associated_cpus);
 337                 const int socket = cpu_to_node(cpu);
 338
 339                 name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern,
 340                                        socket);
 341                 break;
 342         }
 343         case NAME_FMT_GENERIC:
 344                 name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern,
 345                                        atomic_fetch_inc(&pmu_generic_idx));
 346                 break;
 347         default:
 348                 name = NULL;
 349                 break;
 350         }
 351
 352         return name;
 353 }
 354
 355 static int nv_cspmu_init_ops(struct arm_cspmu *cspmu)
 356 {
 357         u32 prodid;
 358         struct nv_cspmu_ctx *ctx;
 359         struct device *dev = cspmu->dev;
 360         struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
 361         const struct nv_cspmu_match *match = nv_cspmu_match;
 362
 363         ctx = devm_kzalloc(dev, sizeof(struct nv_cspmu_ctx), GFP_KERNEL);
 364         if (!ctx)
 365                 return -ENOMEM;
 366
 367         prodid = FIELD_GET(ARM_CSPMU_PMIIDR_PRODUCTID, cspmu->impl.pmiidr);
 368
 369         /* Find matching PMU. */
 370         for (; match->prodid; match++) {
 371                 const u32 prodid_mask = match->prodid_mask;
 372
 373                 if ((match->prodid & prodid_mask) == (prodid & prodid_mask))
 374                         break;
 375         }
 376
 377         ctx->name               = nv_cspmu_format_name(cspmu, match);
 378         ctx->filter_mask        = match->filter_mask;
 379         ctx->filter_default_val = match->filter_default_val;
 380         ctx->event_attr         = match->event_attr;
 381         ctx->format_attr        = match->format_attr;
 382
 383         cspmu->impl.ctx = ctx;
 384
 385         /* NVIDIA specific callbacks. */
 386         impl_ops->event_filter                  = nv_cspmu_event_filter;
 387         impl_ops->get_event_attrs               = nv_cspmu_get_event_attrs;
 388         impl_ops->get_format_attrs              = nv_cspmu_get_format_attrs;
 389         impl_ops->get_name                      = nv_cspmu_get_name;
 390
 391         /* Set others to NULL to use default callback. */
 392         impl_ops->event_type                    = NULL;
 393         impl_ops->event_attr_is_visible         = NULL;
 394         impl_ops->get_identifier                = NULL;
 395         impl_ops->is_cycle_counter_event        = NULL;
 396
 397         return 0;
 398 }
 399
 400 /* Match all NVIDIA Coresight PMU devices */
 401 static const struct arm_cspmu_impl_match nv_cspmu_param = {
 402         .pmiidr_val     = ARM_CSPMU_IMPL_ID_NVIDIA,
 403         .module         = THIS_MODULE,
 404         .impl_init_ops  = nv_cspmu_init_ops
 405 };
 406
 407 static int __init nvidia_cspmu_init(void)
 408 {
 409         int ret;
 410
 411         ret = arm_cspmu_impl_register(&nv_cspmu_param);
 412         if (ret)
 413                 pr_err("nvidia_cspmu backend registration error: %d\n", ret);
 414
 415         return ret;
 416 }
 417
 418 static void __exit nvidia_cspmu_exit(void)
 419 {
 420         arm_cspmu_impl_unregister(&nv_cspmu_param);
 421 }
 422
 423 module_init(nvidia_cspmu_init);
 424 module_exit(nvidia_cspmu_exit);
 425
 426 MODULE_LICENSE("GPL v2");