s390/kernel/perf_pai_crypto.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Performance event support - Processor Activity Instrumentation Facility
   4  *
   5  *  Copyright IBM Corp. 2022
   6  *  Author(s): Thomas Richter <tmricht@linux.ibm.com>
   7  */
   8 #define KMSG_COMPONENT  "pai_crypto"
   9 #define pr_fmt(fmt)     KMSG_COMPONENT ": " fmt
  10
  11 #include <linux/kernel.h>
  12 #include <linux/kernel_stat.h>
  13 #include <linux/percpu.h>
  14 #include <linux/notifier.h>
  15 #include <linux/init.h>
  16 #include <linux/export.h>
  17 #include <linux/io.h>
  18 #include <linux/perf_event.h>
  19
  20 #include <asm/ctl_reg.h>
  21 #include <asm/pai.h>
  22 #include <asm/debug.h>
  23
  24 static debug_info_t *cfm_dbg;
  25 static unsigned int paicrypt_cnt;       /* Size of the mapped counter sets */
  26                                         /* extracted with QPACI instruction */
  27
  28 DEFINE_STATIC_KEY_FALSE(pai_key);
  29
  30 struct pai_userdata {
  31         u16 num;
  32         u64 value;
  33 } __packed;
  34
  35 struct paicrypt_map {
  36         unsigned long *page;            /* Page for CPU to store counters */
  37         struct pai_userdata *save;      /* Page to store no-zero counters */
  38         unsigned int users;             /* # of PAI crypto users */
  39         unsigned int sampler;           /* # of PAI crypto samplers */
  40         unsigned int counter;           /* # of PAI crypto counters */
  41         struct perf_event *event;       /* Perf event for sampling */
  42 };
  43
  44 static DEFINE_PER_CPU(struct paicrypt_map, paicrypt_map);
  45
  46 /* Release the PMU if event is the last perf event */
  47 static DEFINE_MUTEX(pai_reserve_mutex);
  48
  49 /* Adjust usage counters and remove allocated memory when all users are
  50  * gone.
  51  */
  52 static void paicrypt_event_destroy(struct perf_event *event)
  53 {
  54         struct paicrypt_map *cpump = per_cpu_ptr(&paicrypt_map, event->cpu);
  55
  56         cpump->event = NULL;
  57         static_branch_dec(&pai_key);
  58         mutex_lock(&pai_reserve_mutex);
  59         if (event->attr.sample_period)
  60                 cpump->sampler -= 1;
  61         else
  62                 cpump->counter -= 1;
  63         debug_sprintf_event(cfm_dbg, 5, "%s event %#llx cpu %d"
  64                             " sampler %d counter %d\n", __func__,
  65                             event->attr.config, event->cpu, cpump->sampler,
  66                             cpump->counter);
  67         if (!cpump->counter && !cpump->sampler) {
  68                 debug_sprintf_event(cfm_dbg, 4, "%s page %#lx save %p\n",
  69                                     __func__, (unsigned long)cpump->page,
  70                                     cpump->save);
  71                 free_page((unsigned long)cpump->page);
  72                 cpump->page = NULL;
  73                 kvfree(cpump->save);
  74                 cpump->save = NULL;
  75         }
  76         mutex_unlock(&pai_reserve_mutex);
  77 }
  78
  79 static u64 paicrypt_getctr(struct paicrypt_map *cpump, int nr, bool kernel)
  80 {
  81         if (kernel)
  82                 nr += PAI_CRYPTO_MAXCTR;
  83         return cpump->page[nr];
  84 }
  85
  86 /* Read the counter values. Return value from location in CMP. For event
  87  * CRYPTO_ALL sum up all events.
  88  */
  89 static u64 paicrypt_getdata(struct perf_event *event, bool kernel)
  90 {
  91         struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map);
  92         u64 sum = 0;
  93         int i;
  94
  95         if (event->attr.config != PAI_CRYPTO_BASE) {
  96                 return paicrypt_getctr(cpump,
  97                                        event->attr.config - PAI_CRYPTO_BASE,
  98                                        kernel);
  99         }
 100
 101         for (i = 1; i <= paicrypt_cnt; i++) {
 102                 u64 val = paicrypt_getctr(cpump, i, kernel);
 103
 104                 if (!val)
 105                         continue;
 106                 sum += val;
 107         }
 108         return sum;
 109 }
 110
 111 static u64 paicrypt_getall(struct perf_event *event)
 112 {
 113         u64 sum = 0;
 114
 115         if (!event->attr.exclude_kernel)
 116                 sum += paicrypt_getdata(event, true);
 117         if (!event->attr.exclude_user)
 118                 sum += paicrypt_getdata(event, false);
 119
 120         return sum;
 121 }
 122
 123 /* Used to avoid races in checking concurrent access of counting and
 124  * sampling for crypto events
 125  *
 126  * Only one instance of event pai_crypto/CRYPTO_ALL/ for sampling is
 127  * allowed and when this event is running, no counting event is allowed.
 128  * Several counting events are allowed in parallel, but no sampling event
 129  * is allowed while one (or more) counting events are running.
 130  *
 131  * This function is called in process context and it is save to block.
 132  * When the event initialization functions fails, no other call back will
 133  * be invoked.
 134  *
 135  * Allocate the memory for the event.
 136  */
 137 static int paicrypt_busy(struct perf_event_attr *a, struct paicrypt_map *cpump)
 138 {
 139         unsigned int *use_ptr;
 140         int rc = 0;
 141
 142         mutex_lock(&pai_reserve_mutex);
 143         if (a->sample_period) {         /* Sampling requested */
 144                 use_ptr = &cpump->sampler;
 145                 if (cpump->counter || cpump->sampler)
 146                         rc = -EBUSY;    /* ... sampling/counting active */
 147         } else {                        /* Counting requested */
 148                 use_ptr = &cpump->counter;
 149                 if (cpump->sampler)
 150                         rc = -EBUSY;    /* ... and sampling active */
 151         }
 152         if (rc)
 153                 goto unlock;
 154
 155         /* Allocate memory for counter page and counter extraction.
 156          * Only the first counting event has to allocate a page.
 157          */
 158         if (cpump->page)
 159                 goto unlock;
 160
 161         rc = -ENOMEM;
 162         cpump->page = (unsigned long *)get_zeroed_page(GFP_KERNEL);
 163         if (!cpump->page)
 164                 goto unlock;
 165         cpump->save = kvmalloc_array(paicrypt_cnt + 1,
 166                                      sizeof(struct pai_userdata), GFP_KERNEL);
 167         if (!cpump->save) {
 168                 free_page((unsigned long)cpump->page);
 169                 cpump->page = NULL;
 170                 goto unlock;
 171         }
 172         rc = 0;
 173
 174 unlock:
 175         /* If rc is non-zero, do not increment counter/sampler. */
 176         if (!rc)
 177                 *use_ptr += 1;
 178         debug_sprintf_event(cfm_dbg, 5, "%s sample_period %#llx sampler %d"
 179                             " counter %d page %#lx save %p rc %d\n", __func__,
 180                             a->sample_period, cpump->sampler, cpump->counter,
 181                             (unsigned long)cpump->page, cpump->save, rc);
 182         mutex_unlock(&pai_reserve_mutex);
 183         return rc;
 184 }
 185
 186 /* Might be called on different CPU than the one the event is intended for. */
 187 static int paicrypt_event_init(struct perf_event *event)
 188 {
 189         struct perf_event_attr *a = &event->attr;
 190         struct paicrypt_map *cpump;
 191         int rc;
 192
 193         /* PAI crypto PMU registered as PERF_TYPE_RAW, check event type */
 194         if (a->type != PERF_TYPE_RAW && event->pmu->type != a->type)
 195                 return -ENOENT;
 196         /* PAI crypto event must be in valid range */
 197         if (a->config < PAI_CRYPTO_BASE ||
 198             a->config > PAI_CRYPTO_BASE + paicrypt_cnt)
 199                 return -EINVAL;
 200         /* Allow only CPU wide operation, no process context for now. */
 201         if (event->hw.target || event->cpu == -1)
 202                 return -ENOENT;
 203         /* Allow only CRYPTO_ALL for sampling. */
 204         if (a->sample_period && a->config != PAI_CRYPTO_BASE)
 205                 return -EINVAL;
 206
 207         cpump = per_cpu_ptr(&paicrypt_map, event->cpu);
 208         rc = paicrypt_busy(a, cpump);
 209         if (rc)
 210                 return rc;
 211
 212         /* Event initialization sets last_tag to 0. When later on the events
 213          * are deleted and re-added, do not reset the event count value to zero.
 214          * Events are added, deleted and re-added when 2 or more events
 215          * are active at the same time.
 216          */
 217         event->hw.last_tag = 0;
 218         cpump->event = event;
 219         event->destroy = paicrypt_event_destroy;
 220
 221         if (a->sample_period) {
 222                 a->sample_period = 1;
 223                 a->freq = 0;
 224                 /* Register for paicrypt_sched_task() to be called */
 225                 event->attach_state |= PERF_ATTACH_SCHED_CB;
 226                 /* Add raw data which contain the memory mapped counters */
 227                 a->sample_type |= PERF_SAMPLE_RAW;
 228                 /* Turn off inheritance */
 229                 a->inherit = 0;
 230         }
 231
 232         static_branch_inc(&pai_key);
 233         return 0;
 234 }
 235
 236 static void paicrypt_read(struct perf_event *event)
 237 {
 238         u64 prev, new, delta;
 239
 240         prev = local64_read(&event->hw.prev_count);
 241         new = paicrypt_getall(event);
 242         local64_set(&event->hw.prev_count, new);
 243         delta = (prev <= new) ? new - prev
 244                               : (-1ULL - prev) + new + 1;        /* overflow */
 245         local64_add(delta, &event->count);
 246 }
 247
 248 static void paicrypt_start(struct perf_event *event, int flags)
 249 {
 250         u64 sum;
 251
 252         if (!event->hw.last_tag) {
 253                 event->hw.last_tag = 1;
 254                 sum = paicrypt_getall(event);           /* Get current value */
 255                 local64_set(&event->count, 0);
 256                 local64_set(&event->hw.prev_count, sum);
 257         }
 258 }
 259
 260 static int paicrypt_add(struct perf_event *event, int flags)
 261 {
 262         struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map);
 263         unsigned long ccd;
 264
 265         if (cpump->users++ == 0) {
 266                 ccd = virt_to_phys(cpump->page) | PAI_CRYPTO_KERNEL_OFFSET;
 267                 WRITE_ONCE(S390_lowcore.ccd, ccd);
 268                 __ctl_set_bit(0, 50);
 269         }
 270         cpump->event = event;
 271         if (flags & PERF_EF_START && !event->attr.sample_period) {
 272                 /* Only counting needs initial counter value */
 273                 paicrypt_start(event, PERF_EF_RELOAD);
 274         }
 275         event->hw.state = 0;
 276         if (event->attr.sample_period)
 277                 perf_sched_cb_inc(event->pmu);
 278         return 0;
 279 }
 280
 281 static void paicrypt_stop(struct perf_event *event, int flags)
 282 {
 283         paicrypt_read(event);
 284         event->hw.state = PERF_HES_STOPPED;
 285 }
 286
 287 static void paicrypt_del(struct perf_event *event, int flags)
 288 {
 289         struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map);
 290
 291         if (event->attr.sample_period)
 292                 perf_sched_cb_dec(event->pmu);
 293         if (!event->attr.sample_period)
 294                 /* Only counting needs to read counter */
 295                 paicrypt_stop(event, PERF_EF_UPDATE);
 296         if (cpump->users-- == 1) {
 297                 __ctl_clear_bit(0, 50);
 298                 WRITE_ONCE(S390_lowcore.ccd, 0);
 299         }
 300 }
 301
 302 /* Create raw data and save it in buffer. Returns number of bytes copied.
 303  * Saves only positive counter entries of the form
 304  * 2 bytes: Number of counter
 305  * 8 bytes: Value of counter
 306  */
 307 static size_t paicrypt_copy(struct pai_userdata *userdata,
 308                             struct paicrypt_map *cpump,
 309                             bool exclude_user, bool exclude_kernel)
 310 {
 311         int i, outidx = 0;
 312
 313         for (i = 1; i <= paicrypt_cnt; i++) {
 314                 u64 val = 0;
 315
 316                 if (!exclude_kernel)
 317                         val += paicrypt_getctr(cpump, i, true);
 318                 if (!exclude_user)
 319                         val += paicrypt_getctr(cpump, i, false);
 320                 if (val) {
 321                         userdata[outidx].num = i;
 322                         userdata[outidx].value = val;
 323                         outidx++;
 324                 }
 325         }
 326         return outidx * sizeof(struct pai_userdata);
 327 }
 328
 329 static int paicrypt_push_sample(void)
 330 {
 331         struct paicrypt_map *cpump = this_cpu_ptr(&paicrypt_map);
 332         struct perf_event *event = cpump->event;
 333         struct perf_sample_data data;
 334         struct perf_raw_record raw;
 335         struct pt_regs regs;
 336         size_t rawsize;
 337         int overflow;
 338
 339         if (!cpump->event)              /* No event active */
 340                 return 0;
 341         rawsize = paicrypt_copy(cpump->save, cpump,
 342                                 cpump->event->attr.exclude_user,
 343                                 cpump->event->attr.exclude_kernel);
 344         if (!rawsize)                   /* No incremented counters */
 345                 return 0;
 346
 347         /* Setup perf sample */
 348         memset(&regs, 0, sizeof(regs));
 349         memset(&raw, 0, sizeof(raw));
 350         memset(&data, 0, sizeof(data));
 351         perf_sample_data_init(&data, 0, event->hw.last_period);
 352         if (event->attr.sample_type & PERF_SAMPLE_TID) {
 353                 data.tid_entry.pid = task_tgid_nr(current);
 354                 data.tid_entry.tid = task_pid_nr(current);
 355         }
 356         if (event->attr.sample_type & PERF_SAMPLE_TIME)
 357                 data.time = event->clock();
 358         if (event->attr.sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER))
 359                 data.id = event->id;
 360         if (event->attr.sample_type & PERF_SAMPLE_CPU) {
 361                 data.cpu_entry.cpu = smp_processor_id();
 362                 data.cpu_entry.reserved = 0;
 363         }
 364         if (event->attr.sample_type & PERF_SAMPLE_RAW) {
 365                 raw.frag.size = rawsize;
 366                 raw.frag.data = cpump->save;
 367                 raw.size = raw.frag.size;
 368                 data.raw = &raw;
 369                 data.sample_flags |= PERF_SAMPLE_RAW;
 370         }
 371
 372         overflow = perf_event_overflow(event, &data, &regs);
 373         perf_event_update_userpage(event);
 374         /* Clear lowcore page after read */
 375         memset(cpump->page, 0, PAGE_SIZE);
 376         return overflow;
 377 }
 378
 379 /* Called on schedule-in and schedule-out. No access to event structure,
 380  * but for sampling only event CRYPTO_ALL is allowed.
 381  */
 382 static void paicrypt_sched_task(struct perf_event_context *ctx, bool sched_in)
 383 {
 384         /* We started with a clean page on event installation. So read out
 385          * results on schedule_out and if page was dirty, clear values.
 386          */
 387         if (!sched_in)
 388                 paicrypt_push_sample();
 389 }
 390
 391 /* Attribute definitions for paicrypt interface. As with other CPU
 392  * Measurement Facilities, there is one attribute per mapped counter.
 393  * The number of mapped counters may vary per machine generation. Use
 394  * the QUERY PROCESSOR ACTIVITY COUNTER INFORMATION (QPACI) instruction
 395  * to determine the number of mapped counters. The instructions returns
 396  * a positive number, which is the highest number of supported counters.
 397  * All counters less than this number are also supported, there are no
 398  * holes. A returned number of zero means no support for mapped counters.
 399  *
 400  * The identification of the counter is a unique number. The chosen range
 401  * is 0x1000 + offset in mapped kernel page.
 402  * All CPU Measurement Facility counters identifiers must be unique and
 403  * the numbers from 0 to 496 are already used for the CPU Measurement
 404  * Counter facility. Numbers 0xb0000, 0xbc000 and 0xbd000 are already
 405  * used for the CPU Measurement Sampling facility.
 406  */
 407 PMU_FORMAT_ATTR(event, "config:0-63");
 408
 409 static struct attribute *paicrypt_format_attr[] = {
 410         &format_attr_event.attr,
 411         NULL,
 412 };
 413
 414 static struct attribute_group paicrypt_events_group = {
 415         .name = "events",
 416         .attrs = NULL                   /* Filled in attr_event_init() */
 417 };
 418
 419 static struct attribute_group paicrypt_format_group = {
 420         .name = "format",
 421         .attrs = paicrypt_format_attr,
 422 };
 423
 424 static const struct attribute_group *paicrypt_attr_groups[] = {
 425         &paicrypt_events_group,
 426         &paicrypt_format_group,
 427         NULL,
 428 };
 429
 430 /* Performance monitoring unit for mapped counters */
 431 static struct pmu paicrypt = {
 432         .task_ctx_nr  = perf_invalid_context,
 433         .event_init   = paicrypt_event_init,
 434         .add          = paicrypt_add,
 435         .del          = paicrypt_del,
 436         .start        = paicrypt_start,
 437         .stop         = paicrypt_stop,
 438         .read         = paicrypt_read,
 439         .sched_task   = paicrypt_sched_task,
 440         .attr_groups  = paicrypt_attr_groups
 441 };
 442
 443 /* List of symbolic PAI counter names. */
 444 static const char * const paicrypt_ctrnames[] = {
 445         [0] = "CRYPTO_ALL",
 446         [1] = "KM_DEA",
 447         [2] = "KM_TDEA_128",
 448         [3] = "KM_TDEA_192",
 449         [4] = "KM_ENCRYPTED_DEA",
 450         [5] = "KM_ENCRYPTED_TDEA_128",
 451         [6] = "KM_ENCRYPTED_TDEA_192",
 452         [7] = "KM_AES_128",
 453         [8] = "KM_AES_192",
 454         [9] = "KM_AES_256",
 455         [10] = "KM_ENCRYPTED_AES_128",
 456         [11] = "KM_ENCRYPTED_AES_192",
 457         [12] = "KM_ENCRYPTED_AES_256",
 458         [13] = "KM_XTS_AES_128",
 459         [14] = "KM_XTS_AES_256",
 460         [15] = "KM_XTS_ENCRYPTED_AES_128",
 461         [16] = "KM_XTS_ENCRYPTED_AES_256",
 462         [17] = "KMC_DEA",
 463         [18] = "KMC_TDEA_128",
 464         [19] = "KMC_TDEA_192",
 465         [20] = "KMC_ENCRYPTED_DEA",
 466         [21] = "KMC_ENCRYPTED_TDEA_128",
 467         [22] = "KMC_ENCRYPTED_TDEA_192",
 468         [23] = "KMC_AES_128",
 469         [24] = "KMC_AES_192",
 470         [25] = "KMC_AES_256",
 471         [26] = "KMC_ENCRYPTED_AES_128",
 472         [27] = "KMC_ENCRYPTED_AES_192",
 473         [28] = "KMC_ENCRYPTED_AES_256",
 474         [29] = "KMC_PRNG",
 475         [30] = "KMA_GCM_AES_128",
 476         [31] = "KMA_GCM_AES_192",
 477         [32] = "KMA_GCM_AES_256",
 478         [33] = "KMA_GCM_ENCRYPTED_AES_128",
 479         [34] = "KMA_GCM_ENCRYPTED_AES_192",
 480         [35] = "KMA_GCM_ENCRYPTED_AES_256",
 481         [36] = "KMF_DEA",
 482         [37] = "KMF_TDEA_128",
 483         [38] = "KMF_TDEA_192",
 484         [39] = "KMF_ENCRYPTED_DEA",
 485         [40] = "KMF_ENCRYPTED_TDEA_128",
 486         [41] = "KMF_ENCRYPTED_TDEA_192",
 487         [42] = "KMF_AES_128",
 488         [43] = "KMF_AES_192",
 489         [44] = "KMF_AES_256",
 490         [45] = "KMF_ENCRYPTED_AES_128",
 491         [46] = "KMF_ENCRYPTED_AES_192",
 492         [47] = "KMF_ENCRYPTED_AES_256",
 493         [48] = "KMCTR_DEA",
 494         [49] = "KMCTR_TDEA_128",
 495         [50] = "KMCTR_TDEA_192",
 496         [51] = "KMCTR_ENCRYPTED_DEA",
 497         [52] = "KMCTR_ENCRYPTED_TDEA_128",
 498         [53] = "KMCTR_ENCRYPTED_TDEA_192",
 499         [54] = "KMCTR_AES_128",
 500         [55] = "KMCTR_AES_192",
 501         [56] = "KMCTR_AES_256",
 502         [57] = "KMCTR_ENCRYPTED_AES_128",
 503         [58] = "KMCTR_ENCRYPTED_AES_192",
 504         [59] = "KMCTR_ENCRYPTED_AES_256",
 505         [60] = "KMO_DEA",
 506         [61] = "KMO_TDEA_128",
 507         [62] = "KMO_TDEA_192",
 508         [63] = "KMO_ENCRYPTED_DEA",
 509         [64] = "KMO_ENCRYPTED_TDEA_128",
 510         [65] = "KMO_ENCRYPTED_TDEA_192",
 511         [66] = "KMO_AES_128",
 512         [67] = "KMO_AES_192",
 513         [68] = "KMO_AES_256",
 514         [69] = "KMO_ENCRYPTED_AES_128",
 515         [70] = "KMO_ENCRYPTED_AES_192",
 516         [71] = "KMO_ENCRYPTED_AES_256",
 517         [72] = "KIMD_SHA_1",
 518         [73] = "KIMD_SHA_256",
 519         [74] = "KIMD_SHA_512",
 520         [75] = "KIMD_SHA3_224",
 521         [76] = "KIMD_SHA3_256",
 522         [77] = "KIMD_SHA3_384",
 523         [78] = "KIMD_SHA3_512",
 524         [79] = "KIMD_SHAKE_128",
 525         [80] = "KIMD_SHAKE_256",
 526         [81] = "KIMD_GHASH",
 527         [82] = "KLMD_SHA_1",
 528         [83] = "KLMD_SHA_256",
 529         [84] = "KLMD_SHA_512",
 530         [85] = "KLMD_SHA3_224",
 531         [86] = "KLMD_SHA3_256",
 532         [87] = "KLMD_SHA3_384",
 533         [88] = "KLMD_SHA3_512",
 534         [89] = "KLMD_SHAKE_128",
 535         [90] = "KLMD_SHAKE_256",
 536         [91] = "KMAC_DEA",
 537         [92] = "KMAC_TDEA_128",
 538         [93] = "KMAC_TDEA_192",
 539         [94] = "KMAC_ENCRYPTED_DEA",
 540         [95] = "KMAC_ENCRYPTED_TDEA_128",
 541         [96] = "KMAC_ENCRYPTED_TDEA_192",
 542         [97] = "KMAC_AES_128",
 543         [98] = "KMAC_AES_192",
 544         [99] = "KMAC_AES_256",
 545         [100] = "KMAC_ENCRYPTED_AES_128",
 546         [101] = "KMAC_ENCRYPTED_AES_192",
 547         [102] = "KMAC_ENCRYPTED_AES_256",
 548         [103] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_DEA",
 549         [104] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_128",
 550         [105] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_TDEA_192",
 551         [106] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_DEA",
 552         [107] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_128",
 553         [108] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_TDEA_192",
 554         [109] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_128",
 555         [110] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_192",
 556         [111] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_AES_256",
 557         [112] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_128",
 558         [113] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_192",
 559         [114] = "PCC_COMPUTE_LAST_BLOCK_CMAC_USING_ENCRYPTED_AES_256A",
 560         [115] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_128",
 561         [116] = "PCC_COMPUTE_XTS_PARAMETER_USING_AES_256",
 562         [117] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_128",
 563         [118] = "PCC_COMPUTE_XTS_PARAMETER_USING_ENCRYPTED_AES_256",
 564         [119] = "PCC_SCALAR_MULTIPLY_P256",
 565         [120] = "PCC_SCALAR_MULTIPLY_P384",
 566         [121] = "PCC_SCALAR_MULTIPLY_P521",
 567         [122] = "PCC_SCALAR_MULTIPLY_ED25519",
 568         [123] = "PCC_SCALAR_MULTIPLY_ED448",
 569         [124] = "PCC_SCALAR_MULTIPLY_X25519",
 570         [125] = "PCC_SCALAR_MULTIPLY_X448",
 571         [126] = "PRNO_SHA_512_DRNG",
 572         [127] = "PRNO_TRNG_QUERY_RAW_TO_CONDITIONED_RATIO",
 573         [128] = "PRNO_TRNG",
 574         [129] = "KDSA_ECDSA_VERIFY_P256",
 575         [130] = "KDSA_ECDSA_VERIFY_P384",
 576         [131] = "KDSA_ECDSA_VERIFY_P521",
 577         [132] = "KDSA_ECDSA_SIGN_P256",
 578         [133] = "KDSA_ECDSA_SIGN_P384",
 579         [134] = "KDSA_ECDSA_SIGN_P521",
 580         [135] = "KDSA_ENCRYPTED_ECDSA_SIGN_P256",
 581         [136] = "KDSA_ENCRYPTED_ECDSA_SIGN_P384",
 582         [137] = "KDSA_ENCRYPTED_ECDSA_SIGN_P521",
 583         [138] = "KDSA_EDDSA_VERIFY_ED25519",
 584         [139] = "KDSA_EDDSA_VERIFY_ED448",
 585         [140] = "KDSA_EDDSA_SIGN_ED25519",
 586         [141] = "KDSA_EDDSA_SIGN_ED448",
 587         [142] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED25519",
 588         [143] = "KDSA_ENCRYPTED_EDDSA_SIGN_ED448",
 589         [144] = "PCKMO_ENCRYPT_DEA_KEY",
 590         [145] = "PCKMO_ENCRYPT_TDEA_128_KEY",
 591         [146] = "PCKMO_ENCRYPT_TDEA_192_KEY",
 592         [147] = "PCKMO_ENCRYPT_AES_128_KEY",
 593         [148] = "PCKMO_ENCRYPT_AES_192_KEY",
 594         [149] = "PCKMO_ENCRYPT_AES_256_KEY",
 595         [150] = "PCKMO_ENCRYPT_ECC_P256_KEY",
 596         [151] = "PCKMO_ENCRYPT_ECC_P384_KEY",
 597         [152] = "PCKMO_ENCRYPT_ECC_P521_KEY",
 598         [153] = "PCKMO_ENCRYPT_ECC_ED25519_KEY",
 599         [154] = "PCKMO_ENCRYPT_ECC_ED448_KEY",
 600         [155] = "IBM_RESERVED_155",
 601         [156] = "IBM_RESERVED_156",
 602 };
 603
 604 static void __init attr_event_free(struct attribute **attrs, int num)
 605 {
 606         struct perf_pmu_events_attr *pa;
 607         int i;
 608
 609         for (i = 0; i < num; i++) {
 610                 struct device_attribute *dap;
 611
 612                 dap = container_of(attrs[i], struct device_attribute, attr);
 613                 pa = container_of(dap, struct perf_pmu_events_attr, attr);
 614                 kfree(pa);
 615         }
 616         kfree(attrs);
 617 }
 618
 619 static int __init attr_event_init_one(struct attribute **attrs, int num)
 620 {
 621         struct perf_pmu_events_attr *pa;
 622
 623         pa = kzalloc(sizeof(*pa), GFP_KERNEL);
 624         if (!pa)
 625                 return -ENOMEM;
 626
 627         sysfs_attr_init(&pa->attr.attr);
 628         pa->id = PAI_CRYPTO_BASE + num;
 629         pa->attr.attr.name = paicrypt_ctrnames[num];
 630         pa->attr.attr.mode = 0444;
 631         pa->attr.show = cpumf_events_sysfs_show;
 632         pa->attr.store = NULL;
 633         attrs[num] = &pa->attr.attr;
 634         return 0;
 635 }
 636
 637 /* Create PMU sysfs event attributes on the fly. */
 638 static int __init attr_event_init(void)
 639 {
 640         struct attribute **attrs;
 641         int ret, i;
 642
 643         attrs = kmalloc_array(ARRAY_SIZE(paicrypt_ctrnames) + 1, sizeof(*attrs),
 644                               GFP_KERNEL);
 645         if (!attrs)
 646                 return -ENOMEM;
 647         for (i = 0; i < ARRAY_SIZE(paicrypt_ctrnames); i++) {
 648                 ret = attr_event_init_one(attrs, i);
 649                 if (ret) {
 650                         attr_event_free(attrs, i);
 651                         return ret;
 652                 }
 653         }
 654         attrs[i] = NULL;
 655         paicrypt_events_group.attrs = attrs;
 656         return 0;
 657 }
 658
 659 static int __init paicrypt_init(void)
 660 {
 661         struct qpaci_info_block ib;
 662         int rc;
 663
 664         if (!test_facility(196))
 665                 return 0;
 666
 667         qpaci(&ib);
 668         paicrypt_cnt = ib.num_cc;
 669         if (paicrypt_cnt == 0)
 670                 return 0;
 671         if (paicrypt_cnt >= PAI_CRYPTO_MAXCTR)
 672                 paicrypt_cnt = PAI_CRYPTO_MAXCTR - 1;
 673
 674         rc = attr_event_init();         /* Export known PAI crypto events */
 675         if (rc) {
 676                 pr_err("Creation of PMU pai_crypto /sysfs failed\n");
 677                 return rc;
 678         }
 679
 680         /* Setup s390dbf facility */
 681         cfm_dbg = debug_register(KMSG_COMPONENT, 2, 256, 128);
 682         if (!cfm_dbg) {
 683                 pr_err("Registration of s390dbf pai_crypto failed\n");
 684                 return -ENOMEM;
 685         }
 686         debug_register_view(cfm_dbg, &debug_sprintf_view);
 687
 688         rc = perf_pmu_register(&paicrypt, "pai_crypto", -1);
 689         if (rc) {
 690                 pr_err("Registering the pai_crypto PMU failed with rc=%i\n",
 691                        rc);
 692                 debug_unregister_view(cfm_dbg, &debug_sprintf_view);
 693                 debug_unregister(cfm_dbg);
 694                 return rc;
 695         }
 696         return 0;
 697 }
 698
 699 device_initcall(paicrypt_init);