kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/mutex.h>
  11 #include <linux/slab.h>
  12 #include <linux/stacktrace.h>
  13 #include <linux/rculist.h>
  14 #include <linux/tracefs.h>
  15
  16 #include "tracing_map.h"
  17 #include "trace.h"
  18
  19 #define SYNTH_SYSTEM            "synthetic"
  20 #define SYNTH_FIELDS_MAX        16
  21
  22 #define STR_VAR_LEN_MAX         32 /* must be multiple of sizeof(u64) */
  23
  24 struct hist_field;
  25
  26 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  27                                 struct tracing_map_elt *elt,
  28                                 struct ring_buffer_event *rbe,
  29                                 void *event);
  30
  31 #define HIST_FIELD_OPERANDS_MAX 2
  32 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  33 #define HIST_ACTIONS_MAX        8
  34
  35 enum field_op_id {
  36         FIELD_OP_NONE,
  37         FIELD_OP_PLUS,
  38         FIELD_OP_MINUS,
  39         FIELD_OP_UNARY_MINUS,
  40 };
  41
  42 struct hist_var {
  43         char                            *name;
  44         struct hist_trigger_data        *hist_data;
  45         unsigned int                    idx;
  46 };
  47
  48 struct hist_field {
  49         struct ftrace_event_field       *field;
  50         unsigned long                   flags;
  51         hist_field_fn_t                 fn;
  52         unsigned int                    ref;
  53         unsigned int                    size;
  54         unsigned int                    offset;
  55         unsigned int                    is_signed;
  56         const char                      *type;
  57         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
  58         struct hist_trigger_data        *hist_data;
  59         struct hist_var                 var;
  60         enum field_op_id                operator;
  61         char                            *system;
  62         char                            *event_name;
  63         char                            *name;
  64         unsigned int                    var_idx;
  65         unsigned int                    var_ref_idx;
  66         bool                            read_once;
  67 };
  68
  69 static u64 hist_field_none(struct hist_field *field,
  70                            struct tracing_map_elt *elt,
  71                            struct ring_buffer_event *rbe,
  72                            void *event)
  73 {
  74         return 0;
  75 }
  76
  77 static u64 hist_field_counter(struct hist_field *field,
  78                               struct tracing_map_elt *elt,
  79                               struct ring_buffer_event *rbe,
  80                               void *event)
  81 {
  82         return 1;
  83 }
  84
  85 static u64 hist_field_string(struct hist_field *hist_field,
  86                              struct tracing_map_elt *elt,
  87                              struct ring_buffer_event *rbe,
  88                              void *event)
  89 {
  90         char *addr = (char *)(event + hist_field->field->offset);
  91
  92         return (u64)(unsigned long)addr;
  93 }
  94
  95 static u64 hist_field_dynstring(struct hist_field *hist_field,
  96                                 struct tracing_map_elt *elt,
  97                                 struct ring_buffer_event *rbe,
  98                                 void *event)
  99 {
 100         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 101         int str_loc = str_item & 0xffff;
 102         char *addr = (char *)(event + str_loc);
 103
 104         return (u64)(unsigned long)addr;
 105 }
 106
 107 static u64 hist_field_pstring(struct hist_field *hist_field,
 108                               struct tracing_map_elt *elt,
 109                               struct ring_buffer_event *rbe,
 110                               void *event)
 111 {
 112         char **addr = (char **)(event + hist_field->field->offset);
 113
 114         return (u64)(unsigned long)*addr;
 115 }
 116
 117 static u64 hist_field_log2(struct hist_field *hist_field,
 118                            struct tracing_map_elt *elt,
 119                            struct ring_buffer_event *rbe,
 120                            void *event)
 121 {
 122         struct hist_field *operand = hist_field->operands[0];
 123
 124         u64 val = operand->fn(operand, elt, rbe, event);
 125
 126         return (u64) ilog2(roundup_pow_of_two(val));
 127 }
 128
 129 static u64 hist_field_plus(struct hist_field *hist_field,
 130                            struct tracing_map_elt *elt,
 131                            struct ring_buffer_event *rbe,
 132                            void *event)
 133 {
 134         struct hist_field *operand1 = hist_field->operands[0];
 135         struct hist_field *operand2 = hist_field->operands[1];
 136
 137         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 138         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 139
 140         return val1 + val2;
 141 }
 142
 143 static u64 hist_field_minus(struct hist_field *hist_field,
 144                             struct tracing_map_elt *elt,
 145                             struct ring_buffer_event *rbe,
 146                             void *event)
 147 {
 148         struct hist_field *operand1 = hist_field->operands[0];
 149         struct hist_field *operand2 = hist_field->operands[1];
 150
 151         u64 val1 = operand1->fn(operand1, elt, rbe, event);
 152         u64 val2 = operand2->fn(operand2, elt, rbe, event);
 153
 154         return val1 - val2;
 155 }
 156
 157 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 158                                   struct tracing_map_elt *elt,
 159                                   struct ring_buffer_event *rbe,
 160                                   void *event)
 161 {
 162         struct hist_field *operand = hist_field->operands[0];
 163
 164         s64 sval = (s64)operand->fn(operand, elt, rbe, event);
 165         u64 val = (u64)-sval;
 166
 167         return val;
 168 }
 169
 170 #define DEFINE_HIST_FIELD_FN(type)                                      \
 171         static u64 hist_field_##type(struct hist_field *hist_field,     \
 172                                      struct tracing_map_elt *elt,       \
 173                                      struct ring_buffer_event *rbe,     \
 174                                      void *event)                       \
 175 {                                                                       \
 176         type *addr = (type *)(event + hist_field->field->offset);       \
 177                                                                         \
 178         return (u64)(unsigned long)*addr;                               \
 179 }
 180
 181 DEFINE_HIST_FIELD_FN(s64);
 182 DEFINE_HIST_FIELD_FN(u64);
 183 DEFINE_HIST_FIELD_FN(s32);
 184 DEFINE_HIST_FIELD_FN(u32);
 185 DEFINE_HIST_FIELD_FN(s16);
 186 DEFINE_HIST_FIELD_FN(u16);
 187 DEFINE_HIST_FIELD_FN(s8);
 188 DEFINE_HIST_FIELD_FN(u8);
 189
 190 #define for_each_hist_field(i, hist_data)       \
 191         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 192
 193 #define for_each_hist_val_field(i, hist_data)   \
 194         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 195
 196 #define for_each_hist_key_field(i, hist_data)   \
 197         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 198
 199 #define HIST_STACKTRACE_DEPTH   16
 200 #define HIST_STACKTRACE_SIZE    (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
 201 #define HIST_STACKTRACE_SKIP    5
 202
 203 #define HITCOUNT_IDX            0
 204 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 205
 206 enum hist_field_flags {
 207         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 208         HIST_FIELD_FL_KEY               = 1 << 1,
 209         HIST_FIELD_FL_STRING            = 1 << 2,
 210         HIST_FIELD_FL_HEX               = 1 << 3,
 211         HIST_FIELD_FL_SYM               = 1 << 4,
 212         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 213         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 214         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 215         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 216         HIST_FIELD_FL_LOG2              = 1 << 9,
 217         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 218         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 219         HIST_FIELD_FL_VAR               = 1 << 12,
 220         HIST_FIELD_FL_EXPR              = 1 << 13,
 221         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 222         HIST_FIELD_FL_CPU               = 1 << 15,
 223         HIST_FIELD_FL_ALIAS             = 1 << 16,
 224 };
 225
 226 struct var_defs {
 227         unsigned int    n_vars;
 228         char            *name[TRACING_MAP_VARS_MAX];
 229         char            *expr[TRACING_MAP_VARS_MAX];
 230 };
 231
 232 struct hist_trigger_attrs {
 233         char            *keys_str;
 234         char            *vals_str;
 235         char            *sort_key_str;
 236         char            *name;
 237         char            *clock;
 238         bool            pause;
 239         bool            cont;
 240         bool            clear;
 241         bool            ts_in_usecs;
 242         unsigned int    map_bits;
 243
 244         char            *assignment_str[TRACING_MAP_VARS_MAX];
 245         unsigned int    n_assignments;
 246
 247         char            *action_str[HIST_ACTIONS_MAX];
 248         unsigned int    n_actions;
 249
 250         struct var_defs var_defs;
 251 };
 252
 253 struct field_var {
 254         struct hist_field       *var;
 255         struct hist_field       *val;
 256 };
 257
 258 struct field_var_hist {
 259         struct hist_trigger_data        *hist_data;
 260         char                            *cmd;
 261 };
 262
 263 struct hist_trigger_data {
 264         struct hist_field               *fields[HIST_FIELDS_MAX];
 265         unsigned int                    n_vals;
 266         unsigned int                    n_keys;
 267         unsigned int                    n_fields;
 268         unsigned int                    n_vars;
 269         unsigned int                    key_size;
 270         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 271         unsigned int                    n_sort_keys;
 272         struct trace_event_file         *event_file;
 273         struct hist_trigger_attrs       *attrs;
 274         struct tracing_map              *map;
 275         bool                            enable_timestamps;
 276         bool                            remove;
 277         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 278         unsigned int                    n_var_refs;
 279
 280         struct action_data              *actions[HIST_ACTIONS_MAX];
 281         unsigned int                    n_actions;
 282
 283         struct hist_field               *synth_var_refs[SYNTH_FIELDS_MAX];
 284         unsigned int                    n_synth_var_refs;
 285         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 286         unsigned int                    n_field_vars;
 287         unsigned int                    n_field_var_str;
 288         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 289         unsigned int                    n_field_var_hists;
 290
 291         struct field_var                *max_vars[SYNTH_FIELDS_MAX];
 292         unsigned int                    n_max_vars;
 293         unsigned int                    n_max_var_str;
 294 };
 295
 296 struct synth_field {
 297         char *type;
 298         char *name;
 299         size_t size;
 300         bool is_signed;
 301         bool is_string;
 302 };
 303
 304 struct synth_event {
 305         struct list_head                        list;
 306         int                                     ref;
 307         char                                    *name;
 308         struct synth_field                      **fields;
 309         unsigned int                            n_fields;
 310         unsigned int                            n_u64;
 311         struct trace_event_class                class;
 312         struct trace_event_call                 call;
 313         struct tracepoint                       *tp;
 314 };
 315
 316 struct action_data;
 317
 318 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 319                              struct tracing_map_elt *elt, void *rec,
 320                              struct ring_buffer_event *rbe,
 321                              struct action_data *data, u64 *var_ref_vals);
 322
 323 struct action_data {
 324         action_fn_t             fn;
 325         unsigned int            n_params;
 326         char                    *params[SYNTH_FIELDS_MAX];
 327
 328         union {
 329                 struct {
 330                         unsigned int            var_ref_idx;
 331                         char                    *match_event;
 332                         char                    *match_event_system;
 333                         char                    *synth_event_name;
 334                         struct synth_event      *synth_event;
 335                 } onmatch;
 336
 337                 struct {
 338                         char                    *var_str;
 339                         char                    *fn_name;
 340                         unsigned int            max_var_ref_idx;
 341                         struct hist_field       *max_var;
 342                         struct hist_field       *var;
 343                 } onmax;
 344         };
 345 };
 346
 347
 348 static char last_hist_cmd[MAX_FILTER_STR_VAL];
 349 static char hist_err_str[MAX_FILTER_STR_VAL];
 350
 351 static void last_cmd_set(char *str)
 352 {
 353         if (!str)
 354                 return;
 355
 356         strncpy(last_hist_cmd, str, MAX_FILTER_STR_VAL - 1);
 357 }
 358
 359 static void hist_err(char *str, char *var)
 360 {
 361         int maxlen = MAX_FILTER_STR_VAL - 1;
 362
 363         if (!str)
 364                 return;
 365
 366         if (strlen(hist_err_str))
 367                 return;
 368
 369         if (!var)
 370                 var = "";
 371
 372         if (strlen(hist_err_str) + strlen(str) + strlen(var) > maxlen)
 373                 return;
 374
 375         strcat(hist_err_str, str);
 376         strcat(hist_err_str, var);
 377 }
 378
 379 static void hist_err_event(char *str, char *system, char *event, char *var)
 380 {
 381         char err[MAX_FILTER_STR_VAL];
 382
 383         if (system && var)
 384                 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s.%s", system, event, var);
 385         else if (system)
 386                 snprintf(err, MAX_FILTER_STR_VAL, "%s.%s", system, event);
 387         else
 388                 strscpy(err, var, MAX_FILTER_STR_VAL);
 389
 390         hist_err(str, err);
 391 }
 392
 393 static void hist_err_clear(void)
 394 {
 395         hist_err_str[0] = '\0';
 396 }
 397
 398 static bool have_hist_err(void)
 399 {
 400         if (strlen(hist_err_str))
 401                 return true;
 402
 403         return false;
 404 }
 405
 406 static LIST_HEAD(synth_event_list);
 407 static DEFINE_MUTEX(synth_event_mutex);
 408
 409 struct synth_trace_event {
 410         struct trace_entry      ent;
 411         u64                     fields[];
 412 };
 413
 414 static int synth_event_define_fields(struct trace_event_call *call)
 415 {
 416         struct synth_trace_event trace;
 417         int offset = offsetof(typeof(trace), fields);
 418         struct synth_event *event = call->data;
 419         unsigned int i, size, n_u64;
 420         char *name, *type;
 421         bool is_signed;
 422         int ret = 0;
 423
 424         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 425                 size = event->fields[i]->size;
 426                 is_signed = event->fields[i]->is_signed;
 427                 type = event->fields[i]->type;
 428                 name = event->fields[i]->name;
 429                 ret = trace_define_field(call, type, name, offset, size,
 430                                          is_signed, FILTER_OTHER);
 431                 if (ret)
 432                         break;
 433
 434                 if (event->fields[i]->is_string) {
 435                         offset += STR_VAR_LEN_MAX;
 436                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 437                 } else {
 438                         offset += sizeof(u64);
 439                         n_u64++;
 440                 }
 441         }
 442
 443         event->n_u64 = n_u64;
 444
 445         return ret;
 446 }
 447
 448 static bool synth_field_signed(char *type)
 449 {
 450         if (strncmp(type, "u", 1) == 0)
 451                 return false;
 452         if (strcmp(type, "gfp_t") == 0)
 453                 return false;
 454
 455         return true;
 456 }
 457
 458 static int synth_field_is_string(char *type)
 459 {
 460         if (strstr(type, "char[") != NULL)
 461                 return true;
 462
 463         return false;
 464 }
 465
 466 static int synth_field_string_size(char *type)
 467 {
 468         char buf[4], *end, *start;
 469         unsigned int len;
 470         int size, err;
 471
 472         start = strstr(type, "char[");
 473         if (start == NULL)
 474                 return -EINVAL;
 475         start += strlen("char[");
 476
 477         end = strchr(type, ']');
 478         if (!end || end < start)
 479                 return -EINVAL;
 480
 481         len = end - start;
 482         if (len > 3)
 483                 return -EINVAL;
 484
 485         strncpy(buf, start, len);
 486         buf[len] = '\0';
 487
 488         err = kstrtouint(buf, 0, &size);
 489         if (err)
 490                 return err;
 491
 492         if (size > STR_VAR_LEN_MAX)
 493                 return -EINVAL;
 494
 495         return size;
 496 }
 497
 498 static int synth_field_size(char *type)
 499 {
 500         int size = 0;
 501
 502         if (strcmp(type, "s64") == 0)
 503                 size = sizeof(s64);
 504         else if (strcmp(type, "u64") == 0)
 505                 size = sizeof(u64);
 506         else if (strcmp(type, "s32") == 0)
 507                 size = sizeof(s32);
 508         else if (strcmp(type, "u32") == 0)
 509                 size = sizeof(u32);
 510         else if (strcmp(type, "s16") == 0)
 511                 size = sizeof(s16);
 512         else if (strcmp(type, "u16") == 0)
 513                 size = sizeof(u16);
 514         else if (strcmp(type, "s8") == 0)
 515                 size = sizeof(s8);
 516         else if (strcmp(type, "u8") == 0)
 517                 size = sizeof(u8);
 518         else if (strcmp(type, "char") == 0)
 519                 size = sizeof(char);
 520         else if (strcmp(type, "unsigned char") == 0)
 521                 size = sizeof(unsigned char);
 522         else if (strcmp(type, "int") == 0)
 523                 size = sizeof(int);
 524         else if (strcmp(type, "unsigned int") == 0)
 525                 size = sizeof(unsigned int);
 526         else if (strcmp(type, "long") == 0)
 527                 size = sizeof(long);
 528         else if (strcmp(type, "unsigned long") == 0)
 529                 size = sizeof(unsigned long);
 530         else if (strcmp(type, "pid_t") == 0)
 531                 size = sizeof(pid_t);
 532         else if (synth_field_is_string(type))
 533                 size = synth_field_string_size(type);
 534
 535         return size;
 536 }
 537
 538 static const char *synth_field_fmt(char *type)
 539 {
 540         const char *fmt = "%llu";
 541
 542         if (strcmp(type, "s64") == 0)
 543                 fmt = "%lld";
 544         else if (strcmp(type, "u64") == 0)
 545                 fmt = "%llu";
 546         else if (strcmp(type, "s32") == 0)
 547                 fmt = "%d";
 548         else if (strcmp(type, "u32") == 0)
 549                 fmt = "%u";
 550         else if (strcmp(type, "s16") == 0)
 551                 fmt = "%d";
 552         else if (strcmp(type, "u16") == 0)
 553                 fmt = "%u";
 554         else if (strcmp(type, "s8") == 0)
 555                 fmt = "%d";
 556         else if (strcmp(type, "u8") == 0)
 557                 fmt = "%u";
 558         else if (strcmp(type, "char") == 0)
 559                 fmt = "%d";
 560         else if (strcmp(type, "unsigned char") == 0)
 561                 fmt = "%u";
 562         else if (strcmp(type, "int") == 0)
 563                 fmt = "%d";
 564         else if (strcmp(type, "unsigned int") == 0)
 565                 fmt = "%u";
 566         else if (strcmp(type, "long") == 0)
 567                 fmt = "%ld";
 568         else if (strcmp(type, "unsigned long") == 0)
 569                 fmt = "%lu";
 570         else if (strcmp(type, "pid_t") == 0)
 571                 fmt = "%d";
 572         else if (synth_field_is_string(type))
 573                 fmt = "%s";
 574
 575         return fmt;
 576 }
 577
 578 static enum print_line_t print_synth_event(struct trace_iterator *iter,
 579                                            int flags,
 580                                            struct trace_event *event)
 581 {
 582         struct trace_array *tr = iter->tr;
 583         struct trace_seq *s = &iter->seq;
 584         struct synth_trace_event *entry;
 585         struct synth_event *se;
 586         unsigned int i, n_u64;
 587         char print_fmt[32];
 588         const char *fmt;
 589
 590         entry = (struct synth_trace_event *)iter->ent;
 591         se = container_of(event, struct synth_event, call.event);
 592
 593         trace_seq_printf(s, "%s: ", se->name);
 594
 595         for (i = 0, n_u64 = 0; i < se->n_fields; i++) {
 596                 if (trace_seq_has_overflowed(s))
 597                         goto end;
 598
 599                 fmt = synth_field_fmt(se->fields[i]->type);
 600
 601                 /* parameter types */
 602                 if (tr->trace_flags & TRACE_ITER_VERBOSE)
 603                         trace_seq_printf(s, "%s ", fmt);
 604
 605                 snprintf(print_fmt, sizeof(print_fmt), "%%s=%s%%s", fmt);
 606
 607                 /* parameter values */
 608                 if (se->fields[i]->is_string) {
 609                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 610                                          (char *)&entry->fields[n_u64],
 611                                          i == se->n_fields - 1 ? "" : " ");
 612                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 613                 } else {
 614                         trace_seq_printf(s, print_fmt, se->fields[i]->name,
 615                                          entry->fields[n_u64],
 616                                          i == se->n_fields - 1 ? "" : " ");
 617                         n_u64++;
 618                 }
 619         }
 620 end:
 621         trace_seq_putc(s, '\n');
 622
 623         return trace_handle_return(s);
 624 }
 625
 626 static struct trace_event_functions synth_event_funcs = {
 627         .trace          = print_synth_event
 628 };
 629
 630 static notrace void trace_event_raw_event_synth(void *__data,
 631                                                 u64 *var_ref_vals,
 632                                                 unsigned int var_ref_idx)
 633 {
 634         struct trace_event_file *trace_file = __data;
 635         struct synth_trace_event *entry;
 636         struct trace_event_buffer fbuffer;
 637         struct ring_buffer *buffer;
 638         struct synth_event *event;
 639         unsigned int i, n_u64;
 640         int fields_size = 0;
 641
 642         event = trace_file->event_call->data;
 643
 644         if (trace_trigger_soft_disabled(trace_file))
 645                 return;
 646
 647         fields_size = event->n_u64 * sizeof(u64);
 648
 649         /*
 650          * Avoid ring buffer recursion detection, as this event
 651          * is being performed within another event.
 652          */
 653         buffer = trace_file->tr->trace_buffer.buffer;
 654         ring_buffer_nest_start(buffer);
 655
 656         entry = trace_event_buffer_reserve(&fbuffer, trace_file,
 657                                            sizeof(*entry) + fields_size);
 658         if (!entry)
 659                 goto out;
 660
 661         for (i = 0, n_u64 = 0; i < event->n_fields; i++) {
 662                 if (event->fields[i]->is_string) {
 663                         char *str_val = (char *)(long)var_ref_vals[var_ref_idx + i];
 664                         char *str_field = (char *)&entry->fields[n_u64];
 665
 666                         strscpy(str_field, str_val, STR_VAR_LEN_MAX);
 667                         n_u64 += STR_VAR_LEN_MAX / sizeof(u64);
 668                 } else {
 669                         struct synth_field *field = event->fields[i];
 670                         u64 val = var_ref_vals[var_ref_idx + i];
 671
 672                         switch (field->size) {
 673                         case 1:
 674                                 *(u8 *)&entry->fields[n_u64] = (u8)val;
 675                                 break;
 676
 677                         case 2:
 678                                 *(u16 *)&entry->fields[n_u64] = (u16)val;
 679                                 break;
 680
 681                         case 4:
 682                                 *(u32 *)&entry->fields[n_u64] = (u32)val;
 683                                 break;
 684
 685                         default:
 686                                 entry->fields[n_u64] = val;
 687                                 break;
 688                         }
 689                         n_u64++;
 690                 }
 691         }
 692
 693         trace_event_buffer_commit(&fbuffer);
 694 out:
 695         ring_buffer_nest_end(buffer);
 696 }
 697
 698 static void free_synth_event_print_fmt(struct trace_event_call *call)
 699 {
 700         if (call) {
 701                 kfree(call->print_fmt);
 702                 call->print_fmt = NULL;
 703         }
 704 }
 705
 706 static int __set_synth_event_print_fmt(struct synth_event *event,
 707                                        char *buf, int len)
 708 {
 709         const char *fmt;
 710         int pos = 0;
 711         int i;
 712
 713         /* When len=0, we just calculate the needed length */
 714 #define LEN_OR_ZERO (len ? len - pos : 0)
 715
 716         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 717         for (i = 0; i < event->n_fields; i++) {
 718                 fmt = synth_field_fmt(event->fields[i]->type);
 719                 pos += snprintf(buf + pos, LEN_OR_ZERO, "%s=%s%s",
 720                                 event->fields[i]->name, fmt,
 721                                 i == event->n_fields - 1 ? "" : ", ");
 722         }
 723         pos += snprintf(buf + pos, LEN_OR_ZERO, "\"");
 724
 725         for (i = 0; i < event->n_fields; i++) {
 726                 pos += snprintf(buf + pos, LEN_OR_ZERO,
 727                                 ", REC->%s", event->fields[i]->name);
 728         }
 729
 730 #undef LEN_OR_ZERO
 731
 732         /* return the length of print_fmt */
 733         return pos;
 734 }
 735
 736 static int set_synth_event_print_fmt(struct trace_event_call *call)
 737 {
 738         struct synth_event *event = call->data;
 739         char *print_fmt;
 740         int len;
 741
 742         /* First: called with 0 length to calculate the needed length */
 743         len = __set_synth_event_print_fmt(event, NULL, 0);
 744
 745         print_fmt = kmalloc(len + 1, GFP_KERNEL);
 746         if (!print_fmt)
 747                 return -ENOMEM;
 748
 749         /* Second: actually write the @print_fmt */
 750         __set_synth_event_print_fmt(event, print_fmt, len + 1);
 751         call->print_fmt = print_fmt;
 752
 753         return 0;
 754 }
 755
 756 static void free_synth_field(struct synth_field *field)
 757 {
 758         kfree(field->type);
 759         kfree(field->name);
 760         kfree(field);
 761 }
 762
 763 static struct synth_field *parse_synth_field(int argc, char **argv,
 764                                              int *consumed)
 765 {
 766         struct synth_field *field;
 767         const char *prefix = NULL;
 768         char *field_type = argv[0], *field_name;
 769         int len, ret = 0;
 770         char *array;
 771
 772         if (field_type[0] == ';')
 773                 field_type++;
 774
 775         if (!strcmp(field_type, "unsigned")) {
 776                 if (argc < 3)
 777                         return ERR_PTR(-EINVAL);
 778                 prefix = "unsigned ";
 779                 field_type = argv[1];
 780                 field_name = argv[2];
 781                 *consumed = 3;
 782         } else {
 783                 field_name = argv[1];
 784                 *consumed = 2;
 785         }
 786
 787         len = strlen(field_name);
 788         if (field_name[len - 1] == ';')
 789                 field_name[len - 1] = '\0';
 790
 791         field = kzalloc(sizeof(*field), GFP_KERNEL);
 792         if (!field)
 793                 return ERR_PTR(-ENOMEM);
 794
 795         len = strlen(field_type) + 1;
 796         array = strchr(field_name, '[');
 797         if (array)
 798                 len += strlen(array);
 799         if (prefix)
 800                 len += strlen(prefix);
 801         field->type = kzalloc(len, GFP_KERNEL);
 802         if (!field->type) {
 803                 ret = -ENOMEM;
 804                 goto free;
 805         }
 806         if (prefix)
 807                 strcat(field->type, prefix);
 808         strcat(field->type, field_type);
 809         if (array) {
 810                 strcat(field->type, array);
 811                 *array = '\0';
 812         }
 813
 814         field->size = synth_field_size(field->type);
 815         if (!field->size) {
 816                 ret = -EINVAL;
 817                 goto free;
 818         }
 819
 820         if (synth_field_is_string(field->type))
 821                 field->is_string = true;
 822
 823         field->is_signed = synth_field_signed(field->type);
 824
 825         field->name = kstrdup(field_name, GFP_KERNEL);
 826         if (!field->name) {
 827                 ret = -ENOMEM;
 828                 goto free;
 829         }
 830  out:
 831         return field;
 832  free:
 833         free_synth_field(field);
 834         field = ERR_PTR(ret);
 835         goto out;
 836 }
 837
 838 static void free_synth_tracepoint(struct tracepoint *tp)
 839 {
 840         if (!tp)
 841                 return;
 842
 843         kfree(tp->name);
 844         kfree(tp);
 845 }
 846
 847 static struct tracepoint *alloc_synth_tracepoint(char *name)
 848 {
 849         struct tracepoint *tp;
 850
 851         tp = kzalloc(sizeof(*tp), GFP_KERNEL);
 852         if (!tp)
 853                 return ERR_PTR(-ENOMEM);
 854
 855         tp->name = kstrdup(name, GFP_KERNEL);
 856         if (!tp->name) {
 857                 kfree(tp);
 858                 return ERR_PTR(-ENOMEM);
 859         }
 860
 861         return tp;
 862 }
 863
 864 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 865                                     unsigned int var_ref_idx);
 866
 867 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 868                                unsigned int var_ref_idx)
 869 {
 870         struct tracepoint *tp = event->tp;
 871
 872         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 873                 struct tracepoint_func *probe_func_ptr;
 874                 synth_probe_func_t probe_func;
 875                 void *__data;
 876
 877                 if (!(cpu_online(raw_smp_processor_id())))
 878                         return;
 879
 880                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 881                 if (probe_func_ptr) {
 882                         do {
 883                                 probe_func = probe_func_ptr->func;
 884                                 __data = probe_func_ptr->data;
 885                                 probe_func(__data, var_ref_vals, var_ref_idx);
 886                         } while ((++probe_func_ptr)->func);
 887                 }
 888         }
 889 }
 890
 891 static struct synth_event *find_synth_event(const char *name)
 892 {
 893         struct synth_event *event;
 894
 895         list_for_each_entry(event, &synth_event_list, list) {
 896                 if (strcmp(event->name, name) == 0)
 897                         return event;
 898         }
 899
 900         return NULL;
 901 }
 902
 903 static int register_synth_event(struct synth_event *event)
 904 {
 905         struct trace_event_call *call = &event->call;
 906         int ret = 0;
 907
 908         event->call.class = &event->class;
 909         event->class.system = kstrdup(SYNTH_SYSTEM, GFP_KERNEL);
 910         if (!event->class.system) {
 911                 ret = -ENOMEM;
 912                 goto out;
 913         }
 914
 915         event->tp = alloc_synth_tracepoint(event->name);
 916         if (IS_ERR(event->tp)) {
 917                 ret = PTR_ERR(event->tp);
 918                 event->tp = NULL;
 919                 goto out;
 920         }
 921
 922         INIT_LIST_HEAD(&call->class->fields);
 923         call->event.funcs = &synth_event_funcs;
 924         call->class->define_fields = synth_event_define_fields;
 925
 926         ret = register_trace_event(&call->event);
 927         if (!ret) {
 928                 ret = -ENODEV;
 929                 goto out;
 930         }
 931         call->flags = TRACE_EVENT_FL_TRACEPOINT;
 932         call->class->reg = trace_event_reg;
 933         call->class->probe = trace_event_raw_event_synth;
 934         call->data = event;
 935         call->tp = event->tp;
 936
 937         ret = trace_add_event_call_nolock(call);
 938         if (ret) {
 939                 pr_warn("Failed to register synthetic event: %s\n",
 940                         trace_event_name(call));
 941                 goto err;
 942         }
 943
 944         ret = set_synth_event_print_fmt(call);
 945         if (ret < 0) {
 946                 trace_remove_event_call(call);
 947                 goto err;
 948         }
 949  out:
 950         return ret;
 951  err:
 952         unregister_trace_event(&call->event);
 953         goto out;
 954 }
 955
 956 static int unregister_synth_event(struct synth_event *event)
 957 {
 958         struct trace_event_call *call = &event->call;
 959         int ret;
 960
 961         ret = trace_remove_event_call_nolock(call);
 962
 963         return ret;
 964 }
 965
 966 static void free_synth_event(struct synth_event *event)
 967 {
 968         unsigned int i;
 969
 970         if (!event)
 971                 return;
 972
 973         for (i = 0; i < event->n_fields; i++)
 974                 free_synth_field(event->fields[i]);
 975
 976         kfree(event->fields);
 977         kfree(event->name);
 978         kfree(event->class.system);
 979         free_synth_tracepoint(event->tp);
 980         free_synth_event_print_fmt(&event->call);
 981         kfree(event);
 982 }
 983
 984 static struct synth_event *alloc_synth_event(char *event_name, int n_fields,
 985                                              struct synth_field **fields)
 986 {
 987         struct synth_event *event;
 988         unsigned int i;
 989
 990         event = kzalloc(sizeof(*event), GFP_KERNEL);
 991         if (!event) {
 992                 event = ERR_PTR(-ENOMEM);
 993                 goto out;
 994         }
 995
 996         event->name = kstrdup(event_name, GFP_KERNEL);
 997         if (!event->name) {
 998                 kfree(event);
 999                 event = ERR_PTR(-ENOMEM);
1000                 goto out;
1001         }
1002
1003         event->fields = kcalloc(n_fields, sizeof(*event->fields), GFP_KERNEL);
1004         if (!event->fields) {
1005                 free_synth_event(event);
1006                 event = ERR_PTR(-ENOMEM);
1007                 goto out;
1008         }
1009
1010         for (i = 0; i < n_fields; i++)
1011                 event->fields[i] = fields[i];
1012
1013         event->n_fields = n_fields;
1014  out:
1015         return event;
1016 }
1017
1018 static void action_trace(struct hist_trigger_data *hist_data,
1019                          struct tracing_map_elt *elt, void *rec,
1020                          struct ring_buffer_event *rbe,
1021                          struct action_data *data, u64 *var_ref_vals)
1022 {
1023         struct synth_event *event = data->onmatch.synth_event;
1024
1025         trace_synth(event, var_ref_vals, data->onmatch.var_ref_idx);
1026 }
1027
1028 struct hist_var_data {
1029         struct list_head list;
1030         struct hist_trigger_data *hist_data;
1031 };
1032
1033 static void add_or_delete_synth_event(struct synth_event *event, int delete)
1034 {
1035         if (delete)
1036                 free_synth_event(event);
1037         else {
1038                 if (!find_synth_event(event->name))
1039                         list_add(&event->list, &synth_event_list);
1040                 else
1041                         free_synth_event(event);
1042         }
1043 }
1044
1045 static int create_synth_event(int argc, char **argv)
1046 {
1047         struct synth_field *field, *fields[SYNTH_FIELDS_MAX];
1048         struct synth_event *event = NULL;
1049         bool delete_event = false;
1050         int i, consumed = 0, n_fields = 0, ret = 0;
1051         char *name;
1052
1053         mutex_lock(&event_mutex);
1054         mutex_lock(&synth_event_mutex);
1055
1056         /*
1057          * Argument syntax:
1058          *  - Add synthetic event: <event_name> field[;field] ...
1059          *  - Remove synthetic event: !<event_name> field[;field] ...
1060          *      where 'field' = type field_name
1061          */
1062         if (argc < 1) {
1063                 ret = -EINVAL;
1064                 goto out;
1065         }
1066
1067         name = argv[0];
1068         if (name[0] == '!') {
1069                 delete_event = true;
1070                 name++;
1071         }
1072
1073         event = find_synth_event(name);
1074         if (event) {
1075                 if (delete_event) {
1076                         if (event->ref) {
1077                                 event = NULL;
1078                                 ret = -EBUSY;
1079                                 goto out;
1080                         }
1081                         list_del(&event->list);
1082                         goto out;
1083                 }
1084                 event = NULL;
1085                 ret = -EEXIST;
1086                 goto out;
1087         } else if (delete_event) {
1088                 ret = -ENOENT;
1089                 goto out;
1090         }
1091
1092         if (argc < 2) {
1093                 ret = -EINVAL;
1094                 goto out;
1095         }
1096
1097         for (i = 1; i < argc - 1; i++) {
1098                 if (strcmp(argv[i], ";") == 0)
1099                         continue;
1100                 if (n_fields == SYNTH_FIELDS_MAX) {
1101                         ret = -EINVAL;
1102                         goto err;
1103                 }
1104
1105                 field = parse_synth_field(argc - i, &argv[i], &consumed);
1106                 if (IS_ERR(field)) {
1107                         ret = PTR_ERR(field);
1108                         goto err;
1109                 }
1110                 fields[n_fields++] = field;
1111                 i += consumed - 1;
1112         }
1113
1114         if (i < argc && strcmp(argv[i], ";") != 0) {
1115                 ret = -EINVAL;
1116                 goto err;
1117         }
1118
1119         event = alloc_synth_event(name, n_fields, fields);
1120         if (IS_ERR(event)) {
1121                 ret = PTR_ERR(event);
1122                 event = NULL;
1123                 goto err;
1124         }
1125  out:
1126         if (event) {
1127                 if (delete_event) {
1128                         ret = unregister_synth_event(event);
1129                         add_or_delete_synth_event(event, !ret);
1130                 } else {
1131                         ret = register_synth_event(event);
1132                         add_or_delete_synth_event(event, ret);
1133                 }
1134         }
1135         mutex_unlock(&synth_event_mutex);
1136         mutex_unlock(&event_mutex);
1137
1138         return ret;
1139  err:
1140         mutex_unlock(&synth_event_mutex);
1141         mutex_unlock(&event_mutex);
1142
1143         for (i = 0; i < n_fields; i++)
1144                 free_synth_field(fields[i]);
1145         free_synth_event(event);
1146
1147         return ret;
1148 }
1149
1150 static int release_all_synth_events(void)
1151 {
1152         struct synth_event *event, *e;
1153         int ret = 0;
1154
1155         mutex_lock(&event_mutex);
1156         mutex_lock(&synth_event_mutex);
1157
1158         list_for_each_entry(event, &synth_event_list, list) {
1159                 if (event->ref) {
1160                         mutex_unlock(&synth_event_mutex);
1161                         return -EBUSY;
1162                 }
1163         }
1164
1165         list_for_each_entry_safe(event, e, &synth_event_list, list) {
1166                 list_del(&event->list);
1167
1168                 ret = unregister_synth_event(event);
1169                 add_or_delete_synth_event(event, !ret);
1170         }
1171         mutex_unlock(&synth_event_mutex);
1172         mutex_unlock(&event_mutex);
1173
1174         return ret;
1175 }
1176
1177
1178 static void *synth_events_seq_start(struct seq_file *m, loff_t *pos)
1179 {
1180         mutex_lock(&synth_event_mutex);
1181
1182         return seq_list_start(&synth_event_list, *pos);
1183 }
1184
1185 static void *synth_events_seq_next(struct seq_file *m, void *v, loff_t *pos)
1186 {
1187         return seq_list_next(v, &synth_event_list, pos);
1188 }
1189
1190 static void synth_events_seq_stop(struct seq_file *m, void *v)
1191 {
1192         mutex_unlock(&synth_event_mutex);
1193 }
1194
1195 static int synth_events_seq_show(struct seq_file *m, void *v)
1196 {
1197         struct synth_field *field;
1198         struct synth_event *event = v;
1199         unsigned int i;
1200
1201         seq_printf(m, "%s\t", event->name);
1202
1203         for (i = 0; i < event->n_fields; i++) {
1204                 field = event->fields[i];
1205
1206                 /* parameter values */
1207                 seq_printf(m, "%s %s%s", field->type, field->name,
1208                            i == event->n_fields - 1 ? "" : "; ");
1209         }
1210
1211         seq_putc(m, '\n');
1212
1213         return 0;
1214 }
1215
1216 static const struct seq_operations synth_events_seq_op = {
1217         .start  = synth_events_seq_start,
1218         .next   = synth_events_seq_next,
1219         .stop   = synth_events_seq_stop,
1220         .show   = synth_events_seq_show
1221 };
1222
1223 static int synth_events_open(struct inode *inode, struct file *file)
1224 {
1225         int ret;
1226
1227         if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1228                 ret = release_all_synth_events();
1229                 if (ret < 0)
1230                         return ret;
1231         }
1232
1233         return seq_open(file, &synth_events_seq_op);
1234 }
1235
1236 static ssize_t synth_events_write(struct file *file,
1237                                   const char __user *buffer,
1238                                   size_t count, loff_t *ppos)
1239 {
1240         return trace_parse_run_command(file, buffer, count, ppos,
1241                                        create_synth_event);
1242 }
1243
1244 static const struct file_operations synth_events_fops = {
1245         .open           = synth_events_open,
1246         .write          = synth_events_write,
1247         .read           = seq_read,
1248         .llseek         = seq_lseek,
1249         .release        = seq_release,
1250 };
1251
1252 static u64 hist_field_timestamp(struct hist_field *hist_field,
1253                                 struct tracing_map_elt *elt,
1254                                 struct ring_buffer_event *rbe,
1255                                 void *event)
1256 {
1257         struct hist_trigger_data *hist_data = hist_field->hist_data;
1258         struct trace_array *tr = hist_data->event_file->tr;
1259
1260         u64 ts = ring_buffer_event_time_stamp(rbe);
1261
1262         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
1263                 ts = ns2usecs(ts);
1264
1265         return ts;
1266 }
1267
1268 static u64 hist_field_cpu(struct hist_field *hist_field,
1269                           struct tracing_map_elt *elt,
1270                           struct ring_buffer_event *rbe,
1271                           void *event)
1272 {
1273         int cpu = smp_processor_id();
1274
1275         return cpu;
1276 }
1277
1278 /**
1279  * check_field_for_var_ref - Check if a VAR_REF field references a variable
1280  * @hist_field: The VAR_REF field to check
1281  * @var_data: The hist trigger that owns the variable
1282  * @var_idx: The trigger variable identifier
1283  *
1284  * Check the given VAR_REF field to see whether or not it references
1285  * the given variable associated with the given trigger.
1286  *
1287  * Return: The VAR_REF field if it does reference the variable, NULL if not
1288  */
1289 static struct hist_field *
1290 check_field_for_var_ref(struct hist_field *hist_field,
1291                         struct hist_trigger_data *var_data,
1292                         unsigned int var_idx)
1293 {
1294         struct hist_field *found = NULL;
1295
1296         if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF) {
1297                 if (hist_field->var.idx == var_idx &&
1298                     hist_field->var.hist_data == var_data) {
1299                         found = hist_field;
1300                 }
1301         }
1302
1303         return found;
1304 }
1305
1306 static struct hist_field *
1307 check_field_for_var_refs(struct hist_trigger_data *hist_data,
1308                          struct hist_field *hist_field,
1309                          struct hist_trigger_data *var_data,
1310                          unsigned int var_idx,
1311                          unsigned int level)
1312 {
1313         struct hist_field *found = NULL;
1314         unsigned int i;
1315
1316         if (level > 3)
1317                 return found;
1318
1319         if (!hist_field)
1320                 return found;
1321
1322         found = check_field_for_var_ref(hist_field, var_data, var_idx);
1323         if (found)
1324                 return found;
1325
1326         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1327                 struct hist_field *operand;
1328
1329                 operand = hist_field->operands[i];
1330                 found = check_field_for_var_refs(hist_data, operand, var_data,
1331                                                  var_idx, level + 1);
1332                 if (found)
1333                         return found;
1334         }
1335
1336         return found;
1337 }
1338
1339 /**
1340  * find_var_ref - Check if a trigger has a reference to a trigger variable
1341  * @hist_data: The hist trigger that might have a reference to the variable
1342  * @var_data: The hist trigger that owns the variable
1343  * @var_idx: The trigger variable identifier
1344  *
1345  * Check the list of var_refs[] on the first hist trigger to see
1346  * whether any of them are references to the variable on the second
1347  * trigger.
1348  *
1349  * Return: The VAR_REF field referencing the variable if so, NULL if not
1350  */
1351 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
1352                                        struct hist_trigger_data *var_data,
1353                                        unsigned int var_idx)
1354 {
1355         struct hist_field *hist_field, *found = NULL;
1356         unsigned int i;
1357
1358         for_each_hist_field(i, hist_data) {
1359                 hist_field = hist_data->fields[i];
1360                 found = check_field_for_var_refs(hist_data, hist_field,
1361                                                  var_data, var_idx, 0);
1362                 if (found)
1363                         return found;
1364         }
1365
1366         for (i = 0; i < hist_data->n_synth_var_refs; i++) {
1367                 hist_field = hist_data->synth_var_refs[i];
1368                 found = check_field_for_var_refs(hist_data, hist_field,
1369                                                  var_data, var_idx, 0);
1370                 if (found)
1371                         return found;
1372         }
1373
1374         return found;
1375 }
1376
1377 /**
1378  * find_any_var_ref - Check if there is a reference to a given trigger variable
1379  * @hist_data: The hist trigger
1380  * @var_idx: The trigger variable identifier
1381  *
1382  * Check to see whether the given variable is currently referenced by
1383  * any other trigger.
1384  *
1385  * The trigger the variable is defined on is explicitly excluded - the
1386  * assumption being that a self-reference doesn't prevent a trigger
1387  * from being removed.
1388  *
1389  * Return: The VAR_REF field referencing the variable if so, NULL if not
1390  */
1391 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
1392                                            unsigned int var_idx)
1393 {
1394         struct trace_array *tr = hist_data->event_file->tr;
1395         struct hist_field *found = NULL;
1396         struct hist_var_data *var_data;
1397
1398         list_for_each_entry(var_data, &tr->hist_vars, list) {
1399                 if (var_data->hist_data == hist_data)
1400                         continue;
1401                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
1402                 if (found)
1403                         break;
1404         }
1405
1406         return found;
1407 }
1408
1409 /**
1410  * check_var_refs - Check if there is a reference to any of trigger's variables
1411  * @hist_data: The hist trigger
1412  *
1413  * A trigger can define one or more variables.  If any one of them is
1414  * currently referenced by any other trigger, this function will
1415  * determine that.
1416
1417  * Typically used to determine whether or not a trigger can be removed
1418  * - if there are any references to a trigger's variables, it cannot.
1419  *
1420  * Return: True if there is a reference to any of trigger's variables
1421  */
1422 static bool check_var_refs(struct hist_trigger_data *hist_data)
1423 {
1424         struct hist_field *field;
1425         bool found = false;
1426         int i;
1427
1428         for_each_hist_field(i, hist_data) {
1429                 field = hist_data->fields[i];
1430                 if (field && field->flags & HIST_FIELD_FL_VAR) {
1431                         if (find_any_var_ref(hist_data, field->var.idx)) {
1432                                 found = true;
1433                                 break;
1434                         }
1435                 }
1436         }
1437
1438         return found;
1439 }
1440
1441 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1442 {
1443         struct trace_array *tr = hist_data->event_file->tr;
1444         struct hist_var_data *var_data, *found = NULL;
1445
1446         list_for_each_entry(var_data, &tr->hist_vars, list) {
1447                 if (var_data->hist_data == hist_data) {
1448                         found = var_data;
1449                         break;
1450                 }
1451         }
1452
1453         return found;
1454 }
1455
1456 static bool field_has_hist_vars(struct hist_field *hist_field,
1457                                 unsigned int level)
1458 {
1459         int i;
1460
1461         if (level > 3)
1462                 return false;
1463
1464         if (!hist_field)
1465                 return false;
1466
1467         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1468             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1469                 return true;
1470
1471         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1472                 struct hist_field *operand;
1473
1474                 operand = hist_field->operands[i];
1475                 if (field_has_hist_vars(operand, level + 1))
1476                         return true;
1477         }
1478
1479         return false;
1480 }
1481
1482 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1483 {
1484         struct hist_field *hist_field;
1485         int i;
1486
1487         for_each_hist_field(i, hist_data) {
1488                 hist_field = hist_data->fields[i];
1489                 if (field_has_hist_vars(hist_field, 0))
1490                         return true;
1491         }
1492
1493         return false;
1494 }
1495
1496 static int save_hist_vars(struct hist_trigger_data *hist_data)
1497 {
1498         struct trace_array *tr = hist_data->event_file->tr;
1499         struct hist_var_data *var_data;
1500
1501         var_data = find_hist_vars(hist_data);
1502         if (var_data)
1503                 return 0;
1504
1505         if (trace_array_get(tr) < 0)
1506                 return -ENODEV;
1507
1508         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1509         if (!var_data) {
1510                 trace_array_put(tr);
1511                 return -ENOMEM;
1512         }
1513
1514         var_data->hist_data = hist_data;
1515         list_add(&var_data->list, &tr->hist_vars);
1516
1517         return 0;
1518 }
1519
1520 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1521 {
1522         struct trace_array *tr = hist_data->event_file->tr;
1523         struct hist_var_data *var_data;
1524
1525         var_data = find_hist_vars(hist_data);
1526         if (!var_data)
1527                 return;
1528
1529         if (WARN_ON(check_var_refs(hist_data)))
1530                 return;
1531
1532         list_del(&var_data->list);
1533
1534         kfree(var_data);
1535
1536         trace_array_put(tr);
1537 }
1538
1539 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1540                                          const char *var_name)
1541 {
1542         struct hist_field *hist_field, *found = NULL;
1543         int i;
1544
1545         for_each_hist_field(i, hist_data) {
1546                 hist_field = hist_data->fields[i];
1547                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1548                     strcmp(hist_field->var.name, var_name) == 0) {
1549                         found = hist_field;
1550                         break;
1551                 }
1552         }
1553
1554         return found;
1555 }
1556
1557 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1558                                    struct trace_event_file *file,
1559                                    const char *var_name)
1560 {
1561         struct hist_trigger_data *test_data;
1562         struct event_trigger_data *test;
1563         struct hist_field *hist_field;
1564
1565         lockdep_assert_held(&event_mutex);
1566
1567         hist_field = find_var_field(hist_data, var_name);
1568         if (hist_field)
1569                 return hist_field;
1570
1571         list_for_each_entry(test, &file->triggers, list) {
1572                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1573                         test_data = test->private_data;
1574                         hist_field = find_var_field(test_data, var_name);
1575                         if (hist_field)
1576                                 return hist_field;
1577                 }
1578         }
1579
1580         return NULL;
1581 }
1582
1583 static struct trace_event_file *find_var_file(struct trace_array *tr,
1584                                               char *system,
1585                                               char *event_name,
1586                                               char *var_name)
1587 {
1588         struct hist_trigger_data *var_hist_data;
1589         struct hist_var_data *var_data;
1590         struct trace_event_file *file, *found = NULL;
1591
1592         if (system)
1593                 return find_event_file(tr, system, event_name);
1594
1595         list_for_each_entry(var_data, &tr->hist_vars, list) {
1596                 var_hist_data = var_data->hist_data;
1597                 file = var_hist_data->event_file;
1598                 if (file == found)
1599                         continue;
1600
1601                 if (find_var_field(var_hist_data, var_name)) {
1602                         if (found) {
1603                                 hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1604                                 return NULL;
1605                         }
1606
1607                         found = file;
1608                 }
1609         }
1610
1611         return found;
1612 }
1613
1614 static struct hist_field *find_file_var(struct trace_event_file *file,
1615                                         const char *var_name)
1616 {
1617         struct hist_trigger_data *test_data;
1618         struct event_trigger_data *test;
1619         struct hist_field *hist_field;
1620
1621         lockdep_assert_held(&event_mutex);
1622
1623         list_for_each_entry(test, &file->triggers, list) {
1624                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1625                         test_data = test->private_data;
1626                         hist_field = find_var_field(test_data, var_name);
1627                         if (hist_field)
1628                                 return hist_field;
1629                 }
1630         }
1631
1632         return NULL;
1633 }
1634
1635 static struct hist_field *
1636 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1637 {
1638         struct trace_array *tr = hist_data->event_file->tr;
1639         struct hist_field *hist_field, *found = NULL;
1640         struct trace_event_file *file;
1641         unsigned int i;
1642
1643         for (i = 0; i < hist_data->n_actions; i++) {
1644                 struct action_data *data = hist_data->actions[i];
1645
1646                 if (data->fn == action_trace) {
1647                         char *system = data->onmatch.match_event_system;
1648                         char *event_name = data->onmatch.match_event;
1649
1650                         file = find_var_file(tr, system, event_name, var_name);
1651                         if (!file)
1652                                 continue;
1653                         hist_field = find_file_var(file, var_name);
1654                         if (hist_field) {
1655                                 if (found) {
1656                                         hist_err_event("Variable name not unique, need to use fully qualified name (subsys.event.var) for variable: ", system, event_name, var_name);
1657                                         return ERR_PTR(-EINVAL);
1658                                 }
1659
1660                                 found = hist_field;
1661                         }
1662                 }
1663         }
1664         return found;
1665 }
1666
1667 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1668                                          char *system,
1669                                          char *event_name,
1670                                          char *var_name)
1671 {
1672         struct trace_array *tr = hist_data->event_file->tr;
1673         struct hist_field *hist_field = NULL;
1674         struct trace_event_file *file;
1675
1676         if (!system || !event_name) {
1677                 hist_field = find_match_var(hist_data, var_name);
1678                 if (IS_ERR(hist_field))
1679                         return NULL;
1680                 if (hist_field)
1681                         return hist_field;
1682         }
1683
1684         file = find_var_file(tr, system, event_name, var_name);
1685         if (!file)
1686                 return NULL;
1687
1688         hist_field = find_file_var(file, var_name);
1689
1690         return hist_field;
1691 }
1692
1693 struct hist_elt_data {
1694         char *comm;
1695         u64 *var_ref_vals;
1696         char *field_var_str[SYNTH_FIELDS_MAX];
1697 };
1698
1699 static u64 hist_field_var_ref(struct hist_field *hist_field,
1700                               struct tracing_map_elt *elt,
1701                               struct ring_buffer_event *rbe,
1702                               void *event)
1703 {
1704         struct hist_elt_data *elt_data;
1705         u64 var_val = 0;
1706
1707         if (WARN_ON_ONCE(!elt))
1708                 return var_val;
1709
1710         elt_data = elt->private_data;
1711         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1712
1713         return var_val;
1714 }
1715
1716 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1717                              u64 *var_ref_vals, bool self)
1718 {
1719         struct hist_trigger_data *var_data;
1720         struct tracing_map_elt *var_elt;
1721         struct hist_field *hist_field;
1722         unsigned int i, var_idx;
1723         bool resolved = true;
1724         u64 var_val = 0;
1725
1726         for (i = 0; i < hist_data->n_var_refs; i++) {
1727                 hist_field = hist_data->var_refs[i];
1728                 var_idx = hist_field->var.idx;
1729                 var_data = hist_field->var.hist_data;
1730
1731                 if (var_data == NULL) {
1732                         resolved = false;
1733                         break;
1734                 }
1735
1736                 if ((self && var_data != hist_data) ||
1737                     (!self && var_data == hist_data))
1738                         continue;
1739
1740                 var_elt = tracing_map_lookup(var_data->map, key);
1741                 if (!var_elt) {
1742                         resolved = false;
1743                         break;
1744                 }
1745
1746                 if (!tracing_map_var_set(var_elt, var_idx)) {
1747                         resolved = false;
1748                         break;
1749                 }
1750
1751                 if (self || !hist_field->read_once)
1752                         var_val = tracing_map_read_var(var_elt, var_idx);
1753                 else
1754                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1755
1756                 var_ref_vals[i] = var_val;
1757         }
1758
1759         return resolved;
1760 }
1761
1762 static const char *hist_field_name(struct hist_field *field,
1763                                    unsigned int level)
1764 {
1765         const char *field_name = "";
1766
1767         if (level > 1)
1768                 return field_name;
1769
1770         if (field->field)
1771                 field_name = field->field->name;
1772         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1773                  field->flags & HIST_FIELD_FL_ALIAS)
1774                 field_name = hist_field_name(field->operands[0], ++level);
1775         else if (field->flags & HIST_FIELD_FL_CPU)
1776                 field_name = "common_cpu";
1777         else if (field->flags & HIST_FIELD_FL_EXPR ||
1778                  field->flags & HIST_FIELD_FL_VAR_REF) {
1779                 if (field->system) {
1780                         static char full_name[MAX_FILTER_STR_VAL];
1781
1782                         strcat(full_name, field->system);
1783                         strcat(full_name, ".");
1784                         strcat(full_name, field->event_name);
1785                         strcat(full_name, ".");
1786                         strcat(full_name, field->name);
1787                         field_name = full_name;
1788                 } else
1789                         field_name = field->name;
1790         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1791                 field_name = "common_timestamp";
1792
1793         if (field_name == NULL)
1794                 field_name = "";
1795
1796         return field_name;
1797 }
1798
1799 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1800 {
1801         hist_field_fn_t fn = NULL;
1802
1803         switch (field_size) {
1804         case 8:
1805                 if (field_is_signed)
1806                         fn = hist_field_s64;
1807                 else
1808                         fn = hist_field_u64;
1809                 break;
1810         case 4:
1811                 if (field_is_signed)
1812                         fn = hist_field_s32;
1813                 else
1814                         fn = hist_field_u32;
1815                 break;
1816         case 2:
1817                 if (field_is_signed)
1818                         fn = hist_field_s16;
1819                 else
1820                         fn = hist_field_u16;
1821                 break;
1822         case 1:
1823                 if (field_is_signed)
1824                         fn = hist_field_s8;
1825                 else
1826                         fn = hist_field_u8;
1827                 break;
1828         }
1829
1830         return fn;
1831 }
1832
1833 static int parse_map_size(char *str)
1834 {
1835         unsigned long size, map_bits;
1836         int ret;
1837
1838         strsep(&str, "=");
1839         if (!str) {
1840                 ret = -EINVAL;
1841                 goto out;
1842         }
1843
1844         ret = kstrtoul(str, 0, &size);
1845         if (ret)
1846                 goto out;
1847
1848         map_bits = ilog2(roundup_pow_of_two(size));
1849         if (map_bits < TRACING_MAP_BITS_MIN ||
1850             map_bits > TRACING_MAP_BITS_MAX)
1851                 ret = -EINVAL;
1852         else
1853                 ret = map_bits;
1854  out:
1855         return ret;
1856 }
1857
1858 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1859 {
1860         unsigned int i;
1861
1862         if (!attrs)
1863                 return;
1864
1865         for (i = 0; i < attrs->n_assignments; i++)
1866                 kfree(attrs->assignment_str[i]);
1867
1868         for (i = 0; i < attrs->n_actions; i++)
1869                 kfree(attrs->action_str[i]);
1870
1871         kfree(attrs->name);
1872         kfree(attrs->sort_key_str);
1873         kfree(attrs->keys_str);
1874         kfree(attrs->vals_str);
1875         kfree(attrs->clock);
1876         kfree(attrs);
1877 }
1878
1879 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1880 {
1881         int ret = -EINVAL;
1882
1883         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1884                 return ret;
1885
1886         if ((strncmp(str, "onmatch(", strlen("onmatch(")) == 0) ||
1887             (strncmp(str, "onmax(", strlen("onmax(")) == 0)) {
1888                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1889                 if (!attrs->action_str[attrs->n_actions]) {
1890                         ret = -ENOMEM;
1891                         return ret;
1892                 }
1893                 attrs->n_actions++;
1894                 ret = 0;
1895         }
1896
1897         return ret;
1898 }
1899
1900 static int parse_assignment(char *str, struct hist_trigger_attrs *attrs)
1901 {
1902         int ret = 0;
1903
1904         if ((strncmp(str, "key=", strlen("key=")) == 0) ||
1905             (strncmp(str, "keys=", strlen("keys=")) == 0)) {
1906                 attrs->keys_str = kstrdup(str, GFP_KERNEL);
1907                 if (!attrs->keys_str) {
1908                         ret = -ENOMEM;
1909                         goto out;
1910                 }
1911         } else if ((strncmp(str, "val=", strlen("val=")) == 0) ||
1912                  (strncmp(str, "vals=", strlen("vals=")) == 0) ||
1913                  (strncmp(str, "values=", strlen("values=")) == 0)) {
1914                 attrs->vals_str = kstrdup(str, GFP_KERNEL);
1915                 if (!attrs->vals_str) {
1916                         ret = -ENOMEM;
1917                         goto out;
1918                 }
1919         } else if (strncmp(str, "sort=", strlen("sort=")) == 0) {
1920                 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
1921                 if (!attrs->sort_key_str) {
1922                         ret = -ENOMEM;
1923                         goto out;
1924                 }
1925         } else if (strncmp(str, "name=", strlen("name=")) == 0) {
1926                 attrs->name = kstrdup(str, GFP_KERNEL);
1927                 if (!attrs->name) {
1928                         ret = -ENOMEM;
1929                         goto out;
1930                 }
1931         } else if (strncmp(str, "clock=", strlen("clock=")) == 0) {
1932                 strsep(&str, "=");
1933                 if (!str) {
1934                         ret = -EINVAL;
1935                         goto out;
1936                 }
1937
1938                 str = strstrip(str);
1939                 attrs->clock = kstrdup(str, GFP_KERNEL);
1940                 if (!attrs->clock) {
1941                         ret = -ENOMEM;
1942                         goto out;
1943                 }
1944         } else if (strncmp(str, "size=", strlen("size=")) == 0) {
1945                 int map_bits = parse_map_size(str);
1946
1947                 if (map_bits < 0) {
1948                         ret = map_bits;
1949                         goto out;
1950                 }
1951                 attrs->map_bits = map_bits;
1952         } else {
1953                 char *assignment;
1954
1955                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1956                         hist_err("Too many variables defined: ", str);
1957                         ret = -EINVAL;
1958                         goto out;
1959                 }
1960
1961                 assignment = kstrdup(str, GFP_KERNEL);
1962                 if (!assignment) {
1963                         ret = -ENOMEM;
1964                         goto out;
1965                 }
1966
1967                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1968         }
1969  out:
1970         return ret;
1971 }
1972
1973 static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
1974 {
1975         struct hist_trigger_attrs *attrs;
1976         int ret = 0;
1977
1978         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1979         if (!attrs)
1980                 return ERR_PTR(-ENOMEM);
1981
1982         while (trigger_str) {
1983                 char *str = strsep(&trigger_str, ":");
1984
1985                 if (strchr(str, '=')) {
1986                         ret = parse_assignment(str, attrs);
1987                         if (ret)
1988                                 goto free;
1989                 } else if (strcmp(str, "pause") == 0)
1990                         attrs->pause = true;
1991                 else if ((strcmp(str, "cont") == 0) ||
1992                          (strcmp(str, "continue") == 0))
1993                         attrs->cont = true;
1994                 else if (strcmp(str, "clear") == 0)
1995                         attrs->clear = true;
1996                 else {
1997                         ret = parse_action(str, attrs);
1998                         if (ret)
1999                                 goto free;
2000                 }
2001         }
2002
2003         if (!attrs->keys_str) {
2004                 ret = -EINVAL;
2005                 goto free;
2006         }
2007
2008         if (!attrs->clock) {
2009                 attrs->clock = kstrdup("global", GFP_KERNEL);
2010                 if (!attrs->clock) {
2011                         ret = -ENOMEM;
2012                         goto free;
2013                 }
2014         }
2015
2016         return attrs;
2017  free:
2018         destroy_hist_trigger_attrs(attrs);
2019
2020         return ERR_PTR(ret);
2021 }
2022
2023 static inline void save_comm(char *comm, struct task_struct *task)
2024 {
2025         if (!task->pid) {
2026                 strcpy(comm, "<idle>");
2027                 return;
2028         }
2029
2030         if (WARN_ON_ONCE(task->pid < 0)) {
2031                 strcpy(comm, "<XXX>");
2032                 return;
2033         }
2034
2035         memcpy(comm, task->comm, TASK_COMM_LEN);
2036 }
2037
2038 static void hist_elt_data_free(struct hist_elt_data *elt_data)
2039 {
2040         unsigned int i;
2041
2042         for (i = 0; i < SYNTH_FIELDS_MAX; i++)
2043                 kfree(elt_data->field_var_str[i]);
2044
2045         kfree(elt_data->comm);
2046         kfree(elt_data);
2047 }
2048
2049 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
2050 {
2051         struct hist_elt_data *elt_data = elt->private_data;
2052
2053         hist_elt_data_free(elt_data);
2054 }
2055
2056 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
2057 {
2058         struct hist_trigger_data *hist_data = elt->map->private_data;
2059         unsigned int size = TASK_COMM_LEN;
2060         struct hist_elt_data *elt_data;
2061         struct hist_field *key_field;
2062         unsigned int i, n_str;
2063
2064         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
2065         if (!elt_data)
2066                 return -ENOMEM;
2067
2068         for_each_hist_key_field(i, hist_data) {
2069                 key_field = hist_data->fields[i];
2070
2071                 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
2072                         elt_data->comm = kzalloc(size, GFP_KERNEL);
2073                         if (!elt_data->comm) {
2074                                 kfree(elt_data);
2075                                 return -ENOMEM;
2076                         }
2077                         break;
2078                 }
2079         }
2080
2081         n_str = hist_data->n_field_var_str + hist_data->n_max_var_str;
2082
2083         size = STR_VAR_LEN_MAX;
2084
2085         for (i = 0; i < n_str; i++) {
2086                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
2087                 if (!elt_data->field_var_str[i]) {
2088                         hist_elt_data_free(elt_data);
2089                         return -ENOMEM;
2090                 }
2091         }
2092
2093         elt->private_data = elt_data;
2094
2095         return 0;
2096 }
2097
2098 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
2099 {
2100         struct hist_elt_data *elt_data = elt->private_data;
2101
2102         if (elt_data->comm)
2103                 save_comm(elt_data->comm, current);
2104 }
2105
2106 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
2107         .elt_alloc      = hist_trigger_elt_data_alloc,
2108         .elt_free       = hist_trigger_elt_data_free,
2109         .elt_init       = hist_trigger_elt_data_init,
2110 };
2111
2112 static const char *get_hist_field_flags(struct hist_field *hist_field)
2113 {
2114         const char *flags_str = NULL;
2115
2116         if (hist_field->flags & HIST_FIELD_FL_HEX)
2117                 flags_str = "hex";
2118         else if (hist_field->flags & HIST_FIELD_FL_SYM)
2119                 flags_str = "sym";
2120         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
2121                 flags_str = "sym-offset";
2122         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
2123                 flags_str = "execname";
2124         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
2125                 flags_str = "syscall";
2126         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
2127                 flags_str = "log2";
2128         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2129                 flags_str = "usecs";
2130
2131         return flags_str;
2132 }
2133
2134 static void expr_field_str(struct hist_field *field, char *expr)
2135 {
2136         if (field->flags & HIST_FIELD_FL_VAR_REF)
2137                 strcat(expr, "$");
2138
2139         strcat(expr, hist_field_name(field, 0));
2140
2141         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
2142                 const char *flags_str = get_hist_field_flags(field);
2143
2144                 if (flags_str) {
2145                         strcat(expr, ".");
2146                         strcat(expr, flags_str);
2147                 }
2148         }
2149 }
2150
2151 static char *expr_str(struct hist_field *field, unsigned int level)
2152 {
2153         char *expr;
2154
2155         if (level > 1)
2156                 return NULL;
2157
2158         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2159         if (!expr)
2160                 return NULL;
2161
2162         if (!field->operands[0]) {
2163                 expr_field_str(field, expr);
2164                 return expr;
2165         }
2166
2167         if (field->operator == FIELD_OP_UNARY_MINUS) {
2168                 char *subexpr;
2169
2170                 strcat(expr, "-(");
2171                 subexpr = expr_str(field->operands[0], ++level);
2172                 if (!subexpr) {
2173                         kfree(expr);
2174                         return NULL;
2175                 }
2176                 strcat(expr, subexpr);
2177                 strcat(expr, ")");
2178
2179                 kfree(subexpr);
2180
2181                 return expr;
2182         }
2183
2184         expr_field_str(field->operands[0], expr);
2185
2186         switch (field->operator) {
2187         case FIELD_OP_MINUS:
2188                 strcat(expr, "-");
2189                 break;
2190         case FIELD_OP_PLUS:
2191                 strcat(expr, "+");
2192                 break;
2193         default:
2194                 kfree(expr);
2195                 return NULL;
2196         }
2197
2198         expr_field_str(field->operands[1], expr);
2199
2200         return expr;
2201 }
2202
2203 static int contains_operator(char *str)
2204 {
2205         enum field_op_id field_op = FIELD_OP_NONE;
2206         char *op;
2207
2208         op = strpbrk(str, "+-");
2209         if (!op)
2210                 return FIELD_OP_NONE;
2211
2212         switch (*op) {
2213         case '-':
2214                 /*
2215                  * Unfortunately, the modifier ".sym-offset"
2216                  * can confuse things.
2217                  */
2218                 if (op - str >= 4 && !strncmp(op - 4, ".sym-offset", 11))
2219                         return FIELD_OP_NONE;
2220
2221                 if (*str == '-')
2222                         field_op = FIELD_OP_UNARY_MINUS;
2223                 else
2224                         field_op = FIELD_OP_MINUS;
2225                 break;
2226         case '+':
2227                 field_op = FIELD_OP_PLUS;
2228                 break;
2229         default:
2230                 break;
2231         }
2232
2233         return field_op;
2234 }
2235
2236 static void get_hist_field(struct hist_field *hist_field)
2237 {
2238         hist_field->ref++;
2239 }
2240
2241 static void __destroy_hist_field(struct hist_field *hist_field)
2242 {
2243         if (--hist_field->ref > 1)
2244                 return;
2245
2246         kfree(hist_field->var.name);
2247         kfree(hist_field->name);
2248         kfree(hist_field->type);
2249
2250         kfree(hist_field);
2251 }
2252
2253 static void destroy_hist_field(struct hist_field *hist_field,
2254                                unsigned int level)
2255 {
2256         unsigned int i;
2257
2258         if (level > 3)
2259                 return;
2260
2261         if (!hist_field)
2262                 return;
2263
2264         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
2265                 return; /* var refs will be destroyed separately */
2266
2267         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
2268                 destroy_hist_field(hist_field->operands[i], level + 1);
2269
2270         __destroy_hist_field(hist_field);
2271 }
2272
2273 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
2274                                             struct ftrace_event_field *field,
2275                                             unsigned long flags,
2276                                             char *var_name)
2277 {
2278         struct hist_field *hist_field;
2279
2280         if (field && is_function_field(field))
2281                 return NULL;
2282
2283         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2284         if (!hist_field)
2285                 return NULL;
2286
2287         hist_field->ref = 1;
2288
2289         hist_field->hist_data = hist_data;
2290
2291         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
2292                 goto out; /* caller will populate */
2293
2294         if (flags & HIST_FIELD_FL_VAR_REF) {
2295                 hist_field->fn = hist_field_var_ref;
2296                 goto out;
2297         }
2298
2299         if (flags & HIST_FIELD_FL_HITCOUNT) {
2300                 hist_field->fn = hist_field_counter;
2301                 hist_field->size = sizeof(u64);
2302                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2303                 if (!hist_field->type)
2304                         goto free;
2305                 goto out;
2306         }
2307
2308         if (flags & HIST_FIELD_FL_STACKTRACE) {
2309                 hist_field->fn = hist_field_none;
2310                 goto out;
2311         }
2312
2313         if (flags & HIST_FIELD_FL_LOG2) {
2314                 unsigned long fl = flags & ~HIST_FIELD_FL_LOG2;
2315                 hist_field->fn = hist_field_log2;
2316                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
2317                 hist_field->size = hist_field->operands[0]->size;
2318                 hist_field->type = kstrdup(hist_field->operands[0]->type, GFP_KERNEL);
2319                 if (!hist_field->type)
2320                         goto free;
2321                 goto out;
2322         }
2323
2324         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2325                 hist_field->fn = hist_field_timestamp;
2326                 hist_field->size = sizeof(u64);
2327                 hist_field->type = kstrdup("u64", GFP_KERNEL);
2328                 if (!hist_field->type)
2329                         goto free;
2330                 goto out;
2331         }
2332
2333         if (flags & HIST_FIELD_FL_CPU) {
2334                 hist_field->fn = hist_field_cpu;
2335                 hist_field->size = sizeof(int);
2336                 hist_field->type = kstrdup("unsigned int", GFP_KERNEL);
2337                 if (!hist_field->type)
2338                         goto free;
2339                 goto out;
2340         }
2341
2342         if (WARN_ON_ONCE(!field))
2343                 goto out;
2344
2345         /* Pointers to strings are just pointers and dangerous to dereference */
2346         if (is_string_field(field) &&
2347             (field->filter_type != FILTER_PTR_STRING)) {
2348                 flags |= HIST_FIELD_FL_STRING;
2349
2350                 hist_field->size = MAX_FILTER_STR_VAL;
2351                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2352                 if (!hist_field->type)
2353                         goto free;
2354
2355                 if (field->filter_type == FILTER_STATIC_STRING)
2356                         hist_field->fn = hist_field_string;
2357                 else if (field->filter_type == FILTER_DYN_STRING)
2358                         hist_field->fn = hist_field_dynstring;
2359                 else
2360                         hist_field->fn = hist_field_pstring;
2361         } else {
2362                 hist_field->size = field->size;
2363                 hist_field->is_signed = field->is_signed;
2364                 hist_field->type = kstrdup(field->type, GFP_KERNEL);
2365                 if (!hist_field->type)
2366                         goto free;
2367
2368                 hist_field->fn = select_value_fn(field->size,
2369                                                  field->is_signed);
2370                 if (!hist_field->fn) {
2371                         destroy_hist_field(hist_field, 0);
2372                         return NULL;
2373                 }
2374         }
2375  out:
2376         hist_field->field = field;
2377         hist_field->flags = flags;
2378
2379         if (var_name) {
2380                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2381                 if (!hist_field->var.name)
2382                         goto free;
2383         }
2384
2385         return hist_field;
2386  free:
2387         destroy_hist_field(hist_field, 0);
2388         return NULL;
2389 }
2390
2391 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2392 {
2393         unsigned int i;
2394
2395         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2396                 if (hist_data->fields[i]) {
2397                         destroy_hist_field(hist_data->fields[i], 0);
2398                         hist_data->fields[i] = NULL;
2399                 }
2400         }
2401
2402         for (i = 0; i < hist_data->n_var_refs; i++) {
2403                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2404                 __destroy_hist_field(hist_data->var_refs[i]);
2405                 hist_data->var_refs[i] = NULL;
2406         }
2407 }
2408
2409 static int init_var_ref(struct hist_field *ref_field,
2410                         struct hist_field *var_field,
2411                         char *system, char *event_name)
2412 {
2413         int err = 0;
2414
2415         ref_field->var.idx = var_field->var.idx;
2416         ref_field->var.hist_data = var_field->hist_data;
2417         ref_field->size = var_field->size;
2418         ref_field->is_signed = var_field->is_signed;
2419         ref_field->flags |= var_field->flags &
2420                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2421
2422         if (system) {
2423                 ref_field->system = kstrdup(system, GFP_KERNEL);
2424                 if (!ref_field->system)
2425                         return -ENOMEM;
2426         }
2427
2428         if (event_name) {
2429                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2430                 if (!ref_field->event_name) {
2431                         err = -ENOMEM;
2432                         goto free;
2433                 }
2434         }
2435
2436         if (var_field->var.name) {
2437                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2438                 if (!ref_field->name) {
2439                         err = -ENOMEM;
2440                         goto free;
2441                 }
2442         } else if (var_field->name) {
2443                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2444                 if (!ref_field->name) {
2445                         err = -ENOMEM;
2446                         goto free;
2447                 }
2448         }
2449
2450         ref_field->type = kstrdup(var_field->type, GFP_KERNEL);
2451         if (!ref_field->type) {
2452                 err = -ENOMEM;
2453                 goto free;
2454         }
2455  out:
2456         return err;
2457  free:
2458         kfree(ref_field->system);
2459         kfree(ref_field->event_name);
2460         kfree(ref_field->name);
2461
2462         goto out;
2463 }
2464
2465 /**
2466  * create_var_ref - Create a variable reference and attach it to trigger
2467  * @hist_data: The trigger that will be referencing the variable
2468  * @var_field: The VAR field to create a reference to
2469  * @system: The optional system string
2470  * @event_name: The optional event_name string
2471  *
2472  * Given a variable hist_field, create a VAR_REF hist_field that
2473  * represents a reference to it.
2474  *
2475  * This function also adds the reference to the trigger that
2476  * now references the variable.
2477  *
2478  * Return: The VAR_REF field if successful, NULL if not
2479  */
2480 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2481                                          struct hist_field *var_field,
2482                                          char *system, char *event_name)
2483 {
2484         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2485         struct hist_field *ref_field;
2486         int i;
2487
2488         /* Check if the variable already exists */
2489         for (i = 0; i < hist_data->n_var_refs; i++) {
2490                 ref_field = hist_data->var_refs[i];
2491                 if (ref_field->var.idx == var_field->var.idx &&
2492                     ref_field->var.hist_data == var_field->hist_data) {
2493                         get_hist_field(ref_field);
2494                         return ref_field;
2495                 }
2496         }
2497
2498         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2499         if (ref_field) {
2500                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2501                         destroy_hist_field(ref_field, 0);
2502                         return NULL;
2503                 }
2504
2505                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2506                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2507         }
2508
2509         return ref_field;
2510 }
2511
2512 static bool is_var_ref(char *var_name)
2513 {
2514         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2515                 return false;
2516
2517         return true;
2518 }
2519
2520 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2521                                  char *var_name)
2522 {
2523         char *name, *field;
2524         unsigned int i;
2525
2526         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2527                 name = hist_data->attrs->var_defs.name[i];
2528
2529                 if (strcmp(var_name, name) == 0) {
2530                         field = hist_data->attrs->var_defs.expr[i];
2531                         if (contains_operator(field) || is_var_ref(field))
2532                                 continue;
2533                         return field;
2534                 }
2535         }
2536
2537         return NULL;
2538 }
2539
2540 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2541                                  char *system, char *event_name,
2542                                  char *var_name)
2543 {
2544         struct trace_event_call *call;
2545
2546         if (system && event_name) {
2547                 call = hist_data->event_file->event_call;
2548
2549                 if (strcmp(system, call->class->system) != 0)
2550                         return NULL;
2551
2552                 if (strcmp(event_name, trace_event_name(call)) != 0)
2553                         return NULL;
2554         }
2555
2556         if (!!system != !!event_name)
2557                 return NULL;
2558
2559         if (!is_var_ref(var_name))
2560                 return NULL;
2561
2562         var_name++;
2563
2564         return field_name_from_var(hist_data, var_name);
2565 }
2566
2567 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2568                                         char *system, char *event_name,
2569                                         char *var_name)
2570 {
2571         struct hist_field *var_field = NULL, *ref_field = NULL;
2572
2573         if (!is_var_ref(var_name))
2574                 return NULL;
2575
2576         var_name++;
2577
2578         var_field = find_event_var(hist_data, system, event_name, var_name);
2579         if (var_field)
2580                 ref_field = create_var_ref(hist_data, var_field,
2581                                            system, event_name);
2582
2583         if (!ref_field)
2584                 hist_err_event("Couldn't find variable: $",
2585                                system, event_name, var_name);
2586
2587         return ref_field;
2588 }
2589
2590 static struct ftrace_event_field *
2591 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2592             char *field_str, unsigned long *flags)
2593 {
2594         struct ftrace_event_field *field = NULL;
2595         char *field_name, *modifier, *str;
2596
2597         modifier = str = kstrdup(field_str, GFP_KERNEL);
2598         if (!modifier)
2599                 return ERR_PTR(-ENOMEM);
2600
2601         field_name = strsep(&modifier, ".");
2602         if (modifier) {
2603                 if (strcmp(modifier, "hex") == 0)
2604                         *flags |= HIST_FIELD_FL_HEX;
2605                 else if (strcmp(modifier, "sym") == 0)
2606                         *flags |= HIST_FIELD_FL_SYM;
2607                 else if (strcmp(modifier, "sym-offset") == 0)
2608                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2609                 else if ((strcmp(modifier, "execname") == 0) &&
2610                          (strcmp(field_name, "common_pid") == 0))
2611                         *flags |= HIST_FIELD_FL_EXECNAME;
2612                 else if (strcmp(modifier, "syscall") == 0)
2613                         *flags |= HIST_FIELD_FL_SYSCALL;
2614                 else if (strcmp(modifier, "log2") == 0)
2615                         *flags |= HIST_FIELD_FL_LOG2;
2616                 else if (strcmp(modifier, "usecs") == 0)
2617                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2618                 else {
2619                         hist_err("Invalid field modifier: ", modifier);
2620                         field = ERR_PTR(-EINVAL);
2621                         goto out;
2622                 }
2623         }
2624
2625         if (strcmp(field_name, "common_timestamp") == 0) {
2626                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2627                 hist_data->enable_timestamps = true;
2628                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2629                         hist_data->attrs->ts_in_usecs = true;
2630         } else if (strcmp(field_name, "common_cpu") == 0)
2631                 *flags |= HIST_FIELD_FL_CPU;
2632         else {
2633                 field = trace_find_event_field(file->event_call, field_name);
2634                 if (!field || !field->size) {
2635                         /*
2636                          * For backward compatibility, if field_name
2637                          * was "cpu", then we treat this the same as
2638                          * common_cpu.
2639                          */
2640                         if (strcmp(field_name, "cpu") == 0) {
2641                                 *flags |= HIST_FIELD_FL_CPU;
2642                         } else {
2643                                 hist_err("Couldn't find field: ", field_name);
2644                                 field = ERR_PTR(-EINVAL);
2645                                 goto out;
2646                         }
2647                 }
2648         }
2649  out:
2650         kfree(str);
2651
2652         return field;
2653 }
2654
2655 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2656                                        struct hist_field *var_ref,
2657                                        char *var_name)
2658 {
2659         struct hist_field *alias = NULL;
2660         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2661
2662         alias = create_hist_field(hist_data, NULL, flags, var_name);
2663         if (!alias)
2664                 return NULL;
2665
2666         alias->fn = var_ref->fn;
2667         alias->operands[0] = var_ref;
2668
2669         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2670                 destroy_hist_field(alias, 0);
2671                 return NULL;
2672         }
2673
2674         alias->var_ref_idx = var_ref->var_ref_idx;
2675
2676         return alias;
2677 }
2678
2679 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2680                                      struct trace_event_file *file, char *str,
2681                                      unsigned long *flags, char *var_name)
2682 {
2683         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2684         struct ftrace_event_field *field = NULL;
2685         struct hist_field *hist_field = NULL;
2686         int ret = 0;
2687
2688         s = strchr(str, '.');
2689         if (s) {
2690                 s = strchr(++s, '.');
2691                 if (s) {
2692                         ref_system = strsep(&str, ".");
2693                         if (!str) {
2694                                 ret = -EINVAL;
2695                                 goto out;
2696                         }
2697                         ref_event = strsep(&str, ".");
2698                         if (!str) {
2699                                 ret = -EINVAL;
2700                                 goto out;
2701                         }
2702                         ref_var = str;
2703                 }
2704         }
2705
2706         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2707         if (!s) {
2708                 hist_field = parse_var_ref(hist_data, ref_system, ref_event, ref_var);
2709                 if (hist_field) {
2710                         if (var_name) {
2711                                 hist_field = create_alias(hist_data, hist_field, var_name);
2712                                 if (!hist_field) {
2713                                         ret = -ENOMEM;
2714                                         goto out;
2715                                 }
2716                         }
2717                         return hist_field;
2718                 }
2719         } else
2720                 str = s;
2721
2722         field = parse_field(hist_data, file, str, flags);
2723         if (IS_ERR(field)) {
2724                 ret = PTR_ERR(field);
2725                 goto out;
2726         }
2727
2728         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2729         if (!hist_field) {
2730                 ret = -ENOMEM;
2731                 goto out;
2732         }
2733
2734         return hist_field;
2735  out:
2736         return ERR_PTR(ret);
2737 }
2738
2739 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2740                                      struct trace_event_file *file,
2741                                      char *str, unsigned long flags,
2742                                      char *var_name, unsigned int level);
2743
2744 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2745                                       struct trace_event_file *file,
2746                                       char *str, unsigned long flags,
2747                                       char *var_name, unsigned int level)
2748 {
2749         struct hist_field *operand1, *expr = NULL;
2750         unsigned long operand_flags;
2751         int ret = 0;
2752         char *s;
2753
2754         /* we support only -(xxx) i.e. explicit parens required */
2755
2756         if (level > 3) {
2757                 hist_err("Too many subexpressions (3 max): ", str);
2758                 ret = -EINVAL;
2759                 goto free;
2760         }
2761
2762         str++; /* skip leading '-' */
2763
2764         s = strchr(str, '(');
2765         if (s)
2766                 str++;
2767         else {
2768                 ret = -EINVAL;
2769                 goto free;
2770         }
2771
2772         s = strrchr(str, ')');
2773         if (s)
2774                 *s = '\0';
2775         else {
2776                 ret = -EINVAL; /* no closing ')' */
2777                 goto free;
2778         }
2779
2780         flags |= HIST_FIELD_FL_EXPR;
2781         expr = create_hist_field(hist_data, NULL, flags, var_name);
2782         if (!expr) {
2783                 ret = -ENOMEM;
2784                 goto free;
2785         }
2786
2787         operand_flags = 0;
2788         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2789         if (IS_ERR(operand1)) {
2790                 ret = PTR_ERR(operand1);
2791                 goto free;
2792         }
2793         if (operand1->flags & HIST_FIELD_FL_STRING) {
2794                 /* String type can not be the operand of unary operator. */
2795                 destroy_hist_field(operand1, 0);
2796                 ret = -EINVAL;
2797                 goto free;
2798         }
2799
2800         expr->flags |= operand1->flags &
2801                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2802         expr->fn = hist_field_unary_minus;
2803         expr->operands[0] = operand1;
2804         expr->operator = FIELD_OP_UNARY_MINUS;
2805         expr->name = expr_str(expr, 0);
2806         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2807         if (!expr->type) {
2808                 ret = -ENOMEM;
2809                 goto free;
2810         }
2811
2812         return expr;
2813  free:
2814         destroy_hist_field(expr, 0);
2815         return ERR_PTR(ret);
2816 }
2817
2818 static int check_expr_operands(struct hist_field *operand1,
2819                                struct hist_field *operand2)
2820 {
2821         unsigned long operand1_flags = operand1->flags;
2822         unsigned long operand2_flags = operand2->flags;
2823
2824         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2825             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2826                 struct hist_field *var;
2827
2828                 var = find_var_field(operand1->var.hist_data, operand1->name);
2829                 if (!var)
2830                         return -EINVAL;
2831                 operand1_flags = var->flags;
2832         }
2833
2834         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2835             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2836                 struct hist_field *var;
2837
2838                 var = find_var_field(operand2->var.hist_data, operand2->name);
2839                 if (!var)
2840                         return -EINVAL;
2841                 operand2_flags = var->flags;
2842         }
2843
2844         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2845             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2846                 hist_err("Timestamp units in expression don't match", NULL);
2847                 return -EINVAL;
2848         }
2849
2850         return 0;
2851 }
2852
2853 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2854                                      struct trace_event_file *file,
2855                                      char *str, unsigned long flags,
2856                                      char *var_name, unsigned int level)
2857 {
2858         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2859         unsigned long operand_flags;
2860         int field_op, ret = -EINVAL;
2861         char *sep, *operand1_str;
2862
2863         if (level > 3) {
2864                 hist_err("Too many subexpressions (3 max): ", str);
2865                 return ERR_PTR(-EINVAL);
2866         }
2867
2868         field_op = contains_operator(str);
2869
2870         if (field_op == FIELD_OP_NONE)
2871                 return parse_atom(hist_data, file, str, &flags, var_name);
2872
2873         if (field_op == FIELD_OP_UNARY_MINUS)
2874                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2875
2876         switch (field_op) {
2877         case FIELD_OP_MINUS:
2878                 sep = "-";
2879                 break;
2880         case FIELD_OP_PLUS:
2881                 sep = "+";
2882                 break;
2883         default:
2884                 goto free;
2885         }
2886
2887         operand1_str = strsep(&str, sep);
2888         if (!operand1_str || !str)
2889                 goto free;
2890
2891         operand_flags = 0;
2892         operand1 = parse_atom(hist_data, file, operand1_str,
2893                               &operand_flags, NULL);
2894         if (IS_ERR(operand1)) {
2895                 ret = PTR_ERR(operand1);
2896                 operand1 = NULL;
2897                 goto free;
2898         }
2899         if (operand1->flags & HIST_FIELD_FL_STRING) {
2900                 ret = -EINVAL;
2901                 goto free;
2902         }
2903
2904         /* rest of string could be another expression e.g. b+c in a+b+c */
2905         operand_flags = 0;
2906         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2907         if (IS_ERR(operand2)) {
2908                 ret = PTR_ERR(operand2);
2909                 operand2 = NULL;
2910                 goto free;
2911         }
2912         if (operand2->flags & HIST_FIELD_FL_STRING) {
2913                 ret = -EINVAL;
2914                 goto free;
2915         }
2916
2917         ret = check_expr_operands(operand1, operand2);
2918         if (ret)
2919                 goto free;
2920
2921         flags |= HIST_FIELD_FL_EXPR;
2922
2923         flags |= operand1->flags &
2924                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2925
2926         expr = create_hist_field(hist_data, NULL, flags, var_name);
2927         if (!expr) {
2928                 ret = -ENOMEM;
2929                 goto free;
2930         }
2931
2932         operand1->read_once = true;
2933         operand2->read_once = true;
2934
2935         expr->operands[0] = operand1;
2936         expr->operands[1] = operand2;
2937
2938         /* The operand sizes should be the same, so just pick one */
2939         expr->size = operand1->size;
2940
2941         expr->operator = field_op;
2942         expr->name = expr_str(expr, 0);
2943         expr->type = kstrdup(operand1->type, GFP_KERNEL);
2944         if (!expr->type) {
2945                 ret = -ENOMEM;
2946                 goto free;
2947         }
2948
2949         switch (field_op) {
2950         case FIELD_OP_MINUS:
2951                 expr->fn = hist_field_minus;
2952                 break;
2953         case FIELD_OP_PLUS:
2954                 expr->fn = hist_field_plus;
2955                 break;
2956         default:
2957                 ret = -EINVAL;
2958                 goto free;
2959         }
2960
2961         return expr;
2962  free:
2963         destroy_hist_field(operand1, 0);
2964         destroy_hist_field(operand2, 0);
2965         destroy_hist_field(expr, 0);
2966
2967         return ERR_PTR(ret);
2968 }
2969
2970 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2971                                  struct trace_event_file *file)
2972 {
2973         struct event_trigger_data *test;
2974
2975         lockdep_assert_held(&event_mutex);
2976
2977         list_for_each_entry(test, &file->triggers, list) {
2978                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2979                         if (test->private_data == hist_data)
2980                                 return test->filter_str;
2981                 }
2982         }
2983
2984         return NULL;
2985 }
2986
2987 static struct event_command trigger_hist_cmd;
2988 static int event_hist_trigger_func(struct event_command *cmd_ops,
2989                                    struct trace_event_file *file,
2990                                    char *glob, char *cmd, char *param);
2991
2992 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2993                             struct hist_trigger_data *hist_data,
2994                             unsigned int n_keys)
2995 {
2996         struct hist_field *target_hist_field, *hist_field;
2997         unsigned int n, i, j;
2998
2999         if (hist_data->n_fields - hist_data->n_vals != n_keys)
3000                 return false;
3001
3002         i = hist_data->n_vals;
3003         j = target_hist_data->n_vals;
3004
3005         for (n = 0; n < n_keys; n++) {
3006                 hist_field = hist_data->fields[i + n];
3007                 target_hist_field = target_hist_data->fields[j + n];
3008
3009                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
3010                         return false;
3011                 if (hist_field->size != target_hist_field->size)
3012                         return false;
3013                 if (hist_field->is_signed != target_hist_field->is_signed)
3014                         return false;
3015         }
3016
3017         return true;
3018 }
3019
3020 static struct hist_trigger_data *
3021 find_compatible_hist(struct hist_trigger_data *target_hist_data,
3022                      struct trace_event_file *file)
3023 {
3024         struct hist_trigger_data *hist_data;
3025         struct event_trigger_data *test;
3026         unsigned int n_keys;
3027
3028         lockdep_assert_held(&event_mutex);
3029
3030         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
3031
3032         list_for_each_entry(test, &file->triggers, list) {
3033                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
3034                         hist_data = test->private_data;
3035
3036                         if (compatible_keys(target_hist_data, hist_data, n_keys))
3037                                 return hist_data;
3038                 }
3039         }
3040
3041         return NULL;
3042 }
3043
3044 static struct trace_event_file *event_file(struct trace_array *tr,
3045                                            char *system, char *event_name)
3046 {
3047         struct trace_event_file *file;
3048
3049         file = __find_event_file(tr, system, event_name);
3050         if (!file)
3051                 return ERR_PTR(-EINVAL);
3052
3053         return file;
3054 }
3055
3056 static struct hist_field *
3057 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
3058                          char *system, char *event_name, char *field_name)
3059 {
3060         struct hist_field *event_var;
3061         char *synthetic_name;
3062
3063         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3064         if (!synthetic_name)
3065                 return ERR_PTR(-ENOMEM);
3066
3067         strcpy(synthetic_name, "synthetic_");
3068         strcat(synthetic_name, field_name);
3069
3070         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
3071
3072         kfree(synthetic_name);
3073
3074         return event_var;
3075 }
3076
3077 /**
3078  * create_field_var_hist - Automatically create a histogram and var for a field
3079  * @target_hist_data: The target hist trigger
3080  * @subsys_name: Optional subsystem name
3081  * @event_name: Optional event name
3082  * @field_name: The name of the field (and the resulting variable)
3083  *
3084  * Hist trigger actions fetch data from variables, not directly from
3085  * events.  However, for convenience, users are allowed to directly
3086  * specify an event field in an action, which will be automatically
3087  * converted into a variable on their behalf.
3088
3089  * If a user specifies a field on an event that isn't the event the
3090  * histogram currently being defined (the target event histogram), the
3091  * only way that can be accomplished is if a new hist trigger is
3092  * created and the field variable defined on that.
3093  *
3094  * This function creates a new histogram compatible with the target
3095  * event (meaning a histogram with the same key as the target
3096  * histogram), and creates a variable for the specified field, but
3097  * with 'synthetic_' prepended to the variable name in order to avoid
3098  * collision with normal field variables.
3099  *
3100  * Return: The variable created for the field.
3101  */
3102 static struct hist_field *
3103 create_field_var_hist(struct hist_trigger_data *target_hist_data,
3104                       char *subsys_name, char *event_name, char *field_name)
3105 {
3106         struct trace_array *tr = target_hist_data->event_file->tr;
3107         struct hist_field *event_var = ERR_PTR(-EINVAL);
3108         struct hist_trigger_data *hist_data;
3109         unsigned int i, n, first = true;
3110         struct field_var_hist *var_hist;
3111         struct trace_event_file *file;
3112         struct hist_field *key_field;
3113         char *saved_filter;
3114         char *cmd;
3115         int ret;
3116
3117         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
3118                 hist_err_event("onmatch: Too many field variables defined: ",
3119                                subsys_name, event_name, field_name);
3120                 return ERR_PTR(-EINVAL);
3121         }
3122
3123         file = event_file(tr, subsys_name, event_name);
3124
3125         if (IS_ERR(file)) {
3126                 hist_err_event("onmatch: Event file not found: ",
3127                                subsys_name, event_name, field_name);
3128                 ret = PTR_ERR(file);
3129                 return ERR_PTR(ret);
3130         }
3131
3132         /*
3133          * Look for a histogram compatible with target.  We'll use the
3134          * found histogram specification to create a new matching
3135          * histogram with our variable on it.  target_hist_data is not
3136          * yet a registered histogram so we can't use that.
3137          */
3138         hist_data = find_compatible_hist(target_hist_data, file);
3139         if (!hist_data) {
3140                 hist_err_event("onmatch: Matching event histogram not found: ",
3141                                subsys_name, event_name, field_name);
3142                 return ERR_PTR(-EINVAL);
3143         }
3144
3145         /* See if a synthetic field variable has already been created */
3146         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3147                                              event_name, field_name);
3148         if (!IS_ERR_OR_NULL(event_var))
3149                 return event_var;
3150
3151         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3152         if (!var_hist)
3153                 return ERR_PTR(-ENOMEM);
3154
3155         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3156         if (!cmd) {
3157                 kfree(var_hist);
3158                 return ERR_PTR(-ENOMEM);
3159         }
3160
3161         /* Use the same keys as the compatible histogram */
3162         strcat(cmd, "keys=");
3163
3164         for_each_hist_key_field(i, hist_data) {
3165                 key_field = hist_data->fields[i];
3166                 if (!first)
3167                         strcat(cmd, ",");
3168                 strcat(cmd, key_field->field->name);
3169                 first = false;
3170         }
3171
3172         /* Create the synthetic field variable specification */
3173         strcat(cmd, ":synthetic_");
3174         strcat(cmd, field_name);
3175         strcat(cmd, "=");
3176         strcat(cmd, field_name);
3177
3178         /* Use the same filter as the compatible histogram */
3179         saved_filter = find_trigger_filter(hist_data, file);
3180         if (saved_filter) {
3181                 strcat(cmd, " if ");
3182                 strcat(cmd, saved_filter);
3183         }
3184
3185         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3186         if (!var_hist->cmd) {
3187                 kfree(cmd);
3188                 kfree(var_hist);
3189                 return ERR_PTR(-ENOMEM);
3190         }
3191
3192         /* Save the compatible histogram information */
3193         var_hist->hist_data = hist_data;
3194
3195         /* Create the new histogram with our variable */
3196         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
3197                                       "", "hist", cmd);
3198         if (ret) {
3199                 kfree(cmd);
3200                 kfree(var_hist->cmd);
3201                 kfree(var_hist);
3202                 hist_err_event("onmatch: Couldn't create histogram for field: ",
3203                                subsys_name, event_name, field_name);
3204                 return ERR_PTR(ret);
3205         }
3206
3207         kfree(cmd);
3208
3209         /* If we can't find the variable, something went wrong */
3210         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3211                                              event_name, field_name);
3212         if (IS_ERR_OR_NULL(event_var)) {
3213                 kfree(var_hist->cmd);
3214                 kfree(var_hist);
3215                 hist_err_event("onmatch: Couldn't find synthetic variable: ",
3216                                subsys_name, event_name, field_name);
3217                 return ERR_PTR(-EINVAL);
3218         }
3219
3220         n = target_hist_data->n_field_var_hists;
3221         target_hist_data->field_var_hists[n] = var_hist;
3222         target_hist_data->n_field_var_hists++;
3223
3224         return event_var;
3225 }
3226
3227 static struct hist_field *
3228 find_target_event_var(struct hist_trigger_data *hist_data,
3229                       char *subsys_name, char *event_name, char *var_name)
3230 {
3231         struct trace_event_file *file = hist_data->event_file;
3232         struct hist_field *hist_field = NULL;
3233
3234         if (subsys_name) {
3235                 struct trace_event_call *call;
3236
3237                 if (!event_name)
3238                         return NULL;
3239
3240                 call = file->event_call;
3241
3242                 if (strcmp(subsys_name, call->class->system) != 0)
3243                         return NULL;
3244
3245                 if (strcmp(event_name, trace_event_name(call)) != 0)
3246                         return NULL;
3247         }
3248
3249         hist_field = find_var_field(hist_data, var_name);
3250
3251         return hist_field;
3252 }
3253
3254 static inline void __update_field_vars(struct tracing_map_elt *elt,
3255                                        struct ring_buffer_event *rbe,
3256                                        void *rec,
3257                                        struct field_var **field_vars,
3258                                        unsigned int n_field_vars,
3259                                        unsigned int field_var_str_start)
3260 {
3261         struct hist_elt_data *elt_data = elt->private_data;
3262         unsigned int i, j, var_idx;
3263         u64 var_val;
3264
3265         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3266                 struct field_var *field_var = field_vars[i];
3267                 struct hist_field *var = field_var->var;
3268                 struct hist_field *val = field_var->val;
3269
3270                 var_val = val->fn(val, elt, rbe, rec);
3271                 var_idx = var->var.idx;
3272
3273                 if (val->flags & HIST_FIELD_FL_STRING) {
3274                         char *str = elt_data->field_var_str[j++];
3275                         char *val_str = (char *)(uintptr_t)var_val;
3276
3277                         strscpy(str, val_str, STR_VAR_LEN_MAX);
3278                         var_val = (u64)(uintptr_t)str;
3279                 }
3280                 tracing_map_set_var(elt, var_idx, var_val);
3281         }
3282 }
3283
3284 static void update_field_vars(struct hist_trigger_data *hist_data,
3285                               struct tracing_map_elt *elt,
3286                               struct ring_buffer_event *rbe,
3287                               void *rec)
3288 {
3289         __update_field_vars(elt, rbe, rec, hist_data->field_vars,
3290                             hist_data->n_field_vars, 0);
3291 }
3292
3293 static void update_max_vars(struct hist_trigger_data *hist_data,
3294                             struct tracing_map_elt *elt,
3295                             struct ring_buffer_event *rbe,
3296                             void *rec)
3297 {
3298         __update_field_vars(elt, rbe, rec, hist_data->max_vars,
3299                             hist_data->n_max_vars, hist_data->n_field_var_str);
3300 }
3301
3302 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3303                                      struct trace_event_file *file,
3304                                      char *name, int size, const char *type)
3305 {
3306         struct hist_field *var;
3307         int idx;
3308
3309         if (find_var(hist_data, file, name) && !hist_data->remove) {
3310                 var = ERR_PTR(-EINVAL);
3311                 goto out;
3312         }
3313
3314         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3315         if (!var) {
3316                 var = ERR_PTR(-ENOMEM);
3317                 goto out;
3318         }
3319
3320         idx = tracing_map_add_var(hist_data->map);
3321         if (idx < 0) {
3322                 kfree(var);
3323                 var = ERR_PTR(-EINVAL);
3324                 goto out;
3325         }
3326
3327         var->flags = HIST_FIELD_FL_VAR;
3328         var->var.idx = idx;
3329         var->var.hist_data = var->hist_data = hist_data;
3330         var->size = size;
3331         var->var.name = kstrdup(name, GFP_KERNEL);
3332         var->type = kstrdup(type, GFP_KERNEL);
3333         if (!var->var.name || !var->type) {
3334                 kfree(var->var.name);
3335                 kfree(var->type);
3336                 kfree(var);
3337                 var = ERR_PTR(-ENOMEM);
3338         }
3339  out:
3340         return var;
3341 }
3342
3343 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3344                                           struct trace_event_file *file,
3345                                           char *field_name)
3346 {
3347         struct hist_field *val = NULL, *var = NULL;
3348         unsigned long flags = HIST_FIELD_FL_VAR;
3349         struct field_var *field_var;
3350         int ret = 0;
3351
3352         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3353                 hist_err("Too many field variables defined: ", field_name);
3354                 ret = -EINVAL;
3355                 goto err;
3356         }
3357
3358         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3359         if (IS_ERR(val)) {
3360                 hist_err("Couldn't parse field variable: ", field_name);
3361                 ret = PTR_ERR(val);
3362                 goto err;
3363         }
3364
3365         var = create_var(hist_data, file, field_name, val->size, val->type);
3366         if (IS_ERR(var)) {
3367                 hist_err("Couldn't create or find variable: ", field_name);
3368                 kfree(val);
3369                 ret = PTR_ERR(var);
3370                 goto err;
3371         }
3372
3373         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3374         if (!field_var) {
3375                 kfree(val);
3376                 kfree(var);
3377                 ret =  -ENOMEM;
3378                 goto err;
3379         }
3380
3381         field_var->var = var;
3382         field_var->val = val;
3383  out:
3384         return field_var;
3385  err:
3386         field_var = ERR_PTR(ret);
3387         goto out;
3388 }
3389
3390 /**
3391  * create_target_field_var - Automatically create a variable for a field
3392  * @target_hist_data: The target hist trigger
3393  * @subsys_name: Optional subsystem name
3394  * @event_name: Optional event name
3395  * @var_name: The name of the field (and the resulting variable)
3396  *
3397  * Hist trigger actions fetch data from variables, not directly from
3398  * events.  However, for convenience, users are allowed to directly
3399  * specify an event field in an action, which will be automatically
3400  * converted into a variable on their behalf.
3401
3402  * This function creates a field variable with the name var_name on
3403  * the hist trigger currently being defined on the target event.  If
3404  * subsys_name and event_name are specified, this function simply
3405  * verifies that they do in fact match the target event subsystem and
3406  * event name.
3407  *
3408  * Return: The variable created for the field.
3409  */
3410 static struct field_var *
3411 create_target_field_var(struct hist_trigger_data *target_hist_data,
3412                         char *subsys_name, char *event_name, char *var_name)
3413 {
3414         struct trace_event_file *file = target_hist_data->event_file;
3415
3416         if (subsys_name) {
3417                 struct trace_event_call *call;
3418
3419                 if (!event_name)
3420                         return NULL;
3421
3422                 call = file->event_call;
3423
3424                 if (strcmp(subsys_name, call->class->system) != 0)
3425                         return NULL;
3426
3427                 if (strcmp(event_name, trace_event_name(call)) != 0)
3428                         return NULL;
3429         }
3430
3431         return create_field_var(target_hist_data, file, var_name);
3432 }
3433
3434 static void onmax_print(struct seq_file *m,
3435                         struct hist_trigger_data *hist_data,
3436                         struct tracing_map_elt *elt,
3437                         struct action_data *data)
3438 {
3439         unsigned int i, save_var_idx, max_idx = data->onmax.max_var->var.idx;
3440
3441         seq_printf(m, "\n\tmax: %10llu", tracing_map_read_var(elt, max_idx));
3442
3443         for (i = 0; i < hist_data->n_max_vars; i++) {
3444                 struct hist_field *save_val = hist_data->max_vars[i]->val;
3445                 struct hist_field *save_var = hist_data->max_vars[i]->var;
3446                 u64 val;
3447
3448                 save_var_idx = save_var->var.idx;
3449
3450                 val = tracing_map_read_var(elt, save_var_idx);
3451
3452                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3453                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3454                                    (char *)(uintptr_t)(val));
3455                 } else
3456                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3457         }
3458 }
3459
3460 static void onmax_save(struct hist_trigger_data *hist_data,
3461                        struct tracing_map_elt *elt, void *rec,
3462                        struct ring_buffer_event *rbe,
3463                        struct action_data *data, u64 *var_ref_vals)
3464 {
3465         unsigned int max_idx = data->onmax.max_var->var.idx;
3466         unsigned int max_var_ref_idx = data->onmax.max_var_ref_idx;
3467
3468         u64 var_val, max_val;
3469
3470         var_val = var_ref_vals[max_var_ref_idx];
3471         max_val = tracing_map_read_var(elt, max_idx);
3472
3473         if (var_val <= max_val)
3474                 return;
3475
3476         tracing_map_set_var(elt, max_idx, var_val);
3477
3478         update_max_vars(hist_data, elt, rbe, rec);
3479 }
3480
3481 static void onmax_destroy(struct action_data *data)
3482 {
3483         unsigned int i;
3484
3485         destroy_hist_field(data->onmax.max_var, 0);
3486         destroy_hist_field(data->onmax.var, 0);
3487
3488         kfree(data->onmax.var_str);
3489         kfree(data->onmax.fn_name);
3490
3491         for (i = 0; i < data->n_params; i++)
3492                 kfree(data->params[i]);
3493
3494         kfree(data);
3495 }
3496
3497 static int onmax_create(struct hist_trigger_data *hist_data,
3498                         struct action_data *data)
3499 {
3500         struct trace_event_file *file = hist_data->event_file;
3501         struct hist_field *var_field, *ref_field, *max_var;
3502         unsigned int var_ref_idx = hist_data->n_var_refs;
3503         struct field_var *field_var;
3504         char *onmax_var_str, *param;
3505         unsigned int i;
3506         int ret = 0;
3507
3508         onmax_var_str = data->onmax.var_str;
3509         if (onmax_var_str[0] != '$') {
3510                 hist_err("onmax: For onmax(x), x must be a variable: ", onmax_var_str);
3511                 return -EINVAL;
3512         }
3513         onmax_var_str++;
3514
3515         var_field = find_target_event_var(hist_data, NULL, NULL, onmax_var_str);
3516         if (!var_field) {
3517                 hist_err("onmax: Couldn't find onmax variable: ", onmax_var_str);
3518                 return -EINVAL;
3519         }
3520
3521         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3522         if (!ref_field)
3523                 return -ENOMEM;
3524
3525         data->onmax.var = ref_field;
3526
3527         data->fn = onmax_save;
3528         data->onmax.max_var_ref_idx = var_ref_idx;
3529         max_var = create_var(hist_data, file, "max", sizeof(u64), "u64");
3530         if (IS_ERR(max_var)) {
3531                 hist_err("onmax: Couldn't create onmax variable: ", "max");
3532                 ret = PTR_ERR(max_var);
3533                 goto out;
3534         }
3535         data->onmax.max_var = max_var;
3536
3537         for (i = 0; i < data->n_params; i++) {
3538                 param = kstrdup(data->params[i], GFP_KERNEL);
3539                 if (!param) {
3540                         ret = -ENOMEM;
3541                         goto out;
3542                 }
3543
3544                 field_var = create_target_field_var(hist_data, NULL, NULL, param);
3545                 if (IS_ERR(field_var)) {
3546                         hist_err("onmax: Couldn't create field variable: ", param);
3547                         ret = PTR_ERR(field_var);
3548                         kfree(param);
3549                         goto out;
3550                 }
3551
3552                 hist_data->max_vars[hist_data->n_max_vars++] = field_var;
3553                 if (field_var->val->flags & HIST_FIELD_FL_STRING)
3554                         hist_data->n_max_var_str++;
3555
3556                 kfree(param);
3557         }
3558  out:
3559         return ret;
3560 }
3561
3562 static int parse_action_params(char *params, struct action_data *data)
3563 {
3564         char *param, *saved_param;
3565         int ret = 0;
3566
3567         while (params) {
3568                 if (data->n_params >= SYNTH_FIELDS_MAX)
3569                         goto out;
3570
3571                 param = strsep(&params, ",");
3572                 if (!param) {
3573                         ret = -EINVAL;
3574                         goto out;
3575                 }
3576
3577                 param = strstrip(param);
3578                 if (strlen(param) < 2) {
3579                         hist_err("Invalid action param: ", param);
3580                         ret = -EINVAL;
3581                         goto out;
3582                 }
3583
3584                 saved_param = kstrdup(param, GFP_KERNEL);
3585                 if (!saved_param) {
3586                         ret = -ENOMEM;
3587                         goto out;
3588                 }
3589
3590                 data->params[data->n_params++] = saved_param;
3591         }
3592  out:
3593         return ret;
3594 }
3595
3596 static struct action_data *onmax_parse(char *str)
3597 {
3598         char *onmax_fn_name, *onmax_var_str;
3599         struct action_data *data;
3600         int ret = -EINVAL;
3601
3602         data = kzalloc(sizeof(*data), GFP_KERNEL);
3603         if (!data)
3604                 return ERR_PTR(-ENOMEM);
3605
3606         onmax_var_str = strsep(&str, ")");
3607         if (!onmax_var_str || !str) {
3608                 ret = -EINVAL;
3609                 goto free;
3610         }
3611
3612         data->onmax.var_str = kstrdup(onmax_var_str, GFP_KERNEL);
3613         if (!data->onmax.var_str) {
3614                 ret = -ENOMEM;
3615                 goto free;
3616         }
3617
3618         strsep(&str, ".");
3619         if (!str)
3620                 goto free;
3621
3622         onmax_fn_name = strsep(&str, "(");
3623         if (!onmax_fn_name || !str)
3624                 goto free;
3625
3626         if (strncmp(onmax_fn_name, "save", strlen("save")) == 0) {
3627                 char *params = strsep(&str, ")");
3628
3629                 if (!params) {
3630                         ret = -EINVAL;
3631                         goto free;
3632                 }
3633
3634                 ret = parse_action_params(params, data);
3635                 if (ret)
3636                         goto free;
3637         } else
3638                 goto free;
3639
3640         data->onmax.fn_name = kstrdup(onmax_fn_name, GFP_KERNEL);
3641         if (!data->onmax.fn_name) {
3642                 ret = -ENOMEM;
3643                 goto free;
3644         }
3645  out:
3646         return data;
3647  free:
3648         onmax_destroy(data);
3649         data = ERR_PTR(ret);
3650         goto out;
3651 }
3652
3653 static void onmatch_destroy(struct action_data *data)
3654 {
3655         unsigned int i;
3656
3657         mutex_lock(&synth_event_mutex);
3658
3659         kfree(data->onmatch.match_event);
3660         kfree(data->onmatch.match_event_system);
3661         kfree(data->onmatch.synth_event_name);
3662
3663         for (i = 0; i < data->n_params; i++)
3664                 kfree(data->params[i]);
3665
3666         if (data->onmatch.synth_event)
3667                 data->onmatch.synth_event->ref--;
3668
3669         kfree(data);
3670
3671         mutex_unlock(&synth_event_mutex);
3672 }
3673
3674 static void destroy_field_var(struct field_var *field_var)
3675 {
3676         if (!field_var)
3677                 return;
3678
3679         destroy_hist_field(field_var->var, 0);
3680         destroy_hist_field(field_var->val, 0);
3681
3682         kfree(field_var);
3683 }
3684
3685 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3686 {
3687         unsigned int i;
3688
3689         for (i = 0; i < hist_data->n_field_vars; i++)
3690                 destroy_field_var(hist_data->field_vars[i]);
3691 }
3692
3693 static void save_field_var(struct hist_trigger_data *hist_data,
3694                            struct field_var *field_var)
3695 {
3696         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3697
3698         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3699                 hist_data->n_field_var_str++;
3700 }
3701
3702
3703 static void destroy_synth_var_refs(struct hist_trigger_data *hist_data)
3704 {
3705         unsigned int i;
3706
3707         for (i = 0; i < hist_data->n_synth_var_refs; i++)
3708                 destroy_hist_field(hist_data->synth_var_refs[i], 0);
3709 }
3710
3711 static void save_synth_var_ref(struct hist_trigger_data *hist_data,
3712                          struct hist_field *var_ref)
3713 {
3714         hist_data->synth_var_refs[hist_data->n_synth_var_refs++] = var_ref;
3715 }
3716
3717 static int check_synth_field(struct synth_event *event,
3718                              struct hist_field *hist_field,
3719                              unsigned int field_pos)
3720 {
3721         struct synth_field *field;
3722
3723         if (field_pos >= event->n_fields)
3724                 return -EINVAL;
3725
3726         field = event->fields[field_pos];
3727
3728         if (strcmp(field->type, hist_field->type) != 0)
3729                 return -EINVAL;
3730
3731         return 0;
3732 }
3733
3734 static struct hist_field *
3735 onmatch_find_var(struct hist_trigger_data *hist_data, struct action_data *data,
3736                  char *system, char *event, char *var)
3737 {
3738         struct hist_field *hist_field;
3739
3740         var++; /* skip '$' */
3741
3742         hist_field = find_target_event_var(hist_data, system, event, var);
3743         if (!hist_field) {
3744                 if (!system) {
3745                         system = data->onmatch.match_event_system;
3746                         event = data->onmatch.match_event;
3747                 }
3748
3749                 hist_field = find_event_var(hist_data, system, event, var);
3750         }
3751
3752         if (!hist_field)
3753                 hist_err_event("onmatch: Couldn't find onmatch param: $", system, event, var);
3754
3755         return hist_field;
3756 }
3757
3758 static struct hist_field *
3759 onmatch_create_field_var(struct hist_trigger_data *hist_data,
3760                          struct action_data *data, char *system,
3761                          char *event, char *var)
3762 {
3763         struct hist_field *hist_field = NULL;
3764         struct field_var *field_var;
3765
3766         /*
3767          * First try to create a field var on the target event (the
3768          * currently being defined).  This will create a variable for
3769          * unqualified fields on the target event, or if qualified,
3770          * target fields that have qualified names matching the target.
3771          */
3772         field_var = create_target_field_var(hist_data, system, event, var);
3773
3774         if (field_var && !IS_ERR(field_var)) {
3775                 save_field_var(hist_data, field_var);
3776                 hist_field = field_var->var;
3777         } else {
3778                 field_var = NULL;
3779                 /*
3780                  * If no explicit system.event is specfied, default to
3781                  * looking for fields on the onmatch(system.event.xxx)
3782                  * event.
3783                  */
3784                 if (!system) {
3785                         system = data->onmatch.match_event_system;
3786                         event = data->onmatch.match_event;
3787                 }
3788
3789                 if (!event)
3790                         goto free;
3791                 /*
3792                  * At this point, we're looking at a field on another
3793                  * event.  Because we can't modify a hist trigger on
3794                  * another event to add a variable for a field, we need
3795                  * to create a new trigger on that event and create the
3796                  * variable at the same time.
3797                  */
3798                 hist_field = create_field_var_hist(hist_data, system, event, var);
3799                 if (IS_ERR(hist_field))
3800                         goto free;
3801         }
3802  out:
3803         return hist_field;
3804  free:
3805         destroy_field_var(field_var);
3806         hist_field = NULL;
3807         goto out;
3808 }
3809
3810 static int onmatch_create(struct hist_trigger_data *hist_data,
3811                           struct trace_event_file *file,
3812                           struct action_data *data)
3813 {
3814         char *event_name, *param, *system = NULL;
3815         struct hist_field *hist_field, *var_ref;
3816         unsigned int i, var_ref_idx;
3817         unsigned int field_pos = 0;
3818         struct synth_event *event;
3819         int ret = 0;
3820
3821         mutex_lock(&synth_event_mutex);
3822         event = find_synth_event(data->onmatch.synth_event_name);
3823         if (!event) {
3824                 hist_err("onmatch: Couldn't find synthetic event: ", data->onmatch.synth_event_name);
3825                 mutex_unlock(&synth_event_mutex);
3826                 return -EINVAL;
3827         }
3828         event->ref++;
3829         mutex_unlock(&synth_event_mutex);
3830
3831         var_ref_idx = hist_data->n_var_refs;
3832
3833         for (i = 0; i < data->n_params; i++) {
3834                 char *p;
3835
3836                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3837                 if (!param) {
3838                         ret = -ENOMEM;
3839                         goto err;
3840                 }
3841
3842                 system = strsep(&param, ".");
3843                 if (!param) {
3844                         param = (char *)system;
3845                         system = event_name = NULL;
3846                 } else {
3847                         event_name = strsep(&param, ".");
3848                         if (!param) {
3849                                 kfree(p);
3850                                 ret = -EINVAL;
3851                                 goto err;
3852                         }
3853                 }
3854
3855                 if (param[0] == '$')
3856                         hist_field = onmatch_find_var(hist_data, data, system,
3857                                                       event_name, param);
3858                 else
3859                         hist_field = onmatch_create_field_var(hist_data, data,
3860                                                               system,
3861                                                               event_name,
3862                                                               param);
3863
3864                 if (!hist_field) {
3865                         kfree(p);
3866                         ret = -EINVAL;
3867                         goto err;
3868                 }
3869
3870                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3871                         var_ref = create_var_ref(hist_data, hist_field,
3872                                                  system, event_name);
3873                         if (!var_ref) {
3874                                 kfree(p);
3875                                 ret = -ENOMEM;
3876                                 goto err;
3877                         }
3878
3879                         save_synth_var_ref(hist_data, var_ref);
3880                         field_pos++;
3881                         kfree(p);
3882                         continue;
3883                 }
3884
3885                 hist_err_event("onmatch: Param type doesn't match synthetic event field type: ",
3886                                system, event_name, param);
3887                 kfree(p);
3888                 ret = -EINVAL;
3889                 goto err;
3890         }
3891
3892         if (field_pos != event->n_fields) {
3893                 hist_err("onmatch: Param count doesn't match synthetic event field count: ", event->name);
3894                 ret = -EINVAL;
3895                 goto err;
3896         }
3897
3898         data->fn = action_trace;
3899         data->onmatch.synth_event = event;
3900         data->onmatch.var_ref_idx = var_ref_idx;
3901  out:
3902         return ret;
3903  err:
3904         mutex_lock(&synth_event_mutex);
3905         event->ref--;
3906         mutex_unlock(&synth_event_mutex);
3907
3908         goto out;
3909 }
3910
3911 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3912 {
3913         char *match_event, *match_event_system;
3914         char *synth_event_name, *params;
3915         struct action_data *data;
3916         int ret = -EINVAL;
3917
3918         data = kzalloc(sizeof(*data), GFP_KERNEL);
3919         if (!data)
3920                 return ERR_PTR(-ENOMEM);
3921
3922         match_event = strsep(&str, ")");
3923         if (!match_event || !str) {
3924                 hist_err("onmatch: Missing closing paren: ", match_event);
3925                 goto free;
3926         }
3927
3928         match_event_system = strsep(&match_event, ".");
3929         if (!match_event) {
3930                 hist_err("onmatch: Missing subsystem for match event: ", match_event_system);
3931                 goto free;
3932         }
3933
3934         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3935                 hist_err_event("onmatch: Invalid subsystem or event name: ",
3936                                match_event_system, match_event, NULL);
3937                 goto free;
3938         }
3939
3940         data->onmatch.match_event = kstrdup(match_event, GFP_KERNEL);
3941         if (!data->onmatch.match_event) {
3942                 ret = -ENOMEM;
3943                 goto free;
3944         }
3945
3946         data->onmatch.match_event_system = kstrdup(match_event_system, GFP_KERNEL);
3947         if (!data->onmatch.match_event_system) {
3948                 ret = -ENOMEM;
3949                 goto free;
3950         }
3951
3952         strsep(&str, ".");
3953         if (!str) {
3954                 hist_err("onmatch: Missing . after onmatch(): ", str);
3955                 goto free;
3956         }
3957
3958         synth_event_name = strsep(&str, "(");
3959         if (!synth_event_name || !str) {
3960                 hist_err("onmatch: Missing opening paramlist paren: ", synth_event_name);
3961                 goto free;
3962         }
3963
3964         data->onmatch.synth_event_name = kstrdup(synth_event_name, GFP_KERNEL);
3965         if (!data->onmatch.synth_event_name) {
3966                 ret = -ENOMEM;
3967                 goto free;
3968         }
3969
3970         params = strsep(&str, ")");
3971         if (!params || !str || (str && strlen(str))) {
3972                 hist_err("onmatch: Missing closing paramlist paren: ", params);
3973                 goto free;
3974         }
3975
3976         ret = parse_action_params(params, data);
3977         if (ret)
3978                 goto free;
3979  out:
3980         return data;
3981  free:
3982         onmatch_destroy(data);
3983         data = ERR_PTR(ret);
3984         goto out;
3985 }
3986
3987 static int create_hitcount_val(struct hist_trigger_data *hist_data)
3988 {
3989         hist_data->fields[HITCOUNT_IDX] =
3990                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3991         if (!hist_data->fields[HITCOUNT_IDX])
3992                 return -ENOMEM;
3993
3994         hist_data->n_vals++;
3995         hist_data->n_fields++;
3996
3997         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3998                 return -EINVAL;
3999
4000         return 0;
4001 }
4002
4003 static int __create_val_field(struct hist_trigger_data *hist_data,
4004                               unsigned int val_idx,
4005                               struct trace_event_file *file,
4006                               char *var_name, char *field_str,
4007                               unsigned long flags)
4008 {
4009         struct hist_field *hist_field;
4010         int ret = 0;
4011
4012         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
4013         if (IS_ERR(hist_field)) {
4014                 ret = PTR_ERR(hist_field);
4015                 goto out;
4016         }
4017
4018         hist_data->fields[val_idx] = hist_field;
4019
4020         ++hist_data->n_vals;
4021         ++hist_data->n_fields;
4022
4023         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4024                 ret = -EINVAL;
4025  out:
4026         return ret;
4027 }
4028
4029 static int create_val_field(struct hist_trigger_data *hist_data,
4030                             unsigned int val_idx,
4031                             struct trace_event_file *file,
4032                             char *field_str)
4033 {
4034         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4035                 return -EINVAL;
4036
4037         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4038 }
4039
4040 static int create_var_field(struct hist_trigger_data *hist_data,
4041                             unsigned int val_idx,
4042                             struct trace_event_file *file,
4043                             char *var_name, char *expr_str)
4044 {
4045         unsigned long flags = 0;
4046
4047         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4048                 return -EINVAL;
4049
4050         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4051                 hist_err("Variable already defined: ", var_name);
4052                 return -EINVAL;
4053         }
4054
4055         flags |= HIST_FIELD_FL_VAR;
4056         hist_data->n_vars++;
4057         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4058                 return -EINVAL;
4059
4060         return __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4061 }
4062
4063 static int create_val_fields(struct hist_trigger_data *hist_data,
4064                              struct trace_event_file *file)
4065 {
4066         char *fields_str, *field_str;
4067         unsigned int i, j = 1;
4068         int ret;
4069
4070         ret = create_hitcount_val(hist_data);
4071         if (ret)
4072                 goto out;
4073
4074         fields_str = hist_data->attrs->vals_str;
4075         if (!fields_str)
4076                 goto out;
4077
4078         strsep(&fields_str, "=");
4079         if (!fields_str)
4080                 goto out;
4081
4082         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4083                      j < TRACING_MAP_VALS_MAX; i++) {
4084                 field_str = strsep(&fields_str, ",");
4085                 if (!field_str)
4086                         break;
4087
4088                 if (strcmp(field_str, "hitcount") == 0)
4089                         continue;
4090
4091                 ret = create_val_field(hist_data, j++, file, field_str);
4092                 if (ret)
4093                         goto out;
4094         }
4095
4096         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4097                 ret = -EINVAL;
4098  out:
4099         return ret;
4100 }
4101
4102 static int create_key_field(struct hist_trigger_data *hist_data,
4103                             unsigned int key_idx,
4104                             unsigned int key_offset,
4105                             struct trace_event_file *file,
4106                             char *field_str)
4107 {
4108         struct hist_field *hist_field = NULL;
4109
4110         unsigned long flags = 0;
4111         unsigned int key_size;
4112         int ret = 0;
4113
4114         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4115                 return -EINVAL;
4116
4117         flags |= HIST_FIELD_FL_KEY;
4118
4119         if (strcmp(field_str, "stacktrace") == 0) {
4120                 flags |= HIST_FIELD_FL_STACKTRACE;
4121                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4122                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4123         } else {
4124                 hist_field = parse_expr(hist_data, file, field_str, flags,
4125                                         NULL, 0);
4126                 if (IS_ERR(hist_field)) {
4127                         ret = PTR_ERR(hist_field);
4128                         goto out;
4129                 }
4130
4131                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF) {
4132                         hist_err("Using variable references as keys not supported: ", field_str);
4133                         destroy_hist_field(hist_field, 0);
4134                         ret = -EINVAL;
4135                         goto out;
4136                 }
4137
4138                 key_size = hist_field->size;
4139         }
4140
4141         hist_data->fields[key_idx] = hist_field;
4142
4143         key_size = ALIGN(key_size, sizeof(u64));
4144         hist_data->fields[key_idx]->size = key_size;
4145         hist_data->fields[key_idx]->offset = key_offset;
4146
4147         hist_data->key_size += key_size;
4148
4149         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4150                 ret = -EINVAL;
4151                 goto out;
4152         }
4153
4154         hist_data->n_keys++;
4155         hist_data->n_fields++;
4156
4157         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4158                 return -EINVAL;
4159
4160         ret = key_size;
4161  out:
4162         return ret;
4163 }
4164
4165 static int create_key_fields(struct hist_trigger_data *hist_data,
4166                              struct trace_event_file *file)
4167 {
4168         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4169         char *fields_str, *field_str;
4170         int ret = -EINVAL;
4171
4172         fields_str = hist_data->attrs->keys_str;
4173         if (!fields_str)
4174                 goto out;
4175
4176         strsep(&fields_str, "=");
4177         if (!fields_str)
4178                 goto out;
4179
4180         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4181                 field_str = strsep(&fields_str, ",");
4182                 if (!field_str)
4183                         break;
4184                 ret = create_key_field(hist_data, i, key_offset,
4185                                        file, field_str);
4186                 if (ret < 0)
4187                         goto out;
4188                 key_offset += ret;
4189         }
4190         if (fields_str) {
4191                 ret = -EINVAL;
4192                 goto out;
4193         }
4194         ret = 0;
4195  out:
4196         return ret;
4197 }
4198
4199 static int create_var_fields(struct hist_trigger_data *hist_data,
4200                              struct trace_event_file *file)
4201 {
4202         unsigned int i, j = hist_data->n_vals;
4203         int ret = 0;
4204
4205         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4206
4207         for (i = 0; i < n_vars; i++) {
4208                 char *var_name = hist_data->attrs->var_defs.name[i];
4209                 char *expr = hist_data->attrs->var_defs.expr[i];
4210
4211                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4212                 if (ret)
4213                         goto out;
4214         }
4215  out:
4216         return ret;
4217 }
4218
4219 static void free_var_defs(struct hist_trigger_data *hist_data)
4220 {
4221         unsigned int i;
4222
4223         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4224                 kfree(hist_data->attrs->var_defs.name[i]);
4225                 kfree(hist_data->attrs->var_defs.expr[i]);
4226         }
4227
4228         hist_data->attrs->var_defs.n_vars = 0;
4229 }
4230
4231 static int parse_var_defs(struct hist_trigger_data *hist_data)
4232 {
4233         char *s, *str, *var_name, *field_str;
4234         unsigned int i, j, n_vars = 0;
4235         int ret = 0;
4236
4237         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4238                 str = hist_data->attrs->assignment_str[i];
4239                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4240                         field_str = strsep(&str, ",");
4241                         if (!field_str)
4242                                 break;
4243
4244                         var_name = strsep(&field_str, "=");
4245                         if (!var_name || !field_str) {
4246                                 hist_err("Malformed assignment: ", var_name);
4247                                 ret = -EINVAL;
4248                                 goto free;
4249                         }
4250
4251                         if (n_vars == TRACING_MAP_VARS_MAX) {
4252                                 hist_err("Too many variables defined: ", var_name);
4253                                 ret = -EINVAL;
4254                                 goto free;
4255                         }
4256
4257                         s = kstrdup(var_name, GFP_KERNEL);
4258                         if (!s) {
4259                                 ret = -ENOMEM;
4260                                 goto free;
4261                         }
4262                         hist_data->attrs->var_defs.name[n_vars] = s;
4263
4264                         s = kstrdup(field_str, GFP_KERNEL);
4265                         if (!s) {
4266                                 ret = -ENOMEM;
4267                                 goto free;
4268                         }
4269                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4270
4271                         hist_data->attrs->var_defs.n_vars = n_vars;
4272                 }
4273         }
4274
4275         return ret;
4276  free:
4277         free_var_defs(hist_data);
4278
4279         return ret;
4280 }
4281
4282 static int create_hist_fields(struct hist_trigger_data *hist_data,
4283                               struct trace_event_file *file)
4284 {
4285         int ret;
4286
4287         ret = parse_var_defs(hist_data);
4288         if (ret)
4289                 goto out;
4290
4291         ret = create_val_fields(hist_data, file);
4292         if (ret)
4293                 goto out;
4294
4295         ret = create_var_fields(hist_data, file);
4296         if (ret)
4297                 goto out;
4298
4299         ret = create_key_fields(hist_data, file);
4300         if (ret)
4301                 goto out;
4302  out:
4303         free_var_defs(hist_data);
4304
4305         return ret;
4306 }
4307
4308 static int is_descending(const char *str)
4309 {
4310         if (!str)
4311                 return 0;
4312
4313         if (strcmp(str, "descending") == 0)
4314                 return 1;
4315
4316         if (strcmp(str, "ascending") == 0)
4317                 return 0;
4318
4319         return -EINVAL;
4320 }
4321
4322 static int create_sort_keys(struct hist_trigger_data *hist_data)
4323 {
4324         char *fields_str = hist_data->attrs->sort_key_str;
4325         struct tracing_map_sort_key *sort_key;
4326         int descending, ret = 0;
4327         unsigned int i, j, k;
4328
4329         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4330
4331         if (!fields_str)
4332                 goto out;
4333
4334         strsep(&fields_str, "=");
4335         if (!fields_str) {
4336                 ret = -EINVAL;
4337                 goto out;
4338         }
4339
4340         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4341                 struct hist_field *hist_field;
4342                 char *field_str, *field_name;
4343                 const char *test_name;
4344
4345                 sort_key = &hist_data->sort_keys[i];
4346
4347                 field_str = strsep(&fields_str, ",");
4348                 if (!field_str) {
4349                         if (i == 0)
4350                                 ret = -EINVAL;
4351                         break;
4352                 }
4353
4354                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4355                         ret = -EINVAL;
4356                         break;
4357                 }
4358
4359                 field_name = strsep(&field_str, ".");
4360                 if (!field_name) {
4361                         ret = -EINVAL;
4362                         break;
4363                 }
4364
4365                 if (strcmp(field_name, "hitcount") == 0) {
4366                         descending = is_descending(field_str);
4367                         if (descending < 0) {
4368                                 ret = descending;
4369                                 break;
4370                         }
4371                         sort_key->descending = descending;
4372                         continue;
4373                 }
4374
4375                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4376                         unsigned int idx;
4377
4378                         hist_field = hist_data->fields[j];
4379                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4380                                 continue;
4381
4382                         idx = k++;
4383
4384                         test_name = hist_field_name(hist_field, 0);
4385
4386                         if (strcmp(field_name, test_name) == 0) {
4387                                 sort_key->field_idx = idx;
4388                                 descending = is_descending(field_str);
4389                                 if (descending < 0) {
4390                                         ret = descending;
4391                                         goto out;
4392                                 }
4393                                 sort_key->descending = descending;
4394                                 break;
4395                         }
4396                 }
4397                 if (j == hist_data->n_fields) {
4398                         ret = -EINVAL;
4399                         break;
4400                 }
4401         }
4402
4403         hist_data->n_sort_keys = i;
4404  out:
4405         return ret;
4406 }
4407
4408 static void destroy_actions(struct hist_trigger_data *hist_data)
4409 {
4410         unsigned int i;
4411
4412         for (i = 0; i < hist_data->n_actions; i++) {
4413                 struct action_data *data = hist_data->actions[i];
4414
4415                 if (data->fn == action_trace)
4416                         onmatch_destroy(data);
4417                 else if (data->fn == onmax_save)
4418                         onmax_destroy(data);
4419                 else
4420                         kfree(data);
4421         }
4422 }
4423
4424 static int parse_actions(struct hist_trigger_data *hist_data)
4425 {
4426         struct trace_array *tr = hist_data->event_file->tr;
4427         struct action_data *data;
4428         unsigned int i;
4429         int ret = 0;
4430         char *str;
4431
4432         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4433                 str = hist_data->attrs->action_str[i];
4434
4435                 if (strncmp(str, "onmatch(", strlen("onmatch(")) == 0) {
4436                         char *action_str = str + strlen("onmatch(");
4437
4438                         data = onmatch_parse(tr, action_str);
4439                         if (IS_ERR(data)) {
4440                                 ret = PTR_ERR(data);
4441                                 break;
4442                         }
4443                         data->fn = action_trace;
4444                 } else if (strncmp(str, "onmax(", strlen("onmax(")) == 0) {
4445                         char *action_str = str + strlen("onmax(");
4446
4447                         data = onmax_parse(action_str);
4448                         if (IS_ERR(data)) {
4449                                 ret = PTR_ERR(data);
4450                                 break;
4451                         }
4452                         data->fn = onmax_save;
4453                 } else {
4454                         ret = -EINVAL;
4455                         break;
4456                 }
4457
4458                 hist_data->actions[hist_data->n_actions++] = data;
4459         }
4460
4461         return ret;
4462 }
4463
4464 static int create_actions(struct hist_trigger_data *hist_data,
4465                           struct trace_event_file *file)
4466 {
4467         struct action_data *data;
4468         unsigned int i;
4469         int ret = 0;
4470
4471         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4472                 data = hist_data->actions[i];
4473
4474                 if (data->fn == action_trace) {
4475                         ret = onmatch_create(hist_data, file, data);
4476                         if (ret)
4477                                 return ret;
4478                 } else if (data->fn == onmax_save) {
4479                         ret = onmax_create(hist_data, data);
4480                         if (ret)
4481                                 return ret;
4482                 }
4483         }
4484
4485         return ret;
4486 }
4487
4488 static void print_actions(struct seq_file *m,
4489                           struct hist_trigger_data *hist_data,
4490                           struct tracing_map_elt *elt)
4491 {
4492         unsigned int i;
4493
4494         for (i = 0; i < hist_data->n_actions; i++) {
4495                 struct action_data *data = hist_data->actions[i];
4496
4497                 if (data->fn == onmax_save)
4498                         onmax_print(m, hist_data, elt, data);
4499         }
4500 }
4501
4502 static void print_onmax_spec(struct seq_file *m,
4503                              struct hist_trigger_data *hist_data,
4504                              struct action_data *data)
4505 {
4506         unsigned int i;
4507
4508         seq_puts(m, ":onmax(");
4509         seq_printf(m, "%s", data->onmax.var_str);
4510         seq_printf(m, ").%s(", data->onmax.fn_name);
4511
4512         for (i = 0; i < hist_data->n_max_vars; i++) {
4513                 seq_printf(m, "%s", hist_data->max_vars[i]->var->var.name);
4514                 if (i < hist_data->n_max_vars - 1)
4515                         seq_puts(m, ",");
4516         }
4517         seq_puts(m, ")");
4518 }
4519
4520 static void print_onmatch_spec(struct seq_file *m,
4521                                struct hist_trigger_data *hist_data,
4522                                struct action_data *data)
4523 {
4524         unsigned int i;
4525
4526         seq_printf(m, ":onmatch(%s.%s).", data->onmatch.match_event_system,
4527                    data->onmatch.match_event);
4528
4529         seq_printf(m, "%s(", data->onmatch.synth_event->name);
4530
4531         for (i = 0; i < data->n_params; i++) {
4532                 if (i)
4533                         seq_puts(m, ",");
4534                 seq_printf(m, "%s", data->params[i]);
4535         }
4536
4537         seq_puts(m, ")");
4538 }
4539
4540 static bool actions_match(struct hist_trigger_data *hist_data,
4541                           struct hist_trigger_data *hist_data_test)
4542 {
4543         unsigned int i, j;
4544
4545         if (hist_data->n_actions != hist_data_test->n_actions)
4546                 return false;
4547
4548         for (i = 0; i < hist_data->n_actions; i++) {
4549                 struct action_data *data = hist_data->actions[i];
4550                 struct action_data *data_test = hist_data_test->actions[i];
4551
4552                 if (data->fn != data_test->fn)
4553                         return false;
4554
4555                 if (data->n_params != data_test->n_params)
4556                         return false;
4557
4558                 for (j = 0; j < data->n_params; j++) {
4559                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4560                                 return false;
4561                 }
4562
4563                 if (data->fn == action_trace) {
4564                         if (strcmp(data->onmatch.synth_event_name,
4565                                    data_test->onmatch.synth_event_name) != 0)
4566                                 return false;
4567                         if (strcmp(data->onmatch.match_event_system,
4568                                    data_test->onmatch.match_event_system) != 0)
4569                                 return false;
4570                         if (strcmp(data->onmatch.match_event,
4571                                    data_test->onmatch.match_event) != 0)
4572                                 return false;
4573                 } else if (data->fn == onmax_save) {
4574                         if (strcmp(data->onmax.var_str,
4575                                    data_test->onmax.var_str) != 0)
4576                                 return false;
4577                         if (strcmp(data->onmax.fn_name,
4578                                    data_test->onmax.fn_name) != 0)
4579                                 return false;
4580                 }
4581         }
4582
4583         return true;
4584 }
4585
4586
4587 static void print_actions_spec(struct seq_file *m,
4588                                struct hist_trigger_data *hist_data)
4589 {
4590         unsigned int i;
4591
4592         for (i = 0; i < hist_data->n_actions; i++) {
4593                 struct action_data *data = hist_data->actions[i];
4594
4595                 if (data->fn == action_trace)
4596                         print_onmatch_spec(m, hist_data, data);
4597                 else if (data->fn == onmax_save)
4598                         print_onmax_spec(m, hist_data, data);
4599         }
4600 }
4601
4602 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4603 {
4604         unsigned int i;
4605
4606         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4607                 kfree(hist_data->field_var_hists[i]->cmd);
4608                 kfree(hist_data->field_var_hists[i]);
4609         }
4610 }
4611
4612 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4613 {
4614         if (!hist_data)
4615                 return;
4616
4617         destroy_hist_trigger_attrs(hist_data->attrs);
4618         destroy_hist_fields(hist_data);
4619         tracing_map_destroy(hist_data->map);
4620
4621         destroy_actions(hist_data);
4622         destroy_field_vars(hist_data);
4623         destroy_field_var_hists(hist_data);
4624         destroy_synth_var_refs(hist_data);
4625
4626         kfree(hist_data);
4627 }
4628
4629 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4630 {
4631         struct tracing_map *map = hist_data->map;
4632         struct ftrace_event_field *field;
4633         struct hist_field *hist_field;
4634         int i, idx = 0;
4635
4636         for_each_hist_field(i, hist_data) {
4637                 hist_field = hist_data->fields[i];
4638                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4639                         tracing_map_cmp_fn_t cmp_fn;
4640
4641                         field = hist_field->field;
4642
4643                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4644                                 cmp_fn = tracing_map_cmp_none;
4645                         else if (!field)
4646                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4647                                                              hist_field->is_signed);
4648                         else if (is_string_field(field))
4649                                 cmp_fn = tracing_map_cmp_string;
4650                         else
4651                                 cmp_fn = tracing_map_cmp_num(field->size,
4652                                                              field->is_signed);
4653                         idx = tracing_map_add_key_field(map,
4654                                                         hist_field->offset,
4655                                                         cmp_fn);
4656                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4657                         idx = tracing_map_add_sum_field(map);
4658
4659                 if (idx < 0)
4660                         return idx;
4661
4662                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4663                         idx = tracing_map_add_var(map);
4664                         if (idx < 0)
4665                                 return idx;
4666                         hist_field->var.idx = idx;
4667                         hist_field->var.hist_data = hist_data;
4668                 }
4669         }
4670
4671         return 0;
4672 }
4673
4674 static struct hist_trigger_data *
4675 create_hist_data(unsigned int map_bits,
4676                  struct hist_trigger_attrs *attrs,
4677                  struct trace_event_file *file,
4678                  bool remove)
4679 {
4680         const struct tracing_map_ops *map_ops = NULL;
4681         struct hist_trigger_data *hist_data;
4682         int ret = 0;
4683
4684         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4685         if (!hist_data)
4686                 return ERR_PTR(-ENOMEM);
4687
4688         hist_data->attrs = attrs;
4689         hist_data->remove = remove;
4690         hist_data->event_file = file;
4691
4692         ret = parse_actions(hist_data);
4693         if (ret)
4694                 goto free;
4695
4696         ret = create_hist_fields(hist_data, file);
4697         if (ret)
4698                 goto free;
4699
4700         ret = create_sort_keys(hist_data);
4701         if (ret)
4702                 goto free;
4703
4704         map_ops = &hist_trigger_elt_data_ops;
4705
4706         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4707                                             map_ops, hist_data);
4708         if (IS_ERR(hist_data->map)) {
4709                 ret = PTR_ERR(hist_data->map);
4710                 hist_data->map = NULL;
4711                 goto free;
4712         }
4713
4714         ret = create_tracing_map_fields(hist_data);
4715         if (ret)
4716                 goto free;
4717  out:
4718         return hist_data;
4719  free:
4720         hist_data->attrs = NULL;
4721
4722         destroy_hist_data(hist_data);
4723
4724         hist_data = ERR_PTR(ret);
4725
4726         goto out;
4727 }
4728
4729 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4730                                     struct tracing_map_elt *elt, void *rec,
4731                                     struct ring_buffer_event *rbe,
4732                                     u64 *var_ref_vals)
4733 {
4734         struct hist_elt_data *elt_data;
4735         struct hist_field *hist_field;
4736         unsigned int i, var_idx;
4737         u64 hist_val;
4738
4739         elt_data = elt->private_data;
4740         elt_data->var_ref_vals = var_ref_vals;
4741
4742         for_each_hist_val_field(i, hist_data) {
4743                 hist_field = hist_data->fields[i];
4744                 hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4745                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4746                         var_idx = hist_field->var.idx;
4747                         tracing_map_set_var(elt, var_idx, hist_val);
4748                         continue;
4749                 }
4750                 tracing_map_update_sum(elt, i, hist_val);
4751         }
4752
4753         for_each_hist_key_field(i, hist_data) {
4754                 hist_field = hist_data->fields[i];
4755                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4756                         hist_val = hist_field->fn(hist_field, elt, rbe, rec);
4757                         var_idx = hist_field->var.idx;
4758                         tracing_map_set_var(elt, var_idx, hist_val);
4759                 }
4760         }
4761
4762         update_field_vars(hist_data, elt, rbe, rec);
4763 }
4764
4765 static inline void add_to_key(char *compound_key, void *key,
4766                               struct hist_field *key_field, void *rec)
4767 {
4768         size_t size = key_field->size;
4769
4770         if (key_field->flags & HIST_FIELD_FL_STRING) {
4771                 struct ftrace_event_field *field;
4772
4773                 field = key_field->field;
4774                 if (field->filter_type == FILTER_DYN_STRING)
4775                         size = *(u32 *)(rec + field->offset) >> 16;
4776                 else if (field->filter_type == FILTER_STATIC_STRING)
4777                         size = field->size;
4778
4779                 /* ensure NULL-termination */
4780                 if (size > key_field->size - 1)
4781                         size = key_field->size - 1;
4782
4783                 strncpy(compound_key + key_field->offset, (char *)key, size);
4784         } else
4785                 memcpy(compound_key + key_field->offset, key, size);
4786 }
4787
4788 static void
4789 hist_trigger_actions(struct hist_trigger_data *hist_data,
4790                      struct tracing_map_elt *elt, void *rec,
4791                      struct ring_buffer_event *rbe, u64 *var_ref_vals)
4792 {
4793         struct action_data *data;
4794         unsigned int i;
4795
4796         for (i = 0; i < hist_data->n_actions; i++) {
4797                 data = hist_data->actions[i];
4798                 data->fn(hist_data, elt, rec, rbe, data, var_ref_vals);
4799         }
4800 }
4801
4802 static void event_hist_trigger(struct event_trigger_data *data, void *rec,
4803                                struct ring_buffer_event *rbe)
4804 {
4805         struct hist_trigger_data *hist_data = data->private_data;
4806         bool use_compound_key = (hist_data->n_keys > 1);
4807         unsigned long entries[HIST_STACKTRACE_DEPTH];
4808         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4809         char compound_key[HIST_KEY_SIZE_MAX];
4810         struct tracing_map_elt *elt = NULL;
4811         struct stack_trace stacktrace;
4812         struct hist_field *key_field;
4813         u64 field_contents;
4814         void *key = NULL;
4815         unsigned int i;
4816
4817         memset(compound_key, 0, hist_data->key_size);
4818
4819         for_each_hist_key_field(i, hist_data) {
4820                 key_field = hist_data->fields[i];
4821
4822                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4823                         stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
4824                         stacktrace.entries = entries;
4825                         stacktrace.nr_entries = 0;
4826                         stacktrace.skip = HIST_STACKTRACE_SKIP;
4827
4828                         memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
4829                         save_stack_trace(&stacktrace);
4830
4831                         key = entries;
4832                 } else {
4833                         field_contents = key_field->fn(key_field, elt, rbe, rec);
4834                         if (key_field->flags & HIST_FIELD_FL_STRING) {
4835                                 key = (void *)(unsigned long)field_contents;
4836                                 use_compound_key = true;
4837                         } else
4838                                 key = (void *)&field_contents;
4839                 }
4840
4841                 if (use_compound_key)
4842                         add_to_key(compound_key, key, key_field, rec);
4843         }
4844
4845         if (use_compound_key)
4846                 key = compound_key;
4847
4848         if (hist_data->n_var_refs &&
4849             !resolve_var_refs(hist_data, key, var_ref_vals, false))
4850                 return;
4851
4852         elt = tracing_map_insert(hist_data->map, key);
4853         if (!elt)
4854                 return;
4855
4856         hist_trigger_elt_update(hist_data, elt, rec, rbe, var_ref_vals);
4857
4858         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4859                 hist_trigger_actions(hist_data, elt, rec, rbe, var_ref_vals);
4860 }
4861
4862 static void hist_trigger_stacktrace_print(struct seq_file *m,
4863                                           unsigned long *stacktrace_entries,
4864                                           unsigned int max_entries)
4865 {
4866         char str[KSYM_SYMBOL_LEN];
4867         unsigned int spaces = 8;
4868         unsigned int i;
4869
4870         for (i = 0; i < max_entries; i++) {
4871                 if (stacktrace_entries[i] == ULONG_MAX)
4872                         return;
4873
4874                 seq_printf(m, "%*c", 1 + spaces, ' ');
4875                 sprint_symbol(str, stacktrace_entries[i]);
4876                 seq_printf(m, "%s\n", str);
4877         }
4878 }
4879
4880 static void
4881 hist_trigger_entry_print(struct seq_file *m,
4882                          struct hist_trigger_data *hist_data, void *key,
4883                          struct tracing_map_elt *elt)
4884 {
4885         struct hist_field *key_field;
4886         char str[KSYM_SYMBOL_LEN];
4887         bool multiline = false;
4888         const char *field_name;
4889         unsigned int i;
4890         u64 uval;
4891
4892         seq_puts(m, "{ ");
4893
4894         for_each_hist_key_field(i, hist_data) {
4895                 key_field = hist_data->fields[i];
4896
4897                 if (i > hist_data->n_vals)
4898                         seq_puts(m, ", ");
4899
4900                 field_name = hist_field_name(key_field, 0);
4901
4902                 if (key_field->flags & HIST_FIELD_FL_HEX) {
4903                         uval = *(u64 *)(key + key_field->offset);
4904                         seq_printf(m, "%s: %llx", field_name, uval);
4905                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4906                         uval = *(u64 *)(key + key_field->offset);
4907                         sprint_symbol_no_offset(str, uval);
4908                         seq_printf(m, "%s: [%llx] %-45s", field_name,
4909                                    uval, str);
4910                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4911                         uval = *(u64 *)(key + key_field->offset);
4912                         sprint_symbol(str, uval);
4913                         seq_printf(m, "%s: [%llx] %-55s", field_name,
4914                                    uval, str);
4915                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4916                         struct hist_elt_data *elt_data = elt->private_data;
4917                         char *comm;
4918
4919                         if (WARN_ON_ONCE(!elt_data))
4920                                 return;
4921
4922                         comm = elt_data->comm;
4923
4924                         uval = *(u64 *)(key + key_field->offset);
4925                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
4926                                    comm, uval);
4927                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4928                         const char *syscall_name;
4929
4930                         uval = *(u64 *)(key + key_field->offset);
4931                         syscall_name = get_syscall_name(uval);
4932                         if (!syscall_name)
4933                                 syscall_name = "unknown_syscall";
4934
4935                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
4936                                    syscall_name, uval);
4937                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4938                         seq_puts(m, "stacktrace:\n");
4939                         hist_trigger_stacktrace_print(m,
4940                                                       key + key_field->offset,
4941                                                       HIST_STACKTRACE_DEPTH);
4942                         multiline = true;
4943                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4944                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4945                                    *(u64 *)(key + key_field->offset));
4946                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
4947                         seq_printf(m, "%s: %-50s", field_name,
4948                                    (char *)(key + key_field->offset));
4949                 } else {
4950                         uval = *(u64 *)(key + key_field->offset);
4951                         seq_printf(m, "%s: %10llu", field_name, uval);
4952                 }
4953         }
4954
4955         if (!multiline)
4956                 seq_puts(m, " ");
4957
4958         seq_puts(m, "}");
4959
4960         seq_printf(m, " hitcount: %10llu",
4961                    tracing_map_read_sum(elt, HITCOUNT_IDX));
4962
4963         for (i = 1; i < hist_data->n_vals; i++) {
4964                 field_name = hist_field_name(hist_data->fields[i], 0);
4965
4966                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4967                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4968                         continue;
4969
4970                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4971                         seq_printf(m, "  %s: %10llx", field_name,
4972                                    tracing_map_read_sum(elt, i));
4973                 } else {
4974                         seq_printf(m, "  %s: %10llu", field_name,
4975                                    tracing_map_read_sum(elt, i));
4976                 }
4977         }
4978
4979         print_actions(m, hist_data, elt);
4980
4981         seq_puts(m, "\n");
4982 }
4983
4984 static int print_entries(struct seq_file *m,
4985                          struct hist_trigger_data *hist_data)
4986 {
4987         struct tracing_map_sort_entry **sort_entries = NULL;
4988         struct tracing_map *map = hist_data->map;
4989         int i, n_entries;
4990
4991         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4992                                              hist_data->n_sort_keys,
4993                                              &sort_entries);
4994         if (n_entries < 0)
4995                 return n_entries;
4996
4997         for (i = 0; i < n_entries; i++)
4998                 hist_trigger_entry_print(m, hist_data,
4999                                          sort_entries[i]->key,
5000                                          sort_entries[i]->elt);
5001
5002         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5003
5004         return n_entries;
5005 }
5006
5007 static void hist_trigger_show(struct seq_file *m,
5008                               struct event_trigger_data *data, int n)
5009 {
5010         struct hist_trigger_data *hist_data;
5011         int n_entries;
5012
5013         if (n > 0)
5014                 seq_puts(m, "\n\n");
5015
5016         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5017         data->ops->print(m, data->ops, data);
5018         seq_puts(m, "#\n\n");
5019
5020         hist_data = data->private_data;
5021         n_entries = print_entries(m, hist_data);
5022         if (n_entries < 0)
5023                 n_entries = 0;
5024
5025         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5026                    (u64)atomic64_read(&hist_data->map->hits),
5027                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5028 }
5029
5030 static int hist_show(struct seq_file *m, void *v)
5031 {
5032         struct event_trigger_data *data;
5033         struct trace_event_file *event_file;
5034         int n = 0, ret = 0;
5035
5036         mutex_lock(&event_mutex);
5037
5038         event_file = event_file_data(m->private);
5039         if (unlikely(!event_file)) {
5040                 ret = -ENODEV;
5041                 goto out_unlock;
5042         }
5043
5044         list_for_each_entry(data, &event_file->triggers, list) {
5045                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5046                         hist_trigger_show(m, data, n++);
5047         }
5048
5049         if (have_hist_err()) {
5050                 seq_printf(m, "\nERROR: %s\n", hist_err_str);
5051                 seq_printf(m, "  Last command: %s\n", last_hist_cmd);
5052         }
5053
5054  out_unlock:
5055         mutex_unlock(&event_mutex);
5056
5057         return ret;
5058 }
5059
5060 static int event_hist_open(struct inode *inode, struct file *file)
5061 {
5062         return single_open(file, hist_show, file);
5063 }
5064
5065 const struct file_operations event_hist_fops = {
5066         .open = event_hist_open,
5067         .read = seq_read,
5068         .llseek = seq_lseek,
5069         .release = single_release,
5070 };
5071
5072 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5073 {
5074         const char *field_name = hist_field_name(hist_field, 0);
5075
5076         if (hist_field->var.name)
5077                 seq_printf(m, "%s=", hist_field->var.name);
5078
5079         if (hist_field->flags & HIST_FIELD_FL_CPU)
5080                 seq_puts(m, "common_cpu");
5081         else if (field_name) {
5082                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5083                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5084                         seq_putc(m, '$');
5085                 seq_printf(m, "%s", field_name);
5086         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5087                 seq_puts(m, "common_timestamp");
5088
5089         if (hist_field->flags) {
5090                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5091                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5092                         const char *flags = get_hist_field_flags(hist_field);
5093
5094                         if (flags)
5095                                 seq_printf(m, ".%s", flags);
5096                 }
5097         }
5098 }
5099
5100 static int event_hist_trigger_print(struct seq_file *m,
5101                                     struct event_trigger_ops *ops,
5102                                     struct event_trigger_data *data)
5103 {
5104         struct hist_trigger_data *hist_data = data->private_data;
5105         struct hist_field *field;
5106         bool have_var = false;
5107         unsigned int i;
5108
5109         seq_puts(m, "hist:");
5110
5111         if (data->name)
5112                 seq_printf(m, "%s:", data->name);
5113
5114         seq_puts(m, "keys=");
5115
5116         for_each_hist_key_field(i, hist_data) {
5117                 field = hist_data->fields[i];
5118
5119                 if (i > hist_data->n_vals)
5120                         seq_puts(m, ",");
5121
5122                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5123                         seq_puts(m, "stacktrace");
5124                 else
5125                         hist_field_print(m, field);
5126         }
5127
5128         seq_puts(m, ":vals=");
5129
5130         for_each_hist_val_field(i, hist_data) {
5131                 field = hist_data->fields[i];
5132                 if (field->flags & HIST_FIELD_FL_VAR) {
5133                         have_var = true;
5134                         continue;
5135                 }
5136
5137                 if (i == HITCOUNT_IDX)
5138                         seq_puts(m, "hitcount");
5139                 else {
5140                         seq_puts(m, ",");
5141                         hist_field_print(m, field);
5142                 }
5143         }
5144
5145         if (have_var) {
5146                 unsigned int n = 0;
5147
5148                 seq_puts(m, ":");
5149
5150                 for_each_hist_val_field(i, hist_data) {
5151                         field = hist_data->fields[i];
5152
5153                         if (field->flags & HIST_FIELD_FL_VAR) {
5154                                 if (n++)
5155                                         seq_puts(m, ",");
5156                                 hist_field_print(m, field);
5157                         }
5158                 }
5159         }
5160
5161         seq_puts(m, ":sort=");
5162
5163         for (i = 0; i < hist_data->n_sort_keys; i++) {
5164                 struct tracing_map_sort_key *sort_key;
5165                 unsigned int idx, first_key_idx;
5166
5167                 /* skip VAR vals */
5168                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5169
5170                 sort_key = &hist_data->sort_keys[i];
5171                 idx = sort_key->field_idx;
5172
5173                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5174                         return -EINVAL;
5175
5176                 if (i > 0)
5177                         seq_puts(m, ",");
5178
5179                 if (idx == HITCOUNT_IDX)
5180                         seq_puts(m, "hitcount");
5181                 else {
5182                         if (idx >= first_key_idx)
5183                                 idx += hist_data->n_vars;
5184                         hist_field_print(m, hist_data->fields[idx]);
5185                 }
5186
5187                 if (sort_key->descending)
5188                         seq_puts(m, ".descending");
5189         }
5190         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5191         if (hist_data->enable_timestamps)
5192                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5193
5194         print_actions_spec(m, hist_data);
5195
5196         if (data->filter_str)
5197                 seq_printf(m, " if %s", data->filter_str);
5198
5199         if (data->paused)
5200                 seq_puts(m, " [paused]");
5201         else
5202                 seq_puts(m, " [active]");
5203
5204         seq_putc(m, '\n');
5205
5206         return 0;
5207 }
5208
5209 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5210                                    struct event_trigger_data *data)
5211 {
5212         struct hist_trigger_data *hist_data = data->private_data;
5213
5214         if (!data->ref && hist_data->attrs->name)
5215                 save_named_trigger(hist_data->attrs->name, data);
5216
5217         data->ref++;
5218
5219         return 0;
5220 }
5221
5222 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5223 {
5224         struct trace_event_file *file;
5225         unsigned int i;
5226         char *cmd;
5227         int ret;
5228
5229         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5230                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5231                 cmd = hist_data->field_var_hists[i]->cmd;
5232                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5233                                               "!hist", "hist", cmd);
5234         }
5235 }
5236
5237 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5238                                     struct event_trigger_data *data)
5239 {
5240         struct hist_trigger_data *hist_data = data->private_data;
5241
5242         if (WARN_ON_ONCE(data->ref <= 0))
5243                 return;
5244
5245         data->ref--;
5246         if (!data->ref) {
5247                 if (data->name)
5248                         del_named_trigger(data);
5249
5250                 trigger_data_free(data);
5251
5252                 remove_hist_vars(hist_data);
5253
5254                 unregister_field_var_hists(hist_data);
5255
5256                 destroy_hist_data(hist_data);
5257         }
5258 }
5259
5260 static struct event_trigger_ops event_hist_trigger_ops = {
5261         .func                   = event_hist_trigger,
5262         .print                  = event_hist_trigger_print,
5263         .init                   = event_hist_trigger_init,
5264         .free                   = event_hist_trigger_free,
5265 };
5266
5267 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5268                                          struct event_trigger_data *data)
5269 {
5270         data->ref++;
5271
5272         save_named_trigger(data->named_data->name, data);
5273
5274         event_hist_trigger_init(ops, data->named_data);
5275
5276         return 0;
5277 }
5278
5279 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5280                                           struct event_trigger_data *data)
5281 {
5282         if (WARN_ON_ONCE(data->ref <= 0))
5283                 return;
5284
5285         event_hist_trigger_free(ops, data->named_data);
5286
5287         data->ref--;
5288         if (!data->ref) {
5289                 del_named_trigger(data);
5290                 trigger_data_free(data);
5291         }
5292 }
5293
5294 static struct event_trigger_ops event_hist_trigger_named_ops = {
5295         .func                   = event_hist_trigger,
5296         .print                  = event_hist_trigger_print,
5297         .init                   = event_hist_trigger_named_init,
5298         .free                   = event_hist_trigger_named_free,
5299 };
5300
5301 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5302                                                             char *param)
5303 {
5304         return &event_hist_trigger_ops;
5305 }
5306
5307 static void hist_clear(struct event_trigger_data *data)
5308 {
5309         struct hist_trigger_data *hist_data = data->private_data;
5310
5311         if (data->name)
5312                 pause_named_trigger(data);
5313
5314         tracepoint_synchronize_unregister();
5315
5316         tracing_map_clear(hist_data->map);
5317
5318         if (data->name)
5319                 unpause_named_trigger(data);
5320 }
5321
5322 static bool compatible_field(struct ftrace_event_field *field,
5323                              struct ftrace_event_field *test_field)
5324 {
5325         if (field == test_field)
5326                 return true;
5327         if (field == NULL || test_field == NULL)
5328                 return false;
5329         if (strcmp(field->name, test_field->name) != 0)
5330                 return false;
5331         if (strcmp(field->type, test_field->type) != 0)
5332                 return false;
5333         if (field->size != test_field->size)
5334                 return false;
5335         if (field->is_signed != test_field->is_signed)
5336                 return false;
5337
5338         return true;
5339 }
5340
5341 static bool hist_trigger_match(struct event_trigger_data *data,
5342                                struct event_trigger_data *data_test,
5343                                struct event_trigger_data *named_data,
5344                                bool ignore_filter)
5345 {
5346         struct tracing_map_sort_key *sort_key, *sort_key_test;
5347         struct hist_trigger_data *hist_data, *hist_data_test;
5348         struct hist_field *key_field, *key_field_test;
5349         unsigned int i;
5350
5351         if (named_data && (named_data != data_test) &&
5352             (named_data != data_test->named_data))
5353                 return false;
5354
5355         if (!named_data && is_named_trigger(data_test))
5356                 return false;
5357
5358         hist_data = data->private_data;
5359         hist_data_test = data_test->private_data;
5360
5361         if (hist_data->n_vals != hist_data_test->n_vals ||
5362             hist_data->n_fields != hist_data_test->n_fields ||
5363             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5364                 return false;
5365
5366         if (!ignore_filter) {
5367                 if ((data->filter_str && !data_test->filter_str) ||
5368                    (!data->filter_str && data_test->filter_str))
5369                         return false;
5370         }
5371
5372         for_each_hist_field(i, hist_data) {
5373                 key_field = hist_data->fields[i];
5374                 key_field_test = hist_data_test->fields[i];
5375
5376                 if (key_field->flags != key_field_test->flags)
5377                         return false;
5378                 if (!compatible_field(key_field->field, key_field_test->field))
5379                         return false;
5380                 if (key_field->offset != key_field_test->offset)
5381                         return false;
5382                 if (key_field->size != key_field_test->size)
5383                         return false;
5384                 if (key_field->is_signed != key_field_test->is_signed)
5385                         return false;
5386                 if (!!key_field->var.name != !!key_field_test->var.name)
5387                         return false;
5388                 if (key_field->var.name &&
5389                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5390                         return false;
5391         }
5392
5393         for (i = 0; i < hist_data->n_sort_keys; i++) {
5394                 sort_key = &hist_data->sort_keys[i];
5395                 sort_key_test = &hist_data_test->sort_keys[i];
5396
5397                 if (sort_key->field_idx != sort_key_test->field_idx ||
5398                     sort_key->descending != sort_key_test->descending)
5399                         return false;
5400         }
5401
5402         if (!ignore_filter && data->filter_str &&
5403             (strcmp(data->filter_str, data_test->filter_str) != 0))
5404                 return false;
5405
5406         if (!actions_match(hist_data, hist_data_test))
5407                 return false;
5408
5409         return true;
5410 }
5411
5412 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5413                                  struct event_trigger_data *data,
5414                                  struct trace_event_file *file)
5415 {
5416         struct hist_trigger_data *hist_data = data->private_data;
5417         struct event_trigger_data *test, *named_data = NULL;
5418         int ret = 0;
5419
5420         if (hist_data->attrs->name) {
5421                 named_data = find_named_trigger(hist_data->attrs->name);
5422                 if (named_data) {
5423                         if (!hist_trigger_match(data, named_data, named_data,
5424                                                 true)) {
5425                                 hist_err("Named hist trigger doesn't match existing named trigger (includes variables): ", hist_data->attrs->name);
5426                                 ret = -EINVAL;
5427                                 goto out;
5428                         }
5429                 }
5430         }
5431
5432         if (hist_data->attrs->name && !named_data)
5433                 goto new;
5434
5435         lockdep_assert_held(&event_mutex);
5436
5437         list_for_each_entry(test, &file->triggers, list) {
5438                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5439                         if (!hist_trigger_match(data, test, named_data, false))
5440                                 continue;
5441                         if (hist_data->attrs->pause)
5442                                 test->paused = true;
5443                         else if (hist_data->attrs->cont)
5444                                 test->paused = false;
5445                         else if (hist_data->attrs->clear)
5446                                 hist_clear(test);
5447                         else {
5448                                 hist_err("Hist trigger already exists", NULL);
5449                                 ret = -EEXIST;
5450                         }
5451                         goto out;
5452                 }
5453         }
5454  new:
5455         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5456                 hist_err("Can't clear or continue a nonexistent hist trigger", NULL);
5457                 ret = -ENOENT;
5458                 goto out;
5459         }
5460
5461         if (hist_data->attrs->pause)
5462                 data->paused = true;
5463
5464         if (named_data) {
5465                 data->private_data = named_data->private_data;
5466                 set_named_trigger_data(data, named_data);
5467                 data->ops = &event_hist_trigger_named_ops;
5468         }
5469
5470         if (data->ops->init) {
5471                 ret = data->ops->init(data->ops, data);
5472                 if (ret < 0)
5473                         goto out;
5474         }
5475
5476         if (hist_data->enable_timestamps) {
5477                 char *clock = hist_data->attrs->clock;
5478
5479                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5480                 if (ret) {
5481                         hist_err("Couldn't set trace_clock: ", clock);
5482                         goto out;
5483                 }
5484
5485                 tracing_set_time_stamp_abs(file->tr, true);
5486         }
5487
5488         if (named_data)
5489                 destroy_hist_data(hist_data);
5490
5491         ret++;
5492  out:
5493         return ret;
5494 }
5495
5496 static int hist_trigger_enable(struct event_trigger_data *data,
5497                                struct trace_event_file *file)
5498 {
5499         int ret = 0;
5500
5501         list_add_tail_rcu(&data->list, &file->triggers);
5502
5503         update_cond_flag(file);
5504
5505         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5506                 list_del_rcu(&data->list);
5507                 update_cond_flag(file);
5508                 ret--;
5509         }
5510
5511         return ret;
5512 }
5513
5514 static bool have_hist_trigger_match(struct event_trigger_data *data,
5515                                     struct trace_event_file *file)
5516 {
5517         struct hist_trigger_data *hist_data = data->private_data;
5518         struct event_trigger_data *test, *named_data = NULL;
5519         bool match = false;
5520
5521         lockdep_assert_held(&event_mutex);
5522
5523         if (hist_data->attrs->name)
5524                 named_data = find_named_trigger(hist_data->attrs->name);
5525
5526         list_for_each_entry(test, &file->triggers, list) {
5527                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5528                         if (hist_trigger_match(data, test, named_data, false)) {
5529                                 match = true;
5530                                 break;
5531                         }
5532                 }
5533         }
5534
5535         return match;
5536 }
5537
5538 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5539                                     struct trace_event_file *file)
5540 {
5541         struct hist_trigger_data *hist_data = data->private_data;
5542         struct event_trigger_data *test, *named_data = NULL;
5543
5544         lockdep_assert_held(&event_mutex);
5545
5546         if (hist_data->attrs->name)
5547                 named_data = find_named_trigger(hist_data->attrs->name);
5548
5549         list_for_each_entry(test, &file->triggers, list) {
5550                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5551                         if (!hist_trigger_match(data, test, named_data, false))
5552                                 continue;
5553                         hist_data = test->private_data;
5554                         if (check_var_refs(hist_data))
5555                                 return true;
5556                         break;
5557                 }
5558         }
5559
5560         return false;
5561 }
5562
5563 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5564                                     struct event_trigger_data *data,
5565                                     struct trace_event_file *file)
5566 {
5567         struct hist_trigger_data *hist_data = data->private_data;
5568         struct event_trigger_data *test, *named_data = NULL;
5569         bool unregistered = false;
5570
5571         lockdep_assert_held(&event_mutex);
5572
5573         if (hist_data->attrs->name)
5574                 named_data = find_named_trigger(hist_data->attrs->name);
5575
5576         list_for_each_entry(test, &file->triggers, list) {
5577                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5578                         if (!hist_trigger_match(data, test, named_data, false))
5579                                 continue;
5580                         unregistered = true;
5581                         list_del_rcu(&test->list);
5582                         trace_event_trigger_enable_disable(file, 0);
5583                         update_cond_flag(file);
5584                         break;
5585                 }
5586         }
5587
5588         if (unregistered && test->ops->free)
5589                 test->ops->free(test->ops, test);
5590
5591         if (hist_data->enable_timestamps) {
5592                 if (!hist_data->remove || unregistered)
5593                         tracing_set_time_stamp_abs(file->tr, false);
5594         }
5595 }
5596
5597 static bool hist_file_check_refs(struct trace_event_file *file)
5598 {
5599         struct hist_trigger_data *hist_data;
5600         struct event_trigger_data *test;
5601
5602         lockdep_assert_held(&event_mutex);
5603
5604         list_for_each_entry(test, &file->triggers, list) {
5605                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5606                         hist_data = test->private_data;
5607                         if (check_var_refs(hist_data))
5608                                 return true;
5609                 }
5610         }
5611
5612         return false;
5613 }
5614
5615 static void hist_unreg_all(struct trace_event_file *file)
5616 {
5617         struct event_trigger_data *test, *n;
5618         struct hist_trigger_data *hist_data;
5619         struct synth_event *se;
5620         const char *se_name;
5621
5622         if (hist_file_check_refs(file))
5623                 return;
5624
5625         list_for_each_entry_safe(test, n, &file->triggers, list) {
5626                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5627                         hist_data = test->private_data;
5628                         list_del_rcu(&test->list);
5629                         trace_event_trigger_enable_disable(file, 0);
5630
5631                         mutex_lock(&synth_event_mutex);
5632                         se_name = trace_event_name(file->event_call);
5633                         se = find_synth_event(se_name);
5634                         if (se)
5635                                 se->ref--;
5636                         mutex_unlock(&synth_event_mutex);
5637
5638                         update_cond_flag(file);
5639                         if (hist_data->enable_timestamps)
5640                                 tracing_set_time_stamp_abs(file->tr, false);
5641                         if (test->ops->free)
5642                                 test->ops->free(test->ops, test);
5643                 }
5644         }
5645 }
5646
5647 static int event_hist_trigger_func(struct event_command *cmd_ops,
5648                                    struct trace_event_file *file,
5649                                    char *glob, char *cmd, char *param)
5650 {
5651         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5652         struct event_trigger_data *trigger_data;
5653         struct hist_trigger_attrs *attrs;
5654         struct event_trigger_ops *trigger_ops;
5655         struct hist_trigger_data *hist_data;
5656         struct synth_event *se;
5657         const char *se_name;
5658         bool remove = false;
5659         char *trigger, *p;
5660         int ret = 0;
5661
5662         if (glob && strlen(glob)) {
5663                 last_cmd_set(param);
5664                 hist_err_clear();
5665         }
5666
5667         if (!param)
5668                 return -EINVAL;
5669
5670         if (glob[0] == '!')
5671                 remove = true;
5672
5673         /*
5674          * separate the trigger from the filter (k:v [if filter])
5675          * allowing for whitespace in the trigger
5676          */
5677         p = trigger = param;
5678         do {
5679                 p = strstr(p, "if");
5680                 if (!p)
5681                         break;
5682                 if (p == param)
5683                         return -EINVAL;
5684                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5685                         p++;
5686                         continue;
5687                 }
5688                 if (p >= param + strlen(param) - strlen("if") - 1)
5689                         return -EINVAL;
5690                 if (*(p + strlen("if")) != ' ' && *(p + strlen("if")) != '\t') {
5691                         p++;
5692                         continue;
5693                 }
5694                 break;
5695         } while (p);
5696
5697         if (!p)
5698                 param = NULL;
5699         else {
5700                 *(p - 1) = '\0';
5701                 param = strstrip(p);
5702                 trigger = strstrip(trigger);
5703         }
5704
5705         attrs = parse_hist_trigger_attrs(trigger);
5706         if (IS_ERR(attrs))
5707                 return PTR_ERR(attrs);
5708
5709         if (attrs->map_bits)
5710                 hist_trigger_bits = attrs->map_bits;
5711
5712         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5713         if (IS_ERR(hist_data)) {
5714                 destroy_hist_trigger_attrs(attrs);
5715                 return PTR_ERR(hist_data);
5716         }
5717
5718         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5719
5720         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5721         if (!trigger_data) {
5722                 ret = -ENOMEM;
5723                 goto out_free;
5724         }
5725
5726         trigger_data->count = -1;
5727         trigger_data->ops = trigger_ops;
5728         trigger_data->cmd_ops = cmd_ops;
5729
5730         INIT_LIST_HEAD(&trigger_data->list);
5731         RCU_INIT_POINTER(trigger_data->filter, NULL);
5732
5733         trigger_data->private_data = hist_data;
5734
5735         /* if param is non-empty, it's supposed to be a filter */
5736         if (param && cmd_ops->set_filter) {
5737                 ret = cmd_ops->set_filter(param, trigger_data, file);
5738                 if (ret < 0)
5739                         goto out_free;
5740         }
5741
5742         if (remove) {
5743                 if (!have_hist_trigger_match(trigger_data, file))
5744                         goto out_free;
5745
5746                 if (hist_trigger_check_refs(trigger_data, file)) {
5747                         ret = -EBUSY;
5748                         goto out_free;
5749                 }
5750
5751                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5752
5753                 mutex_lock(&synth_event_mutex);
5754                 se_name = trace_event_name(file->event_call);
5755                 se = find_synth_event(se_name);
5756                 if (se)
5757                         se->ref--;
5758                 mutex_unlock(&synth_event_mutex);
5759
5760                 ret = 0;
5761                 goto out_free;
5762         }
5763
5764         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5765         /*
5766          * The above returns on success the # of triggers registered,
5767          * but if it didn't register any it returns zero.  Consider no
5768          * triggers registered a failure too.
5769          */
5770         if (!ret) {
5771                 if (!(attrs->pause || attrs->cont || attrs->clear))
5772                         ret = -ENOENT;
5773                 goto out_free;
5774         } else if (ret < 0)
5775                 goto out_free;
5776
5777         if (get_named_trigger_data(trigger_data))
5778                 goto enable;
5779
5780         if (has_hist_vars(hist_data))
5781                 save_hist_vars(hist_data);
5782
5783         ret = create_actions(hist_data, file);
5784         if (ret)
5785                 goto out_unreg;
5786
5787         ret = tracing_map_init(hist_data->map);
5788         if (ret)
5789                 goto out_unreg;
5790 enable:
5791         ret = hist_trigger_enable(trigger_data, file);
5792         if (ret)
5793                 goto out_unreg;
5794
5795         mutex_lock(&synth_event_mutex);
5796         se_name = trace_event_name(file->event_call);
5797         se = find_synth_event(se_name);
5798         if (se)
5799                 se->ref++;
5800         mutex_unlock(&synth_event_mutex);
5801
5802         /* Just return zero, not the number of registered triggers */
5803         ret = 0;
5804  out:
5805         if (ret == 0)
5806                 hist_err_clear();
5807
5808         return ret;
5809  out_unreg:
5810         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5811  out_free:
5812         if (cmd_ops->set_filter)
5813                 cmd_ops->set_filter(NULL, trigger_data, NULL);
5814
5815         remove_hist_vars(hist_data);
5816
5817         kfree(trigger_data);
5818
5819         destroy_hist_data(hist_data);
5820         goto out;
5821 }
5822
5823 static struct event_command trigger_hist_cmd = {
5824         .name                   = "hist",
5825         .trigger_type           = ETT_EVENT_HIST,
5826         .flags                  = EVENT_CMD_FL_NEEDS_REC,
5827         .func                   = event_hist_trigger_func,
5828         .reg                    = hist_register_trigger,
5829         .unreg                  = hist_unregister_trigger,
5830         .unreg_all              = hist_unreg_all,
5831         .get_trigger_ops        = event_hist_get_trigger_ops,
5832         .set_filter             = set_trigger_filter,
5833 };
5834
5835 __init int register_trigger_hist_cmd(void)
5836 {
5837         int ret;
5838
5839         ret = register_event_command(&trigger_hist_cmd);
5840         WARN_ON(ret < 0);
5841
5842         return ret;
5843 }
5844
5845 static void
5846 hist_enable_trigger(struct event_trigger_data *data, void *rec,
5847                     struct ring_buffer_event *event)
5848 {
5849         struct enable_trigger_data *enable_data = data->private_data;
5850         struct event_trigger_data *test;
5851
5852         list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
5853                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5854                         if (enable_data->enable)
5855                                 test->paused = false;
5856                         else
5857                                 test->paused = true;
5858                 }
5859         }
5860 }
5861
5862 static void
5863 hist_enable_count_trigger(struct event_trigger_data *data, void *rec,
5864                           struct ring_buffer_event *event)
5865 {
5866         if (!data->count)
5867                 return;
5868
5869         if (data->count != -1)
5870                 (data->count)--;
5871
5872         hist_enable_trigger(data, rec, event);
5873 }
5874
5875 static struct event_trigger_ops hist_enable_trigger_ops = {
5876         .func                   = hist_enable_trigger,
5877         .print                  = event_enable_trigger_print,
5878         .init                   = event_trigger_init,
5879         .free                   = event_enable_trigger_free,
5880 };
5881
5882 static struct event_trigger_ops hist_enable_count_trigger_ops = {
5883         .func                   = hist_enable_count_trigger,
5884         .print                  = event_enable_trigger_print,
5885         .init                   = event_trigger_init,
5886         .free                   = event_enable_trigger_free,
5887 };
5888
5889 static struct event_trigger_ops hist_disable_trigger_ops = {
5890         .func                   = hist_enable_trigger,
5891         .print                  = event_enable_trigger_print,
5892         .init                   = event_trigger_init,
5893         .free                   = event_enable_trigger_free,
5894 };
5895
5896 static struct event_trigger_ops hist_disable_count_trigger_ops = {
5897         .func                   = hist_enable_count_trigger,
5898         .print                  = event_enable_trigger_print,
5899         .init                   = event_trigger_init,
5900         .free                   = event_enable_trigger_free,
5901 };
5902
5903 static struct event_trigger_ops *
5904 hist_enable_get_trigger_ops(char *cmd, char *param)
5905 {
5906         struct event_trigger_ops *ops;
5907         bool enable;
5908
5909         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
5910
5911         if (enable)
5912                 ops = param ? &hist_enable_count_trigger_ops :
5913                         &hist_enable_trigger_ops;
5914         else
5915                 ops = param ? &hist_disable_count_trigger_ops :
5916                         &hist_disable_trigger_ops;
5917
5918         return ops;
5919 }
5920
5921 static void hist_enable_unreg_all(struct trace_event_file *file)
5922 {
5923         struct event_trigger_data *test, *n;
5924
5925         list_for_each_entry_safe(test, n, &file->triggers, list) {
5926                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
5927                         list_del_rcu(&test->list);
5928                         update_cond_flag(file);
5929                         trace_event_trigger_enable_disable(file, 0);
5930                         if (test->ops->free)
5931                                 test->ops->free(test->ops, test);
5932                 }
5933         }
5934 }
5935
5936 static struct event_command trigger_hist_enable_cmd = {
5937         .name                   = ENABLE_HIST_STR,
5938         .trigger_type           = ETT_HIST_ENABLE,
5939         .func                   = event_enable_trigger_func,
5940         .reg                    = event_enable_register_trigger,
5941         .unreg                  = event_enable_unregister_trigger,
5942         .unreg_all              = hist_enable_unreg_all,
5943         .get_trigger_ops        = hist_enable_get_trigger_ops,
5944         .set_filter             = set_trigger_filter,
5945 };
5946
5947 static struct event_command trigger_hist_disable_cmd = {
5948         .name                   = DISABLE_HIST_STR,
5949         .trigger_type           = ETT_HIST_ENABLE,
5950         .func                   = event_enable_trigger_func,
5951         .reg                    = event_enable_register_trigger,
5952         .unreg                  = event_enable_unregister_trigger,
5953         .unreg_all              = hist_enable_unreg_all,
5954         .get_trigger_ops        = hist_enable_get_trigger_ops,
5955         .set_filter             = set_trigger_filter,
5956 };
5957
5958 static __init void unregister_trigger_hist_enable_disable_cmds(void)
5959 {
5960         unregister_event_command(&trigger_hist_enable_cmd);
5961         unregister_event_command(&trigger_hist_disable_cmd);
5962 }
5963
5964 __init int register_trigger_hist_enable_disable_cmds(void)
5965 {
5966         int ret;
5967
5968         ret = register_event_command(&trigger_hist_enable_cmd);
5969         if (WARN_ON(ret < 0))
5970                 return ret;
5971         ret = register_event_command(&trigger_hist_disable_cmd);
5972         if (WARN_ON(ret < 0))
5973                 unregister_trigger_hist_enable_disable_cmds();
5974
5975         return ret;
5976 }
5977
5978 static __init int trace_events_hist_init(void)
5979 {
5980         struct dentry *entry = NULL;
5981         struct dentry *d_tracer;
5982         int err = 0;
5983
5984         d_tracer = tracing_init_dentry();
5985         if (IS_ERR(d_tracer)) {
5986                 err = PTR_ERR(d_tracer);
5987                 goto err;
5988         }
5989
5990         entry = tracefs_create_file("synthetic_events", 0644, d_tracer,
5991                                     NULL, &synth_events_fops);
5992         if (!entry) {
5993                 err = -ENODEV;
5994                 goto err;
5995         }
5996
5997         return err;
5998  err:
5999         pr_warn("Could not create tracefs 'synthetic_events' entry\n");
6000
6001         return err;
6002 }
6003
6004 fs_initcall(trace_events_hist_init);