trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),   \
  69         C(INVALID_STR_OPERAND,  "String type can not be an operand in expression"),
  70
  71 #undef C
  72 #define C(a, b)         HIST_ERR_##a
  73
  74 enum { ERRORS };
  75
  76 #undef C
  77 #define C(a, b)         b
  78
  79 static const char *err_text[] = { ERRORS };
  80
  81 struct hist_field;
  82
  83 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  84                                 struct tracing_map_elt *elt,
  85                                 struct trace_buffer *buffer,
  86                                 struct ring_buffer_event *rbe,
  87                                 void *event);
  88
  89 #define HIST_FIELD_OPERANDS_MAX 2
  90 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  91 #define HIST_ACTIONS_MAX        8
  92
  93 enum field_op_id {
  94         FIELD_OP_NONE,
  95         FIELD_OP_PLUS,
  96         FIELD_OP_MINUS,
  97         FIELD_OP_UNARY_MINUS,
  98 };
  99
 100 /*
 101  * A hist_var (histogram variable) contains variable information for
 102  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 103  * flag set.  A hist_var has a variable name e.g. ts0, and is
 104  * associated with a given histogram trigger, as specified by
 105  * hist_data.  The hist_var idx is the unique index assigned to the
 106  * variable by the hist trigger's tracing_map.  The idx is what is
 107  * used to set a variable's value and, by a variable reference, to
 108  * retrieve it.
 109  */
 110 struct hist_var {
 111         char                            *name;
 112         struct hist_trigger_data        *hist_data;
 113         unsigned int                    idx;
 114 };
 115
 116 struct hist_field {
 117         struct ftrace_event_field       *field;
 118         unsigned long                   flags;
 119         hist_field_fn_t                 fn;
 120         unsigned int                    ref;
 121         unsigned int                    size;
 122         unsigned int                    offset;
 123         unsigned int                    is_signed;
 124         unsigned long                   buckets;
 125         const char                      *type;
 126         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 127         struct hist_trigger_data        *hist_data;
 128
 129         /*
 130          * Variable fields contain variable-specific info in var.
 131          */
 132         struct hist_var                 var;
 133         enum field_op_id                operator;
 134         char                            *system;
 135         char                            *event_name;
 136
 137         /*
 138          * The name field is used for EXPR and VAR_REF fields.  VAR
 139          * fields contain the variable name in var.name.
 140          */
 141         char                            *name;
 142
 143         /*
 144          * When a histogram trigger is hit, if it has any references
 145          * to variables, the values of those variables are collected
 146          * into a var_ref_vals array by resolve_var_refs().  The
 147          * current value of each variable is read from the tracing_map
 148          * using the hist field's hist_var.idx and entered into the
 149          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 150          */
 151         unsigned int                    var_ref_idx;
 152         bool                            read_once;
 153
 154         unsigned int                    var_str_idx;
 155 };
 156
 157 static u64 hist_field_none(struct hist_field *field,
 158                            struct tracing_map_elt *elt,
 159                            struct trace_buffer *buffer,
 160                            struct ring_buffer_event *rbe,
 161                            void *event)
 162 {
 163         return 0;
 164 }
 165
 166 static u64 hist_field_counter(struct hist_field *field,
 167                               struct tracing_map_elt *elt,
 168                               struct trace_buffer *buffer,
 169                               struct ring_buffer_event *rbe,
 170                               void *event)
 171 {
 172         return 1;
 173 }
 174
 175 static u64 hist_field_string(struct hist_field *hist_field,
 176                              struct tracing_map_elt *elt,
 177                              struct trace_buffer *buffer,
 178                              struct ring_buffer_event *rbe,
 179                              void *event)
 180 {
 181         char *addr = (char *)(event + hist_field->field->offset);
 182
 183         return (u64)(unsigned long)addr;
 184 }
 185
 186 static u64 hist_field_dynstring(struct hist_field *hist_field,
 187                                 struct tracing_map_elt *elt,
 188                                 struct trace_buffer *buffer,
 189                                 struct ring_buffer_event *rbe,
 190                                 void *event)
 191 {
 192         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 193         int str_loc = str_item & 0xffff;
 194         char *addr = (char *)(event + str_loc);
 195
 196         return (u64)(unsigned long)addr;
 197 }
 198
 199 static u64 hist_field_pstring(struct hist_field *hist_field,
 200                               struct tracing_map_elt *elt,
 201                               struct trace_buffer *buffer,
 202                               struct ring_buffer_event *rbe,
 203                               void *event)
 204 {
 205         char **addr = (char **)(event + hist_field->field->offset);
 206
 207         return (u64)(unsigned long)*addr;
 208 }
 209
 210 static u64 hist_field_log2(struct hist_field *hist_field,
 211                            struct tracing_map_elt *elt,
 212                            struct trace_buffer *buffer,
 213                            struct ring_buffer_event *rbe,
 214                            void *event)
 215 {
 216         struct hist_field *operand = hist_field->operands[0];
 217
 218         u64 val = operand->fn(operand, elt, buffer, rbe, event);
 219
 220         return (u64) ilog2(roundup_pow_of_two(val));
 221 }
 222
 223 static u64 hist_field_bucket(struct hist_field *hist_field,
 224                              struct tracing_map_elt *elt,
 225                              struct trace_buffer *buffer,
 226                              struct ring_buffer_event *rbe,
 227                              void *event)
 228 {
 229         struct hist_field *operand = hist_field->operands[0];
 230         unsigned long buckets = hist_field->buckets;
 231
 232         u64 val = operand->fn(operand, elt, buffer, rbe, event);
 233
 234         if (WARN_ON_ONCE(!buckets))
 235                 return val;
 236
 237         if (val >= LONG_MAX)
 238                 val = div64_ul(val, buckets);
 239         else
 240                 val = (u64)((unsigned long)val / buckets);
 241         return val * buckets;
 242 }
 243
 244 static u64 hist_field_plus(struct hist_field *hist_field,
 245                            struct tracing_map_elt *elt,
 246                            struct trace_buffer *buffer,
 247                            struct ring_buffer_event *rbe,
 248                            void *event)
 249 {
 250         struct hist_field *operand1 = hist_field->operands[0];
 251         struct hist_field *operand2 = hist_field->operands[1];
 252
 253         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 254         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 255
 256         return val1 + val2;
 257 }
 258
 259 static u64 hist_field_minus(struct hist_field *hist_field,
 260                             struct tracing_map_elt *elt,
 261                             struct trace_buffer *buffer,
 262                             struct ring_buffer_event *rbe,
 263                             void *event)
 264 {
 265         struct hist_field *operand1 = hist_field->operands[0];
 266         struct hist_field *operand2 = hist_field->operands[1];
 267
 268         u64 val1 = operand1->fn(operand1, elt, buffer, rbe, event);
 269         u64 val2 = operand2->fn(operand2, elt, buffer, rbe, event);
 270
 271         return val1 - val2;
 272 }
 273
 274 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 275                                   struct tracing_map_elt *elt,
 276                                   struct trace_buffer *buffer,
 277                                   struct ring_buffer_event *rbe,
 278                                   void *event)
 279 {
 280         struct hist_field *operand = hist_field->operands[0];
 281
 282         s64 sval = (s64)operand->fn(operand, elt, buffer, rbe, event);
 283         u64 val = (u64)-sval;
 284
 285         return val;
 286 }
 287
 288 #define DEFINE_HIST_FIELD_FN(type)                                      \
 289         static u64 hist_field_##type(struct hist_field *hist_field,     \
 290                                      struct tracing_map_elt *elt,       \
 291                                      struct trace_buffer *buffer,       \
 292                                      struct ring_buffer_event *rbe,     \
 293                                      void *event)                       \
 294 {                                                                       \
 295         type *addr = (type *)(event + hist_field->field->offset);       \
 296                                                                         \
 297         return (u64)(unsigned long)*addr;                               \
 298 }
 299
 300 DEFINE_HIST_FIELD_FN(s64);
 301 DEFINE_HIST_FIELD_FN(u64);
 302 DEFINE_HIST_FIELD_FN(s32);
 303 DEFINE_HIST_FIELD_FN(u32);
 304 DEFINE_HIST_FIELD_FN(s16);
 305 DEFINE_HIST_FIELD_FN(u16);
 306 DEFINE_HIST_FIELD_FN(s8);
 307 DEFINE_HIST_FIELD_FN(u8);
 308
 309 #define for_each_hist_field(i, hist_data)       \
 310         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 311
 312 #define for_each_hist_val_field(i, hist_data)   \
 313         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 314
 315 #define for_each_hist_key_field(i, hist_data)   \
 316         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 317
 318 #define HITCOUNT_IDX            0
 319 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 320
 321 enum hist_field_flags {
 322         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 323         HIST_FIELD_FL_KEY               = 1 << 1,
 324         HIST_FIELD_FL_STRING            = 1 << 2,
 325         HIST_FIELD_FL_HEX               = 1 << 3,
 326         HIST_FIELD_FL_SYM               = 1 << 4,
 327         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 328         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 329         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 330         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 331         HIST_FIELD_FL_LOG2              = 1 << 9,
 332         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 333         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 334         HIST_FIELD_FL_VAR               = 1 << 12,
 335         HIST_FIELD_FL_EXPR              = 1 << 13,
 336         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 337         HIST_FIELD_FL_CPU               = 1 << 15,
 338         HIST_FIELD_FL_ALIAS             = 1 << 16,
 339         HIST_FIELD_FL_BUCKET            = 1 << 17,
 340 };
 341
 342 struct var_defs {
 343         unsigned int    n_vars;
 344         char            *name[TRACING_MAP_VARS_MAX];
 345         char            *expr[TRACING_MAP_VARS_MAX];
 346 };
 347
 348 struct hist_trigger_attrs {
 349         char            *keys_str;
 350         char            *vals_str;
 351         char            *sort_key_str;
 352         char            *name;
 353         char            *clock;
 354         bool            pause;
 355         bool            cont;
 356         bool            clear;
 357         bool            ts_in_usecs;
 358         unsigned int    map_bits;
 359
 360         char            *assignment_str[TRACING_MAP_VARS_MAX];
 361         unsigned int    n_assignments;
 362
 363         char            *action_str[HIST_ACTIONS_MAX];
 364         unsigned int    n_actions;
 365
 366         struct var_defs var_defs;
 367 };
 368
 369 struct field_var {
 370         struct hist_field       *var;
 371         struct hist_field       *val;
 372 };
 373
 374 struct field_var_hist {
 375         struct hist_trigger_data        *hist_data;
 376         char                            *cmd;
 377 };
 378
 379 struct hist_trigger_data {
 380         struct hist_field               *fields[HIST_FIELDS_MAX];
 381         unsigned int                    n_vals;
 382         unsigned int                    n_keys;
 383         unsigned int                    n_fields;
 384         unsigned int                    n_vars;
 385         unsigned int                    n_var_str;
 386         unsigned int                    key_size;
 387         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 388         unsigned int                    n_sort_keys;
 389         struct trace_event_file         *event_file;
 390         struct hist_trigger_attrs       *attrs;
 391         struct tracing_map              *map;
 392         bool                            enable_timestamps;
 393         bool                            remove;
 394         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 395         unsigned int                    n_var_refs;
 396
 397         struct action_data              *actions[HIST_ACTIONS_MAX];
 398         unsigned int                    n_actions;
 399
 400         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 401         unsigned int                    n_field_vars;
 402         unsigned int                    n_field_var_str;
 403         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 404         unsigned int                    n_field_var_hists;
 405
 406         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 407         unsigned int                    n_save_vars;
 408         unsigned int                    n_save_var_str;
 409 };
 410
 411 struct action_data;
 412
 413 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 414                              struct tracing_map_elt *elt,
 415                              struct trace_buffer *buffer, void *rec,
 416                              struct ring_buffer_event *rbe, void *key,
 417                              struct action_data *data, u64 *var_ref_vals);
 418
 419 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 420
 421 enum handler_id {
 422         HANDLER_ONMATCH = 1,
 423         HANDLER_ONMAX,
 424         HANDLER_ONCHANGE,
 425 };
 426
 427 enum action_id {
 428         ACTION_SAVE = 1,
 429         ACTION_TRACE,
 430         ACTION_SNAPSHOT,
 431 };
 432
 433 struct action_data {
 434         enum handler_id         handler;
 435         enum action_id          action;
 436         char                    *action_name;
 437         action_fn_t             fn;
 438
 439         unsigned int            n_params;
 440         char                    *params[SYNTH_FIELDS_MAX];
 441
 442         /*
 443          * When a histogram trigger is hit, the values of any
 444          * references to variables, including variables being passed
 445          * as parameters to synthetic events, are collected into a
 446          * var_ref_vals array.  This var_ref_idx array is an array of
 447          * indices into the var_ref_vals array, one for each synthetic
 448          * event param, and is passed to the synthetic event
 449          * invocation.
 450          */
 451         unsigned int            var_ref_idx[SYNTH_FIELDS_MAX];
 452         struct synth_event      *synth_event;
 453         bool                    use_trace_keyword;
 454         char                    *synth_event_name;
 455
 456         union {
 457                 struct {
 458                         char                    *event;
 459                         char                    *event_system;
 460                 } match_data;
 461
 462                 struct {
 463                         /*
 464                          * var_str contains the $-unstripped variable
 465                          * name referenced by var_ref, and used when
 466                          * printing the action.  Because var_ref
 467                          * creation is deferred to create_actions(),
 468                          * we need a per-action way to save it until
 469                          * then, thus var_str.
 470                          */
 471                         char                    *var_str;
 472
 473                         /*
 474                          * var_ref refers to the variable being
 475                          * tracked e.g onmax($var).
 476                          */
 477                         struct hist_field       *var_ref;
 478
 479                         /*
 480                          * track_var contains the 'invisible' tracking
 481                          * variable created to keep the current
 482                          * e.g. max value.
 483                          */
 484                         struct hist_field       *track_var;
 485
 486                         check_track_val_fn_t    check_val;
 487                         action_fn_t             save_data;
 488                 } track_data;
 489         };
 490 };
 491
 492 struct track_data {
 493         u64                             track_val;
 494         bool                            updated;
 495
 496         unsigned int                    key_len;
 497         void                            *key;
 498         struct tracing_map_elt          elt;
 499
 500         struct action_data              *action_data;
 501         struct hist_trigger_data        *hist_data;
 502 };
 503
 504 struct hist_elt_data {
 505         char *comm;
 506         u64 *var_ref_vals;
 507         char **field_var_str;
 508         int n_field_var_str;
 509 };
 510
 511 struct snapshot_context {
 512         struct tracing_map_elt  *elt;
 513         void                    *key;
 514 };
 515
 516 static void track_data_free(struct track_data *track_data)
 517 {
 518         struct hist_elt_data *elt_data;
 519
 520         if (!track_data)
 521                 return;
 522
 523         kfree(track_data->key);
 524
 525         elt_data = track_data->elt.private_data;
 526         if (elt_data) {
 527                 kfree(elt_data->comm);
 528                 kfree(elt_data);
 529         }
 530
 531         kfree(track_data);
 532 }
 533
 534 static struct track_data *track_data_alloc(unsigned int key_len,
 535                                            struct action_data *action_data,
 536                                            struct hist_trigger_data *hist_data)
 537 {
 538         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 539         struct hist_elt_data *elt_data;
 540
 541         if (!data)
 542                 return ERR_PTR(-ENOMEM);
 543
 544         data->key = kzalloc(key_len, GFP_KERNEL);
 545         if (!data->key) {
 546                 track_data_free(data);
 547                 return ERR_PTR(-ENOMEM);
 548         }
 549
 550         data->key_len = key_len;
 551         data->action_data = action_data;
 552         data->hist_data = hist_data;
 553
 554         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 555         if (!elt_data) {
 556                 track_data_free(data);
 557                 return ERR_PTR(-ENOMEM);
 558         }
 559
 560         data->elt.private_data = elt_data;
 561
 562         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 563         if (!elt_data->comm) {
 564                 track_data_free(data);
 565                 return ERR_PTR(-ENOMEM);
 566         }
 567
 568         return data;
 569 }
 570
 571 static char last_cmd[MAX_FILTER_STR_VAL];
 572 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 573
 574 static int errpos(char *str)
 575 {
 576         return err_pos(last_cmd, str);
 577 }
 578
 579 static void last_cmd_set(struct trace_event_file *file, char *str)
 580 {
 581         const char *system = NULL, *name = NULL;
 582         struct trace_event_call *call;
 583
 584         if (!str)
 585                 return;
 586
 587         strcpy(last_cmd, "hist:");
 588         strncat(last_cmd, str, MAX_FILTER_STR_VAL - 1 - sizeof("hist:"));
 589
 590         if (file) {
 591                 call = file->event_call;
 592                 system = call->class->system;
 593                 if (system) {
 594                         name = trace_event_name(call);
 595                         if (!name)
 596                                 system = NULL;
 597                 }
 598         }
 599
 600         if (system)
 601                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, "hist:%s:%s", system, name);
 602 }
 603
 604 static void hist_err(struct trace_array *tr, u8 err_type, u8 err_pos)
 605 {
 606         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 607                         err_type, err_pos);
 608 }
 609
 610 static void hist_err_clear(void)
 611 {
 612         last_cmd[0] = '\0';
 613         last_cmd_loc[0] = '\0';
 614 }
 615
 616 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 617                                     unsigned int *var_ref_idx);
 618
 619 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 620                                unsigned int *var_ref_idx)
 621 {
 622         struct tracepoint *tp = event->tp;
 623
 624         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 625                 struct tracepoint_func *probe_func_ptr;
 626                 synth_probe_func_t probe_func;
 627                 void *__data;
 628
 629                 if (!(cpu_online(raw_smp_processor_id())))
 630                         return;
 631
 632                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 633                 if (probe_func_ptr) {
 634                         do {
 635                                 probe_func = probe_func_ptr->func;
 636                                 __data = probe_func_ptr->data;
 637                                 probe_func(__data, var_ref_vals, var_ref_idx);
 638                         } while ((++probe_func_ptr)->func);
 639                 }
 640         }
 641 }
 642
 643 static void action_trace(struct hist_trigger_data *hist_data,
 644                          struct tracing_map_elt *elt,
 645                          struct trace_buffer *buffer, void *rec,
 646                          struct ring_buffer_event *rbe, void *key,
 647                          struct action_data *data, u64 *var_ref_vals)
 648 {
 649         struct synth_event *event = data->synth_event;
 650
 651         trace_synth(event, var_ref_vals, data->var_ref_idx);
 652 }
 653
 654 struct hist_var_data {
 655         struct list_head list;
 656         struct hist_trigger_data *hist_data;
 657 };
 658
 659 static u64 hist_field_timestamp(struct hist_field *hist_field,
 660                                 struct tracing_map_elt *elt,
 661                                 struct trace_buffer *buffer,
 662                                 struct ring_buffer_event *rbe,
 663                                 void *event)
 664 {
 665         struct hist_trigger_data *hist_data = hist_field->hist_data;
 666         struct trace_array *tr = hist_data->event_file->tr;
 667
 668         u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 669
 670         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 671                 ts = ns2usecs(ts);
 672
 673         return ts;
 674 }
 675
 676 static u64 hist_field_cpu(struct hist_field *hist_field,
 677                           struct tracing_map_elt *elt,
 678                           struct trace_buffer *buffer,
 679                           struct ring_buffer_event *rbe,
 680                           void *event)
 681 {
 682         int cpu = smp_processor_id();
 683
 684         return cpu;
 685 }
 686
 687 /**
 688  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 689  * @hist_field: The VAR_REF field to check
 690  * @var_data: The hist trigger that owns the variable
 691  * @var_idx: The trigger variable identifier
 692  *
 693  * Check the given VAR_REF field to see whether or not it references
 694  * the given variable associated with the given trigger.
 695  *
 696  * Return: The VAR_REF field if it does reference the variable, NULL if not
 697  */
 698 static struct hist_field *
 699 check_field_for_var_ref(struct hist_field *hist_field,
 700                         struct hist_trigger_data *var_data,
 701                         unsigned int var_idx)
 702 {
 703         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 704
 705         if (hist_field && hist_field->var.idx == var_idx &&
 706             hist_field->var.hist_data == var_data)
 707                 return hist_field;
 708
 709         return NULL;
 710 }
 711
 712 /**
 713  * find_var_ref - Check if a trigger has a reference to a trigger variable
 714  * @hist_data: The hist trigger that might have a reference to the variable
 715  * @var_data: The hist trigger that owns the variable
 716  * @var_idx: The trigger variable identifier
 717  *
 718  * Check the list of var_refs[] on the first hist trigger to see
 719  * whether any of them are references to the variable on the second
 720  * trigger.
 721  *
 722  * Return: The VAR_REF field referencing the variable if so, NULL if not
 723  */
 724 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 725                                        struct hist_trigger_data *var_data,
 726                                        unsigned int var_idx)
 727 {
 728         struct hist_field *hist_field;
 729         unsigned int i;
 730
 731         for (i = 0; i < hist_data->n_var_refs; i++) {
 732                 hist_field = hist_data->var_refs[i];
 733                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 734                         return hist_field;
 735         }
 736
 737         return NULL;
 738 }
 739
 740 /**
 741  * find_any_var_ref - Check if there is a reference to a given trigger variable
 742  * @hist_data: The hist trigger
 743  * @var_idx: The trigger variable identifier
 744  *
 745  * Check to see whether the given variable is currently referenced by
 746  * any other trigger.
 747  *
 748  * The trigger the variable is defined on is explicitly excluded - the
 749  * assumption being that a self-reference doesn't prevent a trigger
 750  * from being removed.
 751  *
 752  * Return: The VAR_REF field referencing the variable if so, NULL if not
 753  */
 754 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 755                                            unsigned int var_idx)
 756 {
 757         struct trace_array *tr = hist_data->event_file->tr;
 758         struct hist_field *found = NULL;
 759         struct hist_var_data *var_data;
 760
 761         list_for_each_entry(var_data, &tr->hist_vars, list) {
 762                 if (var_data->hist_data == hist_data)
 763                         continue;
 764                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 765                 if (found)
 766                         break;
 767         }
 768
 769         return found;
 770 }
 771
 772 /**
 773  * check_var_refs - Check if there is a reference to any of trigger's variables
 774  * @hist_data: The hist trigger
 775  *
 776  * A trigger can define one or more variables.  If any one of them is
 777  * currently referenced by any other trigger, this function will
 778  * determine that.
 779
 780  * Typically used to determine whether or not a trigger can be removed
 781  * - if there are any references to a trigger's variables, it cannot.
 782  *
 783  * Return: True if there is a reference to any of trigger's variables
 784  */
 785 static bool check_var_refs(struct hist_trigger_data *hist_data)
 786 {
 787         struct hist_field *field;
 788         bool found = false;
 789         int i;
 790
 791         for_each_hist_field(i, hist_data) {
 792                 field = hist_data->fields[i];
 793                 if (field && field->flags & HIST_FIELD_FL_VAR) {
 794                         if (find_any_var_ref(hist_data, field->var.idx)) {
 795                                 found = true;
 796                                 break;
 797                         }
 798                 }
 799         }
 800
 801         return found;
 802 }
 803
 804 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
 805 {
 806         struct trace_array *tr = hist_data->event_file->tr;
 807         struct hist_var_data *var_data, *found = NULL;
 808
 809         list_for_each_entry(var_data, &tr->hist_vars, list) {
 810                 if (var_data->hist_data == hist_data) {
 811                         found = var_data;
 812                         break;
 813                 }
 814         }
 815
 816         return found;
 817 }
 818
 819 static bool field_has_hist_vars(struct hist_field *hist_field,
 820                                 unsigned int level)
 821 {
 822         int i;
 823
 824         if (level > 3)
 825                 return false;
 826
 827         if (!hist_field)
 828                 return false;
 829
 830         if (hist_field->flags & HIST_FIELD_FL_VAR ||
 831             hist_field->flags & HIST_FIELD_FL_VAR_REF)
 832                 return true;
 833
 834         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
 835                 struct hist_field *operand;
 836
 837                 operand = hist_field->operands[i];
 838                 if (field_has_hist_vars(operand, level + 1))
 839                         return true;
 840         }
 841
 842         return false;
 843 }
 844
 845 static bool has_hist_vars(struct hist_trigger_data *hist_data)
 846 {
 847         struct hist_field *hist_field;
 848         int i;
 849
 850         for_each_hist_field(i, hist_data) {
 851                 hist_field = hist_data->fields[i];
 852                 if (field_has_hist_vars(hist_field, 0))
 853                         return true;
 854         }
 855
 856         return false;
 857 }
 858
 859 static int save_hist_vars(struct hist_trigger_data *hist_data)
 860 {
 861         struct trace_array *tr = hist_data->event_file->tr;
 862         struct hist_var_data *var_data;
 863
 864         var_data = find_hist_vars(hist_data);
 865         if (var_data)
 866                 return 0;
 867
 868         if (tracing_check_open_get_tr(tr))
 869                 return -ENODEV;
 870
 871         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
 872         if (!var_data) {
 873                 trace_array_put(tr);
 874                 return -ENOMEM;
 875         }
 876
 877         var_data->hist_data = hist_data;
 878         list_add(&var_data->list, &tr->hist_vars);
 879
 880         return 0;
 881 }
 882
 883 static void remove_hist_vars(struct hist_trigger_data *hist_data)
 884 {
 885         struct trace_array *tr = hist_data->event_file->tr;
 886         struct hist_var_data *var_data;
 887
 888         var_data = find_hist_vars(hist_data);
 889         if (!var_data)
 890                 return;
 891
 892         if (WARN_ON(check_var_refs(hist_data)))
 893                 return;
 894
 895         list_del(&var_data->list);
 896
 897         kfree(var_data);
 898
 899         trace_array_put(tr);
 900 }
 901
 902 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
 903                                          const char *var_name)
 904 {
 905         struct hist_field *hist_field, *found = NULL;
 906         int i;
 907
 908         for_each_hist_field(i, hist_data) {
 909                 hist_field = hist_data->fields[i];
 910                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
 911                     strcmp(hist_field->var.name, var_name) == 0) {
 912                         found = hist_field;
 913                         break;
 914                 }
 915         }
 916
 917         return found;
 918 }
 919
 920 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
 921                                    struct trace_event_file *file,
 922                                    const char *var_name)
 923 {
 924         struct hist_trigger_data *test_data;
 925         struct event_trigger_data *test;
 926         struct hist_field *hist_field;
 927
 928         lockdep_assert_held(&event_mutex);
 929
 930         hist_field = find_var_field(hist_data, var_name);
 931         if (hist_field)
 932                 return hist_field;
 933
 934         list_for_each_entry(test, &file->triggers, list) {
 935                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 936                         test_data = test->private_data;
 937                         hist_field = find_var_field(test_data, var_name);
 938                         if (hist_field)
 939                                 return hist_field;
 940                 }
 941         }
 942
 943         return NULL;
 944 }
 945
 946 static struct trace_event_file *find_var_file(struct trace_array *tr,
 947                                               char *system,
 948                                               char *event_name,
 949                                               char *var_name)
 950 {
 951         struct hist_trigger_data *var_hist_data;
 952         struct hist_var_data *var_data;
 953         struct trace_event_file *file, *found = NULL;
 954
 955         if (system)
 956                 return find_event_file(tr, system, event_name);
 957
 958         list_for_each_entry(var_data, &tr->hist_vars, list) {
 959                 var_hist_data = var_data->hist_data;
 960                 file = var_hist_data->event_file;
 961                 if (file == found)
 962                         continue;
 963
 964                 if (find_var_field(var_hist_data, var_name)) {
 965                         if (found) {
 966                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
 967                                 return NULL;
 968                         }
 969
 970                         found = file;
 971                 }
 972         }
 973
 974         return found;
 975 }
 976
 977 static struct hist_field *find_file_var(struct trace_event_file *file,
 978                                         const char *var_name)
 979 {
 980         struct hist_trigger_data *test_data;
 981         struct event_trigger_data *test;
 982         struct hist_field *hist_field;
 983
 984         lockdep_assert_held(&event_mutex);
 985
 986         list_for_each_entry(test, &file->triggers, list) {
 987                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
 988                         test_data = test->private_data;
 989                         hist_field = find_var_field(test_data, var_name);
 990                         if (hist_field)
 991                                 return hist_field;
 992                 }
 993         }
 994
 995         return NULL;
 996 }
 997
 998 static struct hist_field *
 999 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1000 {
1001         struct trace_array *tr = hist_data->event_file->tr;
1002         struct hist_field *hist_field, *found = NULL;
1003         struct trace_event_file *file;
1004         unsigned int i;
1005
1006         for (i = 0; i < hist_data->n_actions; i++) {
1007                 struct action_data *data = hist_data->actions[i];
1008
1009                 if (data->handler == HANDLER_ONMATCH) {
1010                         char *system = data->match_data.event_system;
1011                         char *event_name = data->match_data.event;
1012
1013                         file = find_var_file(tr, system, event_name, var_name);
1014                         if (!file)
1015                                 continue;
1016                         hist_field = find_file_var(file, var_name);
1017                         if (hist_field) {
1018                                 if (found) {
1019                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1020                                                  errpos(var_name));
1021                                         return ERR_PTR(-EINVAL);
1022                                 }
1023
1024                                 found = hist_field;
1025                         }
1026                 }
1027         }
1028         return found;
1029 }
1030
1031 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1032                                          char *system,
1033                                          char *event_name,
1034                                          char *var_name)
1035 {
1036         struct trace_array *tr = hist_data->event_file->tr;
1037         struct hist_field *hist_field = NULL;
1038         struct trace_event_file *file;
1039
1040         if (!system || !event_name) {
1041                 hist_field = find_match_var(hist_data, var_name);
1042                 if (IS_ERR(hist_field))
1043                         return NULL;
1044                 if (hist_field)
1045                         return hist_field;
1046         }
1047
1048         file = find_var_file(tr, system, event_name, var_name);
1049         if (!file)
1050                 return NULL;
1051
1052         hist_field = find_file_var(file, var_name);
1053
1054         return hist_field;
1055 }
1056
1057 static u64 hist_field_var_ref(struct hist_field *hist_field,
1058                               struct tracing_map_elt *elt,
1059                               struct trace_buffer *buffer,
1060                               struct ring_buffer_event *rbe,
1061                               void *event)
1062 {
1063         struct hist_elt_data *elt_data;
1064         u64 var_val = 0;
1065
1066         if (WARN_ON_ONCE(!elt))
1067                 return var_val;
1068
1069         elt_data = elt->private_data;
1070         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1071
1072         return var_val;
1073 }
1074
1075 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1076                              u64 *var_ref_vals, bool self)
1077 {
1078         struct hist_trigger_data *var_data;
1079         struct tracing_map_elt *var_elt;
1080         struct hist_field *hist_field;
1081         unsigned int i, var_idx;
1082         bool resolved = true;
1083         u64 var_val = 0;
1084
1085         for (i = 0; i < hist_data->n_var_refs; i++) {
1086                 hist_field = hist_data->var_refs[i];
1087                 var_idx = hist_field->var.idx;
1088                 var_data = hist_field->var.hist_data;
1089
1090                 if (var_data == NULL) {
1091                         resolved = false;
1092                         break;
1093                 }
1094
1095                 if ((self && var_data != hist_data) ||
1096                     (!self && var_data == hist_data))
1097                         continue;
1098
1099                 var_elt = tracing_map_lookup(var_data->map, key);
1100                 if (!var_elt) {
1101                         resolved = false;
1102                         break;
1103                 }
1104
1105                 if (!tracing_map_var_set(var_elt, var_idx)) {
1106                         resolved = false;
1107                         break;
1108                 }
1109
1110                 if (self || !hist_field->read_once)
1111                         var_val = tracing_map_read_var(var_elt, var_idx);
1112                 else
1113                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1114
1115                 var_ref_vals[i] = var_val;
1116         }
1117
1118         return resolved;
1119 }
1120
1121 static const char *hist_field_name(struct hist_field *field,
1122                                    unsigned int level)
1123 {
1124         const char *field_name = "";
1125
1126         if (WARN_ON_ONCE(!field))
1127                 return field_name;
1128
1129         if (level > 1)
1130                 return field_name;
1131
1132         if (field->field)
1133                 field_name = field->field->name;
1134         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1135                  field->flags & HIST_FIELD_FL_ALIAS ||
1136                  field->flags & HIST_FIELD_FL_BUCKET)
1137                 field_name = hist_field_name(field->operands[0], ++level);
1138         else if (field->flags & HIST_FIELD_FL_CPU)
1139                 field_name = "common_cpu";
1140         else if (field->flags & HIST_FIELD_FL_EXPR ||
1141                  field->flags & HIST_FIELD_FL_VAR_REF) {
1142                 if (field->system) {
1143                         static char full_name[MAX_FILTER_STR_VAL];
1144
1145                         strcat(full_name, field->system);
1146                         strcat(full_name, ".");
1147                         strcat(full_name, field->event_name);
1148                         strcat(full_name, ".");
1149                         strcat(full_name, field->name);
1150                         field_name = full_name;
1151                 } else
1152                         field_name = field->name;
1153         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1154                 field_name = "common_timestamp";
1155
1156         if (field_name == NULL)
1157                 field_name = "";
1158
1159         return field_name;
1160 }
1161
1162 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
1163 {
1164         hist_field_fn_t fn = NULL;
1165
1166         switch (field_size) {
1167         case 8:
1168                 if (field_is_signed)
1169                         fn = hist_field_s64;
1170                 else
1171                         fn = hist_field_u64;
1172                 break;
1173         case 4:
1174                 if (field_is_signed)
1175                         fn = hist_field_s32;
1176                 else
1177                         fn = hist_field_u32;
1178                 break;
1179         case 2:
1180                 if (field_is_signed)
1181                         fn = hist_field_s16;
1182                 else
1183                         fn = hist_field_u16;
1184                 break;
1185         case 1:
1186                 if (field_is_signed)
1187                         fn = hist_field_s8;
1188                 else
1189                         fn = hist_field_u8;
1190                 break;
1191         }
1192
1193         return fn;
1194 }
1195
1196 static int parse_map_size(char *str)
1197 {
1198         unsigned long size, map_bits;
1199         int ret;
1200
1201         ret = kstrtoul(str, 0, &size);
1202         if (ret)
1203                 goto out;
1204
1205         map_bits = ilog2(roundup_pow_of_two(size));
1206         if (map_bits < TRACING_MAP_BITS_MIN ||
1207             map_bits > TRACING_MAP_BITS_MAX)
1208                 ret = -EINVAL;
1209         else
1210                 ret = map_bits;
1211  out:
1212         return ret;
1213 }
1214
1215 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1216 {
1217         unsigned int i;
1218
1219         if (!attrs)
1220                 return;
1221
1222         for (i = 0; i < attrs->n_assignments; i++)
1223                 kfree(attrs->assignment_str[i]);
1224
1225         for (i = 0; i < attrs->n_actions; i++)
1226                 kfree(attrs->action_str[i]);
1227
1228         kfree(attrs->name);
1229         kfree(attrs->sort_key_str);
1230         kfree(attrs->keys_str);
1231         kfree(attrs->vals_str);
1232         kfree(attrs->clock);
1233         kfree(attrs);
1234 }
1235
1236 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1237 {
1238         int ret = -EINVAL;
1239
1240         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1241                 return ret;
1242
1243         if ((str_has_prefix(str, "onmatch(")) ||
1244             (str_has_prefix(str, "onmax(")) ||
1245             (str_has_prefix(str, "onchange("))) {
1246                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1247                 if (!attrs->action_str[attrs->n_actions]) {
1248                         ret = -ENOMEM;
1249                         return ret;
1250                 }
1251                 attrs->n_actions++;
1252                 ret = 0;
1253         }
1254         return ret;
1255 }
1256
1257 static int parse_assignment(struct trace_array *tr,
1258                             char *str, struct hist_trigger_attrs *attrs)
1259 {
1260         int len, ret = 0;
1261
1262         if ((len = str_has_prefix(str, "key=")) ||
1263             (len = str_has_prefix(str, "keys="))) {
1264                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1265                 if (!attrs->keys_str) {
1266                         ret = -ENOMEM;
1267                         goto out;
1268                 }
1269         } else if ((len = str_has_prefix(str, "val=")) ||
1270                    (len = str_has_prefix(str, "vals=")) ||
1271                    (len = str_has_prefix(str, "values="))) {
1272                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1273                 if (!attrs->vals_str) {
1274                         ret = -ENOMEM;
1275                         goto out;
1276                 }
1277         } else if ((len = str_has_prefix(str, "sort="))) {
1278                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1279                 if (!attrs->sort_key_str) {
1280                         ret = -ENOMEM;
1281                         goto out;
1282                 }
1283         } else if (str_has_prefix(str, "name=")) {
1284                 attrs->name = kstrdup(str, GFP_KERNEL);
1285                 if (!attrs->name) {
1286                         ret = -ENOMEM;
1287                         goto out;
1288                 }
1289         } else if ((len = str_has_prefix(str, "clock="))) {
1290                 str += len;
1291
1292                 str = strstrip(str);
1293                 attrs->clock = kstrdup(str, GFP_KERNEL);
1294                 if (!attrs->clock) {
1295                         ret = -ENOMEM;
1296                         goto out;
1297                 }
1298         } else if ((len = str_has_prefix(str, "size="))) {
1299                 int map_bits = parse_map_size(str + len);
1300
1301                 if (map_bits < 0) {
1302                         ret = map_bits;
1303                         goto out;
1304                 }
1305                 attrs->map_bits = map_bits;
1306         } else {
1307                 char *assignment;
1308
1309                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1310                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1311                         ret = -EINVAL;
1312                         goto out;
1313                 }
1314
1315                 assignment = kstrdup(str, GFP_KERNEL);
1316                 if (!assignment) {
1317                         ret = -ENOMEM;
1318                         goto out;
1319                 }
1320
1321                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1322         }
1323  out:
1324         return ret;
1325 }
1326
1327 static struct hist_trigger_attrs *
1328 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1329 {
1330         struct hist_trigger_attrs *attrs;
1331         int ret = 0;
1332
1333         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1334         if (!attrs)
1335                 return ERR_PTR(-ENOMEM);
1336
1337         while (trigger_str) {
1338                 char *str = strsep(&trigger_str, ":");
1339                 char *rhs;
1340
1341                 rhs = strchr(str, '=');
1342                 if (rhs) {
1343                         if (!strlen(++rhs)) {
1344                                 ret = -EINVAL;
1345                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1346                                 goto free;
1347                         }
1348                         ret = parse_assignment(tr, str, attrs);
1349                         if (ret)
1350                                 goto free;
1351                 } else if (strcmp(str, "pause") == 0)
1352                         attrs->pause = true;
1353                 else if ((strcmp(str, "cont") == 0) ||
1354                          (strcmp(str, "continue") == 0))
1355                         attrs->cont = true;
1356                 else if (strcmp(str, "clear") == 0)
1357                         attrs->clear = true;
1358                 else {
1359                         ret = parse_action(str, attrs);
1360                         if (ret)
1361                                 goto free;
1362                 }
1363         }
1364
1365         if (!attrs->keys_str) {
1366                 ret = -EINVAL;
1367                 goto free;
1368         }
1369
1370         if (!attrs->clock) {
1371                 attrs->clock = kstrdup("global", GFP_KERNEL);
1372                 if (!attrs->clock) {
1373                         ret = -ENOMEM;
1374                         goto free;
1375                 }
1376         }
1377
1378         return attrs;
1379  free:
1380         destroy_hist_trigger_attrs(attrs);
1381
1382         return ERR_PTR(ret);
1383 }
1384
1385 static inline void save_comm(char *comm, struct task_struct *task)
1386 {
1387         if (!task->pid) {
1388                 strcpy(comm, "<idle>");
1389                 return;
1390         }
1391
1392         if (WARN_ON_ONCE(task->pid < 0)) {
1393                 strcpy(comm, "<XXX>");
1394                 return;
1395         }
1396
1397         strncpy(comm, task->comm, TASK_COMM_LEN);
1398 }
1399
1400 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1401 {
1402         unsigned int i;
1403
1404         for (i = 0; i < elt_data->n_field_var_str; i++)
1405                 kfree(elt_data->field_var_str[i]);
1406
1407         kfree(elt_data->field_var_str);
1408
1409         kfree(elt_data->comm);
1410         kfree(elt_data);
1411 }
1412
1413 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1414 {
1415         struct hist_elt_data *elt_data = elt->private_data;
1416
1417         hist_elt_data_free(elt_data);
1418 }
1419
1420 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1421 {
1422         struct hist_trigger_data *hist_data = elt->map->private_data;
1423         unsigned int size = TASK_COMM_LEN;
1424         struct hist_elt_data *elt_data;
1425         struct hist_field *hist_field;
1426         unsigned int i, n_str;
1427
1428         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1429         if (!elt_data)
1430                 return -ENOMEM;
1431
1432         for_each_hist_field(i, hist_data) {
1433                 hist_field = hist_data->fields[i];
1434
1435                 if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1436                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1437                         if (!elt_data->comm) {
1438                                 kfree(elt_data);
1439                                 return -ENOMEM;
1440                         }
1441                         break;
1442                 }
1443         }
1444
1445         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1446                 hist_data->n_var_str;
1447         if (n_str > SYNTH_FIELDS_MAX) {
1448                 hist_elt_data_free(elt_data);
1449                 return -EINVAL;
1450         }
1451
1452         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1453
1454         size = STR_VAR_LEN_MAX;
1455
1456         elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1457         if (!elt_data->field_var_str) {
1458                 hist_elt_data_free(elt_data);
1459                 return -EINVAL;
1460         }
1461         elt_data->n_field_var_str = n_str;
1462
1463         for (i = 0; i < n_str; i++) {
1464                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1465                 if (!elt_data->field_var_str[i]) {
1466                         hist_elt_data_free(elt_data);
1467                         return -ENOMEM;
1468                 }
1469         }
1470
1471         elt->private_data = elt_data;
1472
1473         return 0;
1474 }
1475
1476 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1477 {
1478         struct hist_elt_data *elt_data = elt->private_data;
1479
1480         if (elt_data->comm)
1481                 save_comm(elt_data->comm, current);
1482 }
1483
1484 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1485         .elt_alloc      = hist_trigger_elt_data_alloc,
1486         .elt_free       = hist_trigger_elt_data_free,
1487         .elt_init       = hist_trigger_elt_data_init,
1488 };
1489
1490 static const char *get_hist_field_flags(struct hist_field *hist_field)
1491 {
1492         const char *flags_str = NULL;
1493
1494         if (hist_field->flags & HIST_FIELD_FL_HEX)
1495                 flags_str = "hex";
1496         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1497                 flags_str = "sym";
1498         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1499                 flags_str = "sym-offset";
1500         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1501                 flags_str = "execname";
1502         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1503                 flags_str = "syscall";
1504         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1505                 flags_str = "log2";
1506         else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1507                 flags_str = "buckets";
1508         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1509                 flags_str = "usecs";
1510
1511         return flags_str;
1512 }
1513
1514 static void expr_field_str(struct hist_field *field, char *expr)
1515 {
1516         if (field->flags & HIST_FIELD_FL_VAR_REF)
1517                 strcat(expr, "$");
1518
1519         strcat(expr, hist_field_name(field, 0));
1520
1521         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1522                 const char *flags_str = get_hist_field_flags(field);
1523
1524                 if (flags_str) {
1525                         strcat(expr, ".");
1526                         strcat(expr, flags_str);
1527                 }
1528         }
1529 }
1530
1531 static char *expr_str(struct hist_field *field, unsigned int level)
1532 {
1533         char *expr;
1534
1535         if (level > 1)
1536                 return NULL;
1537
1538         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1539         if (!expr)
1540                 return NULL;
1541
1542         if (!field->operands[0]) {
1543                 expr_field_str(field, expr);
1544                 return expr;
1545         }
1546
1547         if (field->operator == FIELD_OP_UNARY_MINUS) {
1548                 char *subexpr;
1549
1550                 strcat(expr, "-(");
1551                 subexpr = expr_str(field->operands[0], ++level);
1552                 if (!subexpr) {
1553                         kfree(expr);
1554                         return NULL;
1555                 }
1556                 strcat(expr, subexpr);
1557                 strcat(expr, ")");
1558
1559                 kfree(subexpr);
1560
1561                 return expr;
1562         }
1563
1564         expr_field_str(field->operands[0], expr);
1565
1566         switch (field->operator) {
1567         case FIELD_OP_MINUS:
1568                 strcat(expr, "-");
1569                 break;
1570         case FIELD_OP_PLUS:
1571                 strcat(expr, "+");
1572                 break;
1573         default:
1574                 kfree(expr);
1575                 return NULL;
1576         }
1577
1578         expr_field_str(field->operands[1], expr);
1579
1580         return expr;
1581 }
1582
1583 static int contains_operator(char *str)
1584 {
1585         enum field_op_id field_op = FIELD_OP_NONE;
1586         char *op;
1587
1588         op = strpbrk(str, "+-");
1589         if (!op)
1590                 return FIELD_OP_NONE;
1591
1592         switch (*op) {
1593         case '-':
1594                 /*
1595                  * Unfortunately, the modifier ".sym-offset"
1596                  * can confuse things.
1597                  */
1598                 if (op - str >= 4 && !strncmp(op - 4, ".sym-offset", 11))
1599                         return FIELD_OP_NONE;
1600
1601                 if (*str == '-')
1602                         field_op = FIELD_OP_UNARY_MINUS;
1603                 else
1604                         field_op = FIELD_OP_MINUS;
1605                 break;
1606         case '+':
1607                 field_op = FIELD_OP_PLUS;
1608                 break;
1609         default:
1610                 break;
1611         }
1612
1613         return field_op;
1614 }
1615
1616 static void get_hist_field(struct hist_field *hist_field)
1617 {
1618         hist_field->ref++;
1619 }
1620
1621 static void __destroy_hist_field(struct hist_field *hist_field)
1622 {
1623         if (--hist_field->ref > 1)
1624                 return;
1625
1626         kfree(hist_field->var.name);
1627         kfree(hist_field->name);
1628
1629         /* Can likely be a const */
1630         kfree_const(hist_field->type);
1631
1632         kfree(hist_field->system);
1633         kfree(hist_field->event_name);
1634
1635         kfree(hist_field);
1636 }
1637
1638 static void destroy_hist_field(struct hist_field *hist_field,
1639                                unsigned int level)
1640 {
1641         unsigned int i;
1642
1643         if (level > 3)
1644                 return;
1645
1646         if (!hist_field)
1647                 return;
1648
1649         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1650                 return; /* var refs will be destroyed separately */
1651
1652         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1653                 destroy_hist_field(hist_field->operands[i], level + 1);
1654
1655         __destroy_hist_field(hist_field);
1656 }
1657
1658 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1659                                             struct ftrace_event_field *field,
1660                                             unsigned long flags,
1661                                             char *var_name)
1662 {
1663         struct hist_field *hist_field;
1664
1665         if (field && is_function_field(field))
1666                 return NULL;
1667
1668         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1669         if (!hist_field)
1670                 return NULL;
1671
1672         hist_field->ref = 1;
1673
1674         hist_field->hist_data = hist_data;
1675
1676         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1677                 goto out; /* caller will populate */
1678
1679         if (flags & HIST_FIELD_FL_VAR_REF) {
1680                 hist_field->fn = hist_field_var_ref;
1681                 goto out;
1682         }
1683
1684         if (flags & HIST_FIELD_FL_HITCOUNT) {
1685                 hist_field->fn = hist_field_counter;
1686                 hist_field->size = sizeof(u64);
1687                 hist_field->type = "u64";
1688                 goto out;
1689         }
1690
1691         if (flags & HIST_FIELD_FL_STACKTRACE) {
1692                 hist_field->fn = hist_field_none;
1693                 goto out;
1694         }
1695
1696         if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
1697                 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
1698                 hist_field->fn = flags & HIST_FIELD_FL_LOG2 ? hist_field_log2 :
1699                         hist_field_bucket;
1700                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1701                 if (!hist_field->operands[0])
1702                         goto free;
1703                 hist_field->size = hist_field->operands[0]->size;
1704                 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
1705                 if (!hist_field->type)
1706                         goto free;
1707                 goto out;
1708         }
1709
1710         if (flags & HIST_FIELD_FL_TIMESTAMP) {
1711                 hist_field->fn = hist_field_timestamp;
1712                 hist_field->size = sizeof(u64);
1713                 hist_field->type = "u64";
1714                 goto out;
1715         }
1716
1717         if (flags & HIST_FIELD_FL_CPU) {
1718                 hist_field->fn = hist_field_cpu;
1719                 hist_field->size = sizeof(int);
1720                 hist_field->type = "unsigned int";
1721                 goto out;
1722         }
1723
1724         if (WARN_ON_ONCE(!field))
1725                 goto out;
1726
1727         /* Pointers to strings are just pointers and dangerous to dereference */
1728         if (is_string_field(field) &&
1729             (field->filter_type != FILTER_PTR_STRING)) {
1730                 flags |= HIST_FIELD_FL_STRING;
1731
1732                 hist_field->size = MAX_FILTER_STR_VAL;
1733                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
1734                 if (!hist_field->type)
1735                         goto free;
1736
1737                 if (field->filter_type == FILTER_STATIC_STRING) {
1738                         hist_field->fn = hist_field_string;
1739                         hist_field->size = field->size;
1740                 } else if (field->filter_type == FILTER_DYN_STRING)
1741                         hist_field->fn = hist_field_dynstring;
1742                 else
1743                         hist_field->fn = hist_field_pstring;
1744         } else {
1745                 hist_field->size = field->size;
1746                 hist_field->is_signed = field->is_signed;
1747                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
1748                 if (!hist_field->type)
1749                         goto free;
1750
1751                 hist_field->fn = select_value_fn(field->size,
1752                                                  field->is_signed);
1753                 if (!hist_field->fn) {
1754                         destroy_hist_field(hist_field, 0);
1755                         return NULL;
1756                 }
1757         }
1758  out:
1759         hist_field->field = field;
1760         hist_field->flags = flags;
1761
1762         if (var_name) {
1763                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
1764                 if (!hist_field->var.name)
1765                         goto free;
1766         }
1767
1768         return hist_field;
1769  free:
1770         destroy_hist_field(hist_field, 0);
1771         return NULL;
1772 }
1773
1774 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
1775 {
1776         unsigned int i;
1777
1778         for (i = 0; i < HIST_FIELDS_MAX; i++) {
1779                 if (hist_data->fields[i]) {
1780                         destroy_hist_field(hist_data->fields[i], 0);
1781                         hist_data->fields[i] = NULL;
1782                 }
1783         }
1784
1785         for (i = 0; i < hist_data->n_var_refs; i++) {
1786                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
1787                 __destroy_hist_field(hist_data->var_refs[i]);
1788                 hist_data->var_refs[i] = NULL;
1789         }
1790 }
1791
1792 static int init_var_ref(struct hist_field *ref_field,
1793                         struct hist_field *var_field,
1794                         char *system, char *event_name)
1795 {
1796         int err = 0;
1797
1798         ref_field->var.idx = var_field->var.idx;
1799         ref_field->var.hist_data = var_field->hist_data;
1800         ref_field->size = var_field->size;
1801         ref_field->is_signed = var_field->is_signed;
1802         ref_field->flags |= var_field->flags &
1803                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
1804
1805         if (system) {
1806                 ref_field->system = kstrdup(system, GFP_KERNEL);
1807                 if (!ref_field->system)
1808                         return -ENOMEM;
1809         }
1810
1811         if (event_name) {
1812                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
1813                 if (!ref_field->event_name) {
1814                         err = -ENOMEM;
1815                         goto free;
1816                 }
1817         }
1818
1819         if (var_field->var.name) {
1820                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
1821                 if (!ref_field->name) {
1822                         err = -ENOMEM;
1823                         goto free;
1824                 }
1825         } else if (var_field->name) {
1826                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
1827                 if (!ref_field->name) {
1828                         err = -ENOMEM;
1829                         goto free;
1830                 }
1831         }
1832
1833         ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
1834         if (!ref_field->type) {
1835                 err = -ENOMEM;
1836                 goto free;
1837         }
1838  out:
1839         return err;
1840  free:
1841         kfree(ref_field->system);
1842         ref_field->system = NULL;
1843         kfree(ref_field->event_name);
1844         ref_field->event_name = NULL;
1845         kfree(ref_field->name);
1846         ref_field->name = NULL;
1847
1848         goto out;
1849 }
1850
1851 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
1852                             struct hist_field *var_field)
1853 {
1854         struct hist_field *ref_field;
1855         int i;
1856
1857         for (i = 0; i < hist_data->n_var_refs; i++) {
1858                 ref_field = hist_data->var_refs[i];
1859                 if (ref_field->var.idx == var_field->var.idx &&
1860                     ref_field->var.hist_data == var_field->hist_data)
1861                         return i;
1862         }
1863
1864         return -ENOENT;
1865 }
1866
1867 /**
1868  * create_var_ref - Create a variable reference and attach it to trigger
1869  * @hist_data: The trigger that will be referencing the variable
1870  * @var_field: The VAR field to create a reference to
1871  * @system: The optional system string
1872  * @event_name: The optional event_name string
1873  *
1874  * Given a variable hist_field, create a VAR_REF hist_field that
1875  * represents a reference to it.
1876  *
1877  * This function also adds the reference to the trigger that
1878  * now references the variable.
1879  *
1880  * Return: The VAR_REF field if successful, NULL if not
1881  */
1882 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
1883                                          struct hist_field *var_field,
1884                                          char *system, char *event_name)
1885 {
1886         unsigned long flags = HIST_FIELD_FL_VAR_REF;
1887         struct hist_field *ref_field;
1888         int i;
1889
1890         /* Check if the variable already exists */
1891         for (i = 0; i < hist_data->n_var_refs; i++) {
1892                 ref_field = hist_data->var_refs[i];
1893                 if (ref_field->var.idx == var_field->var.idx &&
1894                     ref_field->var.hist_data == var_field->hist_data) {
1895                         get_hist_field(ref_field);
1896                         return ref_field;
1897                 }
1898         }
1899         /* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */
1900         if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX)
1901                 return NULL;
1902         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
1903         if (ref_field) {
1904                 if (init_var_ref(ref_field, var_field, system, event_name)) {
1905                         destroy_hist_field(ref_field, 0);
1906                         return NULL;
1907                 }
1908
1909                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
1910                 ref_field->var_ref_idx = hist_data->n_var_refs++;
1911         }
1912
1913         return ref_field;
1914 }
1915
1916 static bool is_var_ref(char *var_name)
1917 {
1918         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
1919                 return false;
1920
1921         return true;
1922 }
1923
1924 static char *field_name_from_var(struct hist_trigger_data *hist_data,
1925                                  char *var_name)
1926 {
1927         char *name, *field;
1928         unsigned int i;
1929
1930         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
1931                 name = hist_data->attrs->var_defs.name[i];
1932
1933                 if (strcmp(var_name, name) == 0) {
1934                         field = hist_data->attrs->var_defs.expr[i];
1935                         if (contains_operator(field) || is_var_ref(field))
1936                                 continue;
1937                         return field;
1938                 }
1939         }
1940
1941         return NULL;
1942 }
1943
1944 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
1945                                  char *system, char *event_name,
1946                                  char *var_name)
1947 {
1948         struct trace_event_call *call;
1949
1950         if (system && event_name) {
1951                 call = hist_data->event_file->event_call;
1952
1953                 if (strcmp(system, call->class->system) != 0)
1954                         return NULL;
1955
1956                 if (strcmp(event_name, trace_event_name(call)) != 0)
1957                         return NULL;
1958         }
1959
1960         if (!!system != !!event_name)
1961                 return NULL;
1962
1963         if (!is_var_ref(var_name))
1964                 return NULL;
1965
1966         var_name++;
1967
1968         return field_name_from_var(hist_data, var_name);
1969 }
1970
1971 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
1972                                         char *system, char *event_name,
1973                                         char *var_name)
1974 {
1975         struct hist_field *var_field = NULL, *ref_field = NULL;
1976         struct trace_array *tr = hist_data->event_file->tr;
1977
1978         if (!is_var_ref(var_name))
1979                 return NULL;
1980
1981         var_name++;
1982
1983         var_field = find_event_var(hist_data, system, event_name, var_name);
1984         if (var_field)
1985                 ref_field = create_var_ref(hist_data, var_field,
1986                                            system, event_name);
1987
1988         if (!ref_field)
1989                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
1990
1991         return ref_field;
1992 }
1993
1994 static struct ftrace_event_field *
1995 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
1996             char *field_str, unsigned long *flags, unsigned long *buckets)
1997 {
1998         struct ftrace_event_field *field = NULL;
1999         char *field_name, *modifier, *str;
2000         struct trace_array *tr = file->tr;
2001
2002         modifier = str = kstrdup(field_str, GFP_KERNEL);
2003         if (!modifier)
2004                 return ERR_PTR(-ENOMEM);
2005
2006         field_name = strsep(&modifier, ".");
2007         if (modifier) {
2008                 if (strcmp(modifier, "hex") == 0)
2009                         *flags |= HIST_FIELD_FL_HEX;
2010                 else if (strcmp(modifier, "sym") == 0)
2011                         *flags |= HIST_FIELD_FL_SYM;
2012                 else if (strcmp(modifier, "sym-offset") == 0)
2013                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2014                 else if ((strcmp(modifier, "execname") == 0) &&
2015                          (strcmp(field_name, "common_pid") == 0))
2016                         *flags |= HIST_FIELD_FL_EXECNAME;
2017                 else if (strcmp(modifier, "syscall") == 0)
2018                         *flags |= HIST_FIELD_FL_SYSCALL;
2019                 else if (strcmp(modifier, "log2") == 0)
2020                         *flags |= HIST_FIELD_FL_LOG2;
2021                 else if (strcmp(modifier, "usecs") == 0)
2022                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2023                 else if (strncmp(modifier, "bucket", 6) == 0) {
2024                         int ret;
2025
2026                         modifier += 6;
2027
2028                         if (*modifier == 's')
2029                                 modifier++;
2030                         if (*modifier != '=')
2031                                 goto error;
2032                         modifier++;
2033                         ret = kstrtoul(modifier, 0, buckets);
2034                         if (ret || !(*buckets))
2035                                 goto error;
2036                         *flags |= HIST_FIELD_FL_BUCKET;
2037                 } else {
2038  error:
2039                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2040                         field = ERR_PTR(-EINVAL);
2041                         goto out;
2042                 }
2043         }
2044
2045         if (strcmp(field_name, "common_timestamp") == 0) {
2046                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2047                 hist_data->enable_timestamps = true;
2048                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2049                         hist_data->attrs->ts_in_usecs = true;
2050         } else if (strcmp(field_name, "common_cpu") == 0)
2051                 *flags |= HIST_FIELD_FL_CPU;
2052         else {
2053                 field = trace_find_event_field(file->event_call, field_name);
2054                 if (!field || !field->size) {
2055                         /*
2056                          * For backward compatibility, if field_name
2057                          * was "cpu", then we treat this the same as
2058                          * common_cpu. This also works for "CPU".
2059                          */
2060                         if (field && field->filter_type == FILTER_CPU) {
2061                                 *flags |= HIST_FIELD_FL_CPU;
2062                         } else {
2063                                 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2064                                          errpos(field_name));
2065                                 field = ERR_PTR(-EINVAL);
2066                                 goto out;
2067                         }
2068                 }
2069         }
2070  out:
2071         kfree(str);
2072
2073         return field;
2074 }
2075
2076 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2077                                        struct hist_field *var_ref,
2078                                        char *var_name)
2079 {
2080         struct hist_field *alias = NULL;
2081         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2082
2083         alias = create_hist_field(hist_data, NULL, flags, var_name);
2084         if (!alias)
2085                 return NULL;
2086
2087         alias->fn = var_ref->fn;
2088         alias->operands[0] = var_ref;
2089
2090         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2091                 destroy_hist_field(alias, 0);
2092                 return NULL;
2093         }
2094
2095         alias->var_ref_idx = var_ref->var_ref_idx;
2096
2097         return alias;
2098 }
2099
2100 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2101                                      struct trace_event_file *file, char *str,
2102                                      unsigned long *flags, char *var_name)
2103 {
2104         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2105         struct ftrace_event_field *field = NULL;
2106         struct hist_field *hist_field = NULL;
2107         unsigned long buckets = 0;
2108         int ret = 0;
2109
2110         s = strchr(str, '.');
2111         if (s) {
2112                 s = strchr(++s, '.');
2113                 if (s) {
2114                         ref_system = strsep(&str, ".");
2115                         if (!str) {
2116                                 ret = -EINVAL;
2117                                 goto out;
2118                         }
2119                         ref_event = strsep(&str, ".");
2120                         if (!str) {
2121                                 ret = -EINVAL;
2122                                 goto out;
2123                         }
2124                         ref_var = str;
2125                 }
2126         }
2127
2128         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2129         if (!s) {
2130                 hist_field = parse_var_ref(hist_data, ref_system,
2131                                            ref_event, ref_var);
2132                 if (hist_field) {
2133                         if (var_name) {
2134                                 hist_field = create_alias(hist_data, hist_field, var_name);
2135                                 if (!hist_field) {
2136                                         ret = -ENOMEM;
2137                                         goto out;
2138                                 }
2139                         }
2140                         return hist_field;
2141                 }
2142         } else
2143                 str = s;
2144
2145         field = parse_field(hist_data, file, str, flags, &buckets);
2146         if (IS_ERR(field)) {
2147                 ret = PTR_ERR(field);
2148                 goto out;
2149         }
2150
2151         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2152         if (!hist_field) {
2153                 ret = -ENOMEM;
2154                 goto out;
2155         }
2156         hist_field->buckets = buckets;
2157
2158         return hist_field;
2159  out:
2160         return ERR_PTR(ret);
2161 }
2162
2163 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2164                                      struct trace_event_file *file,
2165                                      char *str, unsigned long flags,
2166                                      char *var_name, unsigned int level);
2167
2168 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2169                                       struct trace_event_file *file,
2170                                       char *str, unsigned long flags,
2171                                       char *var_name, unsigned int level)
2172 {
2173         struct hist_field *operand1, *expr = NULL;
2174         unsigned long operand_flags;
2175         int ret = 0;
2176         char *s;
2177
2178         /* we support only -(xxx) i.e. explicit parens required */
2179
2180         if (level > 3) {
2181                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2182                 ret = -EINVAL;
2183                 goto free;
2184         }
2185
2186         str++; /* skip leading '-' */
2187
2188         s = strchr(str, '(');
2189         if (s)
2190                 str++;
2191         else {
2192                 ret = -EINVAL;
2193                 goto free;
2194         }
2195
2196         s = strrchr(str, ')');
2197         if (s)
2198                 *s = '\0';
2199         else {
2200                 ret = -EINVAL; /* no closing ')' */
2201                 goto free;
2202         }
2203
2204         flags |= HIST_FIELD_FL_EXPR;
2205         expr = create_hist_field(hist_data, NULL, flags, var_name);
2206         if (!expr) {
2207                 ret = -ENOMEM;
2208                 goto free;
2209         }
2210
2211         operand_flags = 0;
2212         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2213         if (IS_ERR(operand1)) {
2214                 ret = PTR_ERR(operand1);
2215                 goto free;
2216         }
2217         if (operand1->flags & HIST_FIELD_FL_STRING) {
2218                 /* String type can not be the operand of unary operator. */
2219                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2220                 destroy_hist_field(operand1, 0);
2221                 ret = -EINVAL;
2222                 goto free;
2223         }
2224
2225         expr->flags |= operand1->flags &
2226                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2227         expr->fn = hist_field_unary_minus;
2228         expr->operands[0] = operand1;
2229         expr->size = operand1->size;
2230         expr->is_signed = operand1->is_signed;
2231         expr->operator = FIELD_OP_UNARY_MINUS;
2232         expr->name = expr_str(expr, 0);
2233         expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2234         if (!expr->type) {
2235                 ret = -ENOMEM;
2236                 goto free;
2237         }
2238
2239         return expr;
2240  free:
2241         destroy_hist_field(expr, 0);
2242         return ERR_PTR(ret);
2243 }
2244
2245 static int check_expr_operands(struct trace_array *tr,
2246                                struct hist_field *operand1,
2247                                struct hist_field *operand2)
2248 {
2249         unsigned long operand1_flags = operand1->flags;
2250         unsigned long operand2_flags = operand2->flags;
2251
2252         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2253             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2254                 struct hist_field *var;
2255
2256                 var = find_var_field(operand1->var.hist_data, operand1->name);
2257                 if (!var)
2258                         return -EINVAL;
2259                 operand1_flags = var->flags;
2260         }
2261
2262         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2263             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2264                 struct hist_field *var;
2265
2266                 var = find_var_field(operand2->var.hist_data, operand2->name);
2267                 if (!var)
2268                         return -EINVAL;
2269                 operand2_flags = var->flags;
2270         }
2271
2272         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2273             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2274                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2275                 return -EINVAL;
2276         }
2277
2278         return 0;
2279 }
2280
2281 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2282                                      struct trace_event_file *file,
2283                                      char *str, unsigned long flags,
2284                                      char *var_name, unsigned int level)
2285 {
2286         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2287         unsigned long operand_flags;
2288         int field_op, ret = -EINVAL;
2289         char *sep, *operand1_str;
2290
2291         if (level > 3) {
2292                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2293                 return ERR_PTR(-EINVAL);
2294         }
2295
2296         field_op = contains_operator(str);
2297
2298         if (field_op == FIELD_OP_NONE)
2299                 return parse_atom(hist_data, file, str, &flags, var_name);
2300
2301         if (field_op == FIELD_OP_UNARY_MINUS)
2302                 return parse_unary(hist_data, file, str, flags, var_name, ++level);
2303
2304         switch (field_op) {
2305         case FIELD_OP_MINUS:
2306                 sep = "-";
2307                 break;
2308         case FIELD_OP_PLUS:
2309                 sep = "+";
2310                 break;
2311         default:
2312                 goto free;
2313         }
2314
2315         operand1_str = strsep(&str, sep);
2316         if (!operand1_str || !str)
2317                 goto free;
2318
2319         operand_flags = 0;
2320         operand1 = parse_atom(hist_data, file, operand1_str,
2321                               &operand_flags, NULL);
2322         if (IS_ERR(operand1)) {
2323                 ret = PTR_ERR(operand1);
2324                 operand1 = NULL;
2325                 goto free;
2326         }
2327         if (operand1->flags & HIST_FIELD_FL_STRING) {
2328                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2329                 ret = -EINVAL;
2330                 goto free;
2331         }
2332
2333         /* rest of string could be another expression e.g. b+c in a+b+c */
2334         operand_flags = 0;
2335         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, ++level);
2336         if (IS_ERR(operand2)) {
2337                 ret = PTR_ERR(operand2);
2338                 operand2 = NULL;
2339                 goto free;
2340         }
2341         if (operand2->flags & HIST_FIELD_FL_STRING) {
2342                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2343                 ret = -EINVAL;
2344                 goto free;
2345         }
2346
2347         ret = check_expr_operands(file->tr, operand1, operand2);
2348         if (ret)
2349                 goto free;
2350
2351         flags |= HIST_FIELD_FL_EXPR;
2352
2353         flags |= operand1->flags &
2354                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2355
2356         expr = create_hist_field(hist_data, NULL, flags, var_name);
2357         if (!expr) {
2358                 ret = -ENOMEM;
2359                 goto free;
2360         }
2361
2362         operand1->read_once = true;
2363         operand2->read_once = true;
2364
2365         expr->operands[0] = operand1;
2366         expr->operands[1] = operand2;
2367
2368         /* The operand sizes should be the same, so just pick one */
2369         expr->size = operand1->size;
2370         expr->is_signed = operand1->is_signed;
2371
2372         expr->operator = field_op;
2373         expr->name = expr_str(expr, 0);
2374         expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2375         if (!expr->type) {
2376                 ret = -ENOMEM;
2377                 goto free;
2378         }
2379
2380         switch (field_op) {
2381         case FIELD_OP_MINUS:
2382                 expr->fn = hist_field_minus;
2383                 break;
2384         case FIELD_OP_PLUS:
2385                 expr->fn = hist_field_plus;
2386                 break;
2387         default:
2388                 ret = -EINVAL;
2389                 goto free;
2390         }
2391
2392         return expr;
2393  free:
2394         destroy_hist_field(operand1, 0);
2395         destroy_hist_field(operand2, 0);
2396         destroy_hist_field(expr, 0);
2397
2398         return ERR_PTR(ret);
2399 }
2400
2401 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2402                                  struct trace_event_file *file)
2403 {
2404         struct event_trigger_data *test;
2405
2406         lockdep_assert_held(&event_mutex);
2407
2408         list_for_each_entry(test, &file->triggers, list) {
2409                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2410                         if (test->private_data == hist_data)
2411                                 return test->filter_str;
2412                 }
2413         }
2414
2415         return NULL;
2416 }
2417
2418 static struct event_command trigger_hist_cmd;
2419 static int event_hist_trigger_func(struct event_command *cmd_ops,
2420                                    struct trace_event_file *file,
2421                                    char *glob, char *cmd, char *param);
2422
2423 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2424                             struct hist_trigger_data *hist_data,
2425                             unsigned int n_keys)
2426 {
2427         struct hist_field *target_hist_field, *hist_field;
2428         unsigned int n, i, j;
2429
2430         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2431                 return false;
2432
2433         i = hist_data->n_vals;
2434         j = target_hist_data->n_vals;
2435
2436         for (n = 0; n < n_keys; n++) {
2437                 hist_field = hist_data->fields[i + n];
2438                 target_hist_field = target_hist_data->fields[j + n];
2439
2440                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2441                         return false;
2442                 if (hist_field->size != target_hist_field->size)
2443                         return false;
2444                 if (hist_field->is_signed != target_hist_field->is_signed)
2445                         return false;
2446         }
2447
2448         return true;
2449 }
2450
2451 static struct hist_trigger_data *
2452 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2453                      struct trace_event_file *file)
2454 {
2455         struct hist_trigger_data *hist_data;
2456         struct event_trigger_data *test;
2457         unsigned int n_keys;
2458
2459         lockdep_assert_held(&event_mutex);
2460
2461         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2462
2463         list_for_each_entry(test, &file->triggers, list) {
2464                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2465                         hist_data = test->private_data;
2466
2467                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2468                                 return hist_data;
2469                 }
2470         }
2471
2472         return NULL;
2473 }
2474
2475 static struct trace_event_file *event_file(struct trace_array *tr,
2476                                            char *system, char *event_name)
2477 {
2478         struct trace_event_file *file;
2479
2480         file = __find_event_file(tr, system, event_name);
2481         if (!file)
2482                 return ERR_PTR(-EINVAL);
2483
2484         return file;
2485 }
2486
2487 static struct hist_field *
2488 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2489                          char *system, char *event_name, char *field_name)
2490 {
2491         struct hist_field *event_var;
2492         char *synthetic_name;
2493
2494         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2495         if (!synthetic_name)
2496                 return ERR_PTR(-ENOMEM);
2497
2498         strcpy(synthetic_name, "synthetic_");
2499         strcat(synthetic_name, field_name);
2500
2501         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2502
2503         kfree(synthetic_name);
2504
2505         return event_var;
2506 }
2507
2508 /**
2509  * create_field_var_hist - Automatically create a histogram and var for a field
2510  * @target_hist_data: The target hist trigger
2511  * @subsys_name: Optional subsystem name
2512  * @event_name: Optional event name
2513  * @field_name: The name of the field (and the resulting variable)
2514  *
2515  * Hist trigger actions fetch data from variables, not directly from
2516  * events.  However, for convenience, users are allowed to directly
2517  * specify an event field in an action, which will be automatically
2518  * converted into a variable on their behalf.
2519  *
2520  * If a user specifies a field on an event that isn't the event the
2521  * histogram currently being defined (the target event histogram), the
2522  * only way that can be accomplished is if a new hist trigger is
2523  * created and the field variable defined on that.
2524  *
2525  * This function creates a new histogram compatible with the target
2526  * event (meaning a histogram with the same key as the target
2527  * histogram), and creates a variable for the specified field, but
2528  * with 'synthetic_' prepended to the variable name in order to avoid
2529  * collision with normal field variables.
2530  *
2531  * Return: The variable created for the field.
2532  */
2533 static struct hist_field *
2534 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2535                       char *subsys_name, char *event_name, char *field_name)
2536 {
2537         struct trace_array *tr = target_hist_data->event_file->tr;
2538         struct hist_trigger_data *hist_data;
2539         unsigned int i, n, first = true;
2540         struct field_var_hist *var_hist;
2541         struct trace_event_file *file;
2542         struct hist_field *key_field;
2543         struct hist_field *event_var;
2544         char *saved_filter;
2545         char *cmd;
2546         int ret;
2547
2548         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2549                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2550                 return ERR_PTR(-EINVAL);
2551         }
2552
2553         file = event_file(tr, subsys_name, event_name);
2554
2555         if (IS_ERR(file)) {
2556                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2557                 ret = PTR_ERR(file);
2558                 return ERR_PTR(ret);
2559         }
2560
2561         /*
2562          * Look for a histogram compatible with target.  We'll use the
2563          * found histogram specification to create a new matching
2564          * histogram with our variable on it.  target_hist_data is not
2565          * yet a registered histogram so we can't use that.
2566          */
2567         hist_data = find_compatible_hist(target_hist_data, file);
2568         if (!hist_data) {
2569                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
2570                 return ERR_PTR(-EINVAL);
2571         }
2572
2573         /* See if a synthetic field variable has already been created */
2574         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2575                                              event_name, field_name);
2576         if (!IS_ERR_OR_NULL(event_var))
2577                 return event_var;
2578
2579         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
2580         if (!var_hist)
2581                 return ERR_PTR(-ENOMEM);
2582
2583         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2584         if (!cmd) {
2585                 kfree(var_hist);
2586                 return ERR_PTR(-ENOMEM);
2587         }
2588
2589         /* Use the same keys as the compatible histogram */
2590         strcat(cmd, "keys=");
2591
2592         for_each_hist_key_field(i, hist_data) {
2593                 key_field = hist_data->fields[i];
2594                 if (!first)
2595                         strcat(cmd, ",");
2596                 strcat(cmd, key_field->field->name);
2597                 first = false;
2598         }
2599
2600         /* Create the synthetic field variable specification */
2601         strcat(cmd, ":synthetic_");
2602         strcat(cmd, field_name);
2603         strcat(cmd, "=");
2604         strcat(cmd, field_name);
2605
2606         /* Use the same filter as the compatible histogram */
2607         saved_filter = find_trigger_filter(hist_data, file);
2608         if (saved_filter) {
2609                 strcat(cmd, " if ");
2610                 strcat(cmd, saved_filter);
2611         }
2612
2613         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
2614         if (!var_hist->cmd) {
2615                 kfree(cmd);
2616                 kfree(var_hist);
2617                 return ERR_PTR(-ENOMEM);
2618         }
2619
2620         /* Save the compatible histogram information */
2621         var_hist->hist_data = hist_data;
2622
2623         /* Create the new histogram with our variable */
2624         ret = event_hist_trigger_func(&trigger_hist_cmd, file,
2625                                       "", "hist", cmd);
2626         if (ret) {
2627                 kfree(cmd);
2628                 kfree(var_hist->cmd);
2629                 kfree(var_hist);
2630                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
2631                 return ERR_PTR(ret);
2632         }
2633
2634         kfree(cmd);
2635
2636         /* If we can't find the variable, something went wrong */
2637         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
2638                                              event_name, field_name);
2639         if (IS_ERR_OR_NULL(event_var)) {
2640                 kfree(var_hist->cmd);
2641                 kfree(var_hist);
2642                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
2643                 return ERR_PTR(-EINVAL);
2644         }
2645
2646         n = target_hist_data->n_field_var_hists;
2647         target_hist_data->field_var_hists[n] = var_hist;
2648         target_hist_data->n_field_var_hists++;
2649
2650         return event_var;
2651 }
2652
2653 static struct hist_field *
2654 find_target_event_var(struct hist_trigger_data *hist_data,
2655                       char *subsys_name, char *event_name, char *var_name)
2656 {
2657         struct trace_event_file *file = hist_data->event_file;
2658         struct hist_field *hist_field = NULL;
2659
2660         if (subsys_name) {
2661                 struct trace_event_call *call;
2662
2663                 if (!event_name)
2664                         return NULL;
2665
2666                 call = file->event_call;
2667
2668                 if (strcmp(subsys_name, call->class->system) != 0)
2669                         return NULL;
2670
2671                 if (strcmp(event_name, trace_event_name(call)) != 0)
2672                         return NULL;
2673         }
2674
2675         hist_field = find_var_field(hist_data, var_name);
2676
2677         return hist_field;
2678 }
2679
2680 static inline void __update_field_vars(struct tracing_map_elt *elt,
2681                                        struct trace_buffer *buffer,
2682                                        struct ring_buffer_event *rbe,
2683                                        void *rec,
2684                                        struct field_var **field_vars,
2685                                        unsigned int n_field_vars,
2686                                        unsigned int field_var_str_start)
2687 {
2688         struct hist_elt_data *elt_data = elt->private_data;
2689         unsigned int i, j, var_idx;
2690         u64 var_val;
2691
2692         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
2693                 struct field_var *field_var = field_vars[i];
2694                 struct hist_field *var = field_var->var;
2695                 struct hist_field *val = field_var->val;
2696
2697                 var_val = val->fn(val, elt, buffer, rbe, rec);
2698                 var_idx = var->var.idx;
2699
2700                 if (val->flags & HIST_FIELD_FL_STRING) {
2701                         char *str = elt_data->field_var_str[j++];
2702                         char *val_str = (char *)(uintptr_t)var_val;
2703                         unsigned int size;
2704
2705                         size = min(val->size, STR_VAR_LEN_MAX);
2706                         strscpy(str, val_str, size);
2707                         var_val = (u64)(uintptr_t)str;
2708                 }
2709                 tracing_map_set_var(elt, var_idx, var_val);
2710         }
2711 }
2712
2713 static void update_field_vars(struct hist_trigger_data *hist_data,
2714                               struct tracing_map_elt *elt,
2715                               struct trace_buffer *buffer,
2716                               struct ring_buffer_event *rbe,
2717                               void *rec)
2718 {
2719         __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
2720                             hist_data->n_field_vars, 0);
2721 }
2722
2723 static void save_track_data_vars(struct hist_trigger_data *hist_data,
2724                                  struct tracing_map_elt *elt,
2725                                  struct trace_buffer *buffer,  void *rec,
2726                                  struct ring_buffer_event *rbe, void *key,
2727                                  struct action_data *data, u64 *var_ref_vals)
2728 {
2729         __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
2730                             hist_data->n_save_vars, hist_data->n_field_var_str);
2731 }
2732
2733 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
2734                                      struct trace_event_file *file,
2735                                      char *name, int size, const char *type)
2736 {
2737         struct hist_field *var;
2738         int idx;
2739
2740         if (find_var(hist_data, file, name) && !hist_data->remove) {
2741                 var = ERR_PTR(-EINVAL);
2742                 goto out;
2743         }
2744
2745         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
2746         if (!var) {
2747                 var = ERR_PTR(-ENOMEM);
2748                 goto out;
2749         }
2750
2751         idx = tracing_map_add_var(hist_data->map);
2752         if (idx < 0) {
2753                 kfree(var);
2754                 var = ERR_PTR(-EINVAL);
2755                 goto out;
2756         }
2757
2758         var->ref = 1;
2759         var->flags = HIST_FIELD_FL_VAR;
2760         var->var.idx = idx;
2761         var->var.hist_data = var->hist_data = hist_data;
2762         var->size = size;
2763         var->var.name = kstrdup(name, GFP_KERNEL);
2764         var->type = kstrdup_const(type, GFP_KERNEL);
2765         if (!var->var.name || !var->type) {
2766                 kfree_const(var->type);
2767                 kfree(var->var.name);
2768                 kfree(var);
2769                 var = ERR_PTR(-ENOMEM);
2770         }
2771  out:
2772         return var;
2773 }
2774
2775 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
2776                                           struct trace_event_file *file,
2777                                           char *field_name)
2778 {
2779         struct hist_field *val = NULL, *var = NULL;
2780         unsigned long flags = HIST_FIELD_FL_VAR;
2781         struct trace_array *tr = file->tr;
2782         struct field_var *field_var;
2783         int ret = 0;
2784
2785         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
2786                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2787                 ret = -EINVAL;
2788                 goto err;
2789         }
2790
2791         val = parse_atom(hist_data, file, field_name, &flags, NULL);
2792         if (IS_ERR(val)) {
2793                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
2794                 ret = PTR_ERR(val);
2795                 goto err;
2796         }
2797
2798         var = create_var(hist_data, file, field_name, val->size, val->type);
2799         if (IS_ERR(var)) {
2800                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
2801                 kfree(val);
2802                 ret = PTR_ERR(var);
2803                 goto err;
2804         }
2805
2806         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
2807         if (!field_var) {
2808                 kfree(val);
2809                 kfree(var);
2810                 ret =  -ENOMEM;
2811                 goto err;
2812         }
2813
2814         field_var->var = var;
2815         field_var->val = val;
2816  out:
2817         return field_var;
2818  err:
2819         field_var = ERR_PTR(ret);
2820         goto out;
2821 }
2822
2823 /**
2824  * create_target_field_var - Automatically create a variable for a field
2825  * @target_hist_data: The target hist trigger
2826  * @subsys_name: Optional subsystem name
2827  * @event_name: Optional event name
2828  * @var_name: The name of the field (and the resulting variable)
2829  *
2830  * Hist trigger actions fetch data from variables, not directly from
2831  * events.  However, for convenience, users are allowed to directly
2832  * specify an event field in an action, which will be automatically
2833  * converted into a variable on their behalf.
2834
2835  * This function creates a field variable with the name var_name on
2836  * the hist trigger currently being defined on the target event.  If
2837  * subsys_name and event_name are specified, this function simply
2838  * verifies that they do in fact match the target event subsystem and
2839  * event name.
2840  *
2841  * Return: The variable created for the field.
2842  */
2843 static struct field_var *
2844 create_target_field_var(struct hist_trigger_data *target_hist_data,
2845                         char *subsys_name, char *event_name, char *var_name)
2846 {
2847         struct trace_event_file *file = target_hist_data->event_file;
2848
2849         if (subsys_name) {
2850                 struct trace_event_call *call;
2851
2852                 if (!event_name)
2853                         return NULL;
2854
2855                 call = file->event_call;
2856
2857                 if (strcmp(subsys_name, call->class->system) != 0)
2858                         return NULL;
2859
2860                 if (strcmp(event_name, trace_event_name(call)) != 0)
2861                         return NULL;
2862         }
2863
2864         return create_field_var(target_hist_data, file, var_name);
2865 }
2866
2867 static bool check_track_val_max(u64 track_val, u64 var_val)
2868 {
2869         if (var_val <= track_val)
2870                 return false;
2871
2872         return true;
2873 }
2874
2875 static bool check_track_val_changed(u64 track_val, u64 var_val)
2876 {
2877         if (var_val == track_val)
2878                 return false;
2879
2880         return true;
2881 }
2882
2883 static u64 get_track_val(struct hist_trigger_data *hist_data,
2884                          struct tracing_map_elt *elt,
2885                          struct action_data *data)
2886 {
2887         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2888         u64 track_val;
2889
2890         track_val = tracing_map_read_var(elt, track_var_idx);
2891
2892         return track_val;
2893 }
2894
2895 static void save_track_val(struct hist_trigger_data *hist_data,
2896                            struct tracing_map_elt *elt,
2897                            struct action_data *data, u64 var_val)
2898 {
2899         unsigned int track_var_idx = data->track_data.track_var->var.idx;
2900
2901         tracing_map_set_var(elt, track_var_idx, var_val);
2902 }
2903
2904 static void save_track_data(struct hist_trigger_data *hist_data,
2905                             struct tracing_map_elt *elt,
2906                             struct trace_buffer *buffer, void *rec,
2907                             struct ring_buffer_event *rbe, void *key,
2908                             struct action_data *data, u64 *var_ref_vals)
2909 {
2910         if (data->track_data.save_data)
2911                 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
2912                                            key, data, var_ref_vals);
2913 }
2914
2915 static bool check_track_val(struct tracing_map_elt *elt,
2916                             struct action_data *data,
2917                             u64 var_val)
2918 {
2919         struct hist_trigger_data *hist_data;
2920         u64 track_val;
2921
2922         hist_data = data->track_data.track_var->hist_data;
2923         track_val = get_track_val(hist_data, elt, data);
2924
2925         return data->track_data.check_val(track_val, var_val);
2926 }
2927
2928 #ifdef CONFIG_TRACER_SNAPSHOT
2929 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
2930 {
2931         /* called with tr->max_lock held */
2932         struct track_data *track_data = tr->cond_snapshot->cond_data;
2933         struct hist_elt_data *elt_data, *track_elt_data;
2934         struct snapshot_context *context = cond_data;
2935         struct action_data *action;
2936         u64 track_val;
2937
2938         if (!track_data)
2939                 return false;
2940
2941         action = track_data->action_data;
2942
2943         track_val = get_track_val(track_data->hist_data, context->elt,
2944                                   track_data->action_data);
2945
2946         if (!action->track_data.check_val(track_data->track_val, track_val))
2947                 return false;
2948
2949         track_data->track_val = track_val;
2950         memcpy(track_data->key, context->key, track_data->key_len);
2951
2952         elt_data = context->elt->private_data;
2953         track_elt_data = track_data->elt.private_data;
2954         if (elt_data->comm)
2955                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
2956
2957         track_data->updated = true;
2958
2959         return true;
2960 }
2961
2962 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
2963                                      struct tracing_map_elt *elt,
2964                                      struct trace_buffer *buffer, void *rec,
2965                                      struct ring_buffer_event *rbe, void *key,
2966                                      struct action_data *data,
2967                                      u64 *var_ref_vals)
2968 {
2969         struct trace_event_file *file = hist_data->event_file;
2970         struct snapshot_context context;
2971
2972         context.elt = elt;
2973         context.key = key;
2974
2975         tracing_snapshot_cond(file->tr, &context);
2976 }
2977
2978 static void hist_trigger_print_key(struct seq_file *m,
2979                                    struct hist_trigger_data *hist_data,
2980                                    void *key,
2981                                    struct tracing_map_elt *elt);
2982
2983 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
2984 {
2985         unsigned int i;
2986
2987         if (!hist_data->n_actions)
2988                 return NULL;
2989
2990         for (i = 0; i < hist_data->n_actions; i++) {
2991                 struct action_data *data = hist_data->actions[i];
2992
2993                 if (data->action == ACTION_SNAPSHOT)
2994                         return data;
2995         }
2996
2997         return NULL;
2998 }
2999
3000 static void track_data_snapshot_print(struct seq_file *m,
3001                                       struct hist_trigger_data *hist_data)
3002 {
3003         struct trace_event_file *file = hist_data->event_file;
3004         struct track_data *track_data;
3005         struct action_data *action;
3006
3007         track_data = tracing_cond_snapshot_data(file->tr);
3008         if (!track_data)
3009                 return;
3010
3011         if (!track_data->updated)
3012                 return;
3013
3014         action = snapshot_action(hist_data);
3015         if (!action)
3016                 return;
3017
3018         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3019         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3020                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3021                    action->track_data.var_str, track_data->track_val);
3022
3023         seq_puts(m, "\ttriggered by event with key: ");
3024         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3025         seq_putc(m, '\n');
3026 }
3027 #else
3028 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3029 {
3030         return false;
3031 }
3032 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3033                                      struct tracing_map_elt *elt,
3034                                      struct trace_buffer *buffer, void *rec,
3035                                      struct ring_buffer_event *rbe, void *key,
3036                                      struct action_data *data,
3037                                      u64 *var_ref_vals) {}
3038 static void track_data_snapshot_print(struct seq_file *m,
3039                                       struct hist_trigger_data *hist_data) {}
3040 #endif /* CONFIG_TRACER_SNAPSHOT */
3041
3042 static void track_data_print(struct seq_file *m,
3043                              struct hist_trigger_data *hist_data,
3044                              struct tracing_map_elt *elt,
3045                              struct action_data *data)
3046 {
3047         u64 track_val = get_track_val(hist_data, elt, data);
3048         unsigned int i, save_var_idx;
3049
3050         if (data->handler == HANDLER_ONMAX)
3051                 seq_printf(m, "\n\tmax: %10llu", track_val);
3052         else if (data->handler == HANDLER_ONCHANGE)
3053                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3054
3055         if (data->action == ACTION_SNAPSHOT)
3056                 return;
3057
3058         for (i = 0; i < hist_data->n_save_vars; i++) {
3059                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3060                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3061                 u64 val;
3062
3063                 save_var_idx = save_var->var.idx;
3064
3065                 val = tracing_map_read_var(elt, save_var_idx);
3066
3067                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3068                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3069                                    (char *)(uintptr_t)(val));
3070                 } else
3071                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3072         }
3073 }
3074
3075 static void ontrack_action(struct hist_trigger_data *hist_data,
3076                            struct tracing_map_elt *elt,
3077                            struct trace_buffer *buffer, void *rec,
3078                            struct ring_buffer_event *rbe, void *key,
3079                            struct action_data *data, u64 *var_ref_vals)
3080 {
3081         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3082
3083         if (check_track_val(elt, data, var_val)) {
3084                 save_track_val(hist_data, elt, data, var_val);
3085                 save_track_data(hist_data, elt, buffer, rec, rbe,
3086                                 key, data, var_ref_vals);
3087         }
3088 }
3089
3090 static void action_data_destroy(struct action_data *data)
3091 {
3092         unsigned int i;
3093
3094         lockdep_assert_held(&event_mutex);
3095
3096         kfree(data->action_name);
3097
3098         for (i = 0; i < data->n_params; i++)
3099                 kfree(data->params[i]);
3100
3101         if (data->synth_event)
3102                 data->synth_event->ref--;
3103
3104         kfree(data->synth_event_name);
3105
3106         kfree(data);
3107 }
3108
3109 static void track_data_destroy(struct hist_trigger_data *hist_data,
3110                                struct action_data *data)
3111 {
3112         struct trace_event_file *file = hist_data->event_file;
3113
3114         destroy_hist_field(data->track_data.track_var, 0);
3115
3116         if (data->action == ACTION_SNAPSHOT) {
3117                 struct track_data *track_data;
3118
3119                 track_data = tracing_cond_snapshot_data(file->tr);
3120                 if (track_data && track_data->hist_data == hist_data) {
3121                         tracing_snapshot_cond_disable(file->tr);
3122                         track_data_free(track_data);
3123                 }
3124         }
3125
3126         kfree(data->track_data.var_str);
3127
3128         action_data_destroy(data);
3129 }
3130
3131 static int action_create(struct hist_trigger_data *hist_data,
3132                          struct action_data *data);
3133
3134 static int track_data_create(struct hist_trigger_data *hist_data,
3135                              struct action_data *data)
3136 {
3137         struct hist_field *var_field, *ref_field, *track_var = NULL;
3138         struct trace_event_file *file = hist_data->event_file;
3139         struct trace_array *tr = file->tr;
3140         char *track_data_var_str;
3141         int ret = 0;
3142
3143         track_data_var_str = data->track_data.var_str;
3144         if (track_data_var_str[0] != '$') {
3145                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3146                 return -EINVAL;
3147         }
3148         track_data_var_str++;
3149
3150         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3151         if (!var_field) {
3152                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3153                 return -EINVAL;
3154         }
3155
3156         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3157         if (!ref_field)
3158                 return -ENOMEM;
3159
3160         data->track_data.var_ref = ref_field;
3161
3162         if (data->handler == HANDLER_ONMAX)
3163                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3164         if (IS_ERR(track_var)) {
3165                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3166                 ret = PTR_ERR(track_var);
3167                 goto out;
3168         }
3169
3170         if (data->handler == HANDLER_ONCHANGE)
3171                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3172         if (IS_ERR(track_var)) {
3173                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3174                 ret = PTR_ERR(track_var);
3175                 goto out;
3176         }
3177         data->track_data.track_var = track_var;
3178
3179         ret = action_create(hist_data, data);
3180  out:
3181         return ret;
3182 }
3183
3184 static int parse_action_params(struct trace_array *tr, char *params,
3185                                struct action_data *data)
3186 {
3187         char *param, *saved_param;
3188         bool first_param = true;
3189         int ret = 0;
3190
3191         while (params) {
3192                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3193                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3194                         ret = -EINVAL;
3195                         goto out;
3196                 }
3197
3198                 param = strsep(&params, ",");
3199                 if (!param) {
3200                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3201                         ret = -EINVAL;
3202                         goto out;
3203                 }
3204
3205                 param = strstrip(param);
3206                 if (strlen(param) < 2) {
3207                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3208                         ret = -EINVAL;
3209                         goto out;
3210                 }
3211
3212                 saved_param = kstrdup(param, GFP_KERNEL);
3213                 if (!saved_param) {
3214                         ret = -ENOMEM;
3215                         goto out;
3216                 }
3217
3218                 if (first_param && data->use_trace_keyword) {
3219                         data->synth_event_name = saved_param;
3220                         first_param = false;
3221                         continue;
3222                 }
3223                 first_param = false;
3224
3225                 data->params[data->n_params++] = saved_param;
3226         }
3227  out:
3228         return ret;
3229 }
3230
3231 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3232                         enum handler_id handler)
3233 {
3234         char *action_name;
3235         int ret = 0;
3236
3237         strsep(&str, ".");
3238         if (!str) {
3239                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3240                 ret = -EINVAL;
3241                 goto out;
3242         }
3243
3244         action_name = strsep(&str, "(");
3245         if (!action_name || !str) {
3246                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3247                 ret = -EINVAL;
3248                 goto out;
3249         }
3250
3251         if (str_has_prefix(action_name, "save")) {
3252                 char *params = strsep(&str, ")");
3253
3254                 if (!params) {
3255                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3256                         ret = -EINVAL;
3257                         goto out;
3258                 }
3259
3260                 ret = parse_action_params(tr, params, data);
3261                 if (ret)
3262                         goto out;
3263
3264                 if (handler == HANDLER_ONMAX)
3265                         data->track_data.check_val = check_track_val_max;
3266                 else if (handler == HANDLER_ONCHANGE)
3267                         data->track_data.check_val = check_track_val_changed;
3268                 else {
3269                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3270                         ret = -EINVAL;
3271                         goto out;
3272                 }
3273
3274                 data->track_data.save_data = save_track_data_vars;
3275                 data->fn = ontrack_action;
3276                 data->action = ACTION_SAVE;
3277         } else if (str_has_prefix(action_name, "snapshot")) {
3278                 char *params = strsep(&str, ")");
3279
3280                 if (!str) {
3281                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3282                         ret = -EINVAL;
3283                         goto out;
3284                 }
3285
3286                 if (handler == HANDLER_ONMAX)
3287                         data->track_data.check_val = check_track_val_max;
3288                 else if (handler == HANDLER_ONCHANGE)
3289                         data->track_data.check_val = check_track_val_changed;
3290                 else {
3291                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3292                         ret = -EINVAL;
3293                         goto out;
3294                 }
3295
3296                 data->track_data.save_data = save_track_data_snapshot;
3297                 data->fn = ontrack_action;
3298                 data->action = ACTION_SNAPSHOT;
3299         } else {
3300                 char *params = strsep(&str, ")");
3301
3302                 if (str_has_prefix(action_name, "trace"))
3303                         data->use_trace_keyword = true;
3304
3305                 if (params) {
3306                         ret = parse_action_params(tr, params, data);
3307                         if (ret)
3308                                 goto out;
3309                 }
3310
3311                 if (handler == HANDLER_ONMAX)
3312                         data->track_data.check_val = check_track_val_max;
3313                 else if (handler == HANDLER_ONCHANGE)
3314                         data->track_data.check_val = check_track_val_changed;
3315
3316                 if (handler != HANDLER_ONMATCH) {
3317                         data->track_data.save_data = action_trace;
3318                         data->fn = ontrack_action;
3319                 } else
3320                         data->fn = action_trace;
3321
3322                 data->action = ACTION_TRACE;
3323         }
3324
3325         data->action_name = kstrdup(action_name, GFP_KERNEL);
3326         if (!data->action_name) {
3327                 ret = -ENOMEM;
3328                 goto out;
3329         }
3330
3331         data->handler = handler;
3332  out:
3333         return ret;
3334 }
3335
3336 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3337                                             char *str, enum handler_id handler)
3338 {
3339         struct action_data *data;
3340         int ret = -EINVAL;
3341         char *var_str;
3342
3343         data = kzalloc(sizeof(*data), GFP_KERNEL);
3344         if (!data)
3345                 return ERR_PTR(-ENOMEM);
3346
3347         var_str = strsep(&str, ")");
3348         if (!var_str || !str) {
3349                 ret = -EINVAL;
3350                 goto free;
3351         }
3352
3353         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3354         if (!data->track_data.var_str) {
3355                 ret = -ENOMEM;
3356                 goto free;
3357         }
3358
3359         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3360         if (ret)
3361                 goto free;
3362  out:
3363         return data;
3364  free:
3365         track_data_destroy(hist_data, data);
3366         data = ERR_PTR(ret);
3367         goto out;
3368 }
3369
3370 static void onmatch_destroy(struct action_data *data)
3371 {
3372         kfree(data->match_data.event);
3373         kfree(data->match_data.event_system);
3374
3375         action_data_destroy(data);
3376 }
3377
3378 static void destroy_field_var(struct field_var *field_var)
3379 {
3380         if (!field_var)
3381                 return;
3382
3383         destroy_hist_field(field_var->var, 0);
3384         destroy_hist_field(field_var->val, 0);
3385
3386         kfree(field_var);
3387 }
3388
3389 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3390 {
3391         unsigned int i;
3392
3393         for (i = 0; i < hist_data->n_field_vars; i++)
3394                 destroy_field_var(hist_data->field_vars[i]);
3395
3396         for (i = 0; i < hist_data->n_save_vars; i++)
3397                 destroy_field_var(hist_data->save_vars[i]);
3398 }
3399
3400 static void save_field_var(struct hist_trigger_data *hist_data,
3401                            struct field_var *field_var)
3402 {
3403         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3404
3405         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3406                 hist_data->n_field_var_str++;
3407 }
3408
3409
3410 static int check_synth_field(struct synth_event *event,
3411                              struct hist_field *hist_field,
3412                              unsigned int field_pos)
3413 {
3414         struct synth_field *field;
3415
3416         if (field_pos >= event->n_fields)
3417                 return -EINVAL;
3418
3419         field = event->fields[field_pos];
3420
3421         /*
3422          * A dynamic string synth field can accept static or
3423          * dynamic. A static string synth field can only accept a
3424          * same-sized static string, which is checked for later.
3425          */
3426         if (strstr(hist_field->type, "char[") && field->is_string
3427             && field->is_dynamic)
3428                 return 0;
3429
3430         if (strstr(hist_field->type, "long[") && field->is_stack)
3431                 return 0;
3432
3433         if (strcmp(field->type, hist_field->type) != 0) {
3434                 if (field->size != hist_field->size ||
3435                     (!field->is_string && field->is_signed != hist_field->is_signed))
3436                         return -EINVAL;
3437         }
3438
3439         return 0;
3440 }
3441
3442 static struct hist_field *
3443 trace_action_find_var(struct hist_trigger_data *hist_data,
3444                       struct action_data *data,
3445                       char *system, char *event, char *var)
3446 {
3447         struct trace_array *tr = hist_data->event_file->tr;
3448         struct hist_field *hist_field;
3449
3450         var++; /* skip '$' */
3451
3452         hist_field = find_target_event_var(hist_data, system, event, var);
3453         if (!hist_field) {
3454                 if (!system && data->handler == HANDLER_ONMATCH) {
3455                         system = data->match_data.event_system;
3456                         event = data->match_data.event;
3457                 }
3458
3459                 hist_field = find_event_var(hist_data, system, event, var);
3460         }
3461
3462         if (!hist_field)
3463                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3464
3465         return hist_field;
3466 }
3467
3468 static struct hist_field *
3469 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3470                               struct action_data *data, char *system,
3471                               char *event, char *var)
3472 {
3473         struct hist_field *hist_field = NULL;
3474         struct field_var *field_var;
3475
3476         /*
3477          * First try to create a field var on the target event (the
3478          * currently being defined).  This will create a variable for
3479          * unqualified fields on the target event, or if qualified,
3480          * target fields that have qualified names matching the target.
3481          */
3482         field_var = create_target_field_var(hist_data, system, event, var);
3483
3484         if (field_var && !IS_ERR(field_var)) {
3485                 save_field_var(hist_data, field_var);
3486                 hist_field = field_var->var;
3487         } else {
3488                 field_var = NULL;
3489                 /*
3490                  * If no explicit system.event is specified, default to
3491                  * looking for fields on the onmatch(system.event.xxx)
3492                  * event.
3493                  */
3494                 if (!system && data->handler == HANDLER_ONMATCH) {
3495                         system = data->match_data.event_system;
3496                         event = data->match_data.event;
3497                 }
3498
3499                 if (!event)
3500                         goto free;
3501                 /*
3502                  * At this point, we're looking at a field on another
3503                  * event.  Because we can't modify a hist trigger on
3504                  * another event to add a variable for a field, we need
3505                  * to create a new trigger on that event and create the
3506                  * variable at the same time.
3507                  */
3508                 hist_field = create_field_var_hist(hist_data, system, event, var);
3509                 if (IS_ERR(hist_field))
3510                         goto free;
3511         }
3512  out:
3513         return hist_field;
3514  free:
3515         destroy_field_var(field_var);
3516         hist_field = NULL;
3517         goto out;
3518 }
3519
3520 static int trace_action_create(struct hist_trigger_data *hist_data,
3521                                struct action_data *data)
3522 {
3523         struct trace_array *tr = hist_data->event_file->tr;
3524         char *event_name, *param, *system = NULL;
3525         struct hist_field *hist_field, *var_ref;
3526         unsigned int i;
3527         unsigned int field_pos = 0;
3528         struct synth_event *event;
3529         char *synth_event_name;
3530         int var_ref_idx, ret = 0;
3531
3532         lockdep_assert_held(&event_mutex);
3533
3534         /* Sanity check to avoid out-of-bound write on 'data->var_ref_idx' */
3535         if (data->n_params > SYNTH_FIELDS_MAX)
3536                 return -EINVAL;
3537
3538         if (data->use_trace_keyword)
3539                 synth_event_name = data->synth_event_name;
3540         else
3541                 synth_event_name = data->action_name;
3542
3543         event = find_synth_event(synth_event_name);
3544         if (!event) {
3545                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
3546                 return -EINVAL;
3547         }
3548
3549         event->ref++;
3550
3551         for (i = 0; i < data->n_params; i++) {
3552                 char *p;
3553
3554                 p = param = kstrdup(data->params[i], GFP_KERNEL);
3555                 if (!param) {
3556                         ret = -ENOMEM;
3557                         goto err;
3558                 }
3559
3560                 system = strsep(&param, ".");
3561                 if (!param) {
3562                         param = (char *)system;
3563                         system = event_name = NULL;
3564                 } else {
3565                         event_name = strsep(&param, ".");
3566                         if (!param) {
3567                                 kfree(p);
3568                                 ret = -EINVAL;
3569                                 goto err;
3570                         }
3571                 }
3572
3573                 if (param[0] == '$')
3574                         hist_field = trace_action_find_var(hist_data, data,
3575                                                            system, event_name,
3576                                                            param);
3577                 else
3578                         hist_field = trace_action_create_field_var(hist_data,
3579                                                                    data,
3580                                                                    system,
3581                                                                    event_name,
3582                                                                    param);
3583
3584                 if (!hist_field) {
3585                         kfree(p);
3586                         ret = -EINVAL;
3587                         goto err;
3588                 }
3589
3590                 if (check_synth_field(event, hist_field, field_pos) == 0) {
3591                         var_ref = create_var_ref(hist_data, hist_field,
3592                                                  system, event_name);
3593                         if (!var_ref) {
3594                                 kfree(p);
3595                                 ret = -ENOMEM;
3596                                 goto err;
3597                         }
3598
3599                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
3600                         if (WARN_ON(var_ref_idx < 0)) {
3601                                 kfree(p);
3602                                 ret = var_ref_idx;
3603                                 goto err;
3604                         }
3605
3606                         data->var_ref_idx[i] = var_ref_idx;
3607
3608                         field_pos++;
3609                         kfree(p);
3610                         continue;
3611                 }
3612
3613                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
3614                 kfree(p);
3615                 ret = -EINVAL;
3616                 goto err;
3617         }
3618
3619         if (field_pos != event->n_fields) {
3620                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
3621                 ret = -EINVAL;
3622                 goto err;
3623         }
3624
3625         data->synth_event = event;
3626  out:
3627         return ret;
3628  err:
3629         event->ref--;
3630
3631         goto out;
3632 }
3633
3634 static int action_create(struct hist_trigger_data *hist_data,
3635                          struct action_data *data)
3636 {
3637         struct trace_event_file *file = hist_data->event_file;
3638         struct trace_array *tr = file->tr;
3639         struct track_data *track_data;
3640         struct field_var *field_var;
3641         unsigned int i;
3642         char *param;
3643         int ret = 0;
3644
3645         if (data->action == ACTION_TRACE)
3646                 return trace_action_create(hist_data, data);
3647
3648         if (data->action == ACTION_SNAPSHOT) {
3649                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
3650                 if (IS_ERR(track_data)) {
3651                         ret = PTR_ERR(track_data);
3652                         goto out;
3653                 }
3654
3655                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
3656                                                    cond_snapshot_update);
3657                 if (ret)
3658                         track_data_free(track_data);
3659
3660                 goto out;
3661         }
3662
3663         if (data->action == ACTION_SAVE) {
3664                 if (hist_data->n_save_vars) {
3665                         ret = -EEXIST;
3666                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
3667                         goto out;
3668                 }
3669
3670                 for (i = 0; i < data->n_params; i++) {
3671                         param = kstrdup(data->params[i], GFP_KERNEL);
3672                         if (!param) {
3673                                 ret = -ENOMEM;
3674                                 goto out;
3675                         }
3676
3677                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
3678                         if (IS_ERR(field_var)) {
3679                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
3680                                          errpos(param));
3681                                 ret = PTR_ERR(field_var);
3682                                 kfree(param);
3683                                 goto out;
3684                         }
3685
3686                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
3687                         if (field_var->val->flags & HIST_FIELD_FL_STRING)
3688                                 hist_data->n_save_var_str++;
3689                         kfree(param);
3690                 }
3691         }
3692  out:
3693         return ret;
3694 }
3695
3696 static int onmatch_create(struct hist_trigger_data *hist_data,
3697                           struct action_data *data)
3698 {
3699         return action_create(hist_data, data);
3700 }
3701
3702 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
3703 {
3704         char *match_event, *match_event_system;
3705         struct action_data *data;
3706         int ret = -EINVAL;
3707
3708         data = kzalloc(sizeof(*data), GFP_KERNEL);
3709         if (!data)
3710                 return ERR_PTR(-ENOMEM);
3711
3712         match_event = strsep(&str, ")");
3713         if (!match_event || !str) {
3714                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
3715                 goto free;
3716         }
3717
3718         match_event_system = strsep(&match_event, ".");
3719         if (!match_event) {
3720                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
3721                 goto free;
3722         }
3723
3724         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
3725                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
3726                 goto free;
3727         }
3728
3729         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
3730         if (!data->match_data.event) {
3731                 ret = -ENOMEM;
3732                 goto free;
3733         }
3734
3735         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
3736         if (!data->match_data.event_system) {
3737                 ret = -ENOMEM;
3738                 goto free;
3739         }
3740
3741         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
3742         if (ret)
3743                 goto free;
3744  out:
3745         return data;
3746  free:
3747         onmatch_destroy(data);
3748         data = ERR_PTR(ret);
3749         goto out;
3750 }
3751
3752 static int create_hitcount_val(struct hist_trigger_data *hist_data)
3753 {
3754         hist_data->fields[HITCOUNT_IDX] =
3755                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
3756         if (!hist_data->fields[HITCOUNT_IDX])
3757                 return -ENOMEM;
3758
3759         hist_data->n_vals++;
3760         hist_data->n_fields++;
3761
3762         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
3763                 return -EINVAL;
3764
3765         return 0;
3766 }
3767
3768 static int __create_val_field(struct hist_trigger_data *hist_data,
3769                               unsigned int val_idx,
3770                               struct trace_event_file *file,
3771                               char *var_name, char *field_str,
3772                               unsigned long flags)
3773 {
3774         struct hist_field *hist_field;
3775         int ret = 0;
3776
3777         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, 0);
3778         if (IS_ERR(hist_field)) {
3779                 ret = PTR_ERR(hist_field);
3780                 goto out;
3781         }
3782
3783         hist_data->fields[val_idx] = hist_field;
3784
3785         ++hist_data->n_vals;
3786         ++hist_data->n_fields;
3787
3788         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3789                 ret = -EINVAL;
3790  out:
3791         return ret;
3792 }
3793
3794 static int create_val_field(struct hist_trigger_data *hist_data,
3795                             unsigned int val_idx,
3796                             struct trace_event_file *file,
3797                             char *field_str)
3798 {
3799         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
3800                 return -EINVAL;
3801
3802         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
3803 }
3804
3805 static const char *no_comm = "(no comm)";
3806
3807 static u64 hist_field_execname(struct hist_field *hist_field,
3808                                struct tracing_map_elt *elt,
3809                                struct trace_buffer *buffer,
3810                                struct ring_buffer_event *rbe,
3811                                void *event)
3812 {
3813         struct hist_elt_data *elt_data;
3814
3815         if (WARN_ON_ONCE(!elt))
3816                 return (u64)(unsigned long)no_comm;
3817
3818         elt_data = elt->private_data;
3819
3820         if (WARN_ON_ONCE(!elt_data->comm))
3821                 return (u64)(unsigned long)no_comm;
3822
3823         return (u64)(unsigned long)(elt_data->comm);
3824 }
3825
3826 /* Convert a var that points to common_pid.execname to a string */
3827 static void update_var_execname(struct hist_field *hist_field)
3828 {
3829         hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
3830                 HIST_FIELD_FL_EXECNAME;
3831         hist_field->size = MAX_FILTER_STR_VAL;
3832         hist_field->is_signed = 0;
3833
3834         kfree_const(hist_field->type);
3835         hist_field->type = "char[]";
3836
3837         hist_field->fn = hist_field_execname;
3838 }
3839
3840 static int create_var_field(struct hist_trigger_data *hist_data,
3841                             unsigned int val_idx,
3842                             struct trace_event_file *file,
3843                             char *var_name, char *expr_str)
3844 {
3845         struct trace_array *tr = hist_data->event_file->tr;
3846         unsigned long flags = 0;
3847         int ret;
3848
3849         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
3850                 return -EINVAL;
3851
3852         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
3853                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
3854                 return -EINVAL;
3855         }
3856
3857         flags |= HIST_FIELD_FL_VAR;
3858         hist_data->n_vars++;
3859         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
3860                 return -EINVAL;
3861
3862         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
3863
3864         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
3865                 update_var_execname(hist_data->fields[val_idx]);
3866
3867         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_STRING)
3868                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
3869
3870         return ret;
3871 }
3872
3873 static int create_val_fields(struct hist_trigger_data *hist_data,
3874                              struct trace_event_file *file)
3875 {
3876         char *fields_str, *field_str;
3877         unsigned int i, j = 1;
3878         int ret;
3879
3880         ret = create_hitcount_val(hist_data);
3881         if (ret)
3882                 goto out;
3883
3884         fields_str = hist_data->attrs->vals_str;
3885         if (!fields_str)
3886                 goto out;
3887
3888         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
3889                      j < TRACING_MAP_VALS_MAX; i++) {
3890                 field_str = strsep(&fields_str, ",");
3891                 if (!field_str)
3892                         break;
3893
3894                 if (strcmp(field_str, "hitcount") == 0)
3895                         continue;
3896
3897                 ret = create_val_field(hist_data, j++, file, field_str);
3898                 if (ret)
3899                         goto out;
3900         }
3901
3902         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
3903                 ret = -EINVAL;
3904  out:
3905         return ret;
3906 }
3907
3908 static int create_key_field(struct hist_trigger_data *hist_data,
3909                             unsigned int key_idx,
3910                             unsigned int key_offset,
3911                             struct trace_event_file *file,
3912                             char *field_str)
3913 {
3914         struct trace_array *tr = hist_data->event_file->tr;
3915         struct hist_field *hist_field = NULL;
3916         unsigned long flags = 0;
3917         unsigned int key_size;
3918         int ret = 0;
3919
3920         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
3921                 return -EINVAL;
3922
3923         flags |= HIST_FIELD_FL_KEY;
3924
3925         if (strcmp(field_str, "stacktrace") == 0) {
3926                 flags |= HIST_FIELD_FL_STACKTRACE;
3927                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
3928                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
3929         } else {
3930                 hist_field = parse_expr(hist_data, file, field_str, flags,
3931                                         NULL, 0);
3932                 if (IS_ERR(hist_field)) {
3933                         ret = PTR_ERR(hist_field);
3934                         goto out;
3935                 }
3936
3937                 if (field_has_hist_vars(hist_field, 0)) {
3938                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
3939                         destroy_hist_field(hist_field, 0);
3940                         ret = -EINVAL;
3941                         goto out;
3942                 }
3943
3944                 key_size = hist_field->size;
3945         }
3946
3947         hist_data->fields[key_idx] = hist_field;
3948
3949         key_size = ALIGN(key_size, sizeof(u64));
3950         hist_data->fields[key_idx]->size = key_size;
3951         hist_data->fields[key_idx]->offset = key_offset;
3952
3953         hist_data->key_size += key_size;
3954
3955         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
3956                 ret = -EINVAL;
3957                 goto out;
3958         }
3959
3960         hist_data->n_keys++;
3961         hist_data->n_fields++;
3962
3963         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
3964                 return -EINVAL;
3965
3966         ret = key_size;
3967  out:
3968         return ret;
3969 }
3970
3971 static int create_key_fields(struct hist_trigger_data *hist_data,
3972                              struct trace_event_file *file)
3973 {
3974         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
3975         char *fields_str, *field_str;
3976         int ret = -EINVAL;
3977
3978         fields_str = hist_data->attrs->keys_str;
3979         if (!fields_str)
3980                 goto out;
3981
3982         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
3983                 field_str = strsep(&fields_str, ",");
3984                 if (!field_str)
3985                         break;
3986                 ret = create_key_field(hist_data, i, key_offset,
3987                                        file, field_str);
3988                 if (ret < 0)
3989                         goto out;
3990                 key_offset += ret;
3991         }
3992         if (fields_str) {
3993                 ret = -EINVAL;
3994                 goto out;
3995         }
3996         ret = 0;
3997  out:
3998         return ret;
3999 }
4000
4001 static int create_var_fields(struct hist_trigger_data *hist_data,
4002                              struct trace_event_file *file)
4003 {
4004         unsigned int i, j = hist_data->n_vals;
4005         int ret = 0;
4006
4007         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4008
4009         for (i = 0; i < n_vars; i++) {
4010                 char *var_name = hist_data->attrs->var_defs.name[i];
4011                 char *expr = hist_data->attrs->var_defs.expr[i];
4012
4013                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4014                 if (ret)
4015                         goto out;
4016         }
4017  out:
4018         return ret;
4019 }
4020
4021 static void free_var_defs(struct hist_trigger_data *hist_data)
4022 {
4023         unsigned int i;
4024
4025         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4026                 kfree(hist_data->attrs->var_defs.name[i]);
4027                 kfree(hist_data->attrs->var_defs.expr[i]);
4028         }
4029
4030         hist_data->attrs->var_defs.n_vars = 0;
4031 }
4032
4033 static int parse_var_defs(struct hist_trigger_data *hist_data)
4034 {
4035         struct trace_array *tr = hist_data->event_file->tr;
4036         char *s, *str, *var_name, *field_str;
4037         unsigned int i, j, n_vars = 0;
4038         int ret = 0;
4039
4040         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4041                 str = hist_data->attrs->assignment_str[i];
4042                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4043                         field_str = strsep(&str, ",");
4044                         if (!field_str)
4045                                 break;
4046
4047                         var_name = strsep(&field_str, "=");
4048                         if (!var_name || !field_str) {
4049                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4050                                          errpos(var_name));
4051                                 ret = -EINVAL;
4052                                 goto free;
4053                         }
4054
4055                         if (n_vars == TRACING_MAP_VARS_MAX) {
4056                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4057                                 ret = -EINVAL;
4058                                 goto free;
4059                         }
4060
4061                         s = kstrdup(var_name, GFP_KERNEL);
4062                         if (!s) {
4063                                 ret = -ENOMEM;
4064                                 goto free;
4065                         }
4066                         hist_data->attrs->var_defs.name[n_vars] = s;
4067
4068                         s = kstrdup(field_str, GFP_KERNEL);
4069                         if (!s) {
4070                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4071                                 hist_data->attrs->var_defs.name[n_vars] = NULL;
4072                                 ret = -ENOMEM;
4073                                 goto free;
4074                         }
4075                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4076
4077                         hist_data->attrs->var_defs.n_vars = n_vars;
4078                 }
4079         }
4080
4081         return ret;
4082  free:
4083         free_var_defs(hist_data);
4084
4085         return ret;
4086 }
4087
4088 static int create_hist_fields(struct hist_trigger_data *hist_data,
4089                               struct trace_event_file *file)
4090 {
4091         int ret;
4092
4093         ret = parse_var_defs(hist_data);
4094         if (ret)
4095                 goto out;
4096
4097         ret = create_val_fields(hist_data, file);
4098         if (ret)
4099                 goto out;
4100
4101         ret = create_var_fields(hist_data, file);
4102         if (ret)
4103                 goto out;
4104
4105         ret = create_key_fields(hist_data, file);
4106         if (ret)
4107                 goto out;
4108  out:
4109         free_var_defs(hist_data);
4110
4111         return ret;
4112 }
4113
4114 static int is_descending(struct trace_array *tr, const char *str)
4115 {
4116         if (!str)
4117                 return 0;
4118
4119         if (strcmp(str, "descending") == 0)
4120                 return 1;
4121
4122         if (strcmp(str, "ascending") == 0)
4123                 return 0;
4124
4125         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4126
4127         return -EINVAL;
4128 }
4129
4130 static int create_sort_keys(struct hist_trigger_data *hist_data)
4131 {
4132         struct trace_array *tr = hist_data->event_file->tr;
4133         char *fields_str = hist_data->attrs->sort_key_str;
4134         struct tracing_map_sort_key *sort_key;
4135         int descending, ret = 0;
4136         unsigned int i, j, k;
4137
4138         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4139
4140         if (!fields_str)
4141                 goto out;
4142
4143         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4144                 struct hist_field *hist_field;
4145                 char *field_str, *field_name;
4146                 const char *test_name;
4147
4148                 sort_key = &hist_data->sort_keys[i];
4149
4150                 field_str = strsep(&fields_str, ",");
4151                 if (!field_str)
4152                         break;
4153
4154                 if (!*field_str) {
4155                         ret = -EINVAL;
4156                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4157                         break;
4158                 }
4159
4160                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4161                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4162                         ret = -EINVAL;
4163                         break;
4164                 }
4165
4166                 field_name = strsep(&field_str, ".");
4167                 if (!field_name || !*field_name) {
4168                         ret = -EINVAL;
4169                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4170                         break;
4171                 }
4172
4173                 if (strcmp(field_name, "hitcount") == 0) {
4174                         descending = is_descending(tr, field_str);
4175                         if (descending < 0) {
4176                                 ret = descending;
4177                                 break;
4178                         }
4179                         sort_key->descending = descending;
4180                         continue;
4181                 }
4182
4183                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4184                         unsigned int idx;
4185
4186                         hist_field = hist_data->fields[j];
4187                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4188                                 continue;
4189
4190                         idx = k++;
4191
4192                         test_name = hist_field_name(hist_field, 0);
4193
4194                         if (strcmp(field_name, test_name) == 0) {
4195                                 sort_key->field_idx = idx;
4196                                 descending = is_descending(tr, field_str);
4197                                 if (descending < 0) {
4198                                         ret = descending;
4199                                         goto out;
4200                                 }
4201                                 sort_key->descending = descending;
4202                                 break;
4203                         }
4204                 }
4205                 if (j == hist_data->n_fields) {
4206                         ret = -EINVAL;
4207                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4208                         break;
4209                 }
4210         }
4211
4212         hist_data->n_sort_keys = i;
4213  out:
4214         return ret;
4215 }
4216
4217 static void destroy_actions(struct hist_trigger_data *hist_data)
4218 {
4219         unsigned int i;
4220
4221         for (i = 0; i < hist_data->n_actions; i++) {
4222                 struct action_data *data = hist_data->actions[i];
4223
4224                 if (data->handler == HANDLER_ONMATCH)
4225                         onmatch_destroy(data);
4226                 else if (data->handler == HANDLER_ONMAX ||
4227                          data->handler == HANDLER_ONCHANGE)
4228                         track_data_destroy(hist_data, data);
4229                 else
4230                         kfree(data);
4231         }
4232 }
4233
4234 static int parse_actions(struct hist_trigger_data *hist_data)
4235 {
4236         struct trace_array *tr = hist_data->event_file->tr;
4237         struct action_data *data;
4238         unsigned int i;
4239         int ret = 0;
4240         char *str;
4241         int len;
4242
4243         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4244                 str = hist_data->attrs->action_str[i];
4245
4246                 if ((len = str_has_prefix(str, "onmatch("))) {
4247                         char *action_str = str + len;
4248
4249                         data = onmatch_parse(tr, action_str);
4250                         if (IS_ERR(data)) {
4251                                 ret = PTR_ERR(data);
4252                                 break;
4253                         }
4254                 } else if ((len = str_has_prefix(str, "onmax("))) {
4255                         char *action_str = str + len;
4256
4257                         data = track_data_parse(hist_data, action_str,
4258                                                 HANDLER_ONMAX);
4259                         if (IS_ERR(data)) {
4260                                 ret = PTR_ERR(data);
4261                                 break;
4262                         }
4263                 } else if ((len = str_has_prefix(str, "onchange("))) {
4264                         char *action_str = str + len;
4265
4266                         data = track_data_parse(hist_data, action_str,
4267                                                 HANDLER_ONCHANGE);
4268                         if (IS_ERR(data)) {
4269                                 ret = PTR_ERR(data);
4270                                 break;
4271                         }
4272                 } else {
4273                         ret = -EINVAL;
4274                         break;
4275                 }
4276
4277                 hist_data->actions[hist_data->n_actions++] = data;
4278         }
4279
4280         return ret;
4281 }
4282
4283 static int create_actions(struct hist_trigger_data *hist_data)
4284 {
4285         struct action_data *data;
4286         unsigned int i;
4287         int ret = 0;
4288
4289         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4290                 data = hist_data->actions[i];
4291
4292                 if (data->handler == HANDLER_ONMATCH) {
4293                         ret = onmatch_create(hist_data, data);
4294                         if (ret)
4295                                 break;
4296                 } else if (data->handler == HANDLER_ONMAX ||
4297                            data->handler == HANDLER_ONCHANGE) {
4298                         ret = track_data_create(hist_data, data);
4299                         if (ret)
4300                                 break;
4301                 } else {
4302                         ret = -EINVAL;
4303                         break;
4304                 }
4305         }
4306
4307         return ret;
4308 }
4309
4310 static void print_actions(struct seq_file *m,
4311                           struct hist_trigger_data *hist_data,
4312                           struct tracing_map_elt *elt)
4313 {
4314         unsigned int i;
4315
4316         for (i = 0; i < hist_data->n_actions; i++) {
4317                 struct action_data *data = hist_data->actions[i];
4318
4319                 if (data->action == ACTION_SNAPSHOT)
4320                         continue;
4321
4322                 if (data->handler == HANDLER_ONMAX ||
4323                     data->handler == HANDLER_ONCHANGE)
4324                         track_data_print(m, hist_data, elt, data);
4325         }
4326 }
4327
4328 static void print_action_spec(struct seq_file *m,
4329                               struct hist_trigger_data *hist_data,
4330                               struct action_data *data)
4331 {
4332         unsigned int i;
4333
4334         if (data->action == ACTION_SAVE) {
4335                 for (i = 0; i < hist_data->n_save_vars; i++) {
4336                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4337                         if (i < hist_data->n_save_vars - 1)
4338                                 seq_puts(m, ",");
4339                 }
4340         } else if (data->action == ACTION_TRACE) {
4341                 if (data->use_trace_keyword)
4342                         seq_printf(m, "%s", data->synth_event_name);
4343                 for (i = 0; i < data->n_params; i++) {
4344                         if (i || data->use_trace_keyword)
4345                                 seq_puts(m, ",");
4346                         seq_printf(m, "%s", data->params[i]);
4347                 }
4348         }
4349 }
4350
4351 static void print_track_data_spec(struct seq_file *m,
4352                                   struct hist_trigger_data *hist_data,
4353                                   struct action_data *data)
4354 {
4355         if (data->handler == HANDLER_ONMAX)
4356                 seq_puts(m, ":onmax(");
4357         else if (data->handler == HANDLER_ONCHANGE)
4358                 seq_puts(m, ":onchange(");
4359         seq_printf(m, "%s", data->track_data.var_str);
4360         seq_printf(m, ").%s(", data->action_name);
4361
4362         print_action_spec(m, hist_data, data);
4363
4364         seq_puts(m, ")");
4365 }
4366
4367 static void print_onmatch_spec(struct seq_file *m,
4368                                struct hist_trigger_data *hist_data,
4369                                struct action_data *data)
4370 {
4371         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4372                    data->match_data.event);
4373
4374         seq_printf(m, "%s(", data->action_name);
4375
4376         print_action_spec(m, hist_data, data);
4377
4378         seq_puts(m, ")");
4379 }
4380
4381 static bool actions_match(struct hist_trigger_data *hist_data,
4382                           struct hist_trigger_data *hist_data_test)
4383 {
4384         unsigned int i, j;
4385
4386         if (hist_data->n_actions != hist_data_test->n_actions)
4387                 return false;
4388
4389         for (i = 0; i < hist_data->n_actions; i++) {
4390                 struct action_data *data = hist_data->actions[i];
4391                 struct action_data *data_test = hist_data_test->actions[i];
4392                 char *action_name, *action_name_test;
4393
4394                 if (data->handler != data_test->handler)
4395                         return false;
4396                 if (data->action != data_test->action)
4397                         return false;
4398
4399                 if (data->n_params != data_test->n_params)
4400                         return false;
4401
4402                 for (j = 0; j < data->n_params; j++) {
4403                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4404                                 return false;
4405                 }
4406
4407                 if (data->use_trace_keyword)
4408                         action_name = data->synth_event_name;
4409                 else
4410                         action_name = data->action_name;
4411
4412                 if (data_test->use_trace_keyword)
4413                         action_name_test = data_test->synth_event_name;
4414                 else
4415                         action_name_test = data_test->action_name;
4416
4417                 if (strcmp(action_name, action_name_test) != 0)
4418                         return false;
4419
4420                 if (data->handler == HANDLER_ONMATCH) {
4421                         if (strcmp(data->match_data.event_system,
4422                                    data_test->match_data.event_system) != 0)
4423                                 return false;
4424                         if (strcmp(data->match_data.event,
4425                                    data_test->match_data.event) != 0)
4426                                 return false;
4427                 } else if (data->handler == HANDLER_ONMAX ||
4428                            data->handler == HANDLER_ONCHANGE) {
4429                         if (strcmp(data->track_data.var_str,
4430                                    data_test->track_data.var_str) != 0)
4431                                 return false;
4432                 }
4433         }
4434
4435         return true;
4436 }
4437
4438
4439 static void print_actions_spec(struct seq_file *m,
4440                                struct hist_trigger_data *hist_data)
4441 {
4442         unsigned int i;
4443
4444         for (i = 0; i < hist_data->n_actions; i++) {
4445                 struct action_data *data = hist_data->actions[i];
4446
4447                 if (data->handler == HANDLER_ONMATCH)
4448                         print_onmatch_spec(m, hist_data, data);
4449                 else if (data->handler == HANDLER_ONMAX ||
4450                          data->handler == HANDLER_ONCHANGE)
4451                         print_track_data_spec(m, hist_data, data);
4452         }
4453 }
4454
4455 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
4456 {
4457         unsigned int i;
4458
4459         for (i = 0; i < hist_data->n_field_var_hists; i++) {
4460                 kfree(hist_data->field_var_hists[i]->cmd);
4461                 kfree(hist_data->field_var_hists[i]);
4462         }
4463 }
4464
4465 static void destroy_hist_data(struct hist_trigger_data *hist_data)
4466 {
4467         if (!hist_data)
4468                 return;
4469
4470         destroy_hist_trigger_attrs(hist_data->attrs);
4471         destroy_hist_fields(hist_data);
4472         tracing_map_destroy(hist_data->map);
4473
4474         destroy_actions(hist_data);
4475         destroy_field_vars(hist_data);
4476         destroy_field_var_hists(hist_data);
4477
4478         kfree(hist_data);
4479 }
4480
4481 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
4482 {
4483         struct tracing_map *map = hist_data->map;
4484         struct ftrace_event_field *field;
4485         struct hist_field *hist_field;
4486         int i, idx = 0;
4487
4488         for_each_hist_field(i, hist_data) {
4489                 hist_field = hist_data->fields[i];
4490                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
4491                         tracing_map_cmp_fn_t cmp_fn;
4492
4493                         field = hist_field->field;
4494
4495                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
4496                                 cmp_fn = tracing_map_cmp_none;
4497                         else if (!field || hist_field->flags & HIST_FIELD_FL_CPU)
4498                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
4499                                                              hist_field->is_signed);
4500                         else if (is_string_field(field))
4501                                 cmp_fn = tracing_map_cmp_string;
4502                         else
4503                                 cmp_fn = tracing_map_cmp_num(field->size,
4504                                                              field->is_signed);
4505                         idx = tracing_map_add_key_field(map,
4506                                                         hist_field->offset,
4507                                                         cmp_fn);
4508                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
4509                         idx = tracing_map_add_sum_field(map);
4510
4511                 if (idx < 0)
4512                         return idx;
4513
4514                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4515                         idx = tracing_map_add_var(map);
4516                         if (idx < 0)
4517                                 return idx;
4518                         hist_field->var.idx = idx;
4519                         hist_field->var.hist_data = hist_data;
4520                 }
4521         }
4522
4523         return 0;
4524 }
4525
4526 static struct hist_trigger_data *
4527 create_hist_data(unsigned int map_bits,
4528                  struct hist_trigger_attrs *attrs,
4529                  struct trace_event_file *file,
4530                  bool remove)
4531 {
4532         const struct tracing_map_ops *map_ops = NULL;
4533         struct hist_trigger_data *hist_data;
4534         int ret = 0;
4535
4536         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
4537         if (!hist_data)
4538                 return ERR_PTR(-ENOMEM);
4539
4540         hist_data->attrs = attrs;
4541         hist_data->remove = remove;
4542         hist_data->event_file = file;
4543
4544         ret = parse_actions(hist_data);
4545         if (ret)
4546                 goto free;
4547
4548         ret = create_hist_fields(hist_data, file);
4549         if (ret)
4550                 goto free;
4551
4552         ret = create_sort_keys(hist_data);
4553         if (ret)
4554                 goto free;
4555
4556         map_ops = &hist_trigger_elt_data_ops;
4557
4558         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
4559                                             map_ops, hist_data);
4560         if (IS_ERR(hist_data->map)) {
4561                 ret = PTR_ERR(hist_data->map);
4562                 hist_data->map = NULL;
4563                 goto free;
4564         }
4565
4566         ret = create_tracing_map_fields(hist_data);
4567         if (ret)
4568                 goto free;
4569  out:
4570         return hist_data;
4571  free:
4572         hist_data->attrs = NULL;
4573
4574         destroy_hist_data(hist_data);
4575
4576         hist_data = ERR_PTR(ret);
4577
4578         goto out;
4579 }
4580
4581 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
4582                                     struct tracing_map_elt *elt,
4583                                     struct trace_buffer *buffer, void *rec,
4584                                     struct ring_buffer_event *rbe,
4585                                     u64 *var_ref_vals)
4586 {
4587         struct hist_elt_data *elt_data;
4588         struct hist_field *hist_field;
4589         unsigned int i, var_idx;
4590         u64 hist_val;
4591
4592         elt_data = elt->private_data;
4593         elt_data->var_ref_vals = var_ref_vals;
4594
4595         for_each_hist_val_field(i, hist_data) {
4596                 hist_field = hist_data->fields[i];
4597                 hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4598                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4599                         var_idx = hist_field->var.idx;
4600
4601                         if (hist_field->flags & HIST_FIELD_FL_STRING) {
4602                                 unsigned int str_start, var_str_idx, idx;
4603                                 char *str, *val_str;
4604                                 unsigned int size;
4605
4606                                 str_start = hist_data->n_field_var_str +
4607                                         hist_data->n_save_var_str;
4608                                 var_str_idx = hist_field->var_str_idx;
4609                                 idx = str_start + var_str_idx;
4610
4611                                 str = elt_data->field_var_str[idx];
4612                                 val_str = (char *)(uintptr_t)hist_val;
4613
4614                                 size = min(hist_field->size, STR_VAR_LEN_MAX);
4615                                 strscpy(str, val_str, size);
4616
4617                                 hist_val = (u64)(uintptr_t)str;
4618                         }
4619                         tracing_map_set_var(elt, var_idx, hist_val);
4620                         continue;
4621                 }
4622                 tracing_map_update_sum(elt, i, hist_val);
4623         }
4624
4625         for_each_hist_key_field(i, hist_data) {
4626                 hist_field = hist_data->fields[i];
4627                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
4628                         hist_val = hist_field->fn(hist_field, elt, buffer, rbe, rec);
4629                         var_idx = hist_field->var.idx;
4630                         tracing_map_set_var(elt, var_idx, hist_val);
4631                 }
4632         }
4633
4634         update_field_vars(hist_data, elt, buffer, rbe, rec);
4635 }
4636
4637 static inline void add_to_key(char *compound_key, void *key,
4638                               struct hist_field *key_field, void *rec)
4639 {
4640         size_t size = key_field->size;
4641
4642         if (key_field->flags & HIST_FIELD_FL_STRING) {
4643                 struct ftrace_event_field *field;
4644
4645                 field = key_field->field;
4646                 if (field->filter_type == FILTER_DYN_STRING)
4647                         size = *(u32 *)(rec + field->offset) >> 16;
4648                 else if (field->filter_type == FILTER_STATIC_STRING)
4649                         size = field->size;
4650
4651                 /* ensure NULL-termination */
4652                 if (size > key_field->size - 1)
4653                         size = key_field->size - 1;
4654
4655                 strncpy(compound_key + key_field->offset, (char *)key, size);
4656         } else
4657                 memcpy(compound_key + key_field->offset, key, size);
4658 }
4659
4660 static void
4661 hist_trigger_actions(struct hist_trigger_data *hist_data,
4662                      struct tracing_map_elt *elt,
4663                      struct trace_buffer *buffer, void *rec,
4664                      struct ring_buffer_event *rbe, void *key,
4665                      u64 *var_ref_vals)
4666 {
4667         struct action_data *data;
4668         unsigned int i;
4669
4670         for (i = 0; i < hist_data->n_actions; i++) {
4671                 data = hist_data->actions[i];
4672                 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
4673         }
4674 }
4675
4676 static void event_hist_trigger(struct event_trigger_data *data,
4677                                struct trace_buffer *buffer, void *rec,
4678                                struct ring_buffer_event *rbe)
4679 {
4680         struct hist_trigger_data *hist_data = data->private_data;
4681         bool use_compound_key = (hist_data->n_keys > 1);
4682         unsigned long entries[HIST_STACKTRACE_DEPTH];
4683         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
4684         char compound_key[HIST_KEY_SIZE_MAX];
4685         struct tracing_map_elt *elt = NULL;
4686         struct hist_field *key_field;
4687         u64 field_contents;
4688         void *key = NULL;
4689         unsigned int i;
4690
4691         if (unlikely(!rbe))
4692                 return;
4693
4694         memset(compound_key, 0, hist_data->key_size);
4695
4696         for_each_hist_key_field(i, hist_data) {
4697                 key_field = hist_data->fields[i];
4698
4699                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4700                         memset(entries, 0, HIST_STACKTRACE_SIZE);
4701                         stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
4702                                          HIST_STACKTRACE_SKIP);
4703                         key = entries;
4704                 } else {
4705                         field_contents = key_field->fn(key_field, elt, buffer, rbe, rec);
4706                         if (key_field->flags & HIST_FIELD_FL_STRING) {
4707                                 key = (void *)(unsigned long)field_contents;
4708                                 use_compound_key = true;
4709                         } else
4710                                 key = (void *)&field_contents;
4711                 }
4712
4713                 if (use_compound_key)
4714                         add_to_key(compound_key, key, key_field, rec);
4715         }
4716
4717         if (use_compound_key)
4718                 key = compound_key;
4719
4720         if (hist_data->n_var_refs &&
4721             !resolve_var_refs(hist_data, key, var_ref_vals, false))
4722                 return;
4723
4724         elt = tracing_map_insert(hist_data->map, key);
4725         if (!elt)
4726                 return;
4727
4728         hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
4729
4730         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
4731                 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
4732 }
4733
4734 static void hist_trigger_stacktrace_print(struct seq_file *m,
4735                                           unsigned long *stacktrace_entries,
4736                                           unsigned int max_entries)
4737 {
4738         char str[KSYM_SYMBOL_LEN];
4739         unsigned int spaces = 8;
4740         unsigned int i;
4741
4742         for (i = 0; i < max_entries; i++) {
4743                 if (!stacktrace_entries[i])
4744                         return;
4745
4746                 seq_printf(m, "%*c", 1 + spaces, ' ');
4747                 sprint_symbol(str, stacktrace_entries[i]);
4748                 seq_printf(m, "%s\n", str);
4749         }
4750 }
4751
4752 static void hist_trigger_print_key(struct seq_file *m,
4753                                    struct hist_trigger_data *hist_data,
4754                                    void *key,
4755                                    struct tracing_map_elt *elt)
4756 {
4757         struct hist_field *key_field;
4758         char str[KSYM_SYMBOL_LEN];
4759         bool multiline = false;
4760         const char *field_name;
4761         unsigned int i;
4762         u64 uval;
4763
4764         seq_puts(m, "{ ");
4765
4766         for_each_hist_key_field(i, hist_data) {
4767                 key_field = hist_data->fields[i];
4768
4769                 if (i > hist_data->n_vals)
4770                         seq_puts(m, ", ");
4771
4772                 field_name = hist_field_name(key_field, 0);
4773
4774                 if (key_field->flags & HIST_FIELD_FL_HEX) {
4775                         uval = *(u64 *)(key + key_field->offset);
4776                         seq_printf(m, "%s: %llx", field_name, uval);
4777                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
4778                         uval = *(u64 *)(key + key_field->offset);
4779                         sprint_symbol_no_offset(str, uval);
4780                         seq_printf(m, "%s: [%llx] %-45s", field_name,
4781                                    uval, str);
4782                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
4783                         uval = *(u64 *)(key + key_field->offset);
4784                         sprint_symbol(str, uval);
4785                         seq_printf(m, "%s: [%llx] %-55s", field_name,
4786                                    uval, str);
4787                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
4788                         struct hist_elt_data *elt_data = elt->private_data;
4789                         char *comm;
4790
4791                         if (WARN_ON_ONCE(!elt_data))
4792                                 return;
4793
4794                         comm = elt_data->comm;
4795
4796                         uval = *(u64 *)(key + key_field->offset);
4797                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
4798                                    comm, uval);
4799                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
4800                         const char *syscall_name;
4801
4802                         uval = *(u64 *)(key + key_field->offset);
4803                         syscall_name = get_syscall_name(uval);
4804                         if (!syscall_name)
4805                                 syscall_name = "unknown_syscall";
4806
4807                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
4808                                    syscall_name, uval);
4809                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
4810                         seq_puts(m, "stacktrace:\n");
4811                         hist_trigger_stacktrace_print(m,
4812                                                       key + key_field->offset,
4813                                                       HIST_STACKTRACE_DEPTH);
4814                         multiline = true;
4815                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
4816                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
4817                                    *(u64 *)(key + key_field->offset));
4818                 } else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
4819                         unsigned long buckets = key_field->buckets;
4820                         uval = *(u64 *)(key + key_field->offset);
4821                         seq_printf(m, "%s: ~ %llu-%llu", field_name,
4822                                    uval, uval + buckets -1);
4823                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
4824                         seq_printf(m, "%s: %-50s", field_name,
4825                                    (char *)(key + key_field->offset));
4826                 } else {
4827                         uval = *(u64 *)(key + key_field->offset);
4828                         seq_printf(m, "%s: %10llu", field_name, uval);
4829                 }
4830         }
4831
4832         if (!multiline)
4833                 seq_puts(m, " ");
4834
4835         seq_puts(m, "}");
4836 }
4837
4838 static void hist_trigger_entry_print(struct seq_file *m,
4839                                      struct hist_trigger_data *hist_data,
4840                                      void *key,
4841                                      struct tracing_map_elt *elt)
4842 {
4843         const char *field_name;
4844         unsigned int i;
4845
4846         hist_trigger_print_key(m, hist_data, key, elt);
4847
4848         seq_printf(m, " hitcount: %10llu",
4849                    tracing_map_read_sum(elt, HITCOUNT_IDX));
4850
4851         for (i = 1; i < hist_data->n_vals; i++) {
4852                 field_name = hist_field_name(hist_data->fields[i], 0);
4853
4854                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_VAR ||
4855                     hist_data->fields[i]->flags & HIST_FIELD_FL_EXPR)
4856                         continue;
4857
4858                 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
4859                         seq_printf(m, "  %s: %10llx", field_name,
4860                                    tracing_map_read_sum(elt, i));
4861                 } else {
4862                         seq_printf(m, "  %s: %10llu", field_name,
4863                                    tracing_map_read_sum(elt, i));
4864                 }
4865         }
4866
4867         print_actions(m, hist_data, elt);
4868
4869         seq_puts(m, "\n");
4870 }
4871
4872 static int print_entries(struct seq_file *m,
4873                          struct hist_trigger_data *hist_data)
4874 {
4875         struct tracing_map_sort_entry **sort_entries = NULL;
4876         struct tracing_map *map = hist_data->map;
4877         int i, n_entries;
4878
4879         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
4880                                              hist_data->n_sort_keys,
4881                                              &sort_entries);
4882         if (n_entries < 0)
4883                 return n_entries;
4884
4885         for (i = 0; i < n_entries; i++)
4886                 hist_trigger_entry_print(m, hist_data,
4887                                          sort_entries[i]->key,
4888                                          sort_entries[i]->elt);
4889
4890         tracing_map_destroy_sort_entries(sort_entries, n_entries);
4891
4892         return n_entries;
4893 }
4894
4895 static void hist_trigger_show(struct seq_file *m,
4896                               struct event_trigger_data *data, int n)
4897 {
4898         struct hist_trigger_data *hist_data;
4899         int n_entries;
4900
4901         if (n > 0)
4902                 seq_puts(m, "\n\n");
4903
4904         seq_puts(m, "# event histogram\n#\n# trigger info: ");
4905         data->ops->print(m, data->ops, data);
4906         seq_puts(m, "#\n\n");
4907
4908         hist_data = data->private_data;
4909         n_entries = print_entries(m, hist_data);
4910         if (n_entries < 0)
4911                 n_entries = 0;
4912
4913         track_data_snapshot_print(m, hist_data);
4914
4915         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
4916                    (u64)atomic64_read(&hist_data->map->hits),
4917                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
4918 }
4919
4920 static int hist_show(struct seq_file *m, void *v)
4921 {
4922         struct event_trigger_data *data;
4923         struct trace_event_file *event_file;
4924         int n = 0, ret = 0;
4925
4926         mutex_lock(&event_mutex);
4927
4928         event_file = event_file_data(m->private);
4929         if (unlikely(!event_file)) {
4930                 ret = -ENODEV;
4931                 goto out_unlock;
4932         }
4933
4934         list_for_each_entry(data, &event_file->triggers, list) {
4935                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
4936                         hist_trigger_show(m, data, n++);
4937         }
4938
4939  out_unlock:
4940         mutex_unlock(&event_mutex);
4941
4942         return ret;
4943 }
4944
4945 static int event_hist_open(struct inode *inode, struct file *file)
4946 {
4947         int ret;
4948
4949         ret = security_locked_down(LOCKDOWN_TRACEFS);
4950         if (ret)
4951                 return ret;
4952
4953         return single_open(file, hist_show, file);
4954 }
4955
4956 const struct file_operations event_hist_fops = {
4957         .open = event_hist_open,
4958         .read = seq_read,
4959         .llseek = seq_lseek,
4960         .release = single_release,
4961 };
4962
4963 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
4964 static void hist_field_debug_show_flags(struct seq_file *m,
4965                                         unsigned long flags)
4966 {
4967         seq_puts(m, "      flags:\n");
4968
4969         if (flags & HIST_FIELD_FL_KEY)
4970                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
4971         else if (flags & HIST_FIELD_FL_HITCOUNT)
4972                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
4973         else if (flags & HIST_FIELD_FL_VAR)
4974                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
4975         else if (flags & HIST_FIELD_FL_VAR_REF)
4976                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
4977         else
4978                 seq_puts(m, "        VAL: normal u64 value\n");
4979
4980         if (flags & HIST_FIELD_FL_ALIAS)
4981                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
4982 }
4983
4984 static int hist_field_debug_show(struct seq_file *m,
4985                                  struct hist_field *field, unsigned long flags)
4986 {
4987         if ((field->flags & flags) != flags) {
4988                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
4989                 return -EINVAL;
4990         }
4991
4992         hist_field_debug_show_flags(m, field->flags);
4993         if (field->field)
4994                 seq_printf(m, "      ftrace_event_field name: %s\n",
4995                            field->field->name);
4996
4997         if (field->flags & HIST_FIELD_FL_VAR) {
4998                 seq_printf(m, "      var.name: %s\n", field->var.name);
4999                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5000                            field->var.idx);
5001         }
5002
5003         if (field->flags & HIST_FIELD_FL_ALIAS)
5004                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5005                            field->var_ref_idx);
5006
5007         if (field->flags & HIST_FIELD_FL_VAR_REF) {
5008                 seq_printf(m, "      name: %s\n", field->name);
5009                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5010                            field->var.idx);
5011                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
5012                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5013                            field->var_ref_idx);
5014                 if (field->system)
5015                         seq_printf(m, "      system: %s\n", field->system);
5016                 if (field->event_name)
5017                         seq_printf(m, "      event_name: %s\n", field->event_name);
5018         }
5019
5020         seq_printf(m, "      type: %s\n", field->type);
5021         seq_printf(m, "      size: %u\n", field->size);
5022         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5023
5024         return 0;
5025 }
5026
5027 static int field_var_debug_show(struct seq_file *m,
5028                                 struct field_var *field_var, unsigned int i,
5029                                 bool save_vars)
5030 {
5031         const char *vars_name = save_vars ? "save_vars" : "field_vars";
5032         struct hist_field *field;
5033         int ret = 0;
5034
5035         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
5036
5037         field = field_var->var;
5038
5039         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
5040
5041         hist_field_debug_show_flags(m, field->flags);
5042         seq_printf(m, "      var.name: %s\n", field->var.name);
5043         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5044                    field->var.idx);
5045
5046         field = field_var->val;
5047
5048         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
5049         if (field->field)
5050                 seq_printf(m, "      ftrace_event_field name: %s\n",
5051                            field->field->name);
5052         else {
5053                 ret = -EINVAL;
5054                 goto out;
5055         }
5056
5057         seq_printf(m, "      type: %s\n", field->type);
5058         seq_printf(m, "      size: %u\n", field->size);
5059         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5060 out:
5061         return ret;
5062 }
5063
5064 static int hist_action_debug_show(struct seq_file *m,
5065                                   struct action_data *data, int i)
5066 {
5067         int ret = 0;
5068
5069         if (data->handler == HANDLER_ONMAX ||
5070             data->handler == HANDLER_ONCHANGE) {
5071                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
5072                 ret = hist_field_debug_show(m, data->track_data.var_ref,
5073                                             HIST_FIELD_FL_VAR_REF);
5074                 if (ret)
5075                         goto out;
5076
5077                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
5078                 ret = hist_field_debug_show(m, data->track_data.track_var,
5079                                             HIST_FIELD_FL_VAR);
5080                 if (ret)
5081                         goto out;
5082         }
5083
5084         if (data->handler == HANDLER_ONMATCH) {
5085                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
5086                            i, data->match_data.event_system);
5087                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
5088                            i, data->match_data.event);
5089         }
5090 out:
5091         return ret;
5092 }
5093
5094 static int hist_actions_debug_show(struct seq_file *m,
5095                                    struct hist_trigger_data *hist_data)
5096 {
5097         int i, ret = 0;
5098
5099         if (hist_data->n_actions)
5100                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
5101
5102         for (i = 0; i < hist_data->n_actions; i++) {
5103                 struct action_data *action = hist_data->actions[i];
5104
5105                 ret = hist_action_debug_show(m, action, i);
5106                 if (ret)
5107                         goto out;
5108         }
5109
5110         if (hist_data->n_save_vars)
5111                 seq_puts(m, "\n  save action variables (save() params):\n");
5112
5113         for (i = 0; i < hist_data->n_save_vars; i++) {
5114                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
5115                 if (ret)
5116                         goto out;
5117         }
5118 out:
5119         return ret;
5120 }
5121
5122 static void hist_trigger_debug_show(struct seq_file *m,
5123                                     struct event_trigger_data *data, int n)
5124 {
5125         struct hist_trigger_data *hist_data;
5126         int i, ret;
5127
5128         if (n > 0)
5129                 seq_puts(m, "\n\n");
5130
5131         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5132         data->ops->print(m, data->ops, data);
5133         seq_puts(m, "#\n\n");
5134
5135         hist_data = data->private_data;
5136
5137         seq_printf(m, "hist_data: %p\n\n", hist_data);
5138         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5139         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5140         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5141
5142         seq_puts(m, "\n  val fields:\n\n");
5143
5144         seq_puts(m, "    hist_data->fields[0]:\n");
5145         ret = hist_field_debug_show(m, hist_data->fields[0],
5146                                     HIST_FIELD_FL_HITCOUNT);
5147         if (ret)
5148                 return;
5149
5150         for (i = 1; i < hist_data->n_vals; i++) {
5151                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5152                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5153                 if (ret)
5154                         return;
5155         }
5156
5157         seq_puts(m, "\n  key fields:\n");
5158
5159         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5160                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5161                 ret = hist_field_debug_show(m, hist_data->fields[i],
5162                                             HIST_FIELD_FL_KEY);
5163                 if (ret)
5164                         return;
5165         }
5166
5167         if (hist_data->n_var_refs)
5168                 seq_puts(m, "\n  variable reference fields:\n");
5169
5170         for (i = 0; i < hist_data->n_var_refs; i++) {
5171                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5172                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5173                                             HIST_FIELD_FL_VAR_REF);
5174                 if (ret)
5175                         return;
5176         }
5177
5178         if (hist_data->n_field_vars)
5179                 seq_puts(m, "\n  field variables:\n");
5180
5181         for (i = 0; i < hist_data->n_field_vars; i++) {
5182                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5183                 if (ret)
5184                         return;
5185         }
5186
5187         ret = hist_actions_debug_show(m, hist_data);
5188         if (ret)
5189                 return;
5190 }
5191
5192 static int hist_debug_show(struct seq_file *m, void *v)
5193 {
5194         struct event_trigger_data *data;
5195         struct trace_event_file *event_file;
5196         int n = 0, ret = 0;
5197
5198         mutex_lock(&event_mutex);
5199
5200         event_file = event_file_data(m->private);
5201         if (unlikely(!event_file)) {
5202                 ret = -ENODEV;
5203                 goto out_unlock;
5204         }
5205
5206         list_for_each_entry(data, &event_file->triggers, list) {
5207                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5208                         hist_trigger_debug_show(m, data, n++);
5209         }
5210
5211  out_unlock:
5212         mutex_unlock(&event_mutex);
5213
5214         return ret;
5215 }
5216
5217 static int event_hist_debug_open(struct inode *inode, struct file *file)
5218 {
5219         int ret;
5220
5221         ret = security_locked_down(LOCKDOWN_TRACEFS);
5222         if (ret)
5223                 return ret;
5224
5225         return single_open(file, hist_debug_show, file);
5226 }
5227
5228 const struct file_operations event_hist_debug_fops = {
5229         .open = event_hist_debug_open,
5230         .read = seq_read,
5231         .llseek = seq_lseek,
5232         .release = single_release,
5233 };
5234 #endif
5235
5236 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5237 {
5238         const char *field_name = hist_field_name(hist_field, 0);
5239
5240         if (hist_field->var.name)
5241                 seq_printf(m, "%s=", hist_field->var.name);
5242
5243         if (hist_field->flags & HIST_FIELD_FL_CPU)
5244                 seq_puts(m, "common_cpu");
5245         else if (field_name) {
5246                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5247                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5248                         seq_putc(m, '$');
5249                 seq_printf(m, "%s", field_name);
5250         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5251                 seq_puts(m, "common_timestamp");
5252
5253         if (hist_field->flags) {
5254                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5255                     !(hist_field->flags & HIST_FIELD_FL_EXPR)) {
5256                         const char *flags = get_hist_field_flags(hist_field);
5257
5258                         if (flags)
5259                                 seq_printf(m, ".%s", flags);
5260                 }
5261         }
5262         if (hist_field->buckets)
5263                 seq_printf(m, "=%ld", hist_field->buckets);
5264 }
5265
5266 static int event_hist_trigger_print(struct seq_file *m,
5267                                     struct event_trigger_ops *ops,
5268                                     struct event_trigger_data *data)
5269 {
5270         struct hist_trigger_data *hist_data = data->private_data;
5271         struct hist_field *field;
5272         bool have_var = false;
5273         unsigned int i;
5274
5275         seq_puts(m, "hist:");
5276
5277         if (data->name)
5278                 seq_printf(m, "%s:", data->name);
5279
5280         seq_puts(m, "keys=");
5281
5282         for_each_hist_key_field(i, hist_data) {
5283                 field = hist_data->fields[i];
5284
5285                 if (i > hist_data->n_vals)
5286                         seq_puts(m, ",");
5287
5288                 if (field->flags & HIST_FIELD_FL_STACKTRACE)
5289                         seq_puts(m, "stacktrace");
5290                 else
5291                         hist_field_print(m, field);
5292         }
5293
5294         seq_puts(m, ":vals=");
5295
5296         for_each_hist_val_field(i, hist_data) {
5297                 field = hist_data->fields[i];
5298                 if (field->flags & HIST_FIELD_FL_VAR) {
5299                         have_var = true;
5300                         continue;
5301                 }
5302
5303                 if (i == HITCOUNT_IDX)
5304                         seq_puts(m, "hitcount");
5305                 else {
5306                         seq_puts(m, ",");
5307                         hist_field_print(m, field);
5308                 }
5309         }
5310
5311         if (have_var) {
5312                 unsigned int n = 0;
5313
5314                 seq_puts(m, ":");
5315
5316                 for_each_hist_val_field(i, hist_data) {
5317                         field = hist_data->fields[i];
5318
5319                         if (field->flags & HIST_FIELD_FL_VAR) {
5320                                 if (n++)
5321                                         seq_puts(m, ",");
5322                                 hist_field_print(m, field);
5323                         }
5324                 }
5325         }
5326
5327         seq_puts(m, ":sort=");
5328
5329         for (i = 0; i < hist_data->n_sort_keys; i++) {
5330                 struct tracing_map_sort_key *sort_key;
5331                 unsigned int idx, first_key_idx;
5332
5333                 /* skip VAR vals */
5334                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
5335
5336                 sort_key = &hist_data->sort_keys[i];
5337                 idx = sort_key->field_idx;
5338
5339                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
5340                         return -EINVAL;
5341
5342                 if (i > 0)
5343                         seq_puts(m, ",");
5344
5345                 if (idx == HITCOUNT_IDX)
5346                         seq_puts(m, "hitcount");
5347                 else {
5348                         if (idx >= first_key_idx)
5349                                 idx += hist_data->n_vars;
5350                         hist_field_print(m, hist_data->fields[idx]);
5351                 }
5352
5353                 if (sort_key->descending)
5354                         seq_puts(m, ".descending");
5355         }
5356         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
5357         if (hist_data->enable_timestamps)
5358                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
5359
5360         print_actions_spec(m, hist_data);
5361
5362         if (data->filter_str)
5363                 seq_printf(m, " if %s", data->filter_str);
5364
5365         if (data->paused)
5366                 seq_puts(m, " [paused]");
5367         else
5368                 seq_puts(m, " [active]");
5369
5370         seq_putc(m, '\n');
5371
5372         return 0;
5373 }
5374
5375 static int event_hist_trigger_init(struct event_trigger_ops *ops,
5376                                    struct event_trigger_data *data)
5377 {
5378         struct hist_trigger_data *hist_data = data->private_data;
5379
5380         if (!data->ref && hist_data->attrs->name)
5381                 save_named_trigger(hist_data->attrs->name, data);
5382
5383         data->ref++;
5384
5385         return 0;
5386 }
5387
5388 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
5389 {
5390         struct trace_event_file *file;
5391         unsigned int i;
5392         char *cmd;
5393         int ret;
5394
5395         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5396                 file = hist_data->field_var_hists[i]->hist_data->event_file;
5397                 cmd = hist_data->field_var_hists[i]->cmd;
5398                 ret = event_hist_trigger_func(&trigger_hist_cmd, file,
5399                                               "!hist", "hist", cmd);
5400                 WARN_ON_ONCE(ret < 0);
5401         }
5402 }
5403
5404 static void event_hist_trigger_free(struct event_trigger_ops *ops,
5405                                     struct event_trigger_data *data)
5406 {
5407         struct hist_trigger_data *hist_data = data->private_data;
5408
5409         if (WARN_ON_ONCE(data->ref <= 0))
5410                 return;
5411
5412         data->ref--;
5413         if (!data->ref) {
5414                 if (data->name)
5415                         del_named_trigger(data);
5416
5417                 trigger_data_free(data);
5418
5419                 remove_hist_vars(hist_data);
5420
5421                 unregister_field_var_hists(hist_data);
5422
5423                 destroy_hist_data(hist_data);
5424         }
5425 }
5426
5427 static struct event_trigger_ops event_hist_trigger_ops = {
5428         .func                   = event_hist_trigger,
5429         .print                  = event_hist_trigger_print,
5430         .init                   = event_hist_trigger_init,
5431         .free                   = event_hist_trigger_free,
5432 };
5433
5434 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
5435                                          struct event_trigger_data *data)
5436 {
5437         data->ref++;
5438
5439         save_named_trigger(data->named_data->name, data);
5440
5441         event_hist_trigger_init(ops, data->named_data);
5442
5443         return 0;
5444 }
5445
5446 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
5447                                           struct event_trigger_data *data)
5448 {
5449         if (WARN_ON_ONCE(data->ref <= 0))
5450                 return;
5451
5452         event_hist_trigger_free(ops, data->named_data);
5453
5454         data->ref--;
5455         if (!data->ref) {
5456                 del_named_trigger(data);
5457                 trigger_data_free(data);
5458         }
5459 }
5460
5461 static struct event_trigger_ops event_hist_trigger_named_ops = {
5462         .func                   = event_hist_trigger,
5463         .print                  = event_hist_trigger_print,
5464         .init                   = event_hist_trigger_named_init,
5465         .free                   = event_hist_trigger_named_free,
5466 };
5467
5468 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
5469                                                             char *param)
5470 {
5471         return &event_hist_trigger_ops;
5472 }
5473
5474 static void hist_clear(struct event_trigger_data *data)
5475 {
5476         struct hist_trigger_data *hist_data = data->private_data;
5477
5478         if (data->name)
5479                 pause_named_trigger(data);
5480
5481         tracepoint_synchronize_unregister();
5482
5483         tracing_map_clear(hist_data->map);
5484
5485         if (data->name)
5486                 unpause_named_trigger(data);
5487 }
5488
5489 static bool compatible_field(struct ftrace_event_field *field,
5490                              struct ftrace_event_field *test_field)
5491 {
5492         if (field == test_field)
5493                 return true;
5494         if (field == NULL || test_field == NULL)
5495                 return false;
5496         if (strcmp(field->name, test_field->name) != 0)
5497                 return false;
5498         if (strcmp(field->type, test_field->type) != 0)
5499                 return false;
5500         if (field->size != test_field->size)
5501                 return false;
5502         if (field->is_signed != test_field->is_signed)
5503                 return false;
5504
5505         return true;
5506 }
5507
5508 static bool hist_trigger_match(struct event_trigger_data *data,
5509                                struct event_trigger_data *data_test,
5510                                struct event_trigger_data *named_data,
5511                                bool ignore_filter)
5512 {
5513         struct tracing_map_sort_key *sort_key, *sort_key_test;
5514         struct hist_trigger_data *hist_data, *hist_data_test;
5515         struct hist_field *key_field, *key_field_test;
5516         unsigned int i;
5517
5518         if (named_data && (named_data != data_test) &&
5519             (named_data != data_test->named_data))
5520                 return false;
5521
5522         if (!named_data && is_named_trigger(data_test))
5523                 return false;
5524
5525         hist_data = data->private_data;
5526         hist_data_test = data_test->private_data;
5527
5528         if (hist_data->n_vals != hist_data_test->n_vals ||
5529             hist_data->n_fields != hist_data_test->n_fields ||
5530             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
5531                 return false;
5532
5533         if (!ignore_filter) {
5534                 if ((data->filter_str && !data_test->filter_str) ||
5535                    (!data->filter_str && data_test->filter_str))
5536                         return false;
5537         }
5538
5539         for_each_hist_field(i, hist_data) {
5540                 key_field = hist_data->fields[i];
5541                 key_field_test = hist_data_test->fields[i];
5542
5543                 if (key_field->flags != key_field_test->flags)
5544                         return false;
5545                 if (!compatible_field(key_field->field, key_field_test->field))
5546                         return false;
5547                 if (key_field->offset != key_field_test->offset)
5548                         return false;
5549                 if (key_field->size != key_field_test->size)
5550                         return false;
5551                 if (key_field->is_signed != key_field_test->is_signed)
5552                         return false;
5553                 if (!!key_field->var.name != !!key_field_test->var.name)
5554                         return false;
5555                 if (key_field->var.name &&
5556                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
5557                         return false;
5558         }
5559
5560         for (i = 0; i < hist_data->n_sort_keys; i++) {
5561                 sort_key = &hist_data->sort_keys[i];
5562                 sort_key_test = &hist_data_test->sort_keys[i];
5563
5564                 if (sort_key->field_idx != sort_key_test->field_idx ||
5565                     sort_key->descending != sort_key_test->descending)
5566                         return false;
5567         }
5568
5569         if (!ignore_filter && data->filter_str &&
5570             (strcmp(data->filter_str, data_test->filter_str) != 0))
5571                 return false;
5572
5573         if (!actions_match(hist_data, hist_data_test))
5574                 return false;
5575
5576         return true;
5577 }
5578
5579 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
5580                                  struct event_trigger_data *data,
5581                                  struct trace_event_file *file)
5582 {
5583         struct hist_trigger_data *hist_data = data->private_data;
5584         struct event_trigger_data *test, *named_data = NULL;
5585         struct trace_array *tr = file->tr;
5586         int ret = 0;
5587
5588         if (hist_data->attrs->name) {
5589                 named_data = find_named_trigger(hist_data->attrs->name);
5590                 if (named_data) {
5591                         if (!hist_trigger_match(data, named_data, named_data,
5592                                                 true)) {
5593                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
5594                                 ret = -EINVAL;
5595                                 goto out;
5596                         }
5597                 }
5598         }
5599
5600         if (hist_data->attrs->name && !named_data)
5601                 goto new;
5602
5603         lockdep_assert_held(&event_mutex);
5604
5605         list_for_each_entry(test, &file->triggers, list) {
5606                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5607                         if (!hist_trigger_match(data, test, named_data, false))
5608                                 continue;
5609                         if (hist_data->attrs->pause)
5610                                 test->paused = true;
5611                         else if (hist_data->attrs->cont)
5612                                 test->paused = false;
5613                         else if (hist_data->attrs->clear)
5614                                 hist_clear(test);
5615                         else {
5616                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
5617                                 ret = -EEXIST;
5618                         }
5619                         goto out;
5620                 }
5621         }
5622  new:
5623         if (hist_data->attrs->cont || hist_data->attrs->clear) {
5624                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
5625                 ret = -ENOENT;
5626                 goto out;
5627         }
5628
5629         if (hist_data->attrs->pause)
5630                 data->paused = true;
5631
5632         if (named_data) {
5633                 data->private_data = named_data->private_data;
5634                 set_named_trigger_data(data, named_data);
5635                 data->ops = &event_hist_trigger_named_ops;
5636         }
5637
5638         if (data->ops->init) {
5639                 ret = data->ops->init(data->ops, data);
5640                 if (ret < 0)
5641                         goto out;
5642         }
5643
5644         if (hist_data->enable_timestamps) {
5645                 char *clock = hist_data->attrs->clock;
5646
5647                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
5648                 if (ret) {
5649                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
5650                         goto out;
5651                 }
5652
5653                 tracing_set_filter_buffering(file->tr, true);
5654         }
5655
5656         if (named_data)
5657                 destroy_hist_data(hist_data);
5658
5659         ret++;
5660  out:
5661         return ret;
5662 }
5663
5664 static int hist_trigger_enable(struct event_trigger_data *data,
5665                                struct trace_event_file *file)
5666 {
5667         int ret = 0;
5668
5669         list_add_tail_rcu(&data->list, &file->triggers);
5670
5671         update_cond_flag(file);
5672
5673         if (trace_event_trigger_enable_disable(file, 1) < 0) {
5674                 list_del_rcu(&data->list);
5675                 update_cond_flag(file);
5676                 ret--;
5677         }
5678
5679         return ret;
5680 }
5681
5682 static bool have_hist_trigger_match(struct event_trigger_data *data,
5683                                     struct trace_event_file *file)
5684 {
5685         struct hist_trigger_data *hist_data = data->private_data;
5686         struct event_trigger_data *test, *named_data = NULL;
5687         bool match = false;
5688
5689         lockdep_assert_held(&event_mutex);
5690
5691         if (hist_data->attrs->name)
5692                 named_data = find_named_trigger(hist_data->attrs->name);
5693
5694         list_for_each_entry(test, &file->triggers, list) {
5695                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5696                         if (hist_trigger_match(data, test, named_data, false)) {
5697                                 match = true;
5698                                 break;
5699                         }
5700                 }
5701         }
5702
5703         return match;
5704 }
5705
5706 static bool hist_trigger_check_refs(struct event_trigger_data *data,
5707                                     struct trace_event_file *file)
5708 {
5709         struct hist_trigger_data *hist_data = data->private_data;
5710         struct event_trigger_data *test, *named_data = NULL;
5711
5712         lockdep_assert_held(&event_mutex);
5713
5714         if (hist_data->attrs->name)
5715                 named_data = find_named_trigger(hist_data->attrs->name);
5716
5717         list_for_each_entry(test, &file->triggers, list) {
5718                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5719                         if (!hist_trigger_match(data, test, named_data, false))
5720                                 continue;
5721                         hist_data = test->private_data;
5722                         if (check_var_refs(hist_data))
5723                                 return true;
5724                         break;
5725                 }
5726         }
5727
5728         return false;
5729 }
5730
5731 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
5732                                     struct event_trigger_data *data,
5733                                     struct trace_event_file *file)
5734 {
5735         struct hist_trigger_data *hist_data = data->private_data;
5736         struct event_trigger_data *test, *named_data = NULL;
5737         bool unregistered = false;
5738
5739         lockdep_assert_held(&event_mutex);
5740
5741         if (hist_data->attrs->name)
5742                 named_data = find_named_trigger(hist_data->attrs->name);
5743
5744         list_for_each_entry(test, &file->triggers, list) {
5745                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5746                         if (!hist_trigger_match(data, test, named_data, false))
5747                                 continue;
5748                         unregistered = true;
5749                         list_del_rcu(&test->list);
5750                         trace_event_trigger_enable_disable(file, 0);
5751                         update_cond_flag(file);
5752                         break;
5753                 }
5754         }
5755
5756         if (unregistered && test->ops->free)
5757                 test->ops->free(test->ops, test);
5758
5759         if (hist_data->enable_timestamps) {
5760                 if (!hist_data->remove || unregistered)
5761                         tracing_set_filter_buffering(file->tr, false);
5762         }
5763 }
5764
5765 static bool hist_file_check_refs(struct trace_event_file *file)
5766 {
5767         struct hist_trigger_data *hist_data;
5768         struct event_trigger_data *test;
5769
5770         lockdep_assert_held(&event_mutex);
5771
5772         list_for_each_entry(test, &file->triggers, list) {
5773                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5774                         hist_data = test->private_data;
5775                         if (check_var_refs(hist_data))
5776                                 return true;
5777                 }
5778         }
5779
5780         return false;
5781 }
5782
5783 static void hist_unreg_all(struct trace_event_file *file)
5784 {
5785         struct event_trigger_data *test, *n;
5786         struct hist_trigger_data *hist_data;
5787         struct synth_event *se;
5788         const char *se_name;
5789
5790         lockdep_assert_held(&event_mutex);
5791
5792         if (hist_file_check_refs(file))
5793                 return;
5794
5795         list_for_each_entry_safe(test, n, &file->triggers, list) {
5796                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
5797                         hist_data = test->private_data;
5798                         list_del_rcu(&test->list);
5799                         trace_event_trigger_enable_disable(file, 0);
5800
5801                         se_name = trace_event_name(file->event_call);
5802                         se = find_synth_event(se_name);
5803                         if (se)
5804                                 se->ref--;
5805
5806                         update_cond_flag(file);
5807                         if (hist_data->enable_timestamps)
5808                                 tracing_set_filter_buffering(file->tr, false);
5809                         if (test->ops->free)
5810                                 test->ops->free(test->ops, test);
5811                 }
5812         }
5813 }
5814
5815 static int event_hist_trigger_func(struct event_command *cmd_ops,
5816                                    struct trace_event_file *file,
5817                                    char *glob, char *cmd, char *param)
5818 {
5819         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
5820         struct event_trigger_data *trigger_data;
5821         struct hist_trigger_attrs *attrs;
5822         struct event_trigger_ops *trigger_ops;
5823         struct hist_trigger_data *hist_data;
5824         struct synth_event *se;
5825         const char *se_name;
5826         bool remove = false;
5827         char *trigger, *p;
5828         int ret = 0;
5829
5830         lockdep_assert_held(&event_mutex);
5831
5832         if (glob && strlen(glob)) {
5833                 hist_err_clear();
5834                 last_cmd_set(file, param);
5835         }
5836
5837         if (!param)
5838                 return -EINVAL;
5839
5840         if (glob[0] == '!')
5841                 remove = true;
5842
5843         /*
5844          * separate the trigger from the filter (k:v [if filter])
5845          * allowing for whitespace in the trigger
5846          */
5847         p = trigger = param;
5848         do {
5849                 p = strstr(p, "if");
5850                 if (!p)
5851                         break;
5852                 if (p == param)
5853                         return -EINVAL;
5854                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
5855                         p++;
5856                         continue;
5857                 }
5858                 if (p >= param + strlen(param) - (sizeof("if") - 1) - 1)
5859                         return -EINVAL;
5860                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
5861                         p++;
5862                         continue;
5863                 }
5864                 break;
5865         } while (p);
5866
5867         if (!p)
5868                 param = NULL;
5869         else {
5870                 *(p - 1) = '\0';
5871                 param = strstrip(p);
5872                 trigger = strstrip(trigger);
5873         }
5874
5875         attrs = parse_hist_trigger_attrs(file->tr, trigger);
5876         if (IS_ERR(attrs))
5877                 return PTR_ERR(attrs);
5878
5879         if (attrs->map_bits)
5880                 hist_trigger_bits = attrs->map_bits;
5881
5882         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
5883         if (IS_ERR(hist_data)) {
5884                 destroy_hist_trigger_attrs(attrs);
5885                 return PTR_ERR(hist_data);
5886         }
5887
5888         trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
5889
5890         trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
5891         if (!trigger_data) {
5892                 ret = -ENOMEM;
5893                 goto out_free;
5894         }
5895
5896         trigger_data->count = -1;
5897         trigger_data->ops = trigger_ops;
5898         trigger_data->cmd_ops = cmd_ops;
5899
5900         INIT_LIST_HEAD(&trigger_data->list);
5901         RCU_INIT_POINTER(trigger_data->filter, NULL);
5902
5903         trigger_data->private_data = hist_data;
5904
5905         /* if param is non-empty, it's supposed to be a filter */
5906         if (param && cmd_ops->set_filter) {
5907                 ret = cmd_ops->set_filter(param, trigger_data, file);
5908                 if (ret < 0)
5909                         goto out_free;
5910         }
5911
5912         if (remove) {
5913                 if (!have_hist_trigger_match(trigger_data, file))
5914                         goto out_free;
5915
5916                 if (hist_trigger_check_refs(trigger_data, file)) {
5917                         ret = -EBUSY;
5918                         goto out_free;
5919                 }
5920
5921                 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5922                 se_name = trace_event_name(file->event_call);
5923                 se = find_synth_event(se_name);
5924                 if (se)
5925                         se->ref--;
5926                 ret = 0;
5927                 goto out_free;
5928         }
5929
5930         ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
5931         /*
5932          * The above returns on success the # of triggers registered,
5933          * but if it didn't register any it returns zero.  Consider no
5934          * triggers registered a failure too.
5935          */
5936         if (!ret) {
5937                 if (!(attrs->pause || attrs->cont || attrs->clear))
5938                         ret = -ENOENT;
5939                 goto out_free;
5940         } else if (ret < 0)
5941                 goto out_free;
5942
5943         if (get_named_trigger_data(trigger_data))
5944                 goto enable;
5945
5946         ret = create_actions(hist_data);
5947         if (ret)
5948                 goto out_unreg;
5949
5950         if (has_hist_vars(hist_data) || hist_data->n_var_refs) {
5951                 ret = save_hist_vars(hist_data);
5952                 if (ret)
5953                         goto out_unreg;
5954         }
5955
5956         ret = tracing_map_init(hist_data->map);
5957         if (ret)
5958                 goto out_unreg;
5959 enable:
5960         ret = hist_trigger_enable(trigger_data, file);
5961         if (ret)
5962                 goto out_unreg;
5963
5964         se_name = trace_event_name(file->event_call);
5965         se = find_synth_event(se_name);
5966         if (se)
5967                 se->ref++;
5968         /* Just return zero, not the number of registered triggers */
5969         ret = 0;
5970  out:
5971         if (ret == 0 && glob[0])
5972                 hist_err_clear();
5973
5974         return ret;
5975  out_unreg:
5976         cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
5977  out_free:
5978         if (cmd_ops->set_filter)
5979                 cmd_ops->set_filter(NULL, trigger_data, NULL);
5980
5981         remove_hist_vars(hist_data);
5982
5983         kfree(trigger_data);
5984
5985         destroy_hist_data(hist_data);
5986         goto out;
5987 }
5988
5989 static struct event_command trigger_hist_cmd = {
5990         .name                   = "hist",
5991         .trigger_type           = ETT_EVENT_HIST,
5992         .flags                  = EVENT_CMD_FL_NEEDS_REC,
5993         .func                   = event_hist_trigger_func,
5994         .reg                    = hist_register_trigger,
5995         .unreg                  = hist_unregister_trigger,
5996         .unreg_all              = hist_unreg_all,
5997         .get_trigger_ops        = event_hist_get_trigger_ops,
5998         .set_filter             = set_trigger_filter,
5999 };
6000
6001 __init int register_trigger_hist_cmd(void)
6002 {
6003         int ret;
6004
6005         ret = register_event_command(&trigger_hist_cmd);
6006         WARN_ON(ret < 0);
6007
6008         return ret;
6009 }
6010
6011 static void
6012 hist_enable_trigger(struct event_trigger_data *data,
6013                     struct trace_buffer *buffer,  void *rec,
6014                     struct ring_buffer_event *event)
6015 {
6016         struct enable_trigger_data *enable_data = data->private_data;
6017         struct event_trigger_data *test;
6018
6019         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6020                                 lockdep_is_held(&event_mutex)) {
6021                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6022                         if (enable_data->enable)
6023                                 test->paused = false;
6024                         else
6025                                 test->paused = true;
6026                 }
6027         }
6028 }
6029
6030 static void
6031 hist_enable_count_trigger(struct event_trigger_data *data,
6032                           struct trace_buffer *buffer,  void *rec,
6033                           struct ring_buffer_event *event)
6034 {
6035         if (!data->count)
6036                 return;
6037
6038         if (data->count != -1)
6039                 (data->count)--;
6040
6041         hist_enable_trigger(data, buffer, rec, event);
6042 }
6043
6044 static struct event_trigger_ops hist_enable_trigger_ops = {
6045         .func                   = hist_enable_trigger,
6046         .print                  = event_enable_trigger_print,
6047         .init                   = event_trigger_init,
6048         .free                   = event_enable_trigger_free,
6049 };
6050
6051 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6052         .func                   = hist_enable_count_trigger,
6053         .print                  = event_enable_trigger_print,
6054         .init                   = event_trigger_init,
6055         .free                   = event_enable_trigger_free,
6056 };
6057
6058 static struct event_trigger_ops hist_disable_trigger_ops = {
6059         .func                   = hist_enable_trigger,
6060         .print                  = event_enable_trigger_print,
6061         .init                   = event_trigger_init,
6062         .free                   = event_enable_trigger_free,
6063 };
6064
6065 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6066         .func                   = hist_enable_count_trigger,
6067         .print                  = event_enable_trigger_print,
6068         .init                   = event_trigger_init,
6069         .free                   = event_enable_trigger_free,
6070 };
6071
6072 static struct event_trigger_ops *
6073 hist_enable_get_trigger_ops(char *cmd, char *param)
6074 {
6075         struct event_trigger_ops *ops;
6076         bool enable;
6077
6078         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6079
6080         if (enable)
6081                 ops = param ? &hist_enable_count_trigger_ops :
6082                         &hist_enable_trigger_ops;
6083         else
6084                 ops = param ? &hist_disable_count_trigger_ops :
6085                         &hist_disable_trigger_ops;
6086
6087         return ops;
6088 }
6089
6090 static void hist_enable_unreg_all(struct trace_event_file *file)
6091 {
6092         struct event_trigger_data *test, *n;
6093
6094         list_for_each_entry_safe(test, n, &file->triggers, list) {
6095                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6096                         list_del_rcu(&test->list);
6097                         update_cond_flag(file);
6098                         trace_event_trigger_enable_disable(file, 0);
6099                         if (test->ops->free)
6100                                 test->ops->free(test->ops, test);
6101                 }
6102         }
6103 }
6104
6105 static struct event_command trigger_hist_enable_cmd = {
6106         .name                   = ENABLE_HIST_STR,
6107         .trigger_type           = ETT_HIST_ENABLE,
6108         .func                   = event_enable_trigger_func,
6109         .reg                    = event_enable_register_trigger,
6110         .unreg                  = event_enable_unregister_trigger,
6111         .unreg_all              = hist_enable_unreg_all,
6112         .get_trigger_ops        = hist_enable_get_trigger_ops,
6113         .set_filter             = set_trigger_filter,
6114 };
6115
6116 static struct event_command trigger_hist_disable_cmd = {
6117         .name                   = DISABLE_HIST_STR,
6118         .trigger_type           = ETT_HIST_ENABLE,
6119         .func                   = event_enable_trigger_func,
6120         .reg                    = event_enable_register_trigger,
6121         .unreg                  = event_enable_unregister_trigger,
6122         .unreg_all              = hist_enable_unreg_all,
6123         .get_trigger_ops        = hist_enable_get_trigger_ops,
6124         .set_filter             = set_trigger_filter,
6125 };
6126
6127 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6128 {
6129         unregister_event_command(&trigger_hist_enable_cmd);
6130         unregister_event_command(&trigger_hist_disable_cmd);
6131 }
6132
6133 __init int register_trigger_hist_enable_disable_cmds(void)
6134 {
6135         int ret;
6136
6137         ret = register_event_command(&trigger_hist_enable_cmd);
6138         if (WARN_ON(ret < 0))
6139                 return ret;
6140         ret = register_event_command(&trigger_hist_disable_cmd);
6141         if (WARN_ON(ret < 0))
6142                 unregister_trigger_hist_enable_disable_cmds();
6143
6144         return ret;
6145 }