kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),   \
  69         C(INVALID_STR_OPERAND,  "String type can not be an operand in expression"), \
  70         C(EXPECT_NUMBER,        "Expecting numeric literal"),           \
  71         C(UNARY_MINUS_SUBEXPR,  "Unary minus not supported in sub-expressions"), \
  72         C(DIVISION_BY_ZERO,     "Division by zero"),                    \
  73         C(NEED_NOHC_VAL,        "Non-hitcount value is required for 'nohitcount'"),
  74
  75 #undef C
  76 #define C(a, b)         HIST_ERR_##a
  77
  78 enum { ERRORS };
  79
  80 #undef C
  81 #define C(a, b)         b
  82
  83 static const char *err_text[] = { ERRORS };
  84
  85 struct hist_field;
  86
  87 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  88                                 struct tracing_map_elt *elt,
  89                                 struct trace_buffer *buffer,
  90                                 struct ring_buffer_event *rbe,
  91                                 void *event);
  92
  93 #define HIST_FIELD_OPERANDS_MAX 2
  94 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  95 #define HIST_ACTIONS_MAX        8
  96 #define HIST_CONST_DIGITS_MAX   21
  97 #define HIST_DIV_SHIFT          20  /* For optimizing division by constants */
  98
  99 enum field_op_id {
 100         FIELD_OP_NONE,
 101         FIELD_OP_PLUS,
 102         FIELD_OP_MINUS,
 103         FIELD_OP_UNARY_MINUS,
 104         FIELD_OP_DIV,
 105         FIELD_OP_MULT,
 106 };
 107
 108 enum hist_field_fn {
 109         HIST_FIELD_FN_NOP,
 110         HIST_FIELD_FN_VAR_REF,
 111         HIST_FIELD_FN_COUNTER,
 112         HIST_FIELD_FN_CONST,
 113         HIST_FIELD_FN_LOG2,
 114         HIST_FIELD_FN_BUCKET,
 115         HIST_FIELD_FN_TIMESTAMP,
 116         HIST_FIELD_FN_CPU,
 117         HIST_FIELD_FN_STRING,
 118         HIST_FIELD_FN_DYNSTRING,
 119         HIST_FIELD_FN_RELDYNSTRING,
 120         HIST_FIELD_FN_PSTRING,
 121         HIST_FIELD_FN_S64,
 122         HIST_FIELD_FN_U64,
 123         HIST_FIELD_FN_S32,
 124         HIST_FIELD_FN_U32,
 125         HIST_FIELD_FN_S16,
 126         HIST_FIELD_FN_U16,
 127         HIST_FIELD_FN_S8,
 128         HIST_FIELD_FN_U8,
 129         HIST_FIELD_FN_UMINUS,
 130         HIST_FIELD_FN_MINUS,
 131         HIST_FIELD_FN_PLUS,
 132         HIST_FIELD_FN_DIV,
 133         HIST_FIELD_FN_MULT,
 134         HIST_FIELD_FN_DIV_POWER2,
 135         HIST_FIELD_FN_DIV_NOT_POWER2,
 136         HIST_FIELD_FN_DIV_MULT_SHIFT,
 137         HIST_FIELD_FN_EXECNAME,
 138         HIST_FIELD_FN_STACK,
 139 };
 140
 141 /*
 142  * A hist_var (histogram variable) contains variable information for
 143  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 144  * flag set.  A hist_var has a variable name e.g. ts0, and is
 145  * associated with a given histogram trigger, as specified by
 146  * hist_data.  The hist_var idx is the unique index assigned to the
 147  * variable by the hist trigger's tracing_map.  The idx is what is
 148  * used to set a variable's value and, by a variable reference, to
 149  * retrieve it.
 150  */
 151 struct hist_var {
 152         char                            *name;
 153         struct hist_trigger_data        *hist_data;
 154         unsigned int                    idx;
 155 };
 156
 157 struct hist_field {
 158         struct ftrace_event_field       *field;
 159         unsigned long                   flags;
 160         unsigned long                   buckets;
 161         const char                      *type;
 162         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 163         struct hist_trigger_data        *hist_data;
 164         enum hist_field_fn              fn_num;
 165         unsigned int                    ref;
 166         unsigned int                    size;
 167         unsigned int                    offset;
 168         unsigned int                    is_signed;
 169
 170         /*
 171          * Variable fields contain variable-specific info in var.
 172          */
 173         struct hist_var                 var;
 174         enum field_op_id                operator;
 175         char                            *system;
 176         char                            *event_name;
 177
 178         /*
 179          * The name field is used for EXPR and VAR_REF fields.  VAR
 180          * fields contain the variable name in var.name.
 181          */
 182         char                            *name;
 183
 184         /*
 185          * When a histogram trigger is hit, if it has any references
 186          * to variables, the values of those variables are collected
 187          * into a var_ref_vals array by resolve_var_refs().  The
 188          * current value of each variable is read from the tracing_map
 189          * using the hist field's hist_var.idx and entered into the
 190          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 191          */
 192         unsigned int                    var_ref_idx;
 193         bool                            read_once;
 194
 195         unsigned int                    var_str_idx;
 196
 197         /* Numeric literals are represented as u64 */
 198         u64                             constant;
 199         /* Used to optimize division by constants */
 200         u64                             div_multiplier;
 201 };
 202
 203 static u64 hist_fn_call(struct hist_field *hist_field,
 204                         struct tracing_map_elt *elt,
 205                         struct trace_buffer *buffer,
 206                         struct ring_buffer_event *rbe,
 207                         void *event);
 208
 209 static u64 hist_field_const(struct hist_field *field,
 210                            struct tracing_map_elt *elt,
 211                            struct trace_buffer *buffer,
 212                            struct ring_buffer_event *rbe,
 213                            void *event)
 214 {
 215         return field->constant;
 216 }
 217
 218 static u64 hist_field_counter(struct hist_field *field,
 219                               struct tracing_map_elt *elt,
 220                               struct trace_buffer *buffer,
 221                               struct ring_buffer_event *rbe,
 222                               void *event)
 223 {
 224         return 1;
 225 }
 226
 227 static u64 hist_field_string(struct hist_field *hist_field,
 228                              struct tracing_map_elt *elt,
 229                              struct trace_buffer *buffer,
 230                              struct ring_buffer_event *rbe,
 231                              void *event)
 232 {
 233         char *addr = (char *)(event + hist_field->field->offset);
 234
 235         return (u64)(unsigned long)addr;
 236 }
 237
 238 static u64 hist_field_dynstring(struct hist_field *hist_field,
 239                                 struct tracing_map_elt *elt,
 240                                 struct trace_buffer *buffer,
 241                                 struct ring_buffer_event *rbe,
 242                                 void *event)
 243 {
 244         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 245         int str_loc = str_item & 0xffff;
 246         char *addr = (char *)(event + str_loc);
 247
 248         return (u64)(unsigned long)addr;
 249 }
 250
 251 static u64 hist_field_reldynstring(struct hist_field *hist_field,
 252                                    struct tracing_map_elt *elt,
 253                                    struct trace_buffer *buffer,
 254                                    struct ring_buffer_event *rbe,
 255                                    void *event)
 256 {
 257         u32 *item = event + hist_field->field->offset;
 258         u32 str_item = *item;
 259         int str_loc = str_item & 0xffff;
 260         char *addr = (char *)&item[1] + str_loc;
 261
 262         return (u64)(unsigned long)addr;
 263 }
 264
 265 static u64 hist_field_pstring(struct hist_field *hist_field,
 266                               struct tracing_map_elt *elt,
 267                               struct trace_buffer *buffer,
 268                               struct ring_buffer_event *rbe,
 269                               void *event)
 270 {
 271         char **addr = (char **)(event + hist_field->field->offset);
 272
 273         return (u64)(unsigned long)*addr;
 274 }
 275
 276 static u64 hist_field_log2(struct hist_field *hist_field,
 277                            struct tracing_map_elt *elt,
 278                            struct trace_buffer *buffer,
 279                            struct ring_buffer_event *rbe,
 280                            void *event)
 281 {
 282         struct hist_field *operand = hist_field->operands[0];
 283
 284         u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 285
 286         return (u64) ilog2(roundup_pow_of_two(val));
 287 }
 288
 289 static u64 hist_field_bucket(struct hist_field *hist_field,
 290                              struct tracing_map_elt *elt,
 291                              struct trace_buffer *buffer,
 292                              struct ring_buffer_event *rbe,
 293                              void *event)
 294 {
 295         struct hist_field *operand = hist_field->operands[0];
 296         unsigned long buckets = hist_field->buckets;
 297
 298         u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 299
 300         if (WARN_ON_ONCE(!buckets))
 301                 return val;
 302
 303         if (val >= LONG_MAX)
 304                 val = div64_ul(val, buckets);
 305         else
 306                 val = (u64)((unsigned long)val / buckets);
 307         return val * buckets;
 308 }
 309
 310 static u64 hist_field_plus(struct hist_field *hist_field,
 311                            struct tracing_map_elt *elt,
 312                            struct trace_buffer *buffer,
 313                            struct ring_buffer_event *rbe,
 314                            void *event)
 315 {
 316         struct hist_field *operand1 = hist_field->operands[0];
 317         struct hist_field *operand2 = hist_field->operands[1];
 318
 319         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 320         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 321
 322         return val1 + val2;
 323 }
 324
 325 static u64 hist_field_minus(struct hist_field *hist_field,
 326                             struct tracing_map_elt *elt,
 327                             struct trace_buffer *buffer,
 328                             struct ring_buffer_event *rbe,
 329                             void *event)
 330 {
 331         struct hist_field *operand1 = hist_field->operands[0];
 332         struct hist_field *operand2 = hist_field->operands[1];
 333
 334         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 335         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 336
 337         return val1 - val2;
 338 }
 339
 340 static u64 hist_field_div(struct hist_field *hist_field,
 341                            struct tracing_map_elt *elt,
 342                            struct trace_buffer *buffer,
 343                            struct ring_buffer_event *rbe,
 344                            void *event)
 345 {
 346         struct hist_field *operand1 = hist_field->operands[0];
 347         struct hist_field *operand2 = hist_field->operands[1];
 348
 349         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 350         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 351
 352         /* Return -1 for the undefined case */
 353         if (!val2)
 354                 return -1;
 355
 356         /* Use shift if the divisor is a power of 2 */
 357         if (!(val2 & (val2 - 1)))
 358                 return val1 >> __ffs64(val2);
 359
 360         return div64_u64(val1, val2);
 361 }
 362
 363 static u64 div_by_power_of_two(struct hist_field *hist_field,
 364                                 struct tracing_map_elt *elt,
 365                                 struct trace_buffer *buffer,
 366                                 struct ring_buffer_event *rbe,
 367                                 void *event)
 368 {
 369         struct hist_field *operand1 = hist_field->operands[0];
 370         struct hist_field *operand2 = hist_field->operands[1];
 371
 372         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 373
 374         return val1 >> __ffs64(operand2->constant);
 375 }
 376
 377 static u64 div_by_not_power_of_two(struct hist_field *hist_field,
 378                                 struct tracing_map_elt *elt,
 379                                 struct trace_buffer *buffer,
 380                                 struct ring_buffer_event *rbe,
 381                                 void *event)
 382 {
 383         struct hist_field *operand1 = hist_field->operands[0];
 384         struct hist_field *operand2 = hist_field->operands[1];
 385
 386         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 387
 388         return div64_u64(val1, operand2->constant);
 389 }
 390
 391 static u64 div_by_mult_and_shift(struct hist_field *hist_field,
 392                                 struct tracing_map_elt *elt,
 393                                 struct trace_buffer *buffer,
 394                                 struct ring_buffer_event *rbe,
 395                                 void *event)
 396 {
 397         struct hist_field *operand1 = hist_field->operands[0];
 398         struct hist_field *operand2 = hist_field->operands[1];
 399
 400         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 401
 402         /*
 403          * If the divisor is a constant, do a multiplication and shift instead.
 404          *
 405          * Choose Z = some power of 2. If Y <= Z, then:
 406          *     X / Y = (X * (Z / Y)) / Z
 407          *
 408          * (Z / Y) is a constant (mult) which is calculated at parse time, so:
 409          *     X / Y = (X * mult) / Z
 410          *
 411          * The division by Z can be replaced by a shift since Z is a power of 2:
 412          *     X / Y = (X * mult) >> HIST_DIV_SHIFT
 413          *
 414          * As long, as X < Z the results will not be off by more than 1.
 415          */
 416         if (val1 < (1 << HIST_DIV_SHIFT)) {
 417                 u64 mult = operand2->div_multiplier;
 418
 419                 return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
 420         }
 421
 422         return div64_u64(val1, operand2->constant);
 423 }
 424
 425 static u64 hist_field_mult(struct hist_field *hist_field,
 426                            struct tracing_map_elt *elt,
 427                            struct trace_buffer *buffer,
 428                            struct ring_buffer_event *rbe,
 429                            void *event)
 430 {
 431         struct hist_field *operand1 = hist_field->operands[0];
 432         struct hist_field *operand2 = hist_field->operands[1];
 433
 434         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 435         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 436
 437         return val1 * val2;
 438 }
 439
 440 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 441                                   struct tracing_map_elt *elt,
 442                                   struct trace_buffer *buffer,
 443                                   struct ring_buffer_event *rbe,
 444                                   void *event)
 445 {
 446         struct hist_field *operand = hist_field->operands[0];
 447
 448         s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event);
 449         u64 val = (u64)-sval;
 450
 451         return val;
 452 }
 453
 454 #define DEFINE_HIST_FIELD_FN(type)                                      \
 455         static u64 hist_field_##type(struct hist_field *hist_field,     \
 456                                      struct tracing_map_elt *elt,       \
 457                                      struct trace_buffer *buffer,       \
 458                                      struct ring_buffer_event *rbe,     \
 459                                      void *event)                       \
 460 {                                                                       \
 461         type *addr = (type *)(event + hist_field->field->offset);       \
 462                                                                         \
 463         return (u64)(unsigned long)*addr;                               \
 464 }
 465
 466 DEFINE_HIST_FIELD_FN(s64);
 467 DEFINE_HIST_FIELD_FN(u64);
 468 DEFINE_HIST_FIELD_FN(s32);
 469 DEFINE_HIST_FIELD_FN(u32);
 470 DEFINE_HIST_FIELD_FN(s16);
 471 DEFINE_HIST_FIELD_FN(u16);
 472 DEFINE_HIST_FIELD_FN(s8);
 473 DEFINE_HIST_FIELD_FN(u8);
 474
 475 #define for_each_hist_field(i, hist_data)       \
 476         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 477
 478 #define for_each_hist_val_field(i, hist_data)   \
 479         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 480
 481 #define for_each_hist_key_field(i, hist_data)   \
 482         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 483
 484 #define HITCOUNT_IDX            0
 485 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 486
 487 enum hist_field_flags {
 488         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 489         HIST_FIELD_FL_KEY               = 1 << 1,
 490         HIST_FIELD_FL_STRING            = 1 << 2,
 491         HIST_FIELD_FL_HEX               = 1 << 3,
 492         HIST_FIELD_FL_SYM               = 1 << 4,
 493         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 494         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 495         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 496         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 497         HIST_FIELD_FL_LOG2              = 1 << 9,
 498         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 499         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 500         HIST_FIELD_FL_VAR               = 1 << 12,
 501         HIST_FIELD_FL_EXPR              = 1 << 13,
 502         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 503         HIST_FIELD_FL_CPU               = 1 << 15,
 504         HIST_FIELD_FL_ALIAS             = 1 << 16,
 505         HIST_FIELD_FL_BUCKET            = 1 << 17,
 506         HIST_FIELD_FL_CONST             = 1 << 18,
 507         HIST_FIELD_FL_PERCENT           = 1 << 19,
 508         HIST_FIELD_FL_GRAPH             = 1 << 20,
 509 };
 510
 511 struct var_defs {
 512         unsigned int    n_vars;
 513         char            *name[TRACING_MAP_VARS_MAX];
 514         char            *expr[TRACING_MAP_VARS_MAX];
 515 };
 516
 517 struct hist_trigger_attrs {
 518         char            *keys_str;
 519         char            *vals_str;
 520         char            *sort_key_str;
 521         char            *name;
 522         char            *clock;
 523         bool            pause;
 524         bool            cont;
 525         bool            clear;
 526         bool            ts_in_usecs;
 527         bool            no_hitcount;
 528         unsigned int    map_bits;
 529
 530         char            *assignment_str[TRACING_MAP_VARS_MAX];
 531         unsigned int    n_assignments;
 532
 533         char            *action_str[HIST_ACTIONS_MAX];
 534         unsigned int    n_actions;
 535
 536         struct var_defs var_defs;
 537 };
 538
 539 struct field_var {
 540         struct hist_field       *var;
 541         struct hist_field       *val;
 542 };
 543
 544 struct field_var_hist {
 545         struct hist_trigger_data        *hist_data;
 546         char                            *cmd;
 547 };
 548
 549 struct hist_trigger_data {
 550         struct hist_field               *fields[HIST_FIELDS_MAX];
 551         unsigned int                    n_vals;
 552         unsigned int                    n_keys;
 553         unsigned int                    n_fields;
 554         unsigned int                    n_vars;
 555         unsigned int                    n_var_str;
 556         unsigned int                    key_size;
 557         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 558         unsigned int                    n_sort_keys;
 559         struct trace_event_file         *event_file;
 560         struct hist_trigger_attrs       *attrs;
 561         struct tracing_map              *map;
 562         bool                            enable_timestamps;
 563         bool                            remove;
 564         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 565         unsigned int                    n_var_refs;
 566
 567         struct action_data              *actions[HIST_ACTIONS_MAX];
 568         unsigned int                    n_actions;
 569
 570         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 571         unsigned int                    n_field_vars;
 572         unsigned int                    n_field_var_str;
 573         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 574         unsigned int                    n_field_var_hists;
 575
 576         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 577         unsigned int                    n_save_vars;
 578         unsigned int                    n_save_var_str;
 579 };
 580
 581 struct action_data;
 582
 583 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 584                              struct tracing_map_elt *elt,
 585                              struct trace_buffer *buffer, void *rec,
 586                              struct ring_buffer_event *rbe, void *key,
 587                              struct action_data *data, u64 *var_ref_vals);
 588
 589 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 590
 591 enum handler_id {
 592         HANDLER_ONMATCH = 1,
 593         HANDLER_ONMAX,
 594         HANDLER_ONCHANGE,
 595 };
 596
 597 enum action_id {
 598         ACTION_SAVE = 1,
 599         ACTION_TRACE,
 600         ACTION_SNAPSHOT,
 601 };
 602
 603 struct action_data {
 604         enum handler_id         handler;
 605         enum action_id          action;
 606         char                    *action_name;
 607         action_fn_t             fn;
 608
 609         unsigned int            n_params;
 610         char                    *params[SYNTH_FIELDS_MAX];
 611
 612         /*
 613          * When a histogram trigger is hit, the values of any
 614          * references to variables, including variables being passed
 615          * as parameters to synthetic events, are collected into a
 616          * var_ref_vals array.  This var_ref_idx array is an array of
 617          * indices into the var_ref_vals array, one for each synthetic
 618          * event param, and is passed to the synthetic event
 619          * invocation.
 620          */
 621         unsigned int            var_ref_idx[SYNTH_FIELDS_MAX];
 622         struct synth_event      *synth_event;
 623         bool                    use_trace_keyword;
 624         char                    *synth_event_name;
 625
 626         union {
 627                 struct {
 628                         char                    *event;
 629                         char                    *event_system;
 630                 } match_data;
 631
 632                 struct {
 633                         /*
 634                          * var_str contains the $-unstripped variable
 635                          * name referenced by var_ref, and used when
 636                          * printing the action.  Because var_ref
 637                          * creation is deferred to create_actions(),
 638                          * we need a per-action way to save it until
 639                          * then, thus var_str.
 640                          */
 641                         char                    *var_str;
 642
 643                         /*
 644                          * var_ref refers to the variable being
 645                          * tracked e.g onmax($var).
 646                          */
 647                         struct hist_field       *var_ref;
 648
 649                         /*
 650                          * track_var contains the 'invisible' tracking
 651                          * variable created to keep the current
 652                          * e.g. max value.
 653                          */
 654                         struct hist_field       *track_var;
 655
 656                         check_track_val_fn_t    check_val;
 657                         action_fn_t             save_data;
 658                 } track_data;
 659         };
 660 };
 661
 662 struct track_data {
 663         u64                             track_val;
 664         bool                            updated;
 665
 666         unsigned int                    key_len;
 667         void                            *key;
 668         struct tracing_map_elt          elt;
 669
 670         struct action_data              *action_data;
 671         struct hist_trigger_data        *hist_data;
 672 };
 673
 674 struct hist_elt_data {
 675         char *comm;
 676         u64 *var_ref_vals;
 677         char **field_var_str;
 678         int n_field_var_str;
 679 };
 680
 681 struct snapshot_context {
 682         struct tracing_map_elt  *elt;
 683         void                    *key;
 684 };
 685
 686 /*
 687  * Returns the specific division function to use if the divisor
 688  * is constant. This avoids extra branches when the trigger is hit.
 689  */
 690 static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor)
 691 {
 692         u64 div = divisor->constant;
 693
 694         if (!(div & (div - 1)))
 695                 return HIST_FIELD_FN_DIV_POWER2;
 696
 697         /* If the divisor is too large, do a regular division */
 698         if (div > (1 << HIST_DIV_SHIFT))
 699                 return HIST_FIELD_FN_DIV_NOT_POWER2;
 700
 701         divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
 702         return HIST_FIELD_FN_DIV_MULT_SHIFT;
 703 }
 704
 705 static void track_data_free(struct track_data *track_data)
 706 {
 707         struct hist_elt_data *elt_data;
 708
 709         if (!track_data)
 710                 return;
 711
 712         kfree(track_data->key);
 713
 714         elt_data = track_data->elt.private_data;
 715         if (elt_data) {
 716                 kfree(elt_data->comm);
 717                 kfree(elt_data);
 718         }
 719
 720         kfree(track_data);
 721 }
 722
 723 static struct track_data *track_data_alloc(unsigned int key_len,
 724                                            struct action_data *action_data,
 725                                            struct hist_trigger_data *hist_data)
 726 {
 727         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 728         struct hist_elt_data *elt_data;
 729
 730         if (!data)
 731                 return ERR_PTR(-ENOMEM);
 732
 733         data->key = kzalloc(key_len, GFP_KERNEL);
 734         if (!data->key) {
 735                 track_data_free(data);
 736                 return ERR_PTR(-ENOMEM);
 737         }
 738
 739         data->key_len = key_len;
 740         data->action_data = action_data;
 741         data->hist_data = hist_data;
 742
 743         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 744         if (!elt_data) {
 745                 track_data_free(data);
 746                 return ERR_PTR(-ENOMEM);
 747         }
 748
 749         data->elt.private_data = elt_data;
 750
 751         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 752         if (!elt_data->comm) {
 753                 track_data_free(data);
 754                 return ERR_PTR(-ENOMEM);
 755         }
 756
 757         return data;
 758 }
 759
 760 #define HIST_PREFIX "hist:"
 761
 762 static char *last_cmd;
 763 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 764
 765 static int errpos(char *str)
 766 {
 767         if (!str || !last_cmd)
 768                 return 0;
 769
 770         return err_pos(last_cmd, str);
 771 }
 772
 773 static void last_cmd_set(struct trace_event_file *file, char *str)
 774 {
 775         const char *system = NULL, *name = NULL;
 776         struct trace_event_call *call;
 777
 778         if (!str)
 779                 return;
 780
 781         kfree(last_cmd);
 782
 783         last_cmd = kasprintf(GFP_KERNEL, HIST_PREFIX "%s", str);
 784         if (!last_cmd)
 785                 return;
 786
 787         if (file) {
 788                 call = file->event_call;
 789                 system = call->class->system;
 790                 if (system) {
 791                         name = trace_event_name(call);
 792                         if (!name)
 793                                 system = NULL;
 794                 }
 795         }
 796
 797         if (system)
 798                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name);
 799 }
 800
 801 static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos)
 802 {
 803         if (!last_cmd)
 804                 return;
 805
 806         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 807                         err_type, err_pos);
 808 }
 809
 810 static void hist_err_clear(void)
 811 {
 812         if (last_cmd)
 813                 last_cmd[0] = '\0';
 814         last_cmd_loc[0] = '\0';
 815 }
 816
 817 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 818                                     unsigned int *var_ref_idx);
 819
 820 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 821                                unsigned int *var_ref_idx)
 822 {
 823         struct tracepoint *tp = event->tp;
 824
 825         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 826                 struct tracepoint_func *probe_func_ptr;
 827                 synth_probe_func_t probe_func;
 828                 void *__data;
 829
 830                 if (!(cpu_online(raw_smp_processor_id())))
 831                         return;
 832
 833                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 834                 if (probe_func_ptr) {
 835                         do {
 836                                 probe_func = probe_func_ptr->func;
 837                                 __data = probe_func_ptr->data;
 838                                 probe_func(__data, var_ref_vals, var_ref_idx);
 839                         } while ((++probe_func_ptr)->func);
 840                 }
 841         }
 842 }
 843
 844 static void action_trace(struct hist_trigger_data *hist_data,
 845                          struct tracing_map_elt *elt,
 846                          struct trace_buffer *buffer, void *rec,
 847                          struct ring_buffer_event *rbe, void *key,
 848                          struct action_data *data, u64 *var_ref_vals)
 849 {
 850         struct synth_event *event = data->synth_event;
 851
 852         trace_synth(event, var_ref_vals, data->var_ref_idx);
 853 }
 854
 855 struct hist_var_data {
 856         struct list_head list;
 857         struct hist_trigger_data *hist_data;
 858 };
 859
 860 static u64 hist_field_timestamp(struct hist_field *hist_field,
 861                                 struct tracing_map_elt *elt,
 862                                 struct trace_buffer *buffer,
 863                                 struct ring_buffer_event *rbe,
 864                                 void *event)
 865 {
 866         struct hist_trigger_data *hist_data = hist_field->hist_data;
 867         struct trace_array *tr = hist_data->event_file->tr;
 868
 869         u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 870
 871         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 872                 ts = ns2usecs(ts);
 873
 874         return ts;
 875 }
 876
 877 static u64 hist_field_cpu(struct hist_field *hist_field,
 878                           struct tracing_map_elt *elt,
 879                           struct trace_buffer *buffer,
 880                           struct ring_buffer_event *rbe,
 881                           void *event)
 882 {
 883         int cpu = smp_processor_id();
 884
 885         return cpu;
 886 }
 887
 888 /**
 889  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 890  * @hist_field: The VAR_REF field to check
 891  * @var_data: The hist trigger that owns the variable
 892  * @var_idx: The trigger variable identifier
 893  *
 894  * Check the given VAR_REF field to see whether or not it references
 895  * the given variable associated with the given trigger.
 896  *
 897  * Return: The VAR_REF field if it does reference the variable, NULL if not
 898  */
 899 static struct hist_field *
 900 check_field_for_var_ref(struct hist_field *hist_field,
 901                         struct hist_trigger_data *var_data,
 902                         unsigned int var_idx)
 903 {
 904         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 905
 906         if (hist_field && hist_field->var.idx == var_idx &&
 907             hist_field->var.hist_data == var_data)
 908                 return hist_field;
 909
 910         return NULL;
 911 }
 912
 913 /**
 914  * find_var_ref - Check if a trigger has a reference to a trigger variable
 915  * @hist_data: The hist trigger that might have a reference to the variable
 916  * @var_data: The hist trigger that owns the variable
 917  * @var_idx: The trigger variable identifier
 918  *
 919  * Check the list of var_refs[] on the first hist trigger to see
 920  * whether any of them are references to the variable on the second
 921  * trigger.
 922  *
 923  * Return: The VAR_REF field referencing the variable if so, NULL if not
 924  */
 925 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 926                                        struct hist_trigger_data *var_data,
 927                                        unsigned int var_idx)
 928 {
 929         struct hist_field *hist_field;
 930         unsigned int i;
 931
 932         for (i = 0; i < hist_data->n_var_refs; i++) {
 933                 hist_field = hist_data->var_refs[i];
 934                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 935                         return hist_field;
 936         }
 937
 938         return NULL;
 939 }
 940
 941 /**
 942  * find_any_var_ref - Check if there is a reference to a given trigger variable
 943  * @hist_data: The hist trigger
 944  * @var_idx: The trigger variable identifier
 945  *
 946  * Check to see whether the given variable is currently referenced by
 947  * any other trigger.
 948  *
 949  * The trigger the variable is defined on is explicitly excluded - the
 950  * assumption being that a self-reference doesn't prevent a trigger
 951  * from being removed.
 952  *
 953  * Return: The VAR_REF field referencing the variable if so, NULL if not
 954  */
 955 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 956                                            unsigned int var_idx)
 957 {
 958         struct trace_array *tr = hist_data->event_file->tr;
 959         struct hist_field *found = NULL;
 960         struct hist_var_data *var_data;
 961
 962         list_for_each_entry(var_data, &tr->hist_vars, list) {
 963                 if (var_data->hist_data == hist_data)
 964                         continue;
 965                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 966                 if (found)
 967                         break;
 968         }
 969
 970         return found;
 971 }
 972
 973 /**
 974  * check_var_refs - Check if there is a reference to any of trigger's variables
 975  * @hist_data: The hist trigger
 976  *
 977  * A trigger can define one or more variables.  If any one of them is
 978  * currently referenced by any other trigger, this function will
 979  * determine that.
 980  *
 981  * Typically used to determine whether or not a trigger can be removed
 982  * - if there are any references to a trigger's variables, it cannot.
 983  *
 984  * Return: True if there is a reference to any of trigger's variables
 985  */
 986 static bool check_var_refs(struct hist_trigger_data *hist_data)
 987 {
 988         struct hist_field *field;
 989         bool found = false;
 990         int i;
 991
 992         for_each_hist_field(i, hist_data) {
 993                 field = hist_data->fields[i];
 994                 if (field && field->flags & HIST_FIELD_FL_VAR) {
 995                         if (find_any_var_ref(hist_data, field->var.idx)) {
 996                                 found = true;
 997                                 break;
 998                         }
 999                 }
1000         }
1001
1002         return found;
1003 }
1004
1005 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1006 {
1007         struct trace_array *tr = hist_data->event_file->tr;
1008         struct hist_var_data *var_data, *found = NULL;
1009
1010         list_for_each_entry(var_data, &tr->hist_vars, list) {
1011                 if (var_data->hist_data == hist_data) {
1012                         found = var_data;
1013                         break;
1014                 }
1015         }
1016
1017         return found;
1018 }
1019
1020 static bool field_has_hist_vars(struct hist_field *hist_field,
1021                                 unsigned int level)
1022 {
1023         int i;
1024
1025         if (level > 3)
1026                 return false;
1027
1028         if (!hist_field)
1029                 return false;
1030
1031         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1032             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1033                 return true;
1034
1035         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1036                 struct hist_field *operand;
1037
1038                 operand = hist_field->operands[i];
1039                 if (field_has_hist_vars(operand, level + 1))
1040                         return true;
1041         }
1042
1043         return false;
1044 }
1045
1046 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1047 {
1048         struct hist_field *hist_field;
1049         int i;
1050
1051         for_each_hist_field(i, hist_data) {
1052                 hist_field = hist_data->fields[i];
1053                 if (field_has_hist_vars(hist_field, 0))
1054                         return true;
1055         }
1056
1057         return false;
1058 }
1059
1060 static int save_hist_vars(struct hist_trigger_data *hist_data)
1061 {
1062         struct trace_array *tr = hist_data->event_file->tr;
1063         struct hist_var_data *var_data;
1064
1065         var_data = find_hist_vars(hist_data);
1066         if (var_data)
1067                 return 0;
1068
1069         if (tracing_check_open_get_tr(tr))
1070                 return -ENODEV;
1071
1072         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1073         if (!var_data) {
1074                 trace_array_put(tr);
1075                 return -ENOMEM;
1076         }
1077
1078         var_data->hist_data = hist_data;
1079         list_add(&var_data->list, &tr->hist_vars);
1080
1081         return 0;
1082 }
1083
1084 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1085 {
1086         struct trace_array *tr = hist_data->event_file->tr;
1087         struct hist_var_data *var_data;
1088
1089         var_data = find_hist_vars(hist_data);
1090         if (!var_data)
1091                 return;
1092
1093         if (WARN_ON(check_var_refs(hist_data)))
1094                 return;
1095
1096         list_del(&var_data->list);
1097
1098         kfree(var_data);
1099
1100         trace_array_put(tr);
1101 }
1102
1103 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1104                                          const char *var_name)
1105 {
1106         struct hist_field *hist_field, *found = NULL;
1107         int i;
1108
1109         for_each_hist_field(i, hist_data) {
1110                 hist_field = hist_data->fields[i];
1111                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1112                     strcmp(hist_field->var.name, var_name) == 0) {
1113                         found = hist_field;
1114                         break;
1115                 }
1116         }
1117
1118         return found;
1119 }
1120
1121 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1122                                    struct trace_event_file *file,
1123                                    const char *var_name)
1124 {
1125         struct hist_trigger_data *test_data;
1126         struct event_trigger_data *test;
1127         struct hist_field *hist_field;
1128
1129         lockdep_assert_held(&event_mutex);
1130
1131         hist_field = find_var_field(hist_data, var_name);
1132         if (hist_field)
1133                 return hist_field;
1134
1135         list_for_each_entry(test, &file->triggers, list) {
1136                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1137                         test_data = test->private_data;
1138                         hist_field = find_var_field(test_data, var_name);
1139                         if (hist_field)
1140                                 return hist_field;
1141                 }
1142         }
1143
1144         return NULL;
1145 }
1146
1147 static struct trace_event_file *find_var_file(struct trace_array *tr,
1148                                               char *system,
1149                                               char *event_name,
1150                                               char *var_name)
1151 {
1152         struct hist_trigger_data *var_hist_data;
1153         struct hist_var_data *var_data;
1154         struct trace_event_file *file, *found = NULL;
1155
1156         if (system)
1157                 return find_event_file(tr, system, event_name);
1158
1159         list_for_each_entry(var_data, &tr->hist_vars, list) {
1160                 var_hist_data = var_data->hist_data;
1161                 file = var_hist_data->event_file;
1162                 if (file == found)
1163                         continue;
1164
1165                 if (find_var_field(var_hist_data, var_name)) {
1166                         if (found) {
1167                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1168                                 return NULL;
1169                         }
1170
1171                         found = file;
1172                 }
1173         }
1174
1175         return found;
1176 }
1177
1178 static struct hist_field *find_file_var(struct trace_event_file *file,
1179                                         const char *var_name)
1180 {
1181         struct hist_trigger_data *test_data;
1182         struct event_trigger_data *test;
1183         struct hist_field *hist_field;
1184
1185         lockdep_assert_held(&event_mutex);
1186
1187         list_for_each_entry(test, &file->triggers, list) {
1188                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1189                         test_data = test->private_data;
1190                         hist_field = find_var_field(test_data, var_name);
1191                         if (hist_field)
1192                                 return hist_field;
1193                 }
1194         }
1195
1196         return NULL;
1197 }
1198
1199 static struct hist_field *
1200 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1201 {
1202         struct trace_array *tr = hist_data->event_file->tr;
1203         struct hist_field *hist_field, *found = NULL;
1204         struct trace_event_file *file;
1205         unsigned int i;
1206
1207         for (i = 0; i < hist_data->n_actions; i++) {
1208                 struct action_data *data = hist_data->actions[i];
1209
1210                 if (data->handler == HANDLER_ONMATCH) {
1211                         char *system = data->match_data.event_system;
1212                         char *event_name = data->match_data.event;
1213
1214                         file = find_var_file(tr, system, event_name, var_name);
1215                         if (!file)
1216                                 continue;
1217                         hist_field = find_file_var(file, var_name);
1218                         if (hist_field) {
1219                                 if (found) {
1220                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1221                                                  errpos(var_name));
1222                                         return ERR_PTR(-EINVAL);
1223                                 }
1224
1225                                 found = hist_field;
1226                         }
1227                 }
1228         }
1229         return found;
1230 }
1231
1232 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1233                                          char *system,
1234                                          char *event_name,
1235                                          char *var_name)
1236 {
1237         struct trace_array *tr = hist_data->event_file->tr;
1238         struct hist_field *hist_field = NULL;
1239         struct trace_event_file *file;
1240
1241         if (!system || !event_name) {
1242                 hist_field = find_match_var(hist_data, var_name);
1243                 if (IS_ERR(hist_field))
1244                         return NULL;
1245                 if (hist_field)
1246                         return hist_field;
1247         }
1248
1249         file = find_var_file(tr, system, event_name, var_name);
1250         if (!file)
1251                 return NULL;
1252
1253         hist_field = find_file_var(file, var_name);
1254
1255         return hist_field;
1256 }
1257
1258 static u64 hist_field_var_ref(struct hist_field *hist_field,
1259                               struct tracing_map_elt *elt,
1260                               struct trace_buffer *buffer,
1261                               struct ring_buffer_event *rbe,
1262                               void *event)
1263 {
1264         struct hist_elt_data *elt_data;
1265         u64 var_val = 0;
1266
1267         if (WARN_ON_ONCE(!elt))
1268                 return var_val;
1269
1270         elt_data = elt->private_data;
1271         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1272
1273         return var_val;
1274 }
1275
1276 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1277                              u64 *var_ref_vals, bool self)
1278 {
1279         struct hist_trigger_data *var_data;
1280         struct tracing_map_elt *var_elt;
1281         struct hist_field *hist_field;
1282         unsigned int i, var_idx;
1283         bool resolved = true;
1284         u64 var_val = 0;
1285
1286         for (i = 0; i < hist_data->n_var_refs; i++) {
1287                 hist_field = hist_data->var_refs[i];
1288                 var_idx = hist_field->var.idx;
1289                 var_data = hist_field->var.hist_data;
1290
1291                 if (var_data == NULL) {
1292                         resolved = false;
1293                         break;
1294                 }
1295
1296                 if ((self && var_data != hist_data) ||
1297                     (!self && var_data == hist_data))
1298                         continue;
1299
1300                 var_elt = tracing_map_lookup(var_data->map, key);
1301                 if (!var_elt) {
1302                         resolved = false;
1303                         break;
1304                 }
1305
1306                 if (!tracing_map_var_set(var_elt, var_idx)) {
1307                         resolved = false;
1308                         break;
1309                 }
1310
1311                 if (self || !hist_field->read_once)
1312                         var_val = tracing_map_read_var(var_elt, var_idx);
1313                 else
1314                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1315
1316                 var_ref_vals[i] = var_val;
1317         }
1318
1319         return resolved;
1320 }
1321
1322 static const char *hist_field_name(struct hist_field *field,
1323                                    unsigned int level)
1324 {
1325         const char *field_name = "";
1326
1327         if (WARN_ON_ONCE(!field))
1328                 return field_name;
1329
1330         if (level > 1)
1331                 return field_name;
1332
1333         if (field->field)
1334                 field_name = field->field->name;
1335         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1336                  field->flags & HIST_FIELD_FL_ALIAS ||
1337                  field->flags & HIST_FIELD_FL_BUCKET)
1338                 field_name = hist_field_name(field->operands[0], ++level);
1339         else if (field->flags & HIST_FIELD_FL_CPU)
1340                 field_name = "common_cpu";
1341         else if (field->flags & HIST_FIELD_FL_EXPR ||
1342                  field->flags & HIST_FIELD_FL_VAR_REF) {
1343                 if (field->system) {
1344                         static char full_name[MAX_FILTER_STR_VAL];
1345
1346                         strcat(full_name, field->system);
1347                         strcat(full_name, ".");
1348                         strcat(full_name, field->event_name);
1349                         strcat(full_name, ".");
1350                         strcat(full_name, field->name);
1351                         field_name = full_name;
1352                 } else
1353                         field_name = field->name;
1354         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1355                 field_name = "common_timestamp";
1356         else if (field->flags & HIST_FIELD_FL_STACKTRACE) {
1357                 if (field->field)
1358                         field_name = field->field->name;
1359                 else
1360                         field_name = "common_stacktrace";
1361         } else if (field->flags & HIST_FIELD_FL_HITCOUNT)
1362                 field_name = "hitcount";
1363
1364         if (field_name == NULL)
1365                 field_name = "";
1366
1367         return field_name;
1368 }
1369
1370 static enum hist_field_fn select_value_fn(int field_size, int field_is_signed)
1371 {
1372         switch (field_size) {
1373         case 8:
1374                 if (field_is_signed)
1375                         return HIST_FIELD_FN_S64;
1376                 else
1377                         return HIST_FIELD_FN_U64;
1378         case 4:
1379                 if (field_is_signed)
1380                         return HIST_FIELD_FN_S32;
1381                 else
1382                         return HIST_FIELD_FN_U32;
1383         case 2:
1384                 if (field_is_signed)
1385                         return HIST_FIELD_FN_S16;
1386                 else
1387                         return HIST_FIELD_FN_U16;
1388         case 1:
1389                 if (field_is_signed)
1390                         return HIST_FIELD_FN_S8;
1391                 else
1392                         return HIST_FIELD_FN_U8;
1393         }
1394
1395         return HIST_FIELD_FN_NOP;
1396 }
1397
1398 static int parse_map_size(char *str)
1399 {
1400         unsigned long size, map_bits;
1401         int ret;
1402
1403         ret = kstrtoul(str, 0, &size);
1404         if (ret)
1405                 goto out;
1406
1407         map_bits = ilog2(roundup_pow_of_two(size));
1408         if (map_bits < TRACING_MAP_BITS_MIN ||
1409             map_bits > TRACING_MAP_BITS_MAX)
1410                 ret = -EINVAL;
1411         else
1412                 ret = map_bits;
1413  out:
1414         return ret;
1415 }
1416
1417 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1418 {
1419         unsigned int i;
1420
1421         if (!attrs)
1422                 return;
1423
1424         for (i = 0; i < attrs->n_assignments; i++)
1425                 kfree(attrs->assignment_str[i]);
1426
1427         for (i = 0; i < attrs->n_actions; i++)
1428                 kfree(attrs->action_str[i]);
1429
1430         kfree(attrs->name);
1431         kfree(attrs->sort_key_str);
1432         kfree(attrs->keys_str);
1433         kfree(attrs->vals_str);
1434         kfree(attrs->clock);
1435         kfree(attrs);
1436 }
1437
1438 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1439 {
1440         int ret = -EINVAL;
1441
1442         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1443                 return ret;
1444
1445         if ((str_has_prefix(str, "onmatch(")) ||
1446             (str_has_prefix(str, "onmax(")) ||
1447             (str_has_prefix(str, "onchange("))) {
1448                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1449                 if (!attrs->action_str[attrs->n_actions]) {
1450                         ret = -ENOMEM;
1451                         return ret;
1452                 }
1453                 attrs->n_actions++;
1454                 ret = 0;
1455         }
1456         return ret;
1457 }
1458
1459 static int parse_assignment(struct trace_array *tr,
1460                             char *str, struct hist_trigger_attrs *attrs)
1461 {
1462         int len, ret = 0;
1463
1464         if ((len = str_has_prefix(str, "key=")) ||
1465             (len = str_has_prefix(str, "keys="))) {
1466                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1467                 if (!attrs->keys_str) {
1468                         ret = -ENOMEM;
1469                         goto out;
1470                 }
1471         } else if ((len = str_has_prefix(str, "val=")) ||
1472                    (len = str_has_prefix(str, "vals=")) ||
1473                    (len = str_has_prefix(str, "values="))) {
1474                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1475                 if (!attrs->vals_str) {
1476                         ret = -ENOMEM;
1477                         goto out;
1478                 }
1479         } else if ((len = str_has_prefix(str, "sort="))) {
1480                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1481                 if (!attrs->sort_key_str) {
1482                         ret = -ENOMEM;
1483                         goto out;
1484                 }
1485         } else if (str_has_prefix(str, "name=")) {
1486                 attrs->name = kstrdup(str, GFP_KERNEL);
1487                 if (!attrs->name) {
1488                         ret = -ENOMEM;
1489                         goto out;
1490                 }
1491         } else if ((len = str_has_prefix(str, "clock="))) {
1492                 str += len;
1493
1494                 str = strstrip(str);
1495                 attrs->clock = kstrdup(str, GFP_KERNEL);
1496                 if (!attrs->clock) {
1497                         ret = -ENOMEM;
1498                         goto out;
1499                 }
1500         } else if ((len = str_has_prefix(str, "size="))) {
1501                 int map_bits = parse_map_size(str + len);
1502
1503                 if (map_bits < 0) {
1504                         ret = map_bits;
1505                         goto out;
1506                 }
1507                 attrs->map_bits = map_bits;
1508         } else {
1509                 char *assignment;
1510
1511                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1512                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1513                         ret = -EINVAL;
1514                         goto out;
1515                 }
1516
1517                 assignment = kstrdup(str, GFP_KERNEL);
1518                 if (!assignment) {
1519                         ret = -ENOMEM;
1520                         goto out;
1521                 }
1522
1523                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1524         }
1525  out:
1526         return ret;
1527 }
1528
1529 static struct hist_trigger_attrs *
1530 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1531 {
1532         struct hist_trigger_attrs *attrs;
1533         int ret = 0;
1534
1535         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1536         if (!attrs)
1537                 return ERR_PTR(-ENOMEM);
1538
1539         while (trigger_str) {
1540                 char *str = strsep(&trigger_str, ":");
1541                 char *rhs;
1542
1543                 rhs = strchr(str, '=');
1544                 if (rhs) {
1545                         if (!strlen(++rhs)) {
1546                                 ret = -EINVAL;
1547                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1548                                 goto free;
1549                         }
1550                         ret = parse_assignment(tr, str, attrs);
1551                         if (ret)
1552                                 goto free;
1553                 } else if (strcmp(str, "nohitcount") == 0 ||
1554                            strcmp(str, "NOHC") == 0)
1555                         attrs->no_hitcount = true;
1556                 else if (strcmp(str, "pause") == 0)
1557                         attrs->pause = true;
1558                 else if ((strcmp(str, "cont") == 0) ||
1559                          (strcmp(str, "continue") == 0))
1560                         attrs->cont = true;
1561                 else if (strcmp(str, "clear") == 0)
1562                         attrs->clear = true;
1563                 else {
1564                         ret = parse_action(str, attrs);
1565                         if (ret)
1566                                 goto free;
1567                 }
1568         }
1569
1570         if (!attrs->keys_str) {
1571                 ret = -EINVAL;
1572                 goto free;
1573         }
1574
1575         if (!attrs->clock) {
1576                 attrs->clock = kstrdup("global", GFP_KERNEL);
1577                 if (!attrs->clock) {
1578                         ret = -ENOMEM;
1579                         goto free;
1580                 }
1581         }
1582
1583         return attrs;
1584  free:
1585         destroy_hist_trigger_attrs(attrs);
1586
1587         return ERR_PTR(ret);
1588 }
1589
1590 static inline void save_comm(char *comm, struct task_struct *task)
1591 {
1592         if (!task->pid) {
1593                 strcpy(comm, "<idle>");
1594                 return;
1595         }
1596
1597         if (WARN_ON_ONCE(task->pid < 0)) {
1598                 strcpy(comm, "<XXX>");
1599                 return;
1600         }
1601
1602         strncpy(comm, task->comm, TASK_COMM_LEN);
1603 }
1604
1605 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1606 {
1607         unsigned int i;
1608
1609         for (i = 0; i < elt_data->n_field_var_str; i++)
1610                 kfree(elt_data->field_var_str[i]);
1611
1612         kfree(elt_data->field_var_str);
1613
1614         kfree(elt_data->comm);
1615         kfree(elt_data);
1616 }
1617
1618 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1619 {
1620         struct hist_elt_data *elt_data = elt->private_data;
1621
1622         hist_elt_data_free(elt_data);
1623 }
1624
1625 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1626 {
1627         struct hist_trigger_data *hist_data = elt->map->private_data;
1628         unsigned int size = TASK_COMM_LEN;
1629         struct hist_elt_data *elt_data;
1630         struct hist_field *hist_field;
1631         unsigned int i, n_str;
1632
1633         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1634         if (!elt_data)
1635                 return -ENOMEM;
1636
1637         for_each_hist_field(i, hist_data) {
1638                 hist_field = hist_data->fields[i];
1639
1640                 if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1641                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1642                         if (!elt_data->comm) {
1643                                 kfree(elt_data);
1644                                 return -ENOMEM;
1645                         }
1646                         break;
1647                 }
1648         }
1649
1650         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1651                 hist_data->n_var_str;
1652         if (n_str > SYNTH_FIELDS_MAX) {
1653                 hist_elt_data_free(elt_data);
1654                 return -EINVAL;
1655         }
1656
1657         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1658
1659         size = STR_VAR_LEN_MAX;
1660
1661         elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1662         if (!elt_data->field_var_str) {
1663                 hist_elt_data_free(elt_data);
1664                 return -EINVAL;
1665         }
1666         elt_data->n_field_var_str = n_str;
1667
1668         for (i = 0; i < n_str; i++) {
1669                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1670                 if (!elt_data->field_var_str[i]) {
1671                         hist_elt_data_free(elt_data);
1672                         return -ENOMEM;
1673                 }
1674         }
1675
1676         elt->private_data = elt_data;
1677
1678         return 0;
1679 }
1680
1681 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1682 {
1683         struct hist_elt_data *elt_data = elt->private_data;
1684
1685         if (elt_data->comm)
1686                 save_comm(elt_data->comm, current);
1687 }
1688
1689 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1690         .elt_alloc      = hist_trigger_elt_data_alloc,
1691         .elt_free       = hist_trigger_elt_data_free,
1692         .elt_init       = hist_trigger_elt_data_init,
1693 };
1694
1695 static const char *get_hist_field_flags(struct hist_field *hist_field)
1696 {
1697         const char *flags_str = NULL;
1698
1699         if (hist_field->flags & HIST_FIELD_FL_HEX)
1700                 flags_str = "hex";
1701         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1702                 flags_str = "sym";
1703         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1704                 flags_str = "sym-offset";
1705         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1706                 flags_str = "execname";
1707         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1708                 flags_str = "syscall";
1709         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1710                 flags_str = "log2";
1711         else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1712                 flags_str = "buckets";
1713         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1714                 flags_str = "usecs";
1715         else if (hist_field->flags & HIST_FIELD_FL_PERCENT)
1716                 flags_str = "percent";
1717         else if (hist_field->flags & HIST_FIELD_FL_GRAPH)
1718                 flags_str = "graph";
1719         else if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
1720                 flags_str = "stacktrace";
1721
1722         return flags_str;
1723 }
1724
1725 static void expr_field_str(struct hist_field *field, char *expr)
1726 {
1727         if (field->flags & HIST_FIELD_FL_VAR_REF)
1728                 strcat(expr, "$");
1729         else if (field->flags & HIST_FIELD_FL_CONST) {
1730                 char str[HIST_CONST_DIGITS_MAX];
1731
1732                 snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant);
1733                 strcat(expr, str);
1734         }
1735
1736         strcat(expr, hist_field_name(field, 0));
1737
1738         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1739                 const char *flags_str = get_hist_field_flags(field);
1740
1741                 if (flags_str) {
1742                         strcat(expr, ".");
1743                         strcat(expr, flags_str);
1744                 }
1745         }
1746 }
1747
1748 static char *expr_str(struct hist_field *field, unsigned int level)
1749 {
1750         char *expr;
1751
1752         if (level > 1)
1753                 return NULL;
1754
1755         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1756         if (!expr)
1757                 return NULL;
1758
1759         if (!field->operands[0]) {
1760                 expr_field_str(field, expr);
1761                 return expr;
1762         }
1763
1764         if (field->operator == FIELD_OP_UNARY_MINUS) {
1765                 char *subexpr;
1766
1767                 strcat(expr, "-(");
1768                 subexpr = expr_str(field->operands[0], ++level);
1769                 if (!subexpr) {
1770                         kfree(expr);
1771                         return NULL;
1772                 }
1773                 strcat(expr, subexpr);
1774                 strcat(expr, ")");
1775
1776                 kfree(subexpr);
1777
1778                 return expr;
1779         }
1780
1781         expr_field_str(field->operands[0], expr);
1782
1783         switch (field->operator) {
1784         case FIELD_OP_MINUS:
1785                 strcat(expr, "-");
1786                 break;
1787         case FIELD_OP_PLUS:
1788                 strcat(expr, "+");
1789                 break;
1790         case FIELD_OP_DIV:
1791                 strcat(expr, "/");
1792                 break;
1793         case FIELD_OP_MULT:
1794                 strcat(expr, "*");
1795                 break;
1796         default:
1797                 kfree(expr);
1798                 return NULL;
1799         }
1800
1801         expr_field_str(field->operands[1], expr);
1802
1803         return expr;
1804 }
1805
1806 /*
1807  * If field_op != FIELD_OP_NONE, *sep points to the root operator
1808  * of the expression tree to be evaluated.
1809  */
1810 static int contains_operator(char *str, char **sep)
1811 {
1812         enum field_op_id field_op = FIELD_OP_NONE;
1813         char *minus_op, *plus_op, *div_op, *mult_op;
1814
1815
1816         /*
1817          * Report the last occurrence of the operators first, so that the
1818          * expression is evaluated left to right. This is important since
1819          * subtraction and division are not associative.
1820          *
1821          *      e.g
1822          *              64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
1823          *              14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
1824          */
1825
1826         /*
1827          * First, find lower precedence addition and subtraction
1828          * since the expression will be evaluated recursively.
1829          */
1830         minus_op = strrchr(str, '-');
1831         if (minus_op) {
1832                 /*
1833                  * Unary minus is not supported in sub-expressions. If
1834                  * present, it is always the next root operator.
1835                  */
1836                 if (minus_op == str) {
1837                         field_op = FIELD_OP_UNARY_MINUS;
1838                         goto out;
1839                 }
1840
1841                 field_op = FIELD_OP_MINUS;
1842         }
1843
1844         plus_op = strrchr(str, '+');
1845         if (plus_op || minus_op) {
1846                 /*
1847                  * For operators of the same precedence use to rightmost as the
1848                  * root, so that the expression is evaluated left to right.
1849                  */
1850                 if (plus_op > minus_op)
1851                         field_op = FIELD_OP_PLUS;
1852                 goto out;
1853         }
1854
1855         /*
1856          * Multiplication and division have higher precedence than addition and
1857          * subtraction.
1858          */
1859         div_op = strrchr(str, '/');
1860         if (div_op)
1861                 field_op = FIELD_OP_DIV;
1862
1863         mult_op = strrchr(str, '*');
1864         /*
1865          * For operators of the same precedence use to rightmost as the
1866          * root, so that the expression is evaluated left to right.
1867          */
1868         if (mult_op > div_op)
1869                 field_op = FIELD_OP_MULT;
1870
1871 out:
1872         if (sep) {
1873                 switch (field_op) {
1874                 case FIELD_OP_UNARY_MINUS:
1875                 case FIELD_OP_MINUS:
1876                         *sep = minus_op;
1877                         break;
1878                 case FIELD_OP_PLUS:
1879                         *sep = plus_op;
1880                         break;
1881                 case FIELD_OP_DIV:
1882                         *sep = div_op;
1883                         break;
1884                 case FIELD_OP_MULT:
1885                         *sep = mult_op;
1886                         break;
1887                 case FIELD_OP_NONE:
1888                 default:
1889                         *sep = NULL;
1890                         break;
1891                 }
1892         }
1893
1894         return field_op;
1895 }
1896
1897 static void get_hist_field(struct hist_field *hist_field)
1898 {
1899         hist_field->ref++;
1900 }
1901
1902 static void __destroy_hist_field(struct hist_field *hist_field)
1903 {
1904         if (--hist_field->ref > 1)
1905                 return;
1906
1907         kfree(hist_field->var.name);
1908         kfree(hist_field->name);
1909
1910         /* Can likely be a const */
1911         kfree_const(hist_field->type);
1912
1913         kfree(hist_field->system);
1914         kfree(hist_field->event_name);
1915
1916         kfree(hist_field);
1917 }
1918
1919 static void destroy_hist_field(struct hist_field *hist_field,
1920                                unsigned int level)
1921 {
1922         unsigned int i;
1923
1924         if (level > 3)
1925                 return;
1926
1927         if (!hist_field)
1928                 return;
1929
1930         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1931                 return; /* var refs will be destroyed separately */
1932
1933         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1934                 destroy_hist_field(hist_field->operands[i], level + 1);
1935
1936         __destroy_hist_field(hist_field);
1937 }
1938
1939 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1940                                             struct ftrace_event_field *field,
1941                                             unsigned long flags,
1942                                             char *var_name)
1943 {
1944         struct hist_field *hist_field;
1945
1946         if (field && is_function_field(field))
1947                 return NULL;
1948
1949         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1950         if (!hist_field)
1951                 return NULL;
1952
1953         hist_field->ref = 1;
1954
1955         hist_field->hist_data = hist_data;
1956
1957         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1958                 goto out; /* caller will populate */
1959
1960         if (flags & HIST_FIELD_FL_VAR_REF) {
1961                 hist_field->fn_num = HIST_FIELD_FN_VAR_REF;
1962                 goto out;
1963         }
1964
1965         if (flags & HIST_FIELD_FL_HITCOUNT) {
1966                 hist_field->fn_num = HIST_FIELD_FN_COUNTER;
1967                 hist_field->size = sizeof(u64);
1968                 hist_field->type = "u64";
1969                 goto out;
1970         }
1971
1972         if (flags & HIST_FIELD_FL_CONST) {
1973                 hist_field->fn_num = HIST_FIELD_FN_CONST;
1974                 hist_field->size = sizeof(u64);
1975                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1976                 if (!hist_field->type)
1977                         goto free;
1978                 goto out;
1979         }
1980
1981         if (flags & HIST_FIELD_FL_STACKTRACE) {
1982                 if (field)
1983                         hist_field->fn_num = HIST_FIELD_FN_STACK;
1984                 else
1985                         hist_field->fn_num = HIST_FIELD_FN_NOP;
1986                 hist_field->size = HIST_STACKTRACE_SIZE;
1987                 hist_field->type = kstrdup_const("unsigned long[]", GFP_KERNEL);
1988                 if (!hist_field->type)
1989                         goto free;
1990                 goto out;
1991         }
1992
1993         if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
1994                 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
1995                 hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 :
1996                         HIST_FIELD_FN_BUCKET;
1997                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1998                 if (!hist_field->operands[0])
1999                         goto free;
2000                 hist_field->size = hist_field->operands[0]->size;
2001                 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
2002                 if (!hist_field->type)
2003                         goto free;
2004                 goto out;
2005         }
2006
2007         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2008                 hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP;
2009                 hist_field->size = sizeof(u64);
2010                 hist_field->type = "u64";
2011                 goto out;
2012         }
2013
2014         if (flags & HIST_FIELD_FL_CPU) {
2015                 hist_field->fn_num = HIST_FIELD_FN_CPU;
2016                 hist_field->size = sizeof(int);
2017                 hist_field->type = "unsigned int";
2018                 goto out;
2019         }
2020
2021         if (WARN_ON_ONCE(!field))
2022                 goto out;
2023
2024         /* Pointers to strings are just pointers and dangerous to dereference */
2025         if (is_string_field(field) &&
2026             (field->filter_type != FILTER_PTR_STRING)) {
2027                 flags |= HIST_FIELD_FL_STRING;
2028
2029                 hist_field->size = MAX_FILTER_STR_VAL;
2030                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2031                 if (!hist_field->type)
2032                         goto free;
2033
2034                 if (field->filter_type == FILTER_STATIC_STRING) {
2035                         hist_field->fn_num = HIST_FIELD_FN_STRING;
2036                         hist_field->size = field->size;
2037                 } else if (field->filter_type == FILTER_DYN_STRING) {
2038                         hist_field->fn_num = HIST_FIELD_FN_DYNSTRING;
2039                 } else if (field->filter_type == FILTER_RDYN_STRING)
2040                         hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING;
2041                 else
2042                         hist_field->fn_num = HIST_FIELD_FN_PSTRING;
2043         } else {
2044                 hist_field->size = field->size;
2045                 hist_field->is_signed = field->is_signed;
2046                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2047                 if (!hist_field->type)
2048                         goto free;
2049
2050                 hist_field->fn_num = select_value_fn(field->size,
2051                                                      field->is_signed);
2052                 if (hist_field->fn_num == HIST_FIELD_FN_NOP) {
2053                         destroy_hist_field(hist_field, 0);
2054                         return NULL;
2055                 }
2056         }
2057  out:
2058         hist_field->field = field;
2059         hist_field->flags = flags;
2060
2061         if (var_name) {
2062                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2063                 if (!hist_field->var.name)
2064                         goto free;
2065         }
2066
2067         return hist_field;
2068  free:
2069         destroy_hist_field(hist_field, 0);
2070         return NULL;
2071 }
2072
2073 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2074 {
2075         unsigned int i;
2076
2077         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2078                 if (hist_data->fields[i]) {
2079                         destroy_hist_field(hist_data->fields[i], 0);
2080                         hist_data->fields[i] = NULL;
2081                 }
2082         }
2083
2084         for (i = 0; i < hist_data->n_var_refs; i++) {
2085                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2086                 __destroy_hist_field(hist_data->var_refs[i]);
2087                 hist_data->var_refs[i] = NULL;
2088         }
2089 }
2090
2091 static int init_var_ref(struct hist_field *ref_field,
2092                         struct hist_field *var_field,
2093                         char *system, char *event_name)
2094 {
2095         int err = 0;
2096
2097         ref_field->var.idx = var_field->var.idx;
2098         ref_field->var.hist_data = var_field->hist_data;
2099         ref_field->size = var_field->size;
2100         ref_field->is_signed = var_field->is_signed;
2101         ref_field->flags |= var_field->flags &
2102                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2103
2104         if (system) {
2105                 ref_field->system = kstrdup(system, GFP_KERNEL);
2106                 if (!ref_field->system)
2107                         return -ENOMEM;
2108         }
2109
2110         if (event_name) {
2111                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2112                 if (!ref_field->event_name) {
2113                         err = -ENOMEM;
2114                         goto free;
2115                 }
2116         }
2117
2118         if (var_field->var.name) {
2119                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2120                 if (!ref_field->name) {
2121                         err = -ENOMEM;
2122                         goto free;
2123                 }
2124         } else if (var_field->name) {
2125                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2126                 if (!ref_field->name) {
2127                         err = -ENOMEM;
2128                         goto free;
2129                 }
2130         }
2131
2132         ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
2133         if (!ref_field->type) {
2134                 err = -ENOMEM;
2135                 goto free;
2136         }
2137  out:
2138         return err;
2139  free:
2140         kfree(ref_field->system);
2141         ref_field->system = NULL;
2142         kfree(ref_field->event_name);
2143         ref_field->event_name = NULL;
2144         kfree(ref_field->name);
2145         ref_field->name = NULL;
2146
2147         goto out;
2148 }
2149
2150 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2151                             struct hist_field *var_field)
2152 {
2153         struct hist_field *ref_field;
2154         int i;
2155
2156         for (i = 0; i < hist_data->n_var_refs; i++) {
2157                 ref_field = hist_data->var_refs[i];
2158                 if (ref_field->var.idx == var_field->var.idx &&
2159                     ref_field->var.hist_data == var_field->hist_data)
2160                         return i;
2161         }
2162
2163         return -ENOENT;
2164 }
2165
2166 /**
2167  * create_var_ref - Create a variable reference and attach it to trigger
2168  * @hist_data: The trigger that will be referencing the variable
2169  * @var_field: The VAR field to create a reference to
2170  * @system: The optional system string
2171  * @event_name: The optional event_name string
2172  *
2173  * Given a variable hist_field, create a VAR_REF hist_field that
2174  * represents a reference to it.
2175  *
2176  * This function also adds the reference to the trigger that
2177  * now references the variable.
2178  *
2179  * Return: The VAR_REF field if successful, NULL if not
2180  */
2181 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2182                                          struct hist_field *var_field,
2183                                          char *system, char *event_name)
2184 {
2185         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2186         struct hist_field *ref_field;
2187         int i;
2188
2189         /* Check if the variable already exists */
2190         for (i = 0; i < hist_data->n_var_refs; i++) {
2191                 ref_field = hist_data->var_refs[i];
2192                 if (ref_field->var.idx == var_field->var.idx &&
2193                     ref_field->var.hist_data == var_field->hist_data) {
2194                         get_hist_field(ref_field);
2195                         return ref_field;
2196                 }
2197         }
2198         /* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */
2199         if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX)
2200                 return NULL;
2201         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2202         if (ref_field) {
2203                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2204                         destroy_hist_field(ref_field, 0);
2205                         return NULL;
2206                 }
2207
2208                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2209                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2210         }
2211
2212         return ref_field;
2213 }
2214
2215 static bool is_var_ref(char *var_name)
2216 {
2217         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2218                 return false;
2219
2220         return true;
2221 }
2222
2223 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2224                                  char *var_name)
2225 {
2226         char *name, *field;
2227         unsigned int i;
2228
2229         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2230                 name = hist_data->attrs->var_defs.name[i];
2231
2232                 if (strcmp(var_name, name) == 0) {
2233                         field = hist_data->attrs->var_defs.expr[i];
2234                         if (contains_operator(field, NULL) || is_var_ref(field))
2235                                 continue;
2236                         return field;
2237                 }
2238         }
2239
2240         return NULL;
2241 }
2242
2243 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2244                                  char *system, char *event_name,
2245                                  char *var_name)
2246 {
2247         struct trace_event_call *call;
2248
2249         if (system && event_name) {
2250                 call = hist_data->event_file->event_call;
2251
2252                 if (strcmp(system, call->class->system) != 0)
2253                         return NULL;
2254
2255                 if (strcmp(event_name, trace_event_name(call)) != 0)
2256                         return NULL;
2257         }
2258
2259         if (!!system != !!event_name)
2260                 return NULL;
2261
2262         if (!is_var_ref(var_name))
2263                 return NULL;
2264
2265         var_name++;
2266
2267         return field_name_from_var(hist_data, var_name);
2268 }
2269
2270 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2271                                         char *system, char *event_name,
2272                                         char *var_name)
2273 {
2274         struct hist_field *var_field = NULL, *ref_field = NULL;
2275         struct trace_array *tr = hist_data->event_file->tr;
2276
2277         if (!is_var_ref(var_name))
2278                 return NULL;
2279
2280         var_name++;
2281
2282         var_field = find_event_var(hist_data, system, event_name, var_name);
2283         if (var_field)
2284                 ref_field = create_var_ref(hist_data, var_field,
2285                                            system, event_name);
2286
2287         if (!ref_field)
2288                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2289
2290         return ref_field;
2291 }
2292
2293 static struct ftrace_event_field *
2294 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2295             char *field_str, unsigned long *flags, unsigned long *buckets)
2296 {
2297         struct ftrace_event_field *field = NULL;
2298         char *field_name, *modifier, *str;
2299         struct trace_array *tr = file->tr;
2300
2301         modifier = str = kstrdup(field_str, GFP_KERNEL);
2302         if (!modifier)
2303                 return ERR_PTR(-ENOMEM);
2304
2305         field_name = strsep(&modifier, ".");
2306         if (modifier) {
2307                 if (strcmp(modifier, "hex") == 0)
2308                         *flags |= HIST_FIELD_FL_HEX;
2309                 else if (strcmp(modifier, "sym") == 0)
2310                         *flags |= HIST_FIELD_FL_SYM;
2311                 /*
2312                  * 'sym-offset' occurrences in the trigger string are modified
2313                  * to 'symXoffset' to simplify arithmetic expression parsing.
2314                  */
2315                 else if (strcmp(modifier, "symXoffset") == 0)
2316                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2317                 else if ((strcmp(modifier, "execname") == 0) &&
2318                          (strcmp(field_name, "common_pid") == 0))
2319                         *flags |= HIST_FIELD_FL_EXECNAME;
2320                 else if (strcmp(modifier, "syscall") == 0)
2321                         *flags |= HIST_FIELD_FL_SYSCALL;
2322                 else if (strcmp(modifier, "stacktrace") == 0)
2323                         *flags |= HIST_FIELD_FL_STACKTRACE;
2324                 else if (strcmp(modifier, "log2") == 0)
2325                         *flags |= HIST_FIELD_FL_LOG2;
2326                 else if (strcmp(modifier, "usecs") == 0)
2327                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2328                 else if (strncmp(modifier, "bucket", 6) == 0) {
2329                         int ret;
2330
2331                         modifier += 6;
2332
2333                         if (*modifier == 's')
2334                                 modifier++;
2335                         if (*modifier != '=')
2336                                 goto error;
2337                         modifier++;
2338                         ret = kstrtoul(modifier, 0, buckets);
2339                         if (ret || !(*buckets))
2340                                 goto error;
2341                         *flags |= HIST_FIELD_FL_BUCKET;
2342                 } else if (strncmp(modifier, "percent", 7) == 0) {
2343                         if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2344                                 goto error;
2345                         *flags |= HIST_FIELD_FL_PERCENT;
2346                 } else if (strncmp(modifier, "graph", 5) == 0) {
2347                         if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2348                                 goto error;
2349                         *flags |= HIST_FIELD_FL_GRAPH;
2350                 } else {
2351  error:
2352                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2353                         field = ERR_PTR(-EINVAL);
2354                         goto out;
2355                 }
2356         }
2357
2358         if (strcmp(field_name, "common_timestamp") == 0) {
2359                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2360                 hist_data->enable_timestamps = true;
2361                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2362                         hist_data->attrs->ts_in_usecs = true;
2363         } else if (strcmp(field_name, "common_stacktrace") == 0) {
2364                 *flags |= HIST_FIELD_FL_STACKTRACE;
2365         } else if (strcmp(field_name, "common_cpu") == 0)
2366                 *flags |= HIST_FIELD_FL_CPU;
2367         else if (strcmp(field_name, "hitcount") == 0)
2368                 *flags |= HIST_FIELD_FL_HITCOUNT;
2369         else {
2370                 field = trace_find_event_field(file->event_call, field_name);
2371                 if (!field || !field->size) {
2372                         /*
2373                          * For backward compatibility, if field_name
2374                          * was "cpu" or "stacktrace", then we treat this
2375                          * the same as common_cpu and common_stacktrace
2376                          * respectively. This also works for "CPU", and
2377                          * "STACKTRACE".
2378                          */
2379                         if (field && field->filter_type == FILTER_CPU) {
2380                                 *flags |= HIST_FIELD_FL_CPU;
2381                         } else if (field && field->filter_type == FILTER_STACKTRACE) {
2382                                 *flags |= HIST_FIELD_FL_STACKTRACE;
2383                         } else {
2384                                 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2385                                          errpos(field_name));
2386                                 field = ERR_PTR(-EINVAL);
2387                                 goto out;
2388                         }
2389                 }
2390         }
2391  out:
2392         kfree(str);
2393
2394         return field;
2395 }
2396
2397 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2398                                        struct hist_field *var_ref,
2399                                        char *var_name)
2400 {
2401         struct hist_field *alias = NULL;
2402         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2403
2404         alias = create_hist_field(hist_data, NULL, flags, var_name);
2405         if (!alias)
2406                 return NULL;
2407
2408         alias->fn_num = var_ref->fn_num;
2409         alias->operands[0] = var_ref;
2410
2411         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2412                 destroy_hist_field(alias, 0);
2413                 return NULL;
2414         }
2415
2416         alias->var_ref_idx = var_ref->var_ref_idx;
2417
2418         return alias;
2419 }
2420
2421 static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
2422                                       char *str, char *var_name,
2423                                       unsigned long *flags)
2424 {
2425         struct trace_array *tr = hist_data->event_file->tr;
2426         struct hist_field *field = NULL;
2427         u64 constant;
2428
2429         if (kstrtoull(str, 0, &constant)) {
2430                 hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
2431                 return NULL;
2432         }
2433
2434         *flags |= HIST_FIELD_FL_CONST;
2435         field = create_hist_field(hist_data, NULL, *flags, var_name);
2436         if (!field)
2437                 return NULL;
2438
2439         field->constant = constant;
2440
2441         return field;
2442 }
2443
2444 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2445                                      struct trace_event_file *file, char *str,
2446                                      unsigned long *flags, char *var_name)
2447 {
2448         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2449         struct ftrace_event_field *field = NULL;
2450         struct hist_field *hist_field = NULL;
2451         unsigned long buckets = 0;
2452         int ret = 0;
2453
2454         if (isdigit(str[0])) {
2455                 hist_field = parse_const(hist_data, str, var_name, flags);
2456                 if (!hist_field) {
2457                         ret = -EINVAL;
2458                         goto out;
2459                 }
2460                 return hist_field;
2461         }
2462
2463         s = strchr(str, '.');
2464         if (s) {
2465                 s = strchr(++s, '.');
2466                 if (s) {
2467                         ref_system = strsep(&str, ".");
2468                         if (!str) {
2469                                 ret = -EINVAL;
2470                                 goto out;
2471                         }
2472                         ref_event = strsep(&str, ".");
2473                         if (!str) {
2474                                 ret = -EINVAL;
2475                                 goto out;
2476                         }
2477                         ref_var = str;
2478                 }
2479         }
2480
2481         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2482         if (!s) {
2483                 hist_field = parse_var_ref(hist_data, ref_system,
2484                                            ref_event, ref_var);
2485                 if (hist_field) {
2486                         if (var_name) {
2487                                 hist_field = create_alias(hist_data, hist_field, var_name);
2488                                 if (!hist_field) {
2489                                         ret = -ENOMEM;
2490                                         goto out;
2491                                 }
2492                         }
2493                         return hist_field;
2494                 }
2495         } else
2496                 str = s;
2497
2498         field = parse_field(hist_data, file, str, flags, &buckets);
2499         if (IS_ERR(field)) {
2500                 ret = PTR_ERR(field);
2501                 goto out;
2502         }
2503
2504         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2505         if (!hist_field) {
2506                 ret = -ENOMEM;
2507                 goto out;
2508         }
2509         hist_field->buckets = buckets;
2510
2511         return hist_field;
2512  out:
2513         return ERR_PTR(ret);
2514 }
2515
2516 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2517                                      struct trace_event_file *file,
2518                                      char *str, unsigned long flags,
2519                                      char *var_name, unsigned int *n_subexprs);
2520
2521 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2522                                       struct trace_event_file *file,
2523                                       char *str, unsigned long flags,
2524                                       char *var_name, unsigned int *n_subexprs)
2525 {
2526         struct hist_field *operand1, *expr = NULL;
2527         unsigned long operand_flags;
2528         int ret = 0;
2529         char *s;
2530
2531         /* Unary minus operator, increment n_subexprs */
2532         ++*n_subexprs;
2533
2534         /* we support only -(xxx) i.e. explicit parens required */
2535
2536         if (*n_subexprs > 3) {
2537                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2538                 ret = -EINVAL;
2539                 goto free;
2540         }
2541
2542         str++; /* skip leading '-' */
2543
2544         s = strchr(str, '(');
2545         if (s)
2546                 str++;
2547         else {
2548                 ret = -EINVAL;
2549                 goto free;
2550         }
2551
2552         s = strrchr(str, ')');
2553         if (s) {
2554                  /* unary minus not supported in sub-expressions */
2555                 if (*(s+1) != '\0') {
2556                         hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
2557                                  errpos(str));
2558                         ret = -EINVAL;
2559                         goto free;
2560                 }
2561                 *s = '\0';
2562         }
2563         else {
2564                 ret = -EINVAL; /* no closing ')' */
2565                 goto free;
2566         }
2567
2568         flags |= HIST_FIELD_FL_EXPR;
2569         expr = create_hist_field(hist_data, NULL, flags, var_name);
2570         if (!expr) {
2571                 ret = -ENOMEM;
2572                 goto free;
2573         }
2574
2575         operand_flags = 0;
2576         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2577         if (IS_ERR(operand1)) {
2578                 ret = PTR_ERR(operand1);
2579                 goto free;
2580         }
2581         if (operand1->flags & HIST_FIELD_FL_STRING) {
2582                 /* String type can not be the operand of unary operator. */
2583                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2584                 destroy_hist_field(operand1, 0);
2585                 ret = -EINVAL;
2586                 goto free;
2587         }
2588
2589         expr->flags |= operand1->flags &
2590                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2591         expr->fn_num = HIST_FIELD_FN_UMINUS;
2592         expr->operands[0] = operand1;
2593         expr->size = operand1->size;
2594         expr->is_signed = operand1->is_signed;
2595         expr->operator = FIELD_OP_UNARY_MINUS;
2596         expr->name = expr_str(expr, 0);
2597         expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2598         if (!expr->type) {
2599                 ret = -ENOMEM;
2600                 goto free;
2601         }
2602
2603         return expr;
2604  free:
2605         destroy_hist_field(expr, 0);
2606         return ERR_PTR(ret);
2607 }
2608
2609 /*
2610  * If the operands are var refs, return pointers the
2611  * variable(s) referenced in var1 and var2, else NULL.
2612  */
2613 static int check_expr_operands(struct trace_array *tr,
2614                                struct hist_field *operand1,
2615                                struct hist_field *operand2,
2616                                struct hist_field **var1,
2617                                struct hist_field **var2)
2618 {
2619         unsigned long operand1_flags = operand1->flags;
2620         unsigned long operand2_flags = operand2->flags;
2621
2622         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2623             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2624                 struct hist_field *var;
2625
2626                 var = find_var_field(operand1->var.hist_data, operand1->name);
2627                 if (!var)
2628                         return -EINVAL;
2629                 operand1_flags = var->flags;
2630                 *var1 = var;
2631         }
2632
2633         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2634             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2635                 struct hist_field *var;
2636
2637                 var = find_var_field(operand2->var.hist_data, operand2->name);
2638                 if (!var)
2639                         return -EINVAL;
2640                 operand2_flags = var->flags;
2641                 *var2 = var;
2642         }
2643
2644         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2645             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2646                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2647                 return -EINVAL;
2648         }
2649
2650         return 0;
2651 }
2652
2653 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2654                                      struct trace_event_file *file,
2655                                      char *str, unsigned long flags,
2656                                      char *var_name, unsigned int *n_subexprs)
2657 {
2658         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2659         struct hist_field *var1 = NULL, *var2 = NULL;
2660         unsigned long operand_flags, operand2_flags;
2661         int field_op, ret = -EINVAL;
2662         char *sep, *operand1_str;
2663         enum hist_field_fn op_fn;
2664         bool combine_consts;
2665
2666         if (*n_subexprs > 3) {
2667                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2668                 return ERR_PTR(-EINVAL);
2669         }
2670
2671         field_op = contains_operator(str, &sep);
2672
2673         if (field_op == FIELD_OP_NONE)
2674                 return parse_atom(hist_data, file, str, &flags, var_name);
2675
2676         if (field_op == FIELD_OP_UNARY_MINUS)
2677                 return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2678
2679         /* Binary operator found, increment n_subexprs */
2680         ++*n_subexprs;
2681
2682         /* Split the expression string at the root operator */
2683         if (!sep)
2684                 return ERR_PTR(-EINVAL);
2685
2686         *sep = '\0';
2687         operand1_str = str;
2688         str = sep+1;
2689
2690         /* Binary operator requires both operands */
2691         if (*operand1_str == '\0' || *str == '\0')
2692                 return ERR_PTR(-EINVAL);
2693
2694         operand_flags = 0;
2695
2696         /* LHS of string is an expression e.g. a+b in a+b+c */
2697         operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
2698         if (IS_ERR(operand1))
2699                 return ERR_CAST(operand1);
2700
2701         if (operand1->flags & HIST_FIELD_FL_STRING) {
2702                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2703                 ret = -EINVAL;
2704                 goto free_op1;
2705         }
2706
2707         /* RHS of string is another expression e.g. c in a+b+c */
2708         operand_flags = 0;
2709         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2710         if (IS_ERR(operand2)) {
2711                 ret = PTR_ERR(operand2);
2712                 goto free_op1;
2713         }
2714         if (operand2->flags & HIST_FIELD_FL_STRING) {
2715                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2716                 ret = -EINVAL;
2717                 goto free_operands;
2718         }
2719
2720         switch (field_op) {
2721         case FIELD_OP_MINUS:
2722                 op_fn = HIST_FIELD_FN_MINUS;
2723                 break;
2724         case FIELD_OP_PLUS:
2725                 op_fn = HIST_FIELD_FN_PLUS;
2726                 break;
2727         case FIELD_OP_DIV:
2728                 op_fn = HIST_FIELD_FN_DIV;
2729                 break;
2730         case FIELD_OP_MULT:
2731                 op_fn = HIST_FIELD_FN_MULT;
2732                 break;
2733         default:
2734                 ret = -EINVAL;
2735                 goto free_operands;
2736         }
2737
2738         ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
2739         if (ret)
2740                 goto free_operands;
2741
2742         operand_flags = var1 ? var1->flags : operand1->flags;
2743         operand2_flags = var2 ? var2->flags : operand2->flags;
2744
2745         /*
2746          * If both operands are constant, the expression can be
2747          * collapsed to a single constant.
2748          */
2749         combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2750
2751         flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2752
2753         flags |= operand1->flags &
2754                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2755
2756         expr = create_hist_field(hist_data, NULL, flags, var_name);
2757         if (!expr) {
2758                 ret = -ENOMEM;
2759                 goto free_operands;
2760         }
2761
2762         operand1->read_once = true;
2763         operand2->read_once = true;
2764
2765         /* The operands are now owned and free'd by 'expr' */
2766         expr->operands[0] = operand1;
2767         expr->operands[1] = operand2;
2768
2769         if (field_op == FIELD_OP_DIV &&
2770                         operand2_flags & HIST_FIELD_FL_CONST) {
2771                 u64 divisor = var2 ? var2->constant : operand2->constant;
2772
2773                 if (!divisor) {
2774                         hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str));
2775                         ret = -EDOM;
2776                         goto free_expr;
2777                 }
2778
2779                 /*
2780                  * Copy the divisor here so we don't have to look it up
2781                  * later if this is a var ref
2782                  */
2783                 operand2->constant = divisor;
2784                 op_fn = hist_field_get_div_fn(operand2);
2785         }
2786
2787         expr->fn_num = op_fn;
2788
2789         if (combine_consts) {
2790                 if (var1)
2791                         expr->operands[0] = var1;
2792                 if (var2)
2793                         expr->operands[1] = var2;
2794
2795                 expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL);
2796                 expr->fn_num = HIST_FIELD_FN_CONST;
2797
2798                 expr->operands[0] = NULL;
2799                 expr->operands[1] = NULL;
2800
2801                 /*
2802                  * var refs won't be destroyed immediately
2803                  * See: destroy_hist_field()
2804                  */
2805                 destroy_hist_field(operand2, 0);
2806                 destroy_hist_field(operand1, 0);
2807
2808                 expr->name = expr_str(expr, 0);
2809         } else {
2810                 /* The operand sizes should be the same, so just pick one */
2811                 expr->size = operand1->size;
2812                 expr->is_signed = operand1->is_signed;
2813
2814                 expr->operator = field_op;
2815                 expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2816                 if (!expr->type) {
2817                         ret = -ENOMEM;
2818                         goto free_expr;
2819                 }
2820
2821                 expr->name = expr_str(expr, 0);
2822         }
2823
2824         return expr;
2825
2826 free_operands:
2827         destroy_hist_field(operand2, 0);
2828 free_op1:
2829         destroy_hist_field(operand1, 0);
2830         return ERR_PTR(ret);
2831
2832 free_expr:
2833         destroy_hist_field(expr, 0);
2834         return ERR_PTR(ret);
2835 }
2836
2837 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2838                                  struct trace_event_file *file)
2839 {
2840         struct event_trigger_data *test;
2841
2842         lockdep_assert_held(&event_mutex);
2843
2844         list_for_each_entry(test, &file->triggers, list) {
2845                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2846                         if (test->private_data == hist_data)
2847                                 return test->filter_str;
2848                 }
2849         }
2850
2851         return NULL;
2852 }
2853
2854 static struct event_command trigger_hist_cmd;
2855 static int event_hist_trigger_parse(struct event_command *cmd_ops,
2856                                     struct trace_event_file *file,
2857                                     char *glob, char *cmd,
2858                                     char *param_and_filter);
2859
2860 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2861                             struct hist_trigger_data *hist_data,
2862                             unsigned int n_keys)
2863 {
2864         struct hist_field *target_hist_field, *hist_field;
2865         unsigned int n, i, j;
2866
2867         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2868                 return false;
2869
2870         i = hist_data->n_vals;
2871         j = target_hist_data->n_vals;
2872
2873         for (n = 0; n < n_keys; n++) {
2874                 hist_field = hist_data->fields[i + n];
2875                 target_hist_field = target_hist_data->fields[j + n];
2876
2877                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2878                         return false;
2879                 if (hist_field->size != target_hist_field->size)
2880                         return false;
2881                 if (hist_field->is_signed != target_hist_field->is_signed)
2882                         return false;
2883         }
2884
2885         return true;
2886 }
2887
2888 static struct hist_trigger_data *
2889 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2890                      struct trace_event_file *file)
2891 {
2892         struct hist_trigger_data *hist_data;
2893         struct event_trigger_data *test;
2894         unsigned int n_keys;
2895
2896         lockdep_assert_held(&event_mutex);
2897
2898         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2899
2900         list_for_each_entry(test, &file->triggers, list) {
2901                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2902                         hist_data = test->private_data;
2903
2904                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2905                                 return hist_data;
2906                 }
2907         }
2908
2909         return NULL;
2910 }
2911
2912 static struct trace_event_file *event_file(struct trace_array *tr,
2913                                            char *system, char *event_name)
2914 {
2915         struct trace_event_file *file;
2916
2917         file = __find_event_file(tr, system, event_name);
2918         if (!file)
2919                 return ERR_PTR(-EINVAL);
2920
2921         return file;
2922 }
2923
2924 static struct hist_field *
2925 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2926                          char *system, char *event_name, char *field_name)
2927 {
2928         struct hist_field *event_var;
2929         char *synthetic_name;
2930
2931         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2932         if (!synthetic_name)
2933                 return ERR_PTR(-ENOMEM);
2934
2935         strcpy(synthetic_name, "synthetic_");
2936         strcat(synthetic_name, field_name);
2937
2938         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2939
2940         kfree(synthetic_name);
2941
2942         return event_var;
2943 }
2944
2945 /**
2946  * create_field_var_hist - Automatically create a histogram and var for a field
2947  * @target_hist_data: The target hist trigger
2948  * @subsys_name: Optional subsystem name
2949  * @event_name: Optional event name
2950  * @field_name: The name of the field (and the resulting variable)
2951  *
2952  * Hist trigger actions fetch data from variables, not directly from
2953  * events.  However, for convenience, users are allowed to directly
2954  * specify an event field in an action, which will be automatically
2955  * converted into a variable on their behalf.
2956  *
2957  * If a user specifies a field on an event that isn't the event the
2958  * histogram currently being defined (the target event histogram), the
2959  * only way that can be accomplished is if a new hist trigger is
2960  * created and the field variable defined on that.
2961  *
2962  * This function creates a new histogram compatible with the target
2963  * event (meaning a histogram with the same key as the target
2964  * histogram), and creates a variable for the specified field, but
2965  * with 'synthetic_' prepended to the variable name in order to avoid
2966  * collision with normal field variables.
2967  *
2968  * Return: The variable created for the field.
2969  */
2970 static struct hist_field *
2971 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2972                       char *subsys_name, char *event_name, char *field_name)
2973 {
2974         struct trace_array *tr = target_hist_data->event_file->tr;
2975         struct hist_trigger_data *hist_data;
2976         unsigned int i, n, first = true;
2977         struct field_var_hist *var_hist;
2978         struct trace_event_file *file;
2979         struct hist_field *key_field;
2980         struct hist_field *event_var;
2981         char *saved_filter;
2982         char *cmd;
2983         int ret;
2984
2985         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2986                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2987                 return ERR_PTR(-EINVAL);
2988         }
2989
2990         file = event_file(tr, subsys_name, event_name);
2991
2992         if (IS_ERR(file)) {
2993                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2994                 ret = PTR_ERR(file);
2995                 return ERR_PTR(ret);
2996         }
2997
2998         /*
2999          * Look for a histogram compatible with target.  We'll use the
3000          * found histogram specification to create a new matching
3001          * histogram with our variable on it.  target_hist_data is not
3002          * yet a registered histogram so we can't use that.
3003          */
3004         hist_data = find_compatible_hist(target_hist_data, file);
3005         if (!hist_data) {
3006                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
3007                 return ERR_PTR(-EINVAL);
3008         }
3009
3010         /* See if a synthetic field variable has already been created */
3011         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3012                                              event_name, field_name);
3013         if (!IS_ERR_OR_NULL(event_var))
3014                 return event_var;
3015
3016         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3017         if (!var_hist)
3018                 return ERR_PTR(-ENOMEM);
3019
3020         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3021         if (!cmd) {
3022                 kfree(var_hist);
3023                 return ERR_PTR(-ENOMEM);
3024         }
3025
3026         /* Use the same keys as the compatible histogram */
3027         strcat(cmd, "keys=");
3028
3029         for_each_hist_key_field(i, hist_data) {
3030                 key_field = hist_data->fields[i];
3031                 if (!first)
3032                         strcat(cmd, ",");
3033                 strcat(cmd, key_field->field->name);
3034                 first = false;
3035         }
3036
3037         /* Create the synthetic field variable specification */
3038         strcat(cmd, ":synthetic_");
3039         strcat(cmd, field_name);
3040         strcat(cmd, "=");
3041         strcat(cmd, field_name);
3042
3043         /* Use the same filter as the compatible histogram */
3044         saved_filter = find_trigger_filter(hist_data, file);
3045         if (saved_filter) {
3046                 strcat(cmd, " if ");
3047                 strcat(cmd, saved_filter);
3048         }
3049
3050         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3051         if (!var_hist->cmd) {
3052                 kfree(cmd);
3053                 kfree(var_hist);
3054                 return ERR_PTR(-ENOMEM);
3055         }
3056
3057         /* Save the compatible histogram information */
3058         var_hist->hist_data = hist_data;
3059
3060         /* Create the new histogram with our variable */
3061         ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
3062                                        "", "hist", cmd);
3063         if (ret) {
3064                 kfree(cmd);
3065                 kfree(var_hist->cmd);
3066                 kfree(var_hist);
3067                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3068                 return ERR_PTR(ret);
3069         }
3070
3071         kfree(cmd);
3072
3073         /* If we can't find the variable, something went wrong */
3074         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3075                                              event_name, field_name);
3076         if (IS_ERR_OR_NULL(event_var)) {
3077                 kfree(var_hist->cmd);
3078                 kfree(var_hist);
3079                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3080                 return ERR_PTR(-EINVAL);
3081         }
3082
3083         n = target_hist_data->n_field_var_hists;
3084         target_hist_data->field_var_hists[n] = var_hist;
3085         target_hist_data->n_field_var_hists++;
3086
3087         return event_var;
3088 }
3089
3090 static struct hist_field *
3091 find_target_event_var(struct hist_trigger_data *hist_data,
3092                       char *subsys_name, char *event_name, char *var_name)
3093 {
3094         struct trace_event_file *file = hist_data->event_file;
3095         struct hist_field *hist_field = NULL;
3096
3097         if (subsys_name) {
3098                 struct trace_event_call *call;
3099
3100                 if (!event_name)
3101                         return NULL;
3102
3103                 call = file->event_call;
3104
3105                 if (strcmp(subsys_name, call->class->system) != 0)
3106                         return NULL;
3107
3108                 if (strcmp(event_name, trace_event_name(call)) != 0)
3109                         return NULL;
3110         }
3111
3112         hist_field = find_var_field(hist_data, var_name);
3113
3114         return hist_field;
3115 }
3116
3117 static inline void __update_field_vars(struct tracing_map_elt *elt,
3118                                        struct trace_buffer *buffer,
3119                                        struct ring_buffer_event *rbe,
3120                                        void *rec,
3121                                        struct field_var **field_vars,
3122                                        unsigned int n_field_vars,
3123                                        unsigned int field_var_str_start)
3124 {
3125         struct hist_elt_data *elt_data = elt->private_data;
3126         unsigned int i, j, var_idx;
3127         u64 var_val;
3128
3129         /* Make sure stacktrace can fit in the string variable length */
3130         BUILD_BUG_ON((HIST_STACKTRACE_DEPTH + 1) * sizeof(long) >= STR_VAR_LEN_MAX);
3131
3132         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3133                 struct field_var *field_var = field_vars[i];
3134                 struct hist_field *var = field_var->var;
3135                 struct hist_field *val = field_var->val;
3136
3137                 var_val = hist_fn_call(val, elt, buffer, rbe, rec);
3138                 var_idx = var->var.idx;
3139
3140                 if (val->flags & (HIST_FIELD_FL_STRING |
3141                                   HIST_FIELD_FL_STACKTRACE)) {
3142                         char *str = elt_data->field_var_str[j++];
3143                         char *val_str = (char *)(uintptr_t)var_val;
3144                         unsigned int size;
3145
3146                         if (val->flags & HIST_FIELD_FL_STRING) {
3147                                 size = min(val->size, STR_VAR_LEN_MAX);
3148                                 strscpy(str, val_str, size);
3149                         } else {
3150                                 char *stack_start = str + sizeof(unsigned long);
3151                                 int e;
3152
3153                                 e = stack_trace_save((void *)stack_start,
3154                                                      HIST_STACKTRACE_DEPTH,
3155                                                      HIST_STACKTRACE_SKIP);
3156                                 if (e < HIST_STACKTRACE_DEPTH - 1)
3157                                         ((unsigned long *)stack_start)[e] = 0;
3158                                 *((unsigned long *)str) = e;
3159                         }
3160                         var_val = (u64)(uintptr_t)str;
3161                 }
3162                 tracing_map_set_var(elt, var_idx, var_val);
3163         }
3164 }
3165
3166 static void update_field_vars(struct hist_trigger_data *hist_data,
3167                               struct tracing_map_elt *elt,
3168                               struct trace_buffer *buffer,
3169                               struct ring_buffer_event *rbe,
3170                               void *rec)
3171 {
3172         __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
3173                             hist_data->n_field_vars, 0);
3174 }
3175
3176 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3177                                  struct tracing_map_elt *elt,
3178                                  struct trace_buffer *buffer,  void *rec,
3179                                  struct ring_buffer_event *rbe, void *key,
3180                                  struct action_data *data, u64 *var_ref_vals)
3181 {
3182         __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
3183                             hist_data->n_save_vars, hist_data->n_field_var_str);
3184 }
3185
3186 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3187                                      struct trace_event_file *file,
3188                                      char *name, int size, const char *type)
3189 {
3190         struct hist_field *var;
3191         int idx;
3192
3193         if (find_var(hist_data, file, name) && !hist_data->remove) {
3194                 var = ERR_PTR(-EINVAL);
3195                 goto out;
3196         }
3197
3198         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3199         if (!var) {
3200                 var = ERR_PTR(-ENOMEM);
3201                 goto out;
3202         }
3203
3204         idx = tracing_map_add_var(hist_data->map);
3205         if (idx < 0) {
3206                 kfree(var);
3207                 var = ERR_PTR(-EINVAL);
3208                 goto out;
3209         }
3210
3211         var->ref = 1;
3212         var->flags = HIST_FIELD_FL_VAR;
3213         var->var.idx = idx;
3214         var->var.hist_data = var->hist_data = hist_data;
3215         var->size = size;
3216         var->var.name = kstrdup(name, GFP_KERNEL);
3217         var->type = kstrdup_const(type, GFP_KERNEL);
3218         if (!var->var.name || !var->type) {
3219                 kfree_const(var->type);
3220                 kfree(var->var.name);
3221                 kfree(var);
3222                 var = ERR_PTR(-ENOMEM);
3223         }
3224  out:
3225         return var;
3226 }
3227
3228 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3229                                           struct trace_event_file *file,
3230                                           char *field_name)
3231 {
3232         struct hist_field *val = NULL, *var = NULL;
3233         unsigned long flags = HIST_FIELD_FL_VAR;
3234         struct trace_array *tr = file->tr;
3235         struct field_var *field_var;
3236         int ret = 0;
3237
3238         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3239                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3240                 ret = -EINVAL;
3241                 goto err;
3242         }
3243
3244         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3245         if (IS_ERR(val)) {
3246                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3247                 ret = PTR_ERR(val);
3248                 goto err;
3249         }
3250
3251         var = create_var(hist_data, file, field_name, val->size, val->type);
3252         if (IS_ERR(var)) {
3253                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3254                 kfree(val);
3255                 ret = PTR_ERR(var);
3256                 goto err;
3257         }
3258
3259         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3260         if (!field_var) {
3261                 kfree(val);
3262                 kfree(var);
3263                 ret =  -ENOMEM;
3264                 goto err;
3265         }
3266
3267         field_var->var = var;
3268         field_var->val = val;
3269  out:
3270         return field_var;
3271  err:
3272         field_var = ERR_PTR(ret);
3273         goto out;
3274 }
3275
3276 /**
3277  * create_target_field_var - Automatically create a variable for a field
3278  * @target_hist_data: The target hist trigger
3279  * @subsys_name: Optional subsystem name
3280  * @event_name: Optional event name
3281  * @var_name: The name of the field (and the resulting variable)
3282  *
3283  * Hist trigger actions fetch data from variables, not directly from
3284  * events.  However, for convenience, users are allowed to directly
3285  * specify an event field in an action, which will be automatically
3286  * converted into a variable on their behalf.
3287  *
3288  * This function creates a field variable with the name var_name on
3289  * the hist trigger currently being defined on the target event.  If
3290  * subsys_name and event_name are specified, this function simply
3291  * verifies that they do in fact match the target event subsystem and
3292  * event name.
3293  *
3294  * Return: The variable created for the field.
3295  */
3296 static struct field_var *
3297 create_target_field_var(struct hist_trigger_data *target_hist_data,
3298                         char *subsys_name, char *event_name, char *var_name)
3299 {
3300         struct trace_event_file *file = target_hist_data->event_file;
3301
3302         if (subsys_name) {
3303                 struct trace_event_call *call;
3304
3305                 if (!event_name)
3306                         return NULL;
3307
3308                 call = file->event_call;
3309
3310                 if (strcmp(subsys_name, call->class->system) != 0)
3311                         return NULL;
3312
3313                 if (strcmp(event_name, trace_event_name(call)) != 0)
3314                         return NULL;
3315         }
3316
3317         return create_field_var(target_hist_data, file, var_name);
3318 }
3319
3320 static bool check_track_val_max(u64 track_val, u64 var_val)
3321 {
3322         if (var_val <= track_val)
3323                 return false;
3324
3325         return true;
3326 }
3327
3328 static bool check_track_val_changed(u64 track_val, u64 var_val)
3329 {
3330         if (var_val == track_val)
3331                 return false;
3332
3333         return true;
3334 }
3335
3336 static u64 get_track_val(struct hist_trigger_data *hist_data,
3337                          struct tracing_map_elt *elt,
3338                          struct action_data *data)
3339 {
3340         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3341         u64 track_val;
3342
3343         track_val = tracing_map_read_var(elt, track_var_idx);
3344
3345         return track_val;
3346 }
3347
3348 static void save_track_val(struct hist_trigger_data *hist_data,
3349                            struct tracing_map_elt *elt,
3350                            struct action_data *data, u64 var_val)
3351 {
3352         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3353
3354         tracing_map_set_var(elt, track_var_idx, var_val);
3355 }
3356
3357 static void save_track_data(struct hist_trigger_data *hist_data,
3358                             struct tracing_map_elt *elt,
3359                             struct trace_buffer *buffer, void *rec,
3360                             struct ring_buffer_event *rbe, void *key,
3361                             struct action_data *data, u64 *var_ref_vals)
3362 {
3363         if (data->track_data.save_data)
3364                 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3365                                            key, data, var_ref_vals);
3366 }
3367
3368 static bool check_track_val(struct tracing_map_elt *elt,
3369                             struct action_data *data,
3370                             u64 var_val)
3371 {
3372         struct hist_trigger_data *hist_data;
3373         u64 track_val;
3374
3375         hist_data = data->track_data.track_var->hist_data;
3376         track_val = get_track_val(hist_data, elt, data);
3377
3378         return data->track_data.check_val(track_val, var_val);
3379 }
3380
3381 #ifdef CONFIG_TRACER_SNAPSHOT
3382 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3383 {
3384         /* called with tr->max_lock held */
3385         struct track_data *track_data = tr->cond_snapshot->cond_data;
3386         struct hist_elt_data *elt_data, *track_elt_data;
3387         struct snapshot_context *context = cond_data;
3388         struct action_data *action;
3389         u64 track_val;
3390
3391         if (!track_data)
3392                 return false;
3393
3394         action = track_data->action_data;
3395
3396         track_val = get_track_val(track_data->hist_data, context->elt,
3397                                   track_data->action_data);
3398
3399         if (!action->track_data.check_val(track_data->track_val, track_val))
3400                 return false;
3401
3402         track_data->track_val = track_val;
3403         memcpy(track_data->key, context->key, track_data->key_len);
3404
3405         elt_data = context->elt->private_data;
3406         track_elt_data = track_data->elt.private_data;
3407         if (elt_data->comm)
3408                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3409
3410         track_data->updated = true;
3411
3412         return true;
3413 }
3414
3415 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3416                                      struct tracing_map_elt *elt,
3417                                      struct trace_buffer *buffer, void *rec,
3418                                      struct ring_buffer_event *rbe, void *key,
3419                                      struct action_data *data,
3420                                      u64 *var_ref_vals)
3421 {
3422         struct trace_event_file *file = hist_data->event_file;
3423         struct snapshot_context context;
3424
3425         context.elt = elt;
3426         context.key = key;
3427
3428         tracing_snapshot_cond(file->tr, &context);
3429 }
3430
3431 static void hist_trigger_print_key(struct seq_file *m,
3432                                    struct hist_trigger_data *hist_data,
3433                                    void *key,
3434                                    struct tracing_map_elt *elt);
3435
3436 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3437 {
3438         unsigned int i;
3439
3440         if (!hist_data->n_actions)
3441                 return NULL;
3442
3443         for (i = 0; i < hist_data->n_actions; i++) {
3444                 struct action_data *data = hist_data->actions[i];
3445
3446                 if (data->action == ACTION_SNAPSHOT)
3447                         return data;
3448         }
3449
3450         return NULL;
3451 }
3452
3453 static void track_data_snapshot_print(struct seq_file *m,
3454                                       struct hist_trigger_data *hist_data)
3455 {
3456         struct trace_event_file *file = hist_data->event_file;
3457         struct track_data *track_data;
3458         struct action_data *action;
3459
3460         track_data = tracing_cond_snapshot_data(file->tr);
3461         if (!track_data)
3462                 return;
3463
3464         if (!track_data->updated)
3465                 return;
3466
3467         action = snapshot_action(hist_data);
3468         if (!action)
3469                 return;
3470
3471         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3472         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3473                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3474                    action->track_data.var_str, track_data->track_val);
3475
3476         seq_puts(m, "\ttriggered by event with key: ");
3477         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3478         seq_putc(m, '\n');
3479 }
3480 #else
3481 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3482 {
3483         return false;
3484 }
3485 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3486                                      struct tracing_map_elt *elt,
3487                                      struct trace_buffer *buffer, void *rec,
3488                                      struct ring_buffer_event *rbe, void *key,
3489                                      struct action_data *data,
3490                                      u64 *var_ref_vals) {}
3491 static void track_data_snapshot_print(struct seq_file *m,
3492                                       struct hist_trigger_data *hist_data) {}
3493 #endif /* CONFIG_TRACER_SNAPSHOT */
3494
3495 static void track_data_print(struct seq_file *m,
3496                              struct hist_trigger_data *hist_data,
3497                              struct tracing_map_elt *elt,
3498                              struct action_data *data)
3499 {
3500         u64 track_val = get_track_val(hist_data, elt, data);
3501         unsigned int i, save_var_idx;
3502
3503         if (data->handler == HANDLER_ONMAX)
3504                 seq_printf(m, "\n\tmax: %10llu", track_val);
3505         else if (data->handler == HANDLER_ONCHANGE)
3506                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3507
3508         if (data->action == ACTION_SNAPSHOT)
3509                 return;
3510
3511         for (i = 0; i < hist_data->n_save_vars; i++) {
3512                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3513                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3514                 u64 val;
3515
3516                 save_var_idx = save_var->var.idx;
3517
3518                 val = tracing_map_read_var(elt, save_var_idx);
3519
3520                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3521                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3522                                    (char *)(uintptr_t)(val));
3523                 } else
3524                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3525         }
3526 }
3527
3528 static void ontrack_action(struct hist_trigger_data *hist_data,
3529                            struct tracing_map_elt *elt,
3530                            struct trace_buffer *buffer, void *rec,
3531                            struct ring_buffer_event *rbe, void *key,
3532                            struct action_data *data, u64 *var_ref_vals)
3533 {
3534         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3535
3536         if (check_track_val(elt, data, var_val)) {
3537                 save_track_val(hist_data, elt, data, var_val);
3538                 save_track_data(hist_data, elt, buffer, rec, rbe,
3539                                 key, data, var_ref_vals);
3540         }
3541 }
3542
3543 static void action_data_destroy(struct action_data *data)
3544 {
3545         unsigned int i;
3546
3547         lockdep_assert_held(&event_mutex);
3548
3549         kfree(data->action_name);
3550
3551         for (i = 0; i < data->n_params; i++)
3552                 kfree(data->params[i]);
3553
3554         if (data->synth_event)
3555                 data->synth_event->ref--;
3556
3557         kfree(data->synth_event_name);
3558
3559         kfree(data);
3560 }
3561
3562 static void track_data_destroy(struct hist_trigger_data *hist_data,
3563                                struct action_data *data)
3564 {
3565         struct trace_event_file *file = hist_data->event_file;
3566
3567         destroy_hist_field(data->track_data.track_var, 0);
3568
3569         if (data->action == ACTION_SNAPSHOT) {
3570                 struct track_data *track_data;
3571
3572                 track_data = tracing_cond_snapshot_data(file->tr);
3573                 if (track_data && track_data->hist_data == hist_data) {
3574                         tracing_snapshot_cond_disable(file->tr);
3575                         track_data_free(track_data);
3576                 }
3577         }
3578
3579         kfree(data->track_data.var_str);
3580
3581         action_data_destroy(data);
3582 }
3583
3584 static int action_create(struct hist_trigger_data *hist_data,
3585                          struct action_data *data);
3586
3587 static int track_data_create(struct hist_trigger_data *hist_data,
3588                              struct action_data *data)
3589 {
3590         struct hist_field *var_field, *ref_field, *track_var = NULL;
3591         struct trace_event_file *file = hist_data->event_file;
3592         struct trace_array *tr = file->tr;
3593         char *track_data_var_str;
3594         int ret = 0;
3595
3596         track_data_var_str = data->track_data.var_str;
3597         if (track_data_var_str[0] != '$') {
3598                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3599                 return -EINVAL;
3600         }
3601         track_data_var_str++;
3602
3603         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3604         if (!var_field) {
3605                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3606                 return -EINVAL;
3607         }
3608
3609         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3610         if (!ref_field)
3611                 return -ENOMEM;
3612
3613         data->track_data.var_ref = ref_field;
3614
3615         if (data->handler == HANDLER_ONMAX)
3616                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3617         if (IS_ERR(track_var)) {
3618                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3619                 ret = PTR_ERR(track_var);
3620                 goto out;
3621         }
3622
3623         if (data->handler == HANDLER_ONCHANGE)
3624                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3625         if (IS_ERR(track_var)) {
3626                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3627                 ret = PTR_ERR(track_var);
3628                 goto out;
3629         }
3630         data->track_data.track_var = track_var;
3631
3632         ret = action_create(hist_data, data);
3633  out:
3634         return ret;
3635 }
3636
3637 static int parse_action_params(struct trace_array *tr, char *params,
3638                                struct action_data *data)
3639 {
3640         char *param, *saved_param;
3641         bool first_param = true;
3642         int ret = 0;
3643
3644         while (params) {
3645                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3646                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3647                         ret = -EINVAL;
3648                         goto out;
3649                 }
3650
3651                 param = strsep(&params, ",");
3652                 if (!param) {
3653                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3654                         ret = -EINVAL;
3655                         goto out;
3656                 }
3657
3658                 param = strstrip(param);
3659                 if (strlen(param) < 2) {
3660                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3661                         ret = -EINVAL;
3662                         goto out;
3663                 }
3664
3665                 saved_param = kstrdup(param, GFP_KERNEL);
3666                 if (!saved_param) {
3667                         ret = -ENOMEM;
3668                         goto out;
3669                 }
3670
3671                 if (first_param && data->use_trace_keyword) {
3672                         data->synth_event_name = saved_param;
3673                         first_param = false;
3674                         continue;
3675                 }
3676                 first_param = false;
3677
3678                 data->params[data->n_params++] = saved_param;
3679         }
3680  out:
3681         return ret;
3682 }
3683
3684 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3685                         enum handler_id handler)
3686 {
3687         char *action_name;
3688         int ret = 0;
3689
3690         strsep(&str, ".");
3691         if (!str) {
3692                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3693                 ret = -EINVAL;
3694                 goto out;
3695         }
3696
3697         action_name = strsep(&str, "(");
3698         if (!action_name || !str) {
3699                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3700                 ret = -EINVAL;
3701                 goto out;
3702         }
3703
3704         if (str_has_prefix(action_name, "save")) {
3705                 char *params = strsep(&str, ")");
3706
3707                 if (!params) {
3708                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3709                         ret = -EINVAL;
3710                         goto out;
3711                 }
3712
3713                 ret = parse_action_params(tr, params, data);
3714                 if (ret)
3715                         goto out;
3716
3717                 if (handler == HANDLER_ONMAX)
3718                         data->track_data.check_val = check_track_val_max;
3719                 else if (handler == HANDLER_ONCHANGE)
3720                         data->track_data.check_val = check_track_val_changed;
3721                 else {
3722                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3723                         ret = -EINVAL;
3724                         goto out;
3725                 }
3726
3727                 data->track_data.save_data = save_track_data_vars;
3728                 data->fn = ontrack_action;
3729                 data->action = ACTION_SAVE;
3730         } else if (str_has_prefix(action_name, "snapshot")) {
3731                 char *params = strsep(&str, ")");
3732
3733                 if (!str) {
3734                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3735                         ret = -EINVAL;
3736                         goto out;
3737                 }
3738
3739                 if (handler == HANDLER_ONMAX)
3740                         data->track_data.check_val = check_track_val_max;
3741                 else if (handler == HANDLER_ONCHANGE)
3742                         data->track_data.check_val = check_track_val_changed;
3743                 else {
3744                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3745                         ret = -EINVAL;
3746                         goto out;
3747                 }
3748
3749                 data->track_data.save_data = save_track_data_snapshot;
3750                 data->fn = ontrack_action;
3751                 data->action = ACTION_SNAPSHOT;
3752         } else {
3753                 char *params = strsep(&str, ")");
3754
3755                 if (str_has_prefix(action_name, "trace"))
3756                         data->use_trace_keyword = true;
3757
3758                 if (params) {
3759                         ret = parse_action_params(tr, params, data);
3760                         if (ret)
3761                                 goto out;
3762                 }
3763
3764                 if (handler == HANDLER_ONMAX)
3765                         data->track_data.check_val = check_track_val_max;
3766                 else if (handler == HANDLER_ONCHANGE)
3767                         data->track_data.check_val = check_track_val_changed;
3768
3769                 if (handler != HANDLER_ONMATCH) {
3770                         data->track_data.save_data = action_trace;
3771                         data->fn = ontrack_action;
3772                 } else
3773                         data->fn = action_trace;
3774
3775                 data->action = ACTION_TRACE;
3776         }
3777
3778         data->action_name = kstrdup(action_name, GFP_KERNEL);
3779         if (!data->action_name) {
3780                 ret = -ENOMEM;
3781                 goto out;
3782         }
3783
3784         data->handler = handler;
3785  out:
3786         return ret;
3787 }
3788
3789 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3790                                             char *str, enum handler_id handler)
3791 {
3792         struct action_data *data;
3793         int ret = -EINVAL;
3794         char *var_str;
3795
3796         data = kzalloc(sizeof(*data), GFP_KERNEL);
3797         if (!data)
3798                 return ERR_PTR(-ENOMEM);
3799
3800         var_str = strsep(&str, ")");
3801         if (!var_str || !str) {
3802                 ret = -EINVAL;
3803                 goto free;
3804         }
3805
3806         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3807         if (!data->track_data.var_str) {
3808                 ret = -ENOMEM;
3809                 goto free;
3810         }
3811
3812         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3813         if (ret)
3814                 goto free;
3815  out:
3816         return data;
3817  free:
3818         track_data_destroy(hist_data, data);
3819         data = ERR_PTR(ret);
3820         goto out;
3821 }
3822
3823 static void onmatch_destroy(struct action_data *data)
3824 {
3825         kfree(data->match_data.event);
3826         kfree(data->match_data.event_system);
3827
3828         action_data_destroy(data);
3829 }
3830
3831 static void destroy_field_var(struct field_var *field_var)
3832 {
3833         if (!field_var)
3834                 return;
3835
3836         destroy_hist_field(field_var->var, 0);
3837         destroy_hist_field(field_var->val, 0);
3838
3839         kfree(field_var);
3840 }
3841
3842 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3843 {
3844         unsigned int i;
3845
3846         for (i = 0; i < hist_data->n_field_vars; i++)
3847                 destroy_field_var(hist_data->field_vars[i]);
3848
3849         for (i = 0; i < hist_data->n_save_vars; i++)
3850                 destroy_field_var(hist_data->save_vars[i]);
3851 }
3852
3853 static void save_field_var(struct hist_trigger_data *hist_data,
3854                            struct field_var *field_var)
3855 {
3856         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3857
3858         /* Stack traces are saved in the string storage too */
3859         if (field_var->val->flags & (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
3860                 hist_data->n_field_var_str++;
3861 }
3862
3863
3864 static int check_synth_field(struct synth_event *event,
3865                              struct hist_field *hist_field,
3866                              unsigned int field_pos)
3867 {
3868         struct synth_field *field;
3869
3870         if (field_pos >= event->n_fields)
3871                 return -EINVAL;
3872
3873         field = event->fields[field_pos];
3874
3875         /*
3876          * A dynamic string synth field can accept static or
3877          * dynamic. A static string synth field can only accept a
3878          * same-sized static string, which is checked for later.
3879          */
3880         if (strstr(hist_field->type, "char[") && field->is_string
3881             && field->is_dynamic)
3882                 return 0;
3883
3884         if (strstr(hist_field->type, "long[") && field->is_stack)
3885                 return 0;
3886
3887         if (strcmp(field->type, hist_field->type) != 0) {
3888                 if (field->size != hist_field->size ||
3889                     (!field->is_string && field->is_signed != hist_field->is_signed))
3890                         return -EINVAL;
3891         }
3892
3893         return 0;
3894 }
3895
3896 static struct hist_field *
3897 trace_action_find_var(struct hist_trigger_data *hist_data,
3898                       struct action_data *data,
3899                       char *system, char *event, char *var)
3900 {
3901         struct trace_array *tr = hist_data->event_file->tr;
3902         struct hist_field *hist_field;
3903
3904         var++; /* skip '$' */
3905
3906         hist_field = find_target_event_var(hist_data, system, event, var);
3907         if (!hist_field) {
3908                 if (!system && data->handler == HANDLER_ONMATCH) {
3909                         system = data->match_data.event_system;
3910                         event = data->match_data.event;
3911                 }
3912
3913                 hist_field = find_event_var(hist_data, system, event, var);
3914         }
3915
3916         if (!hist_field)
3917                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3918
3919         return hist_field;
3920 }
3921
3922 static struct hist_field *
3923 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3924                               struct action_data *data, char *system,
3925                               char *event, char *var)
3926 {
3927         struct hist_field *hist_field = NULL;
3928         struct field_var *field_var;
3929
3930         /*
3931          * First try to create a field var on the target event (the
3932          * currently being defined).  This will create a variable for
3933          * unqualified fields on the target event, or if qualified,
3934          * target fields that have qualified names matching the target.
3935          */
3936         field_var = create_target_field_var(hist_data, system, event, var);
3937
3938         if (field_var && !IS_ERR(field_var)) {
3939                 save_field_var(hist_data, field_var);
3940                 hist_field = field_var->var;
3941         } else {
3942                 field_var = NULL;
3943                 /*
3944                  * If no explicit system.event is specified, default to
3945                  * looking for fields on the onmatch(system.event.xxx)
3946                  * event.
3947                  */
3948                 if (!system && data->handler == HANDLER_ONMATCH) {
3949                         system = data->match_data.event_system;
3950                         event = data->match_data.event;
3951                 }
3952
3953                 if (!event)
3954                         goto free;
3955                 /*
3956                  * At this point, we're looking at a field on another
3957                  * event.  Because we can't modify a hist trigger on
3958                  * another event to add a variable for a field, we need
3959                  * to create a new trigger on that event and create the
3960                  * variable at the same time.
3961                  */
3962                 hist_field = create_field_var_hist(hist_data, system, event, var);
3963                 if (IS_ERR(hist_field))
3964                         goto free;
3965         }
3966  out:
3967         return hist_field;
3968  free:
3969         destroy_field_var(field_var);
3970         hist_field = NULL;
3971         goto out;
3972 }
3973
3974 static int trace_action_create(struct hist_trigger_data *hist_data,
3975                                struct action_data *data)
3976 {
3977         struct trace_array *tr = hist_data->event_file->tr;
3978         char *event_name, *param, *system = NULL;
3979         struct hist_field *hist_field, *var_ref;
3980         unsigned int i;
3981         unsigned int field_pos = 0;
3982         struct synth_event *event;
3983         char *synth_event_name;
3984         int var_ref_idx, ret = 0;
3985
3986         lockdep_assert_held(&event_mutex);
3987
3988         /* Sanity check to avoid out-of-bound write on 'data->var_ref_idx' */
3989         if (data->n_params > SYNTH_FIELDS_MAX)
3990                 return -EINVAL;
3991
3992         if (data->use_trace_keyword)
3993                 synth_event_name = data->synth_event_name;
3994         else
3995                 synth_event_name = data->action_name;
3996
3997         event = find_synth_event(synth_event_name);
3998         if (!event) {
3999                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
4000                 return -EINVAL;
4001         }
4002
4003         event->ref++;
4004
4005         for (i = 0; i < data->n_params; i++) {
4006                 char *p;
4007
4008                 p = param = kstrdup(data->params[i], GFP_KERNEL);
4009                 if (!param) {
4010                         ret = -ENOMEM;
4011                         goto err;
4012                 }
4013
4014                 system = strsep(&param, ".");
4015                 if (!param) {
4016                         param = (char *)system;
4017                         system = event_name = NULL;
4018                 } else {
4019                         event_name = strsep(&param, ".");
4020                         if (!param) {
4021                                 kfree(p);
4022                                 ret = -EINVAL;
4023                                 goto err;
4024                         }
4025                 }
4026
4027                 if (param[0] == '$')
4028                         hist_field = trace_action_find_var(hist_data, data,
4029                                                            system, event_name,
4030                                                            param);
4031                 else
4032                         hist_field = trace_action_create_field_var(hist_data,
4033                                                                    data,
4034                                                                    system,
4035                                                                    event_name,
4036                                                                    param);
4037
4038                 if (!hist_field) {
4039                         kfree(p);
4040                         ret = -EINVAL;
4041                         goto err;
4042                 }
4043
4044                 if (check_synth_field(event, hist_field, field_pos) == 0) {
4045                         var_ref = create_var_ref(hist_data, hist_field,
4046                                                  system, event_name);
4047                         if (!var_ref) {
4048                                 kfree(p);
4049                                 ret = -ENOMEM;
4050                                 goto err;
4051                         }
4052
4053                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
4054                         if (WARN_ON(var_ref_idx < 0)) {
4055                                 kfree(p);
4056                                 ret = var_ref_idx;
4057                                 goto err;
4058                         }
4059
4060                         data->var_ref_idx[i] = var_ref_idx;
4061
4062                         field_pos++;
4063                         kfree(p);
4064                         continue;
4065                 }
4066
4067                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4068                 kfree(p);
4069                 ret = -EINVAL;
4070                 goto err;
4071         }
4072
4073         if (field_pos != event->n_fields) {
4074                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4075                 ret = -EINVAL;
4076                 goto err;
4077         }
4078
4079         data->synth_event = event;
4080  out:
4081         return ret;
4082  err:
4083         event->ref--;
4084
4085         goto out;
4086 }
4087
4088 static int action_create(struct hist_trigger_data *hist_data,
4089                          struct action_data *data)
4090 {
4091         struct trace_event_file *file = hist_data->event_file;
4092         struct trace_array *tr = file->tr;
4093         struct track_data *track_data;
4094         struct field_var *field_var;
4095         unsigned int i;
4096         char *param;
4097         int ret = 0;
4098
4099         if (data->action == ACTION_TRACE)
4100                 return trace_action_create(hist_data, data);
4101
4102         if (data->action == ACTION_SNAPSHOT) {
4103                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4104                 if (IS_ERR(track_data)) {
4105                         ret = PTR_ERR(track_data);
4106                         goto out;
4107                 }
4108
4109                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
4110                                                    cond_snapshot_update);
4111                 if (ret)
4112                         track_data_free(track_data);
4113
4114                 goto out;
4115         }
4116
4117         if (data->action == ACTION_SAVE) {
4118                 if (hist_data->n_save_vars) {
4119                         ret = -EEXIST;
4120                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4121                         goto out;
4122                 }
4123
4124                 for (i = 0; i < data->n_params; i++) {
4125                         param = kstrdup(data->params[i], GFP_KERNEL);
4126                         if (!param) {
4127                                 ret = -ENOMEM;
4128                                 goto out;
4129                         }
4130
4131                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
4132                         if (IS_ERR(field_var)) {
4133                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4134                                          errpos(param));
4135                                 ret = PTR_ERR(field_var);
4136                                 kfree(param);
4137                                 goto out;
4138                         }
4139
4140                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4141                         if (field_var->val->flags &
4142                             (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
4143                                 hist_data->n_save_var_str++;
4144                         kfree(param);
4145                 }
4146         }
4147  out:
4148         return ret;
4149 }
4150
4151 static int onmatch_create(struct hist_trigger_data *hist_data,
4152                           struct action_data *data)
4153 {
4154         return action_create(hist_data, data);
4155 }
4156
4157 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4158 {
4159         char *match_event, *match_event_system;
4160         struct action_data *data;
4161         int ret = -EINVAL;
4162
4163         data = kzalloc(sizeof(*data), GFP_KERNEL);
4164         if (!data)
4165                 return ERR_PTR(-ENOMEM);
4166
4167         match_event = strsep(&str, ")");
4168         if (!match_event || !str) {
4169                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4170                 goto free;
4171         }
4172
4173         match_event_system = strsep(&match_event, ".");
4174         if (!match_event) {
4175                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4176                 goto free;
4177         }
4178
4179         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4180                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4181                 goto free;
4182         }
4183
4184         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4185         if (!data->match_data.event) {
4186                 ret = -ENOMEM;
4187                 goto free;
4188         }
4189
4190         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4191         if (!data->match_data.event_system) {
4192                 ret = -ENOMEM;
4193                 goto free;
4194         }
4195
4196         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4197         if (ret)
4198                 goto free;
4199  out:
4200         return data;
4201  free:
4202         onmatch_destroy(data);
4203         data = ERR_PTR(ret);
4204         goto out;
4205 }
4206
4207 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4208 {
4209         hist_data->fields[HITCOUNT_IDX] =
4210                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4211         if (!hist_data->fields[HITCOUNT_IDX])
4212                 return -ENOMEM;
4213
4214         hist_data->n_vals++;
4215         hist_data->n_fields++;
4216
4217         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4218                 return -EINVAL;
4219
4220         return 0;
4221 }
4222
4223 static int __create_val_field(struct hist_trigger_data *hist_data,
4224                               unsigned int val_idx,
4225                               struct trace_event_file *file,
4226                               char *var_name, char *field_str,
4227                               unsigned long flags)
4228 {
4229         struct hist_field *hist_field;
4230         int ret = 0, n_subexprs = 0;
4231
4232         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
4233         if (IS_ERR(hist_field)) {
4234                 ret = PTR_ERR(hist_field);
4235                 goto out;
4236         }
4237
4238         /* values and variables should not have some modifiers */
4239         if (hist_field->flags & HIST_FIELD_FL_VAR) {
4240                 /* Variable */
4241                 if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT |
4242                                          HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2))
4243                         goto err;
4244         } else {
4245                 /* Value */
4246                 if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT |
4247                                          HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 |
4248                                          HIST_FIELD_FL_SYM | HIST_FIELD_FL_SYM_OFFSET |
4249                                          HIST_FIELD_FL_SYSCALL | HIST_FIELD_FL_STACKTRACE))
4250                         goto err;
4251         }
4252
4253         hist_data->fields[val_idx] = hist_field;
4254
4255         ++hist_data->n_vals;
4256         ++hist_data->n_fields;
4257
4258         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4259                 ret = -EINVAL;
4260  out:
4261         return ret;
4262  err:
4263         hist_err(file->tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str));
4264         return -EINVAL;
4265 }
4266
4267 static int create_val_field(struct hist_trigger_data *hist_data,
4268                             unsigned int val_idx,
4269                             struct trace_event_file *file,
4270                             char *field_str)
4271 {
4272         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4273                 return -EINVAL;
4274
4275         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4276 }
4277
4278 static const char no_comm[] = "(no comm)";
4279
4280 static u64 hist_field_execname(struct hist_field *hist_field,
4281                                struct tracing_map_elt *elt,
4282                                struct trace_buffer *buffer,
4283                                struct ring_buffer_event *rbe,
4284                                void *event)
4285 {
4286         struct hist_elt_data *elt_data;
4287
4288         if (WARN_ON_ONCE(!elt))
4289                 return (u64)(unsigned long)no_comm;
4290
4291         elt_data = elt->private_data;
4292
4293         if (WARN_ON_ONCE(!elt_data->comm))
4294                 return (u64)(unsigned long)no_comm;
4295
4296         return (u64)(unsigned long)(elt_data->comm);
4297 }
4298
4299 static u64 hist_field_stack(struct hist_field *hist_field,
4300                             struct tracing_map_elt *elt,
4301                             struct trace_buffer *buffer,
4302                             struct ring_buffer_event *rbe,
4303                             void *event)
4304 {
4305         u32 str_item = *(u32 *)(event + hist_field->field->offset);
4306         int str_loc = str_item & 0xffff;
4307         char *addr = (char *)(event + str_loc);
4308
4309         return (u64)(unsigned long)addr;
4310 }
4311
4312 static u64 hist_fn_call(struct hist_field *hist_field,
4313                         struct tracing_map_elt *elt,
4314                         struct trace_buffer *buffer,
4315                         struct ring_buffer_event *rbe,
4316                         void *event)
4317 {
4318         switch (hist_field->fn_num) {
4319         case HIST_FIELD_FN_VAR_REF:
4320                 return hist_field_var_ref(hist_field, elt, buffer, rbe, event);
4321         case HIST_FIELD_FN_COUNTER:
4322                 return hist_field_counter(hist_field, elt, buffer, rbe, event);
4323         case HIST_FIELD_FN_CONST:
4324                 return hist_field_const(hist_field, elt, buffer, rbe, event);
4325         case HIST_FIELD_FN_LOG2:
4326                 return hist_field_log2(hist_field, elt, buffer, rbe, event);
4327         case HIST_FIELD_FN_BUCKET:
4328                 return hist_field_bucket(hist_field, elt, buffer, rbe, event);
4329         case HIST_FIELD_FN_TIMESTAMP:
4330                 return hist_field_timestamp(hist_field, elt, buffer, rbe, event);
4331         case HIST_FIELD_FN_CPU:
4332                 return hist_field_cpu(hist_field, elt, buffer, rbe, event);
4333         case HIST_FIELD_FN_STRING:
4334                 return hist_field_string(hist_field, elt, buffer, rbe, event);
4335         case HIST_FIELD_FN_DYNSTRING:
4336                 return hist_field_dynstring(hist_field, elt, buffer, rbe, event);
4337         case HIST_FIELD_FN_RELDYNSTRING:
4338                 return hist_field_reldynstring(hist_field, elt, buffer, rbe, event);
4339         case HIST_FIELD_FN_PSTRING:
4340                 return hist_field_pstring(hist_field, elt, buffer, rbe, event);
4341         case HIST_FIELD_FN_S64:
4342                 return hist_field_s64(hist_field, elt, buffer, rbe, event);
4343         case HIST_FIELD_FN_U64:
4344                 return hist_field_u64(hist_field, elt, buffer, rbe, event);
4345         case HIST_FIELD_FN_S32:
4346                 return hist_field_s32(hist_field, elt, buffer, rbe, event);
4347         case HIST_FIELD_FN_U32:
4348                 return hist_field_u32(hist_field, elt, buffer, rbe, event);
4349         case HIST_FIELD_FN_S16:
4350                 return hist_field_s16(hist_field, elt, buffer, rbe, event);
4351         case HIST_FIELD_FN_U16:
4352                 return hist_field_u16(hist_field, elt, buffer, rbe, event);
4353         case HIST_FIELD_FN_S8:
4354                 return hist_field_s8(hist_field, elt, buffer, rbe, event);
4355         case HIST_FIELD_FN_U8:
4356                 return hist_field_u8(hist_field, elt, buffer, rbe, event);
4357         case HIST_FIELD_FN_UMINUS:
4358                 return hist_field_unary_minus(hist_field, elt, buffer, rbe, event);
4359         case HIST_FIELD_FN_MINUS:
4360                 return hist_field_minus(hist_field, elt, buffer, rbe, event);
4361         case HIST_FIELD_FN_PLUS:
4362                 return hist_field_plus(hist_field, elt, buffer, rbe, event);
4363         case HIST_FIELD_FN_DIV:
4364                 return hist_field_div(hist_field, elt, buffer, rbe, event);
4365         case HIST_FIELD_FN_MULT:
4366                 return hist_field_mult(hist_field, elt, buffer, rbe, event);
4367         case HIST_FIELD_FN_DIV_POWER2:
4368                 return div_by_power_of_two(hist_field, elt, buffer, rbe, event);
4369         case HIST_FIELD_FN_DIV_NOT_POWER2:
4370                 return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event);
4371         case HIST_FIELD_FN_DIV_MULT_SHIFT:
4372                 return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event);
4373         case HIST_FIELD_FN_EXECNAME:
4374                 return hist_field_execname(hist_field, elt, buffer, rbe, event);
4375         case HIST_FIELD_FN_STACK:
4376                 return hist_field_stack(hist_field, elt, buffer, rbe, event);
4377         default:
4378                 return 0;
4379         }
4380 }
4381
4382 /* Convert a var that points to common_pid.execname to a string */
4383 static void update_var_execname(struct hist_field *hist_field)
4384 {
4385         hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
4386                 HIST_FIELD_FL_EXECNAME;
4387         hist_field->size = MAX_FILTER_STR_VAL;
4388         hist_field->is_signed = 0;
4389
4390         kfree_const(hist_field->type);
4391         hist_field->type = "char[]";
4392
4393         hist_field->fn_num = HIST_FIELD_FN_EXECNAME;
4394 }
4395
4396 static int create_var_field(struct hist_trigger_data *hist_data,
4397                             unsigned int val_idx,
4398                             struct trace_event_file *file,
4399                             char *var_name, char *expr_str)
4400 {
4401         struct trace_array *tr = hist_data->event_file->tr;
4402         unsigned long flags = 0;
4403         int ret;
4404
4405         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4406                 return -EINVAL;
4407
4408         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4409                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4410                 return -EINVAL;
4411         }
4412
4413         flags |= HIST_FIELD_FL_VAR;
4414         hist_data->n_vars++;
4415         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4416                 return -EINVAL;
4417
4418         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4419
4420         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4421                 update_var_execname(hist_data->fields[val_idx]);
4422
4423         if (!ret && hist_data->fields[val_idx]->flags &
4424             (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
4425                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4426
4427         return ret;
4428 }
4429
4430 static int create_val_fields(struct hist_trigger_data *hist_data,
4431                              struct trace_event_file *file)
4432 {
4433         unsigned int i, j = 1, n_hitcount = 0;
4434         char *fields_str, *field_str;
4435         int ret;
4436
4437         ret = create_hitcount_val(hist_data);
4438         if (ret)
4439                 goto out;
4440
4441         fields_str = hist_data->attrs->vals_str;
4442         if (!fields_str)
4443                 goto out;
4444
4445         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4446                      j < TRACING_MAP_VALS_MAX; i++) {
4447                 field_str = strsep(&fields_str, ",");
4448                 if (!field_str)
4449                         break;
4450
4451                 if (strcmp(field_str, "hitcount") == 0) {
4452                         if (!n_hitcount++)
4453                                 continue;
4454                 }
4455
4456                 ret = create_val_field(hist_data, j++, file, field_str);
4457                 if (ret)
4458                         goto out;
4459         }
4460
4461         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4462                 ret = -EINVAL;
4463  out:
4464         /* There is only raw hitcount but nohitcount suppresses it. */
4465         if (j == 1 && hist_data->attrs->no_hitcount) {
4466                 hist_err(hist_data->event_file->tr, HIST_ERR_NEED_NOHC_VAL, 0);
4467                 ret = -ENOENT;
4468         }
4469
4470         return ret;
4471 }
4472
4473 static int create_key_field(struct hist_trigger_data *hist_data,
4474                             unsigned int key_idx,
4475                             unsigned int key_offset,
4476                             struct trace_event_file *file,
4477                             char *field_str)
4478 {
4479         struct trace_array *tr = hist_data->event_file->tr;
4480         struct hist_field *hist_field = NULL;
4481         unsigned long flags = 0;
4482         unsigned int key_size;
4483         int ret = 0, n_subexprs = 0;
4484
4485         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4486                 return -EINVAL;
4487
4488         flags |= HIST_FIELD_FL_KEY;
4489
4490         if (strcmp(field_str, "stacktrace") == 0) {
4491                 flags |= HIST_FIELD_FL_STACKTRACE;
4492                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4493                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4494         } else {
4495                 hist_field = parse_expr(hist_data, file, field_str, flags,
4496                                         NULL, &n_subexprs);
4497                 if (IS_ERR(hist_field)) {
4498                         ret = PTR_ERR(hist_field);
4499                         goto out;
4500                 }
4501
4502                 if (field_has_hist_vars(hist_field, 0)) {
4503                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4504                         destroy_hist_field(hist_field, 0);
4505                         ret = -EINVAL;
4506                         goto out;
4507                 }
4508
4509                 key_size = hist_field->size;
4510         }
4511
4512         hist_data->fields[key_idx] = hist_field;
4513
4514         key_size = ALIGN(key_size, sizeof(u64));
4515         hist_data->fields[key_idx]->size = key_size;
4516         hist_data->fields[key_idx]->offset = key_offset;
4517
4518         hist_data->key_size += key_size;
4519
4520         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4521                 ret = -EINVAL;
4522                 goto out;
4523         }
4524
4525         hist_data->n_keys++;
4526         hist_data->n_fields++;
4527
4528         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4529                 return -EINVAL;
4530
4531         ret = key_size;
4532  out:
4533         return ret;
4534 }
4535
4536 static int create_key_fields(struct hist_trigger_data *hist_data,
4537                              struct trace_event_file *file)
4538 {
4539         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4540         char *fields_str, *field_str;
4541         int ret = -EINVAL;
4542
4543         fields_str = hist_data->attrs->keys_str;
4544         if (!fields_str)
4545                 goto out;
4546
4547         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4548                 field_str = strsep(&fields_str, ",");
4549                 if (!field_str)
4550                         break;
4551                 ret = create_key_field(hist_data, i, key_offset,
4552                                        file, field_str);
4553                 if (ret < 0)
4554                         goto out;
4555                 key_offset += ret;
4556         }
4557         if (fields_str) {
4558                 ret = -EINVAL;
4559                 goto out;
4560         }
4561         ret = 0;
4562  out:
4563         return ret;
4564 }
4565
4566 static int create_var_fields(struct hist_trigger_data *hist_data,
4567                              struct trace_event_file *file)
4568 {
4569         unsigned int i, j = hist_data->n_vals;
4570         int ret = 0;
4571
4572         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4573
4574         for (i = 0; i < n_vars; i++) {
4575                 char *var_name = hist_data->attrs->var_defs.name[i];
4576                 char *expr = hist_data->attrs->var_defs.expr[i];
4577
4578                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4579                 if (ret)
4580                         goto out;
4581         }
4582  out:
4583         return ret;
4584 }
4585
4586 static void free_var_defs(struct hist_trigger_data *hist_data)
4587 {
4588         unsigned int i;
4589
4590         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4591                 kfree(hist_data->attrs->var_defs.name[i]);
4592                 kfree(hist_data->attrs->var_defs.expr[i]);
4593         }
4594
4595         hist_data->attrs->var_defs.n_vars = 0;
4596 }
4597
4598 static int parse_var_defs(struct hist_trigger_data *hist_data)
4599 {
4600         struct trace_array *tr = hist_data->event_file->tr;
4601         char *s, *str, *var_name, *field_str;
4602         unsigned int i, j, n_vars = 0;
4603         int ret = 0;
4604
4605         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4606                 str = hist_data->attrs->assignment_str[i];
4607                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4608                         field_str = strsep(&str, ",");
4609                         if (!field_str)
4610                                 break;
4611
4612                         var_name = strsep(&field_str, "=");
4613                         if (!var_name || !field_str) {
4614                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4615                                          errpos(var_name));
4616                                 ret = -EINVAL;
4617                                 goto free;
4618                         }
4619
4620                         if (n_vars == TRACING_MAP_VARS_MAX) {
4621                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4622                                 ret = -EINVAL;
4623                                 goto free;
4624                         }
4625
4626                         s = kstrdup(var_name, GFP_KERNEL);
4627                         if (!s) {
4628                                 ret = -ENOMEM;
4629                                 goto free;
4630                         }
4631                         hist_data->attrs->var_defs.name[n_vars] = s;
4632
4633                         s = kstrdup(field_str, GFP_KERNEL);
4634                         if (!s) {
4635                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4636                                 hist_data->attrs->var_defs.name[n_vars] = NULL;
4637                                 ret = -ENOMEM;
4638                                 goto free;
4639                         }
4640                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4641
4642                         hist_data->attrs->var_defs.n_vars = n_vars;
4643                 }
4644         }
4645
4646         return ret;
4647  free:
4648         free_var_defs(hist_data);
4649
4650         return ret;
4651 }
4652
4653 static int create_hist_fields(struct hist_trigger_data *hist_data,
4654                               struct trace_event_file *file)
4655 {
4656         int ret;
4657
4658         ret = parse_var_defs(hist_data);
4659         if (ret)
4660                 return ret;
4661
4662         ret = create_val_fields(hist_data, file);
4663         if (ret)
4664                 goto out;
4665
4666         ret = create_var_fields(hist_data, file);
4667         if (ret)
4668                 goto out;
4669
4670         ret = create_key_fields(hist_data, file);
4671
4672  out:
4673         free_var_defs(hist_data);
4674
4675         return ret;
4676 }
4677
4678 static int is_descending(struct trace_array *tr, const char *str)
4679 {
4680         if (!str)
4681                 return 0;
4682
4683         if (strcmp(str, "descending") == 0)
4684                 return 1;
4685
4686         if (strcmp(str, "ascending") == 0)
4687                 return 0;
4688
4689         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4690
4691         return -EINVAL;
4692 }
4693
4694 static int create_sort_keys(struct hist_trigger_data *hist_data)
4695 {
4696         struct trace_array *tr = hist_data->event_file->tr;
4697         char *fields_str = hist_data->attrs->sort_key_str;
4698         struct tracing_map_sort_key *sort_key;
4699         int descending, ret = 0;
4700         unsigned int i, j, k;
4701
4702         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4703
4704         if (!fields_str)
4705                 goto out;
4706
4707         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4708                 struct hist_field *hist_field;
4709                 char *field_str, *field_name;
4710                 const char *test_name;
4711
4712                 sort_key = &hist_data->sort_keys[i];
4713
4714                 field_str = strsep(&fields_str, ",");
4715                 if (!field_str)
4716                         break;
4717
4718                 if (!*field_str) {
4719                         ret = -EINVAL;
4720                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4721                         break;
4722                 }
4723
4724                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4725                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4726                         ret = -EINVAL;
4727                         break;
4728                 }
4729
4730                 field_name = strsep(&field_str, ".");
4731                 if (!field_name || !*field_name) {
4732                         ret = -EINVAL;
4733                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4734                         break;
4735                 }
4736
4737                 if (strcmp(field_name, "hitcount") == 0) {
4738                         descending = is_descending(tr, field_str);
4739                         if (descending < 0) {
4740                                 ret = descending;
4741                                 break;
4742                         }
4743                         sort_key->descending = descending;
4744                         continue;
4745                 }
4746
4747                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4748                         unsigned int idx;
4749
4750                         hist_field = hist_data->fields[j];
4751                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4752                                 continue;
4753
4754                         idx = k++;
4755
4756                         test_name = hist_field_name(hist_field, 0);
4757
4758                         if (strcmp(field_name, test_name) == 0) {
4759                                 sort_key->field_idx = idx;
4760                                 descending = is_descending(tr, field_str);
4761                                 if (descending < 0) {
4762                                         ret = descending;
4763                                         goto out;
4764                                 }
4765                                 sort_key->descending = descending;
4766                                 break;
4767                         }
4768                 }
4769                 if (j == hist_data->n_fields) {
4770                         ret = -EINVAL;
4771                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4772                         break;
4773                 }
4774         }
4775
4776         hist_data->n_sort_keys = i;
4777  out:
4778         return ret;
4779 }
4780
4781 static void destroy_actions(struct hist_trigger_data *hist_data)
4782 {
4783         unsigned int i;
4784
4785         for (i = 0; i < hist_data->n_actions; i++) {
4786                 struct action_data *data = hist_data->actions[i];
4787
4788                 if (data->handler == HANDLER_ONMATCH)
4789                         onmatch_destroy(data);
4790                 else if (data->handler == HANDLER_ONMAX ||
4791                          data->handler == HANDLER_ONCHANGE)
4792                         track_data_destroy(hist_data, data);
4793                 else
4794                         kfree(data);
4795         }
4796 }
4797
4798 static int parse_actions(struct hist_trigger_data *hist_data)
4799 {
4800         struct trace_array *tr = hist_data->event_file->tr;
4801         struct action_data *data;
4802         unsigned int i;
4803         int ret = 0;
4804         char *str;
4805         int len;
4806
4807         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4808                 str = hist_data->attrs->action_str[i];
4809
4810                 if ((len = str_has_prefix(str, "onmatch("))) {
4811                         char *action_str = str + len;
4812
4813                         data = onmatch_parse(tr, action_str);
4814                         if (IS_ERR(data)) {
4815                                 ret = PTR_ERR(data);
4816                                 break;
4817                         }
4818                 } else if ((len = str_has_prefix(str, "onmax("))) {
4819                         char *action_str = str + len;
4820
4821                         data = track_data_parse(hist_data, action_str,
4822                                                 HANDLER_ONMAX);
4823                         if (IS_ERR(data)) {
4824                                 ret = PTR_ERR(data);
4825                                 break;
4826                         }
4827                 } else if ((len = str_has_prefix(str, "onchange("))) {
4828                         char *action_str = str + len;
4829
4830                         data = track_data_parse(hist_data, action_str,
4831                                                 HANDLER_ONCHANGE);
4832                         if (IS_ERR(data)) {
4833                                 ret = PTR_ERR(data);
4834                                 break;
4835                         }
4836                 } else {
4837                         ret = -EINVAL;
4838                         break;
4839                 }
4840
4841                 hist_data->actions[hist_data->n_actions++] = data;
4842         }
4843
4844         return ret;
4845 }
4846
4847 static int create_actions(struct hist_trigger_data *hist_data)
4848 {
4849         struct action_data *data;
4850         unsigned int i;
4851         int ret = 0;
4852
4853         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4854                 data = hist_data->actions[i];
4855
4856                 if (data->handler == HANDLER_ONMATCH) {
4857                         ret = onmatch_create(hist_data, data);
4858                         if (ret)
4859                                 break;
4860                 } else if (data->handler == HANDLER_ONMAX ||
4861                            data->handler == HANDLER_ONCHANGE) {
4862                         ret = track_data_create(hist_data, data);
4863                         if (ret)
4864                                 break;
4865                 } else {
4866                         ret = -EINVAL;
4867                         break;
4868                 }
4869         }
4870
4871         return ret;
4872 }
4873
4874 static void print_actions(struct seq_file *m,
4875                           struct hist_trigger_data *hist_data,
4876                           struct tracing_map_elt *elt)
4877 {
4878         unsigned int i;
4879
4880         for (i = 0; i < hist_data->n_actions; i++) {
4881                 struct action_data *data = hist_data->actions[i];
4882
4883                 if (data->action == ACTION_SNAPSHOT)
4884                         continue;
4885
4886                 if (data->handler == HANDLER_ONMAX ||
4887                     data->handler == HANDLER_ONCHANGE)
4888                         track_data_print(m, hist_data, elt, data);
4889         }
4890 }
4891
4892 static void print_action_spec(struct seq_file *m,
4893                               struct hist_trigger_data *hist_data,
4894                               struct action_data *data)
4895 {
4896         unsigned int i;
4897
4898         if (data->action == ACTION_SAVE) {
4899                 for (i = 0; i < hist_data->n_save_vars; i++) {
4900                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4901                         if (i < hist_data->n_save_vars - 1)
4902                                 seq_puts(m, ",");
4903                 }
4904         } else if (data->action == ACTION_TRACE) {
4905                 if (data->use_trace_keyword)
4906                         seq_printf(m, "%s", data->synth_event_name);
4907                 for (i = 0; i < data->n_params; i++) {
4908                         if (i || data->use_trace_keyword)
4909                                 seq_puts(m, ",");
4910                         seq_printf(m, "%s", data->params[i]);
4911                 }
4912         }
4913 }
4914
4915 static void print_track_data_spec(struct seq_file *m,
4916                                   struct hist_trigger_data *hist_data,
4917                                   struct action_data *data)
4918 {
4919         if (data->handler == HANDLER_ONMAX)
4920                 seq_puts(m, ":onmax(");
4921         else if (data->handler == HANDLER_ONCHANGE)
4922                 seq_puts(m, ":onchange(");
4923         seq_printf(m, "%s", data->track_data.var_str);
4924         seq_printf(m, ").%s(", data->action_name);
4925
4926         print_action_spec(m, hist_data, data);
4927
4928         seq_puts(m, ")");
4929 }
4930
4931 static void print_onmatch_spec(struct seq_file *m,
4932                                struct hist_trigger_data *hist_data,
4933                                struct action_data *data)
4934 {
4935         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4936                    data->match_data.event);
4937
4938         seq_printf(m, "%s(", data->action_name);
4939
4940         print_action_spec(m, hist_data, data);
4941
4942         seq_puts(m, ")");
4943 }
4944
4945 static bool actions_match(struct hist_trigger_data *hist_data,
4946                           struct hist_trigger_data *hist_data_test)
4947 {
4948         unsigned int i, j;
4949
4950         if (hist_data->n_actions != hist_data_test->n_actions)
4951                 return false;
4952
4953         for (i = 0; i < hist_data->n_actions; i++) {
4954                 struct action_data *data = hist_data->actions[i];
4955                 struct action_data *data_test = hist_data_test->actions[i];
4956                 char *action_name, *action_name_test;
4957
4958                 if (data->handler != data_test->handler)
4959                         return false;
4960                 if (data->action != data_test->action)
4961                         return false;
4962
4963                 if (data->n_params != data_test->n_params)
4964                         return false;
4965
4966                 for (j = 0; j < data->n_params; j++) {
4967                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4968                                 return false;
4969                 }
4970
4971                 if (data->use_trace_keyword)
4972                         action_name = data->synth_event_name;
4973                 else
4974                         action_name = data->action_name;
4975
4976                 if (data_test->use_trace_keyword)
4977                         action_name_test = data_test->synth_event_name;
4978                 else
4979                         action_name_test = data_test->action_name;
4980
4981                 if (strcmp(action_name, action_name_test) != 0)
4982                         return false;
4983
4984                 if (data->handler == HANDLER_ONMATCH) {
4985                         if (strcmp(data->match_data.event_system,
4986                                    data_test->match_data.event_system) != 0)
4987                                 return false;
4988                         if (strcmp(data->match_data.event,
4989                                    data_test->match_data.event) != 0)
4990                                 return false;
4991                 } else if (data->handler == HANDLER_ONMAX ||
4992                            data->handler == HANDLER_ONCHANGE) {
4993                         if (strcmp(data->track_data.var_str,
4994                                    data_test->track_data.var_str) != 0)
4995                                 return false;
4996                 }
4997         }
4998
4999         return true;
5000 }
5001
5002
5003 static void print_actions_spec(struct seq_file *m,
5004                                struct hist_trigger_data *hist_data)
5005 {
5006         unsigned int i;
5007
5008         for (i = 0; i < hist_data->n_actions; i++) {
5009                 struct action_data *data = hist_data->actions[i];
5010
5011                 if (data->handler == HANDLER_ONMATCH)
5012                         print_onmatch_spec(m, hist_data, data);
5013                 else if (data->handler == HANDLER_ONMAX ||
5014                          data->handler == HANDLER_ONCHANGE)
5015                         print_track_data_spec(m, hist_data, data);
5016         }
5017 }
5018
5019 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
5020 {
5021         unsigned int i;
5022
5023         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5024                 kfree(hist_data->field_var_hists[i]->cmd);
5025                 kfree(hist_data->field_var_hists[i]);
5026         }
5027 }
5028
5029 static void destroy_hist_data(struct hist_trigger_data *hist_data)
5030 {
5031         if (!hist_data)
5032                 return;
5033
5034         destroy_hist_trigger_attrs(hist_data->attrs);
5035         destroy_hist_fields(hist_data);
5036         tracing_map_destroy(hist_data->map);
5037
5038         destroy_actions(hist_data);
5039         destroy_field_vars(hist_data);
5040         destroy_field_var_hists(hist_data);
5041
5042         kfree(hist_data);
5043 }
5044
5045 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5046 {
5047         struct tracing_map *map = hist_data->map;
5048         struct ftrace_event_field *field;
5049         struct hist_field *hist_field;
5050         int i, idx = 0;
5051
5052         for_each_hist_field(i, hist_data) {
5053                 hist_field = hist_data->fields[i];
5054                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
5055                         tracing_map_cmp_fn_t cmp_fn;
5056
5057                         field = hist_field->field;
5058
5059                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5060                                 cmp_fn = tracing_map_cmp_none;
5061                         else if (!field || hist_field->flags & HIST_FIELD_FL_CPU)
5062                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
5063                                                              hist_field->is_signed);
5064                         else if (is_string_field(field))
5065                                 cmp_fn = tracing_map_cmp_string;
5066                         else
5067                                 cmp_fn = tracing_map_cmp_num(field->size,
5068                                                              field->is_signed);
5069                         idx = tracing_map_add_key_field(map,
5070                                                         hist_field->offset,
5071                                                         cmp_fn);
5072                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5073                         idx = tracing_map_add_sum_field(map);
5074
5075                 if (idx < 0)
5076                         return idx;
5077
5078                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5079                         idx = tracing_map_add_var(map);
5080                         if (idx < 0)
5081                                 return idx;
5082                         hist_field->var.idx = idx;
5083                         hist_field->var.hist_data = hist_data;
5084                 }
5085         }
5086
5087         return 0;
5088 }
5089
5090 static struct hist_trigger_data *
5091 create_hist_data(unsigned int map_bits,
5092                  struct hist_trigger_attrs *attrs,
5093                  struct trace_event_file *file,
5094                  bool remove)
5095 {
5096         const struct tracing_map_ops *map_ops = NULL;
5097         struct hist_trigger_data *hist_data;
5098         int ret = 0;
5099
5100         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5101         if (!hist_data)
5102                 return ERR_PTR(-ENOMEM);
5103
5104         hist_data->attrs = attrs;
5105         hist_data->remove = remove;
5106         hist_data->event_file = file;
5107
5108         ret = parse_actions(hist_data);
5109         if (ret)
5110                 goto free;
5111
5112         ret = create_hist_fields(hist_data, file);
5113         if (ret)
5114                 goto free;
5115
5116         ret = create_sort_keys(hist_data);
5117         if (ret)
5118                 goto free;
5119
5120         map_ops = &hist_trigger_elt_data_ops;
5121
5122         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5123                                             map_ops, hist_data);
5124         if (IS_ERR(hist_data->map)) {
5125                 ret = PTR_ERR(hist_data->map);
5126                 hist_data->map = NULL;
5127                 goto free;
5128         }
5129
5130         ret = create_tracing_map_fields(hist_data);
5131         if (ret)
5132                 goto free;
5133  out:
5134         return hist_data;
5135  free:
5136         hist_data->attrs = NULL;
5137
5138         destroy_hist_data(hist_data);
5139
5140         hist_data = ERR_PTR(ret);
5141
5142         goto out;
5143 }
5144
5145 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5146                                     struct tracing_map_elt *elt,
5147                                     struct trace_buffer *buffer, void *rec,
5148                                     struct ring_buffer_event *rbe,
5149                                     u64 *var_ref_vals)
5150 {
5151         struct hist_elt_data *elt_data;
5152         struct hist_field *hist_field;
5153         unsigned int i, var_idx;
5154         u64 hist_val;
5155
5156         elt_data = elt->private_data;
5157         elt_data->var_ref_vals = var_ref_vals;
5158
5159         for_each_hist_val_field(i, hist_data) {
5160                 hist_field = hist_data->fields[i];
5161                 hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5162                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5163                         var_idx = hist_field->var.idx;
5164
5165                         if (hist_field->flags &
5166                             (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) {
5167                                 unsigned int str_start, var_str_idx, idx;
5168                                 char *str, *val_str;
5169                                 unsigned int size;
5170
5171                                 str_start = hist_data->n_field_var_str +
5172                                         hist_data->n_save_var_str;
5173                                 var_str_idx = hist_field->var_str_idx;
5174                                 idx = str_start + var_str_idx;
5175
5176                                 str = elt_data->field_var_str[idx];
5177                                 val_str = (char *)(uintptr_t)hist_val;
5178
5179                                 if (hist_field->flags & HIST_FIELD_FL_STRING) {
5180                                         size = min(hist_field->size, STR_VAR_LEN_MAX);
5181                                         strscpy(str, val_str, size);
5182                                 } else {
5183                                         char *stack_start = str + sizeof(unsigned long);
5184                                         int e;
5185
5186                                         e = stack_trace_save((void *)stack_start,
5187                                                              HIST_STACKTRACE_DEPTH,
5188                                                              HIST_STACKTRACE_SKIP);
5189                                         if (e < HIST_STACKTRACE_DEPTH - 1)
5190                                                 ((unsigned long *)stack_start)[e] = 0;
5191                                         *((unsigned long *)str) = e;
5192                                 }
5193                                 hist_val = (u64)(uintptr_t)str;
5194                         }
5195                         tracing_map_set_var(elt, var_idx, hist_val);
5196                         continue;
5197                 }
5198                 tracing_map_update_sum(elt, i, hist_val);
5199         }
5200
5201         for_each_hist_key_field(i, hist_data) {
5202                 hist_field = hist_data->fields[i];
5203                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5204                         hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5205                         var_idx = hist_field->var.idx;
5206                         tracing_map_set_var(elt, var_idx, hist_val);
5207                 }
5208         }
5209
5210         update_field_vars(hist_data, elt, buffer, rbe, rec);
5211 }
5212
5213 static inline void add_to_key(char *compound_key, void *key,
5214                               struct hist_field *key_field, void *rec)
5215 {
5216         size_t size = key_field->size;
5217
5218         if (key_field->flags & HIST_FIELD_FL_STRING) {
5219                 struct ftrace_event_field *field;
5220
5221                 field = key_field->field;
5222                 if (field->filter_type == FILTER_DYN_STRING ||
5223                     field->filter_type == FILTER_RDYN_STRING)
5224                         size = *(u32 *)(rec + field->offset) >> 16;
5225                 else if (field->filter_type == FILTER_STATIC_STRING)
5226                         size = field->size;
5227
5228                 /* ensure NULL-termination */
5229                 if (size > key_field->size - 1)
5230                         size = key_field->size - 1;
5231
5232                 strncpy(compound_key + key_field->offset, (char *)key, size);
5233         } else
5234                 memcpy(compound_key + key_field->offset, key, size);
5235 }
5236
5237 static void
5238 hist_trigger_actions(struct hist_trigger_data *hist_data,
5239                      struct tracing_map_elt *elt,
5240                      struct trace_buffer *buffer, void *rec,
5241                      struct ring_buffer_event *rbe, void *key,
5242                      u64 *var_ref_vals)
5243 {
5244         struct action_data *data;
5245         unsigned int i;
5246
5247         for (i = 0; i < hist_data->n_actions; i++) {
5248                 data = hist_data->actions[i];
5249                 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
5250         }
5251 }
5252
5253 static void event_hist_trigger(struct event_trigger_data *data,
5254                                struct trace_buffer *buffer, void *rec,
5255                                struct ring_buffer_event *rbe)
5256 {
5257         struct hist_trigger_data *hist_data = data->private_data;
5258         bool use_compound_key = (hist_data->n_keys > 1);
5259         unsigned long entries[HIST_STACKTRACE_DEPTH];
5260         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5261         char compound_key[HIST_KEY_SIZE_MAX];
5262         struct tracing_map_elt *elt = NULL;
5263         struct hist_field *key_field;
5264         u64 field_contents;
5265         void *key = NULL;
5266         unsigned int i;
5267
5268         if (unlikely(!rbe))
5269                 return;
5270
5271         memset(compound_key, 0, hist_data->key_size);
5272
5273         for_each_hist_key_field(i, hist_data) {
5274                 key_field = hist_data->fields[i];
5275
5276                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5277                         memset(entries, 0, HIST_STACKTRACE_SIZE);
5278                         if (key_field->field) {
5279                                 unsigned long *stack, n_entries;
5280
5281                                 field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5282                                 stack = (unsigned long *)(long)field_contents;
5283                                 n_entries = *stack;
5284                                 memcpy(entries, ++stack, n_entries * sizeof(unsigned long));
5285                         } else {
5286                                 stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5287                                                  HIST_STACKTRACE_SKIP);
5288                         }
5289                         key = entries;
5290                 } else {
5291                         field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5292                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5293                                 key = (void *)(unsigned long)field_contents;
5294                                 use_compound_key = true;
5295                         } else
5296                                 key = (void *)&field_contents;
5297                 }
5298
5299                 if (use_compound_key)
5300                         add_to_key(compound_key, key, key_field, rec);
5301         }
5302
5303         if (use_compound_key)
5304                 key = compound_key;
5305
5306         if (hist_data->n_var_refs &&
5307             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5308                 return;
5309
5310         elt = tracing_map_insert(hist_data->map, key);
5311         if (!elt)
5312                 return;
5313
5314         hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
5315
5316         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5317                 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
5318 }
5319
5320 static void hist_trigger_stacktrace_print(struct seq_file *m,
5321                                           unsigned long *stacktrace_entries,
5322                                           unsigned int max_entries)
5323 {
5324         unsigned int spaces = 8;
5325         unsigned int i;
5326
5327         for (i = 0; i < max_entries; i++) {
5328                 if (!stacktrace_entries[i])
5329                         return;
5330
5331                 seq_printf(m, "%*c", 1 + spaces, ' ');
5332                 seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
5333         }
5334 }
5335
5336 static void hist_trigger_print_key(struct seq_file *m,
5337                                    struct hist_trigger_data *hist_data,
5338                                    void *key,
5339                                    struct tracing_map_elt *elt)
5340 {
5341         struct hist_field *key_field;
5342         bool multiline = false;
5343         const char *field_name;
5344         unsigned int i;
5345         u64 uval;
5346
5347         seq_puts(m, "{ ");
5348
5349         for_each_hist_key_field(i, hist_data) {
5350                 key_field = hist_data->fields[i];
5351
5352                 if (i > hist_data->n_vals)
5353                         seq_puts(m, ", ");
5354
5355                 field_name = hist_field_name(key_field, 0);
5356
5357                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5358                         uval = *(u64 *)(key + key_field->offset);
5359                         seq_printf(m, "%s: %llx", field_name, uval);
5360                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5361                         uval = *(u64 *)(key + key_field->offset);
5362                         seq_printf(m, "%s: [%llx] %-45ps", field_name,
5363                                    uval, (void *)(uintptr_t)uval);
5364                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5365                         uval = *(u64 *)(key + key_field->offset);
5366                         seq_printf(m, "%s: [%llx] %-55pS", field_name,
5367                                    uval, (void *)(uintptr_t)uval);
5368                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5369                         struct hist_elt_data *elt_data = elt->private_data;
5370                         char *comm;
5371
5372                         if (WARN_ON_ONCE(!elt_data))
5373                                 return;
5374
5375                         comm = elt_data->comm;
5376
5377                         uval = *(u64 *)(key + key_field->offset);
5378                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5379                                    comm, uval);
5380                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5381                         const char *syscall_name;
5382
5383                         uval = *(u64 *)(key + key_field->offset);
5384                         syscall_name = get_syscall_name(uval);
5385                         if (!syscall_name)
5386                                 syscall_name = "unknown_syscall";
5387
5388                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5389                                    syscall_name, uval);
5390                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5391                         if (key_field->field)
5392                                 seq_printf(m, "%s.stacktrace", key_field->field->name);
5393                         else
5394                                 seq_puts(m, "common_stacktrace:\n");
5395                         hist_trigger_stacktrace_print(m,
5396                                                       key + key_field->offset,
5397                                                       HIST_STACKTRACE_DEPTH);
5398                         multiline = true;
5399                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5400                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5401                                    *(u64 *)(key + key_field->offset));
5402                 } else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5403                         unsigned long buckets = key_field->buckets;
5404                         uval = *(u64 *)(key + key_field->offset);
5405                         seq_printf(m, "%s: ~ %llu-%llu", field_name,
5406                                    uval, uval + buckets -1);
5407                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5408                         seq_printf(m, "%s: %-50s", field_name,
5409                                    (char *)(key + key_field->offset));
5410                 } else {
5411                         uval = *(u64 *)(key + key_field->offset);
5412                         seq_printf(m, "%s: %10llu", field_name, uval);
5413                 }
5414         }
5415
5416         if (!multiline)
5417                 seq_puts(m, " ");
5418
5419         seq_puts(m, "}");
5420 }
5421
5422 /* Get the 100 times of the percentage of @val in @total */
5423 static inline unsigned int __get_percentage(u64 val, u64 total)
5424 {
5425         if (!total)
5426                 goto div0;
5427
5428         if (val < (U64_MAX / 10000))
5429                 return (unsigned int)div64_ul(val * 10000, total);
5430
5431         total = div64_u64(total, 10000);
5432         if (!total)
5433                 goto div0;
5434
5435         return (unsigned int)div64_ul(val, total);
5436 div0:
5437         return val ? UINT_MAX : 0;
5438 }
5439
5440 #define BAR_CHAR '#'
5441
5442 static inline const char *__fill_bar_str(char *buf, int size, u64 val, u64 max)
5443 {
5444         unsigned int len = __get_percentage(val, max);
5445         int i;
5446
5447         if (len == UINT_MAX) {
5448                 snprintf(buf, size, "[ERROR]");
5449                 return buf;
5450         }
5451
5452         len = len * size / 10000;
5453         for (i = 0; i < len && i < size; i++)
5454                 buf[i] = BAR_CHAR;
5455         while (i < size)
5456                 buf[i++] = ' ';
5457         buf[size] = '\0';
5458
5459         return buf;
5460 }
5461
5462 struct hist_val_stat {
5463         u64 max;
5464         u64 total;
5465 };
5466
5467 static void hist_trigger_print_val(struct seq_file *m, unsigned int idx,
5468                                    const char *field_name, unsigned long flags,
5469                                    struct hist_val_stat *stats,
5470                                    struct tracing_map_elt *elt)
5471 {
5472         u64 val = tracing_map_read_sum(elt, idx);
5473         unsigned int pc;
5474         char bar[21];
5475
5476         if (flags & HIST_FIELD_FL_PERCENT) {
5477                 pc = __get_percentage(val, stats[idx].total);
5478                 if (pc == UINT_MAX)
5479                         seq_printf(m, " %s (%%):[ERROR]", field_name);
5480                 else
5481                         seq_printf(m, " %s (%%): %3u.%02u", field_name,
5482                                         pc / 100, pc % 100);
5483         } else if (flags & HIST_FIELD_FL_GRAPH) {
5484                 seq_printf(m, " %s: %20s", field_name,
5485                            __fill_bar_str(bar, 20, val, stats[idx].max));
5486         } else if (flags & HIST_FIELD_FL_HEX) {
5487                 seq_printf(m, " %s: %10llx", field_name, val);
5488         } else {
5489                 seq_printf(m, " %s: %10llu", field_name, val);
5490         }
5491 }
5492
5493 static void hist_trigger_entry_print(struct seq_file *m,
5494                                      struct hist_trigger_data *hist_data,
5495                                      struct hist_val_stat *stats,
5496                                      void *key,
5497                                      struct tracing_map_elt *elt)
5498 {
5499         const char *field_name;
5500         unsigned int i = HITCOUNT_IDX;
5501         unsigned long flags;
5502
5503         hist_trigger_print_key(m, hist_data, key, elt);
5504
5505         /* At first, show the raw hitcount if !nohitcount */
5506         if (!hist_data->attrs->no_hitcount)
5507                 hist_trigger_print_val(m, i, "hitcount", 0, stats, elt);
5508
5509         for (i = 1; i < hist_data->n_vals; i++) {
5510                 field_name = hist_field_name(hist_data->fields[i], 0);
5511                 flags = hist_data->fields[i]->flags;
5512                 if (flags & HIST_FIELD_FL_VAR || flags & HIST_FIELD_FL_EXPR)
5513                         continue;
5514
5515                 seq_puts(m, " ");
5516                 hist_trigger_print_val(m, i, field_name, flags, stats, elt);
5517         }
5518
5519         print_actions(m, hist_data, elt);
5520
5521         seq_puts(m, "\n");
5522 }
5523
5524 static int print_entries(struct seq_file *m,
5525                          struct hist_trigger_data *hist_data)
5526 {
5527         struct tracing_map_sort_entry **sort_entries = NULL;
5528         struct tracing_map *map = hist_data->map;
5529         int i, j, n_entries;
5530         struct hist_val_stat *stats = NULL;
5531         u64 val;
5532
5533         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5534                                              hist_data->n_sort_keys,
5535                                              &sort_entries);
5536         if (n_entries < 0)
5537                 return n_entries;
5538
5539         /* Calculate the max and the total for each field if needed. */
5540         for (j = 0; j < hist_data->n_vals; j++) {
5541                 if (!(hist_data->fields[j]->flags &
5542                         (HIST_FIELD_FL_PERCENT | HIST_FIELD_FL_GRAPH)))
5543                         continue;
5544                 if (!stats) {
5545                         stats = kcalloc(hist_data->n_vals, sizeof(*stats),
5546                                        GFP_KERNEL);
5547                         if (!stats) {
5548                                 n_entries = -ENOMEM;
5549                                 goto out;
5550                         }
5551                 }
5552                 for (i = 0; i < n_entries; i++) {
5553                         val = tracing_map_read_sum(sort_entries[i]->elt, j);
5554                         stats[j].total += val;
5555                         if (stats[j].max < val)
5556                                 stats[j].max = val;
5557                 }
5558         }
5559
5560         for (i = 0; i < n_entries; i++)
5561                 hist_trigger_entry_print(m, hist_data, stats,
5562                                          sort_entries[i]->key,
5563                                          sort_entries[i]->elt);
5564
5565         kfree(stats);
5566 out:
5567         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5568
5569         return n_entries;
5570 }
5571
5572 static void hist_trigger_show(struct seq_file *m,
5573                               struct event_trigger_data *data, int n)
5574 {
5575         struct hist_trigger_data *hist_data;
5576         int n_entries;
5577
5578         if (n > 0)
5579                 seq_puts(m, "\n\n");
5580
5581         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5582         data->ops->print(m, data);
5583         seq_puts(m, "#\n\n");
5584
5585         hist_data = data->private_data;
5586         n_entries = print_entries(m, hist_data);
5587         if (n_entries < 0)
5588                 n_entries = 0;
5589
5590         track_data_snapshot_print(m, hist_data);
5591
5592         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5593                    (u64)atomic64_read(&hist_data->map->hits),
5594                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5595 }
5596
5597 static int hist_show(struct seq_file *m, void *v)
5598 {
5599         struct event_trigger_data *data;
5600         struct trace_event_file *event_file;
5601         int n = 0, ret = 0;
5602
5603         mutex_lock(&event_mutex);
5604
5605         event_file = event_file_data(m->private);
5606         if (unlikely(!event_file)) {
5607                 ret = -ENODEV;
5608                 goto out_unlock;
5609         }
5610
5611         list_for_each_entry(data, &event_file->triggers, list) {
5612                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5613                         hist_trigger_show(m, data, n++);
5614         }
5615
5616  out_unlock:
5617         mutex_unlock(&event_mutex);
5618
5619         return ret;
5620 }
5621
5622 static int event_hist_open(struct inode *inode, struct file *file)
5623 {
5624         int ret;
5625
5626         ret = tracing_open_file_tr(inode, file);
5627         if (ret)
5628                 return ret;
5629
5630         /* Clear private_data to avoid warning in single_open() */
5631         file->private_data = NULL;
5632         return single_open(file, hist_show, file);
5633 }
5634
5635 const struct file_operations event_hist_fops = {
5636         .open = event_hist_open,
5637         .read = seq_read,
5638         .llseek = seq_lseek,
5639         .release = tracing_single_release_file_tr,
5640 };
5641
5642 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
5643 static void hist_field_debug_show_flags(struct seq_file *m,
5644                                         unsigned long flags)
5645 {
5646         seq_puts(m, "      flags:\n");
5647
5648         if (flags & HIST_FIELD_FL_KEY)
5649                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
5650         else if (flags & HIST_FIELD_FL_HITCOUNT)
5651                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
5652         else if (flags & HIST_FIELD_FL_VAR)
5653                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
5654         else if (flags & HIST_FIELD_FL_VAR_REF)
5655                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
5656         else
5657                 seq_puts(m, "        VAL: normal u64 value\n");
5658
5659         if (flags & HIST_FIELD_FL_ALIAS)
5660                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
5661         else if (flags & HIST_FIELD_FL_CONST)
5662                 seq_puts(m, "        HIST_FIELD_FL_CONST\n");
5663 }
5664
5665 static int hist_field_debug_show(struct seq_file *m,
5666                                  struct hist_field *field, unsigned long flags)
5667 {
5668         if ((field->flags & flags) != flags) {
5669                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
5670                 return -EINVAL;
5671         }
5672
5673         hist_field_debug_show_flags(m, field->flags);
5674         if (field->field)
5675                 seq_printf(m, "      ftrace_event_field name: %s\n",
5676                            field->field->name);
5677
5678         if (field->flags & HIST_FIELD_FL_VAR) {
5679                 seq_printf(m, "      var.name: %s\n", field->var.name);
5680                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5681                            field->var.idx);
5682         }
5683
5684         if (field->flags & HIST_FIELD_FL_CONST)
5685                 seq_printf(m, "      constant: %llu\n", field->constant);
5686
5687         if (field->flags & HIST_FIELD_FL_ALIAS)
5688                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5689                            field->var_ref_idx);
5690
5691         if (field->flags & HIST_FIELD_FL_VAR_REF) {
5692                 seq_printf(m, "      name: %s\n", field->name);
5693                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5694                            field->var.idx);
5695                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
5696                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5697                            field->var_ref_idx);
5698                 if (field->system)
5699                         seq_printf(m, "      system: %s\n", field->system);
5700                 if (field->event_name)
5701                         seq_printf(m, "      event_name: %s\n", field->event_name);
5702         }
5703
5704         seq_printf(m, "      type: %s\n", field->type);
5705         seq_printf(m, "      size: %u\n", field->size);
5706         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5707
5708         return 0;
5709 }
5710
5711 static int field_var_debug_show(struct seq_file *m,
5712                                 struct field_var *field_var, unsigned int i,
5713                                 bool save_vars)
5714 {
5715         const char *vars_name = save_vars ? "save_vars" : "field_vars";
5716         struct hist_field *field;
5717         int ret = 0;
5718
5719         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
5720
5721         field = field_var->var;
5722
5723         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
5724
5725         hist_field_debug_show_flags(m, field->flags);
5726         seq_printf(m, "      var.name: %s\n", field->var.name);
5727         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5728                    field->var.idx);
5729
5730         field = field_var->val;
5731
5732         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
5733         if (field->field)
5734                 seq_printf(m, "      ftrace_event_field name: %s\n",
5735                            field->field->name);
5736         else {
5737                 ret = -EINVAL;
5738                 goto out;
5739         }
5740
5741         seq_printf(m, "      type: %s\n", field->type);
5742         seq_printf(m, "      size: %u\n", field->size);
5743         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5744 out:
5745         return ret;
5746 }
5747
5748 static int hist_action_debug_show(struct seq_file *m,
5749                                   struct action_data *data, int i)
5750 {
5751         int ret = 0;
5752
5753         if (data->handler == HANDLER_ONMAX ||
5754             data->handler == HANDLER_ONCHANGE) {
5755                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
5756                 ret = hist_field_debug_show(m, data->track_data.var_ref,
5757                                             HIST_FIELD_FL_VAR_REF);
5758                 if (ret)
5759                         goto out;
5760
5761                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
5762                 ret = hist_field_debug_show(m, data->track_data.track_var,
5763                                             HIST_FIELD_FL_VAR);
5764                 if (ret)
5765                         goto out;
5766         }
5767
5768         if (data->handler == HANDLER_ONMATCH) {
5769                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
5770                            i, data->match_data.event_system);
5771                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
5772                            i, data->match_data.event);
5773         }
5774 out:
5775         return ret;
5776 }
5777
5778 static int hist_actions_debug_show(struct seq_file *m,
5779                                    struct hist_trigger_data *hist_data)
5780 {
5781         int i, ret = 0;
5782
5783         if (hist_data->n_actions)
5784                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
5785
5786         for (i = 0; i < hist_data->n_actions; i++) {
5787                 struct action_data *action = hist_data->actions[i];
5788
5789                 ret = hist_action_debug_show(m, action, i);
5790                 if (ret)
5791                         goto out;
5792         }
5793
5794         if (hist_data->n_save_vars)
5795                 seq_puts(m, "\n  save action variables (save() params):\n");
5796
5797         for (i = 0; i < hist_data->n_save_vars; i++) {
5798                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
5799                 if (ret)
5800                         goto out;
5801         }
5802 out:
5803         return ret;
5804 }
5805
5806 static void hist_trigger_debug_show(struct seq_file *m,
5807                                     struct event_trigger_data *data, int n)
5808 {
5809         struct hist_trigger_data *hist_data;
5810         int i, ret;
5811
5812         if (n > 0)
5813                 seq_puts(m, "\n\n");
5814
5815         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5816         data->ops->print(m, data);
5817         seq_puts(m, "#\n\n");
5818
5819         hist_data = data->private_data;
5820
5821         seq_printf(m, "hist_data: %p\n\n", hist_data);
5822         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5823         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5824         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5825
5826         seq_puts(m, "\n  val fields:\n\n");
5827
5828         seq_puts(m, "    hist_data->fields[0]:\n");
5829         ret = hist_field_debug_show(m, hist_data->fields[0],
5830                                     HIST_FIELD_FL_HITCOUNT);
5831         if (ret)
5832                 return;
5833
5834         for (i = 1; i < hist_data->n_vals; i++) {
5835                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5836                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5837                 if (ret)
5838                         return;
5839         }
5840
5841         seq_puts(m, "\n  key fields:\n");
5842
5843         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5844                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5845                 ret = hist_field_debug_show(m, hist_data->fields[i],
5846                                             HIST_FIELD_FL_KEY);
5847                 if (ret)
5848                         return;
5849         }
5850
5851         if (hist_data->n_var_refs)
5852                 seq_puts(m, "\n  variable reference fields:\n");
5853
5854         for (i = 0; i < hist_data->n_var_refs; i++) {
5855                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5856                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5857                                             HIST_FIELD_FL_VAR_REF);
5858                 if (ret)
5859                         return;
5860         }
5861
5862         if (hist_data->n_field_vars)
5863                 seq_puts(m, "\n  field variables:\n");
5864
5865         for (i = 0; i < hist_data->n_field_vars; i++) {
5866                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5867                 if (ret)
5868                         return;
5869         }
5870
5871         ret = hist_actions_debug_show(m, hist_data);
5872         if (ret)
5873                 return;
5874 }
5875
5876 static int hist_debug_show(struct seq_file *m, void *v)
5877 {
5878         struct event_trigger_data *data;
5879         struct trace_event_file *event_file;
5880         int n = 0, ret = 0;
5881
5882         mutex_lock(&event_mutex);
5883
5884         event_file = event_file_data(m->private);
5885         if (unlikely(!event_file)) {
5886                 ret = -ENODEV;
5887                 goto out_unlock;
5888         }
5889
5890         list_for_each_entry(data, &event_file->triggers, list) {
5891                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5892                         hist_trigger_debug_show(m, data, n++);
5893         }
5894
5895  out_unlock:
5896         mutex_unlock(&event_mutex);
5897
5898         return ret;
5899 }
5900
5901 static int event_hist_debug_open(struct inode *inode, struct file *file)
5902 {
5903         int ret;
5904
5905         ret = tracing_open_file_tr(inode, file);
5906         if (ret)
5907                 return ret;
5908
5909         /* Clear private_data to avoid warning in single_open() */
5910         file->private_data = NULL;
5911         return single_open(file, hist_debug_show, file);
5912 }
5913
5914 const struct file_operations event_hist_debug_fops = {
5915         .open = event_hist_debug_open,
5916         .read = seq_read,
5917         .llseek = seq_lseek,
5918         .release = tracing_single_release_file_tr,
5919 };
5920 #endif
5921
5922 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5923 {
5924         const char *field_name = hist_field_name(hist_field, 0);
5925
5926         if (hist_field->var.name)
5927                 seq_printf(m, "%s=", hist_field->var.name);
5928
5929         if (hist_field->flags & HIST_FIELD_FL_CPU)
5930                 seq_puts(m, "common_cpu");
5931         else if (hist_field->flags & HIST_FIELD_FL_CONST)
5932                 seq_printf(m, "%llu", hist_field->constant);
5933         else if (field_name) {
5934                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5935                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5936                         seq_putc(m, '$');
5937                 seq_printf(m, "%s", field_name);
5938         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5939                 seq_puts(m, "common_timestamp");
5940
5941         if (hist_field->flags) {
5942                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5943                     !(hist_field->flags & HIST_FIELD_FL_EXPR) &&
5944                     !(hist_field->flags & HIST_FIELD_FL_STACKTRACE)) {
5945                         const char *flags = get_hist_field_flags(hist_field);
5946
5947                         if (flags)
5948                                 seq_printf(m, ".%s", flags);
5949                 }
5950         }
5951         if (hist_field->buckets)
5952                 seq_printf(m, "=%ld", hist_field->buckets);
5953 }
5954
5955 static int event_hist_trigger_print(struct seq_file *m,
5956                                     struct event_trigger_data *data)
5957 {
5958         struct hist_trigger_data *hist_data = data->private_data;
5959         struct hist_field *field;
5960         bool have_var = false;
5961         bool show_val = false;
5962         unsigned int i;
5963
5964         seq_puts(m, HIST_PREFIX);
5965
5966         if (data->name)
5967                 seq_printf(m, "%s:", data->name);
5968
5969         seq_puts(m, "keys=");
5970
5971         for_each_hist_key_field(i, hist_data) {
5972                 field = hist_data->fields[i];
5973
5974                 if (i > hist_data->n_vals)
5975                         seq_puts(m, ",");
5976
5977                 if (field->flags & HIST_FIELD_FL_STACKTRACE) {
5978                         if (field->field)
5979                                 seq_printf(m, "%s.stacktrace", field->field->name);
5980                         else
5981                                 seq_puts(m, "common_stacktrace");
5982                 } else
5983                         hist_field_print(m, field);
5984         }
5985
5986         seq_puts(m, ":vals=");
5987
5988         for_each_hist_val_field(i, hist_data) {
5989                 field = hist_data->fields[i];
5990                 if (field->flags & HIST_FIELD_FL_VAR) {
5991                         have_var = true;
5992                         continue;
5993                 }
5994
5995                 if (i == HITCOUNT_IDX) {
5996                         if (hist_data->attrs->no_hitcount)
5997                                 continue;
5998                         seq_puts(m, "hitcount");
5999                 } else {
6000                         if (show_val)
6001                                 seq_puts(m, ",");
6002                         hist_field_print(m, field);
6003                 }
6004                 show_val = true;
6005         }
6006
6007         if (have_var) {
6008                 unsigned int n = 0;
6009
6010                 seq_puts(m, ":");
6011
6012                 for_each_hist_val_field(i, hist_data) {
6013                         field = hist_data->fields[i];
6014
6015                         if (field->flags & HIST_FIELD_FL_VAR) {
6016                                 if (n++)
6017                                         seq_puts(m, ",");
6018                                 hist_field_print(m, field);
6019                         }
6020                 }
6021         }
6022
6023         seq_puts(m, ":sort=");
6024
6025         for (i = 0; i < hist_data->n_sort_keys; i++) {
6026                 struct tracing_map_sort_key *sort_key;
6027                 unsigned int idx, first_key_idx;
6028
6029                 /* skip VAR vals */
6030                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
6031
6032                 sort_key = &hist_data->sort_keys[i];
6033                 idx = sort_key->field_idx;
6034
6035                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
6036                         return -EINVAL;
6037
6038                 if (i > 0)
6039                         seq_puts(m, ",");
6040
6041                 if (idx == HITCOUNT_IDX)
6042                         seq_puts(m, "hitcount");
6043                 else {
6044                         if (idx >= first_key_idx)
6045                                 idx += hist_data->n_vars;
6046                         hist_field_print(m, hist_data->fields[idx]);
6047                 }
6048
6049                 if (sort_key->descending)
6050                         seq_puts(m, ".descending");
6051         }
6052         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
6053         if (hist_data->enable_timestamps)
6054                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
6055         if (hist_data->attrs->no_hitcount)
6056                 seq_puts(m, ":nohitcount");
6057
6058         print_actions_spec(m, hist_data);
6059
6060         if (data->filter_str)
6061                 seq_printf(m, " if %s", data->filter_str);
6062
6063         if (data->paused)
6064                 seq_puts(m, " [paused]");
6065         else
6066                 seq_puts(m, " [active]");
6067
6068         seq_putc(m, '\n');
6069
6070         return 0;
6071 }
6072
6073 static int event_hist_trigger_init(struct event_trigger_data *data)
6074 {
6075         struct hist_trigger_data *hist_data = data->private_data;
6076
6077         if (!data->ref && hist_data->attrs->name)
6078                 save_named_trigger(hist_data->attrs->name, data);
6079
6080         data->ref++;
6081
6082         return 0;
6083 }
6084
6085 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
6086 {
6087         struct trace_event_file *file;
6088         unsigned int i;
6089         char *cmd;
6090         int ret;
6091
6092         for (i = 0; i < hist_data->n_field_var_hists; i++) {
6093                 file = hist_data->field_var_hists[i]->hist_data->event_file;
6094                 cmd = hist_data->field_var_hists[i]->cmd;
6095                 ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
6096                                                "!hist", "hist", cmd);
6097                 WARN_ON_ONCE(ret < 0);
6098         }
6099 }
6100
6101 static void event_hist_trigger_free(struct event_trigger_data *data)
6102 {
6103         struct hist_trigger_data *hist_data = data->private_data;
6104
6105         if (WARN_ON_ONCE(data->ref <= 0))
6106                 return;
6107
6108         data->ref--;
6109         if (!data->ref) {
6110                 if (data->name)
6111                         del_named_trigger(data);
6112
6113                 trigger_data_free(data);
6114
6115                 remove_hist_vars(hist_data);
6116
6117                 unregister_field_var_hists(hist_data);
6118
6119                 destroy_hist_data(hist_data);
6120         }
6121 }
6122
6123 static struct event_trigger_ops event_hist_trigger_ops = {
6124         .trigger                = event_hist_trigger,
6125         .print                  = event_hist_trigger_print,
6126         .init                   = event_hist_trigger_init,
6127         .free                   = event_hist_trigger_free,
6128 };
6129
6130 static int event_hist_trigger_named_init(struct event_trigger_data *data)
6131 {
6132         data->ref++;
6133
6134         save_named_trigger(data->named_data->name, data);
6135
6136         event_hist_trigger_init(data->named_data);
6137
6138         return 0;
6139 }
6140
6141 static void event_hist_trigger_named_free(struct event_trigger_data *data)
6142 {
6143         if (WARN_ON_ONCE(data->ref <= 0))
6144                 return;
6145
6146         event_hist_trigger_free(data->named_data);
6147
6148         data->ref--;
6149         if (!data->ref) {
6150                 del_named_trigger(data);
6151                 trigger_data_free(data);
6152         }
6153 }
6154
6155 static struct event_trigger_ops event_hist_trigger_named_ops = {
6156         .trigger                = event_hist_trigger,
6157         .print                  = event_hist_trigger_print,
6158         .init                   = event_hist_trigger_named_init,
6159         .free                   = event_hist_trigger_named_free,
6160 };
6161
6162 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
6163                                                             char *param)
6164 {
6165         return &event_hist_trigger_ops;
6166 }
6167
6168 static void hist_clear(struct event_trigger_data *data)
6169 {
6170         struct hist_trigger_data *hist_data = data->private_data;
6171
6172         if (data->name)
6173                 pause_named_trigger(data);
6174
6175         tracepoint_synchronize_unregister();
6176
6177         tracing_map_clear(hist_data->map);
6178
6179         if (data->name)
6180                 unpause_named_trigger(data);
6181 }
6182
6183 static bool compatible_field(struct ftrace_event_field *field,
6184                              struct ftrace_event_field *test_field)
6185 {
6186         if (field == test_field)
6187                 return true;
6188         if (field == NULL || test_field == NULL)
6189                 return false;
6190         if (strcmp(field->name, test_field->name) != 0)
6191                 return false;
6192         if (strcmp(field->type, test_field->type) != 0)
6193                 return false;
6194         if (field->size != test_field->size)
6195                 return false;
6196         if (field->is_signed != test_field->is_signed)
6197                 return false;
6198
6199         return true;
6200 }
6201
6202 static bool hist_trigger_match(struct event_trigger_data *data,
6203                                struct event_trigger_data *data_test,
6204                                struct event_trigger_data *named_data,
6205                                bool ignore_filter)
6206 {
6207         struct tracing_map_sort_key *sort_key, *sort_key_test;
6208         struct hist_trigger_data *hist_data, *hist_data_test;
6209         struct hist_field *key_field, *key_field_test;
6210         unsigned int i;
6211
6212         if (named_data && (named_data != data_test) &&
6213             (named_data != data_test->named_data))
6214                 return false;
6215
6216         if (!named_data && is_named_trigger(data_test))
6217                 return false;
6218
6219         hist_data = data->private_data;
6220         hist_data_test = data_test->private_data;
6221
6222         if (hist_data->n_vals != hist_data_test->n_vals ||
6223             hist_data->n_fields != hist_data_test->n_fields ||
6224             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
6225                 return false;
6226
6227         if (!ignore_filter) {
6228                 if ((data->filter_str && !data_test->filter_str) ||
6229                    (!data->filter_str && data_test->filter_str))
6230                         return false;
6231         }
6232
6233         for_each_hist_field(i, hist_data) {
6234                 key_field = hist_data->fields[i];
6235                 key_field_test = hist_data_test->fields[i];
6236
6237                 if (key_field->flags != key_field_test->flags)
6238                         return false;
6239                 if (!compatible_field(key_field->field, key_field_test->field))
6240                         return false;
6241                 if (key_field->offset != key_field_test->offset)
6242                         return false;
6243                 if (key_field->size != key_field_test->size)
6244                         return false;
6245                 if (key_field->is_signed != key_field_test->is_signed)
6246                         return false;
6247                 if (!!key_field->var.name != !!key_field_test->var.name)
6248                         return false;
6249                 if (key_field->var.name &&
6250                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
6251                         return false;
6252         }
6253
6254         for (i = 0; i < hist_data->n_sort_keys; i++) {
6255                 sort_key = &hist_data->sort_keys[i];
6256                 sort_key_test = &hist_data_test->sort_keys[i];
6257
6258                 if (sort_key->field_idx != sort_key_test->field_idx ||
6259                     sort_key->descending != sort_key_test->descending)
6260                         return false;
6261         }
6262
6263         if (!ignore_filter && data->filter_str &&
6264             (strcmp(data->filter_str, data_test->filter_str) != 0))
6265                 return false;
6266
6267         if (!actions_match(hist_data, hist_data_test))
6268                 return false;
6269
6270         return true;
6271 }
6272
6273 static bool existing_hist_update_only(char *glob,
6274                                       struct event_trigger_data *data,
6275                                       struct trace_event_file *file)
6276 {
6277         struct hist_trigger_data *hist_data = data->private_data;
6278         struct event_trigger_data *test, *named_data = NULL;
6279         bool updated = false;
6280
6281         if (!hist_data->attrs->pause && !hist_data->attrs->cont &&
6282             !hist_data->attrs->clear)
6283                 goto out;
6284
6285         if (hist_data->attrs->name) {
6286                 named_data = find_named_trigger(hist_data->attrs->name);
6287                 if (named_data) {
6288                         if (!hist_trigger_match(data, named_data, named_data,
6289                                                 true))
6290                                 goto out;
6291                 }
6292         }
6293
6294         if (hist_data->attrs->name && !named_data)
6295                 goto out;
6296
6297         list_for_each_entry(test, &file->triggers, list) {
6298                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6299                         if (!hist_trigger_match(data, test, named_data, false))
6300                                 continue;
6301                         if (hist_data->attrs->pause)
6302                                 test->paused = true;
6303                         else if (hist_data->attrs->cont)
6304                                 test->paused = false;
6305                         else if (hist_data->attrs->clear)
6306                                 hist_clear(test);
6307                         updated = true;
6308                         goto out;
6309                 }
6310         }
6311  out:
6312         return updated;
6313 }
6314
6315 static int hist_register_trigger(char *glob,
6316                                  struct event_trigger_data *data,
6317                                  struct trace_event_file *file)
6318 {
6319         struct hist_trigger_data *hist_data = data->private_data;
6320         struct event_trigger_data *test, *named_data = NULL;
6321         struct trace_array *tr = file->tr;
6322         int ret = 0;
6323
6324         if (hist_data->attrs->name) {
6325                 named_data = find_named_trigger(hist_data->attrs->name);
6326                 if (named_data) {
6327                         if (!hist_trigger_match(data, named_data, named_data,
6328                                                 true)) {
6329                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
6330                                 ret = -EINVAL;
6331                                 goto out;
6332                         }
6333                 }
6334         }
6335
6336         if (hist_data->attrs->name && !named_data)
6337                 goto new;
6338
6339         lockdep_assert_held(&event_mutex);
6340
6341         list_for_each_entry(test, &file->triggers, list) {
6342                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6343                         if (hist_trigger_match(data, test, named_data, false)) {
6344                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
6345                                 ret = -EEXIST;
6346                                 goto out;
6347                         }
6348                 }
6349         }
6350  new:
6351         if (hist_data->attrs->cont || hist_data->attrs->clear) {
6352                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
6353                 ret = -ENOENT;
6354                 goto out;
6355         }
6356
6357         if (hist_data->attrs->pause)
6358                 data->paused = true;
6359
6360         if (named_data) {
6361                 data->private_data = named_data->private_data;
6362                 set_named_trigger_data(data, named_data);
6363                 data->ops = &event_hist_trigger_named_ops;
6364         }
6365
6366         if (data->ops->init) {
6367                 ret = data->ops->init(data);
6368                 if (ret < 0)
6369                         goto out;
6370         }
6371
6372         if (hist_data->enable_timestamps) {
6373                 char *clock = hist_data->attrs->clock;
6374
6375                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
6376                 if (ret) {
6377                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
6378                         goto out;
6379                 }
6380
6381                 tracing_set_filter_buffering(file->tr, true);
6382         }
6383
6384         if (named_data)
6385                 destroy_hist_data(hist_data);
6386  out:
6387         return ret;
6388 }
6389
6390 static int hist_trigger_enable(struct event_trigger_data *data,
6391                                struct trace_event_file *file)
6392 {
6393         int ret = 0;
6394
6395         list_add_tail_rcu(&data->list, &file->triggers);
6396
6397         update_cond_flag(file);
6398
6399         if (trace_event_trigger_enable_disable(file, 1) < 0) {
6400                 list_del_rcu(&data->list);
6401                 update_cond_flag(file);
6402                 ret--;
6403         }
6404
6405         return ret;
6406 }
6407
6408 static bool have_hist_trigger_match(struct event_trigger_data *data,
6409                                     struct trace_event_file *file)
6410 {
6411         struct hist_trigger_data *hist_data = data->private_data;
6412         struct event_trigger_data *test, *named_data = NULL;
6413         bool match = false;
6414
6415         lockdep_assert_held(&event_mutex);
6416
6417         if (hist_data->attrs->name)
6418                 named_data = find_named_trigger(hist_data->attrs->name);
6419
6420         list_for_each_entry(test, &file->triggers, list) {
6421                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6422                         if (hist_trigger_match(data, test, named_data, false)) {
6423                                 match = true;
6424                                 break;
6425                         }
6426                 }
6427         }
6428
6429         return match;
6430 }
6431
6432 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6433                                     struct trace_event_file *file)
6434 {
6435         struct hist_trigger_data *hist_data = data->private_data;
6436         struct event_trigger_data *test, *named_data = NULL;
6437
6438         lockdep_assert_held(&event_mutex);
6439
6440         if (hist_data->attrs->name)
6441                 named_data = find_named_trigger(hist_data->attrs->name);
6442
6443         list_for_each_entry(test, &file->triggers, list) {
6444                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6445                         if (!hist_trigger_match(data, test, named_data, false))
6446                                 continue;
6447                         hist_data = test->private_data;
6448                         if (check_var_refs(hist_data))
6449                                 return true;
6450                         break;
6451                 }
6452         }
6453
6454         return false;
6455 }
6456
6457 static void hist_unregister_trigger(char *glob,
6458                                     struct event_trigger_data *data,
6459                                     struct trace_event_file *file)
6460 {
6461         struct event_trigger_data *test = NULL, *iter, *named_data = NULL;
6462         struct hist_trigger_data *hist_data = data->private_data;
6463
6464         lockdep_assert_held(&event_mutex);
6465
6466         if (hist_data->attrs->name)
6467                 named_data = find_named_trigger(hist_data->attrs->name);
6468
6469         list_for_each_entry(iter, &file->triggers, list) {
6470                 if (iter->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6471                         if (!hist_trigger_match(data, iter, named_data, false))
6472                                 continue;
6473                         test = iter;
6474                         list_del_rcu(&test->list);
6475                         trace_event_trigger_enable_disable(file, 0);
6476                         update_cond_flag(file);
6477                         break;
6478                 }
6479         }
6480
6481         if (test && test->ops->free)
6482                 test->ops->free(test);
6483
6484         if (hist_data->enable_timestamps) {
6485                 if (!hist_data->remove || test)
6486                         tracing_set_filter_buffering(file->tr, false);
6487         }
6488 }
6489
6490 static bool hist_file_check_refs(struct trace_event_file *file)
6491 {
6492         struct hist_trigger_data *hist_data;
6493         struct event_trigger_data *test;
6494
6495         lockdep_assert_held(&event_mutex);
6496
6497         list_for_each_entry(test, &file->triggers, list) {
6498                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6499                         hist_data = test->private_data;
6500                         if (check_var_refs(hist_data))
6501                                 return true;
6502                 }
6503         }
6504
6505         return false;
6506 }
6507
6508 static void hist_unreg_all(struct trace_event_file *file)
6509 {
6510         struct event_trigger_data *test, *n;
6511         struct hist_trigger_data *hist_data;
6512         struct synth_event *se;
6513         const char *se_name;
6514
6515         lockdep_assert_held(&event_mutex);
6516
6517         if (hist_file_check_refs(file))
6518                 return;
6519
6520         list_for_each_entry_safe(test, n, &file->triggers, list) {
6521                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6522                         hist_data = test->private_data;
6523                         list_del_rcu(&test->list);
6524                         trace_event_trigger_enable_disable(file, 0);
6525
6526                         se_name = trace_event_name(file->event_call);
6527                         se = find_synth_event(se_name);
6528                         if (se)
6529                                 se->ref--;
6530
6531                         update_cond_flag(file);
6532                         if (hist_data->enable_timestamps)
6533                                 tracing_set_filter_buffering(file->tr, false);
6534                         if (test->ops->free)
6535                                 test->ops->free(test);
6536                 }
6537         }
6538 }
6539
6540 static int event_hist_trigger_parse(struct event_command *cmd_ops,
6541                                     struct trace_event_file *file,
6542                                     char *glob, char *cmd,
6543                                     char *param_and_filter)
6544 {
6545         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6546         struct event_trigger_data *trigger_data;
6547         struct hist_trigger_attrs *attrs;
6548         struct hist_trigger_data *hist_data;
6549         char *param, *filter, *p, *start;
6550         struct synth_event *se;
6551         const char *se_name;
6552         bool remove;
6553         int ret = 0;
6554
6555         lockdep_assert_held(&event_mutex);
6556
6557         if (WARN_ON(!glob))
6558                 return -EINVAL;
6559
6560         if (glob[0]) {
6561                 hist_err_clear();
6562                 last_cmd_set(file, param_and_filter);
6563         }
6564
6565         remove = event_trigger_check_remove(glob);
6566
6567         if (event_trigger_empty_param(param_and_filter))
6568                 return -EINVAL;
6569
6570         /*
6571          * separate the trigger from the filter (k:v [if filter])
6572          * allowing for whitespace in the trigger
6573          */
6574         p = param = param_and_filter;
6575         do {
6576                 p = strstr(p, "if");
6577                 if (!p)
6578                         break;
6579                 if (p == param_and_filter)
6580                         return -EINVAL;
6581                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6582                         p++;
6583                         continue;
6584                 }
6585                 if (p >= param_and_filter + strlen(param_and_filter) - (sizeof("if") - 1) - 1)
6586                         return -EINVAL;
6587                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6588                         p++;
6589                         continue;
6590                 }
6591                 break;
6592         } while (1);
6593
6594         if (!p)
6595                 filter = NULL;
6596         else {
6597                 *(p - 1) = '\0';
6598                 filter = strstrip(p);
6599                 param = strstrip(param);
6600         }
6601
6602         /*
6603          * To simplify arithmetic expression parsing, replace occurrences of
6604          * '.sym-offset' modifier with '.symXoffset'
6605          */
6606         start = strstr(param, ".sym-offset");
6607         while (start) {
6608                 *(start + 4) = 'X';
6609                 start = strstr(start + 11, ".sym-offset");
6610         }
6611
6612         attrs = parse_hist_trigger_attrs(file->tr, param);
6613         if (IS_ERR(attrs))
6614                 return PTR_ERR(attrs);
6615
6616         if (attrs->map_bits)
6617                 hist_trigger_bits = attrs->map_bits;
6618
6619         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6620         if (IS_ERR(hist_data)) {
6621                 destroy_hist_trigger_attrs(attrs);
6622                 return PTR_ERR(hist_data);
6623         }
6624
6625         trigger_data = event_trigger_alloc(cmd_ops, cmd, param, hist_data);
6626         if (!trigger_data) {
6627                 ret = -ENOMEM;
6628                 goto out_free;
6629         }
6630
6631         ret = event_trigger_set_filter(cmd_ops, file, filter, trigger_data);
6632         if (ret < 0)
6633                 goto out_free;
6634
6635         if (remove) {
6636                 if (!have_hist_trigger_match(trigger_data, file))
6637                         goto out_free;
6638
6639                 if (hist_trigger_check_refs(trigger_data, file)) {
6640                         ret = -EBUSY;
6641                         goto out_free;
6642                 }
6643
6644                 event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6645                 se_name = trace_event_name(file->event_call);
6646                 se = find_synth_event(se_name);
6647                 if (se)
6648                         se->ref--;
6649                 ret = 0;
6650                 goto out_free;
6651         }
6652
6653         if (existing_hist_update_only(glob, trigger_data, file))
6654                 goto out_free;
6655
6656         ret = event_trigger_register(cmd_ops, file, glob, trigger_data);
6657         if (ret < 0)
6658                 goto out_free;
6659
6660         if (get_named_trigger_data(trigger_data))
6661                 goto enable;
6662
6663         ret = create_actions(hist_data);
6664         if (ret)
6665                 goto out_unreg;
6666
6667         if (has_hist_vars(hist_data) || hist_data->n_var_refs) {
6668                 ret = save_hist_vars(hist_data);
6669                 if (ret)
6670                         goto out_unreg;
6671         }
6672
6673         ret = tracing_map_init(hist_data->map);
6674         if (ret)
6675                 goto out_unreg;
6676 enable:
6677         ret = hist_trigger_enable(trigger_data, file);
6678         if (ret)
6679                 goto out_unreg;
6680
6681         se_name = trace_event_name(file->event_call);
6682         se = find_synth_event(se_name);
6683         if (se)
6684                 se->ref++;
6685  out:
6686         if (ret == 0 && glob[0])
6687                 hist_err_clear();
6688
6689         return ret;
6690  out_unreg:
6691         event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6692  out_free:
6693         event_trigger_reset_filter(cmd_ops, trigger_data);
6694
6695         remove_hist_vars(hist_data);
6696
6697         kfree(trigger_data);
6698
6699         destroy_hist_data(hist_data);
6700         goto out;
6701 }
6702
6703 static struct event_command trigger_hist_cmd = {
6704         .name                   = "hist",
6705         .trigger_type           = ETT_EVENT_HIST,
6706         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6707         .parse                  = event_hist_trigger_parse,
6708         .reg                    = hist_register_trigger,
6709         .unreg                  = hist_unregister_trigger,
6710         .unreg_all              = hist_unreg_all,
6711         .get_trigger_ops        = event_hist_get_trigger_ops,
6712         .set_filter             = set_trigger_filter,
6713 };
6714
6715 __init int register_trigger_hist_cmd(void)
6716 {
6717         int ret;
6718
6719         ret = register_event_command(&trigger_hist_cmd);
6720         WARN_ON(ret < 0);
6721
6722         return ret;
6723 }
6724
6725 static void
6726 hist_enable_trigger(struct event_trigger_data *data,
6727                     struct trace_buffer *buffer,  void *rec,
6728                     struct ring_buffer_event *event)
6729 {
6730         struct enable_trigger_data *enable_data = data->private_data;
6731         struct event_trigger_data *test;
6732
6733         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6734                                 lockdep_is_held(&event_mutex)) {
6735                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6736                         if (enable_data->enable)
6737                                 test->paused = false;
6738                         else
6739                                 test->paused = true;
6740                 }
6741         }
6742 }
6743
6744 static void
6745 hist_enable_count_trigger(struct event_trigger_data *data,
6746                           struct trace_buffer *buffer,  void *rec,
6747                           struct ring_buffer_event *event)
6748 {
6749         if (!data->count)
6750                 return;
6751
6752         if (data->count != -1)
6753                 (data->count)--;
6754
6755         hist_enable_trigger(data, buffer, rec, event);
6756 }
6757
6758 static struct event_trigger_ops hist_enable_trigger_ops = {
6759         .trigger                = hist_enable_trigger,
6760         .print                  = event_enable_trigger_print,
6761         .init                   = event_trigger_init,
6762         .free                   = event_enable_trigger_free,
6763 };
6764
6765 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6766         .trigger                = hist_enable_count_trigger,
6767         .print                  = event_enable_trigger_print,
6768         .init                   = event_trigger_init,
6769         .free                   = event_enable_trigger_free,
6770 };
6771
6772 static struct event_trigger_ops hist_disable_trigger_ops = {
6773         .trigger                = hist_enable_trigger,
6774         .print                  = event_enable_trigger_print,
6775         .init                   = event_trigger_init,
6776         .free                   = event_enable_trigger_free,
6777 };
6778
6779 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6780         .trigger                = hist_enable_count_trigger,
6781         .print                  = event_enable_trigger_print,
6782         .init                   = event_trigger_init,
6783         .free                   = event_enable_trigger_free,
6784 };
6785
6786 static struct event_trigger_ops *
6787 hist_enable_get_trigger_ops(char *cmd, char *param)
6788 {
6789         struct event_trigger_ops *ops;
6790         bool enable;
6791
6792         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6793
6794         if (enable)
6795                 ops = param ? &hist_enable_count_trigger_ops :
6796                         &hist_enable_trigger_ops;
6797         else
6798                 ops = param ? &hist_disable_count_trigger_ops :
6799                         &hist_disable_trigger_ops;
6800
6801         return ops;
6802 }
6803
6804 static void hist_enable_unreg_all(struct trace_event_file *file)
6805 {
6806         struct event_trigger_data *test, *n;
6807
6808         list_for_each_entry_safe(test, n, &file->triggers, list) {
6809                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6810                         list_del_rcu(&test->list);
6811                         update_cond_flag(file);
6812                         trace_event_trigger_enable_disable(file, 0);
6813                         if (test->ops->free)
6814                                 test->ops->free(test);
6815                 }
6816         }
6817 }
6818
6819 static struct event_command trigger_hist_enable_cmd = {
6820         .name                   = ENABLE_HIST_STR,
6821         .trigger_type           = ETT_HIST_ENABLE,
6822         .parse                  = event_enable_trigger_parse,
6823         .reg                    = event_enable_register_trigger,
6824         .unreg                  = event_enable_unregister_trigger,
6825         .unreg_all              = hist_enable_unreg_all,
6826         .get_trigger_ops        = hist_enable_get_trigger_ops,
6827         .set_filter             = set_trigger_filter,
6828 };
6829
6830 static struct event_command trigger_hist_disable_cmd = {
6831         .name                   = DISABLE_HIST_STR,
6832         .trigger_type           = ETT_HIST_ENABLE,
6833         .parse                  = event_enable_trigger_parse,
6834         .reg                    = event_enable_register_trigger,
6835         .unreg                  = event_enable_unregister_trigger,
6836         .unreg_all              = hist_enable_unreg_all,
6837         .get_trigger_ops        = hist_enable_get_trigger_ops,
6838         .set_filter             = set_trigger_filter,
6839 };
6840
6841 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6842 {
6843         unregister_event_command(&trigger_hist_enable_cmd);
6844         unregister_event_command(&trigger_hist_disable_cmd);
6845 }
6846
6847 __init int register_trigger_hist_enable_disable_cmds(void)
6848 {
6849         int ret;
6850
6851         ret = register_event_command(&trigger_hist_enable_cmd);
6852         if (WARN_ON(ret < 0))
6853                 return ret;
6854         ret = register_event_command(&trigger_hist_disable_cmd);
6855         if (WARN_ON(ret < 0))
6856                 unregister_trigger_hist_enable_disable_cmds();
6857
6858         return ret;
6859 }