kernel/trace/trace_events_hist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * trace_events_hist - trace event hist triggers
   4  *
   5  * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/kallsyms.h>
  10 #include <linux/security.h>
  11 #include <linux/mutex.h>
  12 #include <linux/slab.h>
  13 #include <linux/stacktrace.h>
  14 #include <linux/rculist.h>
  15 #include <linux/tracefs.h>
  16
  17 /* for gfp flag names */
  18 #include <linux/trace_events.h>
  19 #include <trace/events/mmflags.h>
  20
  21 #include "tracing_map.h"
  22 #include "trace_synth.h"
  23
  24 #define ERRORS                                                          \
  25         C(NONE,                 "No error"),                            \
  26         C(DUPLICATE_VAR,        "Variable already defined"),            \
  27         C(VAR_NOT_UNIQUE,       "Variable name not unique, need to use fully qualified name (subsys.event.var) for variable"), \
  28         C(TOO_MANY_VARS,        "Too many variables defined"),          \
  29         C(MALFORMED_ASSIGNMENT, "Malformed assignment"),                \
  30         C(NAMED_MISMATCH,       "Named hist trigger doesn't match existing named trigger (includes variables)"), \
  31         C(TRIGGER_EEXIST,       "Hist trigger already exists"),         \
  32         C(TRIGGER_ENOENT_CLEAR, "Can't clear or continue a nonexistent hist trigger"), \
  33         C(SET_CLOCK_FAIL,       "Couldn't set trace_clock"),            \
  34         C(BAD_FIELD_MODIFIER,   "Invalid field modifier"),              \
  35         C(TOO_MANY_SUBEXPR,     "Too many subexpressions (3 max)"),     \
  36         C(TIMESTAMP_MISMATCH,   "Timestamp units in expression don't match"), \
  37         C(TOO_MANY_FIELD_VARS,  "Too many field variables defined"),    \
  38         C(EVENT_FILE_NOT_FOUND, "Event file not found"),                \
  39         C(HIST_NOT_FOUND,       "Matching event histogram not found"),  \
  40         C(HIST_CREATE_FAIL,     "Couldn't create histogram for field"), \
  41         C(SYNTH_VAR_NOT_FOUND,  "Couldn't find synthetic variable"),    \
  42         C(SYNTH_EVENT_NOT_FOUND,"Couldn't find synthetic event"),       \
  43         C(SYNTH_TYPE_MISMATCH,  "Param type doesn't match synthetic event field type"), \
  44         C(SYNTH_COUNT_MISMATCH, "Param count doesn't match synthetic event field count"), \
  45         C(FIELD_VAR_PARSE_FAIL, "Couldn't parse field variable"),       \
  46         C(VAR_CREATE_FIND_FAIL, "Couldn't create or find variable"),    \
  47         C(ONX_NOT_VAR,          "For onmax(x) or onchange(x), x must be a variable"), \
  48         C(ONX_VAR_NOT_FOUND,    "Couldn't find onmax or onchange variable"), \
  49         C(ONX_VAR_CREATE_FAIL,  "Couldn't create onmax or onchange variable"), \
  50         C(FIELD_VAR_CREATE_FAIL,"Couldn't create field variable"),      \
  51         C(TOO_MANY_PARAMS,      "Too many action params"),              \
  52         C(PARAM_NOT_FOUND,      "Couldn't find param"),                 \
  53         C(INVALID_PARAM,        "Invalid action param"),                \
  54         C(ACTION_NOT_FOUND,     "No action found"),                     \
  55         C(NO_SAVE_PARAMS,       "No params found for save()"),          \
  56         C(TOO_MANY_SAVE_ACTIONS,"Can't have more than one save() action per hist"), \
  57         C(ACTION_MISMATCH,      "Handler doesn't support action"),      \
  58         C(NO_CLOSING_PAREN,     "No closing paren found"),              \
  59         C(SUBSYS_NOT_FOUND,     "Missing subsystem"),                   \
  60         C(INVALID_SUBSYS_EVENT, "Invalid subsystem or event name"),     \
  61         C(INVALID_REF_KEY,      "Using variable references in keys not supported"), \
  62         C(VAR_NOT_FOUND,        "Couldn't find variable"),              \
  63         C(FIELD_NOT_FOUND,      "Couldn't find field"),                 \
  64         C(EMPTY_ASSIGNMENT,     "Empty assignment"),                    \
  65         C(INVALID_SORT_MODIFIER,"Invalid sort modifier"),               \
  66         C(EMPTY_SORT_FIELD,     "Empty sort field"),                    \
  67         C(TOO_MANY_SORT_FIELDS, "Too many sort fields (Max = 2)"),      \
  68         C(INVALID_SORT_FIELD,   "Sort field must be a key or a val"),   \
  69         C(INVALID_STR_OPERAND,  "String type can not be an operand in expression"), \
  70         C(EXPECT_NUMBER,        "Expecting numeric literal"),           \
  71         C(UNARY_MINUS_SUBEXPR,  "Unary minus not supported in sub-expressions"), \
  72         C(DIVISION_BY_ZERO,     "Division by zero"),                    \
  73         C(NEED_NOHC_VAL,        "Non-hitcount value is required for 'nohitcount'"),
  74
  75 #undef C
  76 #define C(a, b)         HIST_ERR_##a
  77
  78 enum { ERRORS };
  79
  80 #undef C
  81 #define C(a, b)         b
  82
  83 static const char *err_text[] = { ERRORS };
  84
  85 struct hist_field;
  86
  87 typedef u64 (*hist_field_fn_t) (struct hist_field *field,
  88                                 struct tracing_map_elt *elt,
  89                                 struct trace_buffer *buffer,
  90                                 struct ring_buffer_event *rbe,
  91                                 void *event);
  92
  93 #define HIST_FIELD_OPERANDS_MAX 2
  94 #define HIST_FIELDS_MAX         (TRACING_MAP_FIELDS_MAX + TRACING_MAP_VARS_MAX)
  95 #define HIST_ACTIONS_MAX        8
  96 #define HIST_CONST_DIGITS_MAX   21
  97 #define HIST_DIV_SHIFT          20  /* For optimizing division by constants */
  98
  99 enum field_op_id {
 100         FIELD_OP_NONE,
 101         FIELD_OP_PLUS,
 102         FIELD_OP_MINUS,
 103         FIELD_OP_UNARY_MINUS,
 104         FIELD_OP_DIV,
 105         FIELD_OP_MULT,
 106 };
 107
 108 enum hist_field_fn {
 109         HIST_FIELD_FN_NOP,
 110         HIST_FIELD_FN_VAR_REF,
 111         HIST_FIELD_FN_COUNTER,
 112         HIST_FIELD_FN_CONST,
 113         HIST_FIELD_FN_LOG2,
 114         HIST_FIELD_FN_BUCKET,
 115         HIST_FIELD_FN_TIMESTAMP,
 116         HIST_FIELD_FN_CPU,
 117         HIST_FIELD_FN_STRING,
 118         HIST_FIELD_FN_DYNSTRING,
 119         HIST_FIELD_FN_RELDYNSTRING,
 120         HIST_FIELD_FN_PSTRING,
 121         HIST_FIELD_FN_S64,
 122         HIST_FIELD_FN_U64,
 123         HIST_FIELD_FN_S32,
 124         HIST_FIELD_FN_U32,
 125         HIST_FIELD_FN_S16,
 126         HIST_FIELD_FN_U16,
 127         HIST_FIELD_FN_S8,
 128         HIST_FIELD_FN_U8,
 129         HIST_FIELD_FN_UMINUS,
 130         HIST_FIELD_FN_MINUS,
 131         HIST_FIELD_FN_PLUS,
 132         HIST_FIELD_FN_DIV,
 133         HIST_FIELD_FN_MULT,
 134         HIST_FIELD_FN_DIV_POWER2,
 135         HIST_FIELD_FN_DIV_NOT_POWER2,
 136         HIST_FIELD_FN_DIV_MULT_SHIFT,
 137         HIST_FIELD_FN_EXECNAME,
 138         HIST_FIELD_FN_STACK,
 139 };
 140
 141 /*
 142  * A hist_var (histogram variable) contains variable information for
 143  * hist_fields having the HIST_FIELD_FL_VAR or HIST_FIELD_FL_VAR_REF
 144  * flag set.  A hist_var has a variable name e.g. ts0, and is
 145  * associated with a given histogram trigger, as specified by
 146  * hist_data.  The hist_var idx is the unique index assigned to the
 147  * variable by the hist trigger's tracing_map.  The idx is what is
 148  * used to set a variable's value and, by a variable reference, to
 149  * retrieve it.
 150  */
 151 struct hist_var {
 152         char                            *name;
 153         struct hist_trigger_data        *hist_data;
 154         unsigned int                    idx;
 155 };
 156
 157 struct hist_field {
 158         struct ftrace_event_field       *field;
 159         unsigned long                   flags;
 160         unsigned long                   buckets;
 161         const char                      *type;
 162         struct hist_field               *operands[HIST_FIELD_OPERANDS_MAX];
 163         struct hist_trigger_data        *hist_data;
 164         enum hist_field_fn              fn_num;
 165         unsigned int                    ref;
 166         unsigned int                    size;
 167         unsigned int                    offset;
 168         unsigned int                    is_signed;
 169
 170         /*
 171          * Variable fields contain variable-specific info in var.
 172          */
 173         struct hist_var                 var;
 174         enum field_op_id                operator;
 175         char                            *system;
 176         char                            *event_name;
 177
 178         /*
 179          * The name field is used for EXPR and VAR_REF fields.  VAR
 180          * fields contain the variable name in var.name.
 181          */
 182         char                            *name;
 183
 184         /*
 185          * When a histogram trigger is hit, if it has any references
 186          * to variables, the values of those variables are collected
 187          * into a var_ref_vals array by resolve_var_refs().  The
 188          * current value of each variable is read from the tracing_map
 189          * using the hist field's hist_var.idx and entered into the
 190          * var_ref_idx entry i.e. var_ref_vals[var_ref_idx].
 191          */
 192         unsigned int                    var_ref_idx;
 193         bool                            read_once;
 194
 195         unsigned int                    var_str_idx;
 196
 197         /* Numeric literals are represented as u64 */
 198         u64                             constant;
 199         /* Used to optimize division by constants */
 200         u64                             div_multiplier;
 201 };
 202
 203 static u64 hist_fn_call(struct hist_field *hist_field,
 204                         struct tracing_map_elt *elt,
 205                         struct trace_buffer *buffer,
 206                         struct ring_buffer_event *rbe,
 207                         void *event);
 208
 209 static u64 hist_field_const(struct hist_field *field,
 210                            struct tracing_map_elt *elt,
 211                            struct trace_buffer *buffer,
 212                            struct ring_buffer_event *rbe,
 213                            void *event)
 214 {
 215         return field->constant;
 216 }
 217
 218 static u64 hist_field_counter(struct hist_field *field,
 219                               struct tracing_map_elt *elt,
 220                               struct trace_buffer *buffer,
 221                               struct ring_buffer_event *rbe,
 222                               void *event)
 223 {
 224         return 1;
 225 }
 226
 227 static u64 hist_field_string(struct hist_field *hist_field,
 228                              struct tracing_map_elt *elt,
 229                              struct trace_buffer *buffer,
 230                              struct ring_buffer_event *rbe,
 231                              void *event)
 232 {
 233         char *addr = (char *)(event + hist_field->field->offset);
 234
 235         return (u64)(unsigned long)addr;
 236 }
 237
 238 static u64 hist_field_dynstring(struct hist_field *hist_field,
 239                                 struct tracing_map_elt *elt,
 240                                 struct trace_buffer *buffer,
 241                                 struct ring_buffer_event *rbe,
 242                                 void *event)
 243 {
 244         u32 str_item = *(u32 *)(event + hist_field->field->offset);
 245         int str_loc = str_item & 0xffff;
 246         char *addr = (char *)(event + str_loc);
 247
 248         return (u64)(unsigned long)addr;
 249 }
 250
 251 static u64 hist_field_reldynstring(struct hist_field *hist_field,
 252                                    struct tracing_map_elt *elt,
 253                                    struct trace_buffer *buffer,
 254                                    struct ring_buffer_event *rbe,
 255                                    void *event)
 256 {
 257         u32 *item = event + hist_field->field->offset;
 258         u32 str_item = *item;
 259         int str_loc = str_item & 0xffff;
 260         char *addr = (char *)&item[1] + str_loc;
 261
 262         return (u64)(unsigned long)addr;
 263 }
 264
 265 static u64 hist_field_pstring(struct hist_field *hist_field,
 266                               struct tracing_map_elt *elt,
 267                               struct trace_buffer *buffer,
 268                               struct ring_buffer_event *rbe,
 269                               void *event)
 270 {
 271         char **addr = (char **)(event + hist_field->field->offset);
 272
 273         return (u64)(unsigned long)*addr;
 274 }
 275
 276 static u64 hist_field_log2(struct hist_field *hist_field,
 277                            struct tracing_map_elt *elt,
 278                            struct trace_buffer *buffer,
 279                            struct ring_buffer_event *rbe,
 280                            void *event)
 281 {
 282         struct hist_field *operand = hist_field->operands[0];
 283
 284         u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 285
 286         return (u64) ilog2(roundup_pow_of_two(val));
 287 }
 288
 289 static u64 hist_field_bucket(struct hist_field *hist_field,
 290                              struct tracing_map_elt *elt,
 291                              struct trace_buffer *buffer,
 292                              struct ring_buffer_event *rbe,
 293                              void *event)
 294 {
 295         struct hist_field *operand = hist_field->operands[0];
 296         unsigned long buckets = hist_field->buckets;
 297
 298         u64 val = hist_fn_call(operand, elt, buffer, rbe, event);
 299
 300         if (WARN_ON_ONCE(!buckets))
 301                 return val;
 302
 303         if (val >= LONG_MAX)
 304                 val = div64_ul(val, buckets);
 305         else
 306                 val = (u64)((unsigned long)val / buckets);
 307         return val * buckets;
 308 }
 309
 310 static u64 hist_field_plus(struct hist_field *hist_field,
 311                            struct tracing_map_elt *elt,
 312                            struct trace_buffer *buffer,
 313                            struct ring_buffer_event *rbe,
 314                            void *event)
 315 {
 316         struct hist_field *operand1 = hist_field->operands[0];
 317         struct hist_field *operand2 = hist_field->operands[1];
 318
 319         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 320         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 321
 322         return val1 + val2;
 323 }
 324
 325 static u64 hist_field_minus(struct hist_field *hist_field,
 326                             struct tracing_map_elt *elt,
 327                             struct trace_buffer *buffer,
 328                             struct ring_buffer_event *rbe,
 329                             void *event)
 330 {
 331         struct hist_field *operand1 = hist_field->operands[0];
 332         struct hist_field *operand2 = hist_field->operands[1];
 333
 334         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 335         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 336
 337         return val1 - val2;
 338 }
 339
 340 static u64 hist_field_div(struct hist_field *hist_field,
 341                            struct tracing_map_elt *elt,
 342                            struct trace_buffer *buffer,
 343                            struct ring_buffer_event *rbe,
 344                            void *event)
 345 {
 346         struct hist_field *operand1 = hist_field->operands[0];
 347         struct hist_field *operand2 = hist_field->operands[1];
 348
 349         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 350         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 351
 352         /* Return -1 for the undefined case */
 353         if (!val2)
 354                 return -1;
 355
 356         /* Use shift if the divisor is a power of 2 */
 357         if (!(val2 & (val2 - 1)))
 358                 return val1 >> __ffs64(val2);
 359
 360         return div64_u64(val1, val2);
 361 }
 362
 363 static u64 div_by_power_of_two(struct hist_field *hist_field,
 364                                 struct tracing_map_elt *elt,
 365                                 struct trace_buffer *buffer,
 366                                 struct ring_buffer_event *rbe,
 367                                 void *event)
 368 {
 369         struct hist_field *operand1 = hist_field->operands[0];
 370         struct hist_field *operand2 = hist_field->operands[1];
 371
 372         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 373
 374         return val1 >> __ffs64(operand2->constant);
 375 }
 376
 377 static u64 div_by_not_power_of_two(struct hist_field *hist_field,
 378                                 struct tracing_map_elt *elt,
 379                                 struct trace_buffer *buffer,
 380                                 struct ring_buffer_event *rbe,
 381                                 void *event)
 382 {
 383         struct hist_field *operand1 = hist_field->operands[0];
 384         struct hist_field *operand2 = hist_field->operands[1];
 385
 386         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 387
 388         return div64_u64(val1, operand2->constant);
 389 }
 390
 391 static u64 div_by_mult_and_shift(struct hist_field *hist_field,
 392                                 struct tracing_map_elt *elt,
 393                                 struct trace_buffer *buffer,
 394                                 struct ring_buffer_event *rbe,
 395                                 void *event)
 396 {
 397         struct hist_field *operand1 = hist_field->operands[0];
 398         struct hist_field *operand2 = hist_field->operands[1];
 399
 400         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 401
 402         /*
 403          * If the divisor is a constant, do a multiplication and shift instead.
 404          *
 405          * Choose Z = some power of 2. If Y <= Z, then:
 406          *     X / Y = (X * (Z / Y)) / Z
 407          *
 408          * (Z / Y) is a constant (mult) which is calculated at parse time, so:
 409          *     X / Y = (X * mult) / Z
 410          *
 411          * The division by Z can be replaced by a shift since Z is a power of 2:
 412          *     X / Y = (X * mult) >> HIST_DIV_SHIFT
 413          *
 414          * As long, as X < Z the results will not be off by more than 1.
 415          */
 416         if (val1 < (1 << HIST_DIV_SHIFT)) {
 417                 u64 mult = operand2->div_multiplier;
 418
 419                 return (val1 * mult + ((1 << HIST_DIV_SHIFT) - 1)) >> HIST_DIV_SHIFT;
 420         }
 421
 422         return div64_u64(val1, operand2->constant);
 423 }
 424
 425 static u64 hist_field_mult(struct hist_field *hist_field,
 426                            struct tracing_map_elt *elt,
 427                            struct trace_buffer *buffer,
 428                            struct ring_buffer_event *rbe,
 429                            void *event)
 430 {
 431         struct hist_field *operand1 = hist_field->operands[0];
 432         struct hist_field *operand2 = hist_field->operands[1];
 433
 434         u64 val1 = hist_fn_call(operand1, elt, buffer, rbe, event);
 435         u64 val2 = hist_fn_call(operand2, elt, buffer, rbe, event);
 436
 437         return val1 * val2;
 438 }
 439
 440 static u64 hist_field_unary_minus(struct hist_field *hist_field,
 441                                   struct tracing_map_elt *elt,
 442                                   struct trace_buffer *buffer,
 443                                   struct ring_buffer_event *rbe,
 444                                   void *event)
 445 {
 446         struct hist_field *operand = hist_field->operands[0];
 447
 448         s64 sval = (s64)hist_fn_call(operand, elt, buffer, rbe, event);
 449         u64 val = (u64)-sval;
 450
 451         return val;
 452 }
 453
 454 #define DEFINE_HIST_FIELD_FN(type)                                      \
 455         static u64 hist_field_##type(struct hist_field *hist_field,     \
 456                                      struct tracing_map_elt *elt,       \
 457                                      struct trace_buffer *buffer,       \
 458                                      struct ring_buffer_event *rbe,     \
 459                                      void *event)                       \
 460 {                                                                       \
 461         type *addr = (type *)(event + hist_field->field->offset);       \
 462                                                                         \
 463         return (u64)(unsigned long)*addr;                               \
 464 }
 465
 466 DEFINE_HIST_FIELD_FN(s64);
 467 DEFINE_HIST_FIELD_FN(u64);
 468 DEFINE_HIST_FIELD_FN(s32);
 469 DEFINE_HIST_FIELD_FN(u32);
 470 DEFINE_HIST_FIELD_FN(s16);
 471 DEFINE_HIST_FIELD_FN(u16);
 472 DEFINE_HIST_FIELD_FN(s8);
 473 DEFINE_HIST_FIELD_FN(u8);
 474
 475 #define for_each_hist_field(i, hist_data)       \
 476         for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
 477
 478 #define for_each_hist_val_field(i, hist_data)   \
 479         for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
 480
 481 #define for_each_hist_key_field(i, hist_data)   \
 482         for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
 483
 484 #define HITCOUNT_IDX            0
 485 #define HIST_KEY_SIZE_MAX       (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
 486
 487 enum hist_field_flags {
 488         HIST_FIELD_FL_HITCOUNT          = 1 << 0,
 489         HIST_FIELD_FL_KEY               = 1 << 1,
 490         HIST_FIELD_FL_STRING            = 1 << 2,
 491         HIST_FIELD_FL_HEX               = 1 << 3,
 492         HIST_FIELD_FL_SYM               = 1 << 4,
 493         HIST_FIELD_FL_SYM_OFFSET        = 1 << 5,
 494         HIST_FIELD_FL_EXECNAME          = 1 << 6,
 495         HIST_FIELD_FL_SYSCALL           = 1 << 7,
 496         HIST_FIELD_FL_STACKTRACE        = 1 << 8,
 497         HIST_FIELD_FL_LOG2              = 1 << 9,
 498         HIST_FIELD_FL_TIMESTAMP         = 1 << 10,
 499         HIST_FIELD_FL_TIMESTAMP_USECS   = 1 << 11,
 500         HIST_FIELD_FL_VAR               = 1 << 12,
 501         HIST_FIELD_FL_EXPR              = 1 << 13,
 502         HIST_FIELD_FL_VAR_REF           = 1 << 14,
 503         HIST_FIELD_FL_CPU               = 1 << 15,
 504         HIST_FIELD_FL_ALIAS             = 1 << 16,
 505         HIST_FIELD_FL_BUCKET            = 1 << 17,
 506         HIST_FIELD_FL_CONST             = 1 << 18,
 507         HIST_FIELD_FL_PERCENT           = 1 << 19,
 508         HIST_FIELD_FL_GRAPH             = 1 << 20,
 509 };
 510
 511 struct var_defs {
 512         unsigned int    n_vars;
 513         char            *name[TRACING_MAP_VARS_MAX];
 514         char            *expr[TRACING_MAP_VARS_MAX];
 515 };
 516
 517 struct hist_trigger_attrs {
 518         char            *keys_str;
 519         char            *vals_str;
 520         char            *sort_key_str;
 521         char            *name;
 522         char            *clock;
 523         bool            pause;
 524         bool            cont;
 525         bool            clear;
 526         bool            ts_in_usecs;
 527         bool            no_hitcount;
 528         unsigned int    map_bits;
 529
 530         char            *assignment_str[TRACING_MAP_VARS_MAX];
 531         unsigned int    n_assignments;
 532
 533         char            *action_str[HIST_ACTIONS_MAX];
 534         unsigned int    n_actions;
 535
 536         struct var_defs var_defs;
 537 };
 538
 539 struct field_var {
 540         struct hist_field       *var;
 541         struct hist_field       *val;
 542 };
 543
 544 struct field_var_hist {
 545         struct hist_trigger_data        *hist_data;
 546         char                            *cmd;
 547 };
 548
 549 struct hist_trigger_data {
 550         struct hist_field               *fields[HIST_FIELDS_MAX];
 551         unsigned int                    n_vals;
 552         unsigned int                    n_keys;
 553         unsigned int                    n_fields;
 554         unsigned int                    n_vars;
 555         unsigned int                    n_var_str;
 556         unsigned int                    key_size;
 557         struct tracing_map_sort_key     sort_keys[TRACING_MAP_SORT_KEYS_MAX];
 558         unsigned int                    n_sort_keys;
 559         struct trace_event_file         *event_file;
 560         struct hist_trigger_attrs       *attrs;
 561         struct tracing_map              *map;
 562         bool                            enable_timestamps;
 563         bool                            remove;
 564         struct hist_field               *var_refs[TRACING_MAP_VARS_MAX];
 565         unsigned int                    n_var_refs;
 566
 567         struct action_data              *actions[HIST_ACTIONS_MAX];
 568         unsigned int                    n_actions;
 569
 570         struct field_var                *field_vars[SYNTH_FIELDS_MAX];
 571         unsigned int                    n_field_vars;
 572         unsigned int                    n_field_var_str;
 573         struct field_var_hist           *field_var_hists[SYNTH_FIELDS_MAX];
 574         unsigned int                    n_field_var_hists;
 575
 576         struct field_var                *save_vars[SYNTH_FIELDS_MAX];
 577         unsigned int                    n_save_vars;
 578         unsigned int                    n_save_var_str;
 579 };
 580
 581 struct action_data;
 582
 583 typedef void (*action_fn_t) (struct hist_trigger_data *hist_data,
 584                              struct tracing_map_elt *elt,
 585                              struct trace_buffer *buffer, void *rec,
 586                              struct ring_buffer_event *rbe, void *key,
 587                              struct action_data *data, u64 *var_ref_vals);
 588
 589 typedef bool (*check_track_val_fn_t) (u64 track_val, u64 var_val);
 590
 591 enum handler_id {
 592         HANDLER_ONMATCH = 1,
 593         HANDLER_ONMAX,
 594         HANDLER_ONCHANGE,
 595 };
 596
 597 enum action_id {
 598         ACTION_SAVE = 1,
 599         ACTION_TRACE,
 600         ACTION_SNAPSHOT,
 601 };
 602
 603 struct action_data {
 604         enum handler_id         handler;
 605         enum action_id          action;
 606         char                    *action_name;
 607         action_fn_t             fn;
 608
 609         unsigned int            n_params;
 610         char                    *params[SYNTH_FIELDS_MAX];
 611
 612         /*
 613          * When a histogram trigger is hit, the values of any
 614          * references to variables, including variables being passed
 615          * as parameters to synthetic events, are collected into a
 616          * var_ref_vals array.  This var_ref_idx array is an array of
 617          * indices into the var_ref_vals array, one for each synthetic
 618          * event param, and is passed to the synthetic event
 619          * invocation.
 620          */
 621         unsigned int            var_ref_idx[SYNTH_FIELDS_MAX];
 622         struct synth_event      *synth_event;
 623         bool                    use_trace_keyword;
 624         char                    *synth_event_name;
 625
 626         union {
 627                 struct {
 628                         char                    *event;
 629                         char                    *event_system;
 630                 } match_data;
 631
 632                 struct {
 633                         /*
 634                          * var_str contains the $-unstripped variable
 635                          * name referenced by var_ref, and used when
 636                          * printing the action.  Because var_ref
 637                          * creation is deferred to create_actions(),
 638                          * we need a per-action way to save it until
 639                          * then, thus var_str.
 640                          */
 641                         char                    *var_str;
 642
 643                         /*
 644                          * var_ref refers to the variable being
 645                          * tracked e.g onmax($var).
 646                          */
 647                         struct hist_field       *var_ref;
 648
 649                         /*
 650                          * track_var contains the 'invisible' tracking
 651                          * variable created to keep the current
 652                          * e.g. max value.
 653                          */
 654                         struct hist_field       *track_var;
 655
 656                         check_track_val_fn_t    check_val;
 657                         action_fn_t             save_data;
 658                 } track_data;
 659         };
 660 };
 661
 662 struct track_data {
 663         u64                             track_val;
 664         bool                            updated;
 665
 666         unsigned int                    key_len;
 667         void                            *key;
 668         struct tracing_map_elt          elt;
 669
 670         struct action_data              *action_data;
 671         struct hist_trigger_data        *hist_data;
 672 };
 673
 674 struct hist_elt_data {
 675         char *comm;
 676         u64 *var_ref_vals;
 677         char **field_var_str;
 678         int n_field_var_str;
 679 };
 680
 681 struct snapshot_context {
 682         struct tracing_map_elt  *elt;
 683         void                    *key;
 684 };
 685
 686 /*
 687  * Returns the specific division function to use if the divisor
 688  * is constant. This avoids extra branches when the trigger is hit.
 689  */
 690 static enum hist_field_fn hist_field_get_div_fn(struct hist_field *divisor)
 691 {
 692         u64 div = divisor->constant;
 693
 694         if (!(div & (div - 1)))
 695                 return HIST_FIELD_FN_DIV_POWER2;
 696
 697         /* If the divisor is too large, do a regular division */
 698         if (div > (1 << HIST_DIV_SHIFT))
 699                 return HIST_FIELD_FN_DIV_NOT_POWER2;
 700
 701         divisor->div_multiplier = div64_u64((u64)(1 << HIST_DIV_SHIFT), div);
 702         return HIST_FIELD_FN_DIV_MULT_SHIFT;
 703 }
 704
 705 static void track_data_free(struct track_data *track_data)
 706 {
 707         struct hist_elt_data *elt_data;
 708
 709         if (!track_data)
 710                 return;
 711
 712         kfree(track_data->key);
 713
 714         elt_data = track_data->elt.private_data;
 715         if (elt_data) {
 716                 kfree(elt_data->comm);
 717                 kfree(elt_data);
 718         }
 719
 720         kfree(track_data);
 721 }
 722
 723 static struct track_data *track_data_alloc(unsigned int key_len,
 724                                            struct action_data *action_data,
 725                                            struct hist_trigger_data *hist_data)
 726 {
 727         struct track_data *data = kzalloc(sizeof(*data), GFP_KERNEL);
 728         struct hist_elt_data *elt_data;
 729
 730         if (!data)
 731                 return ERR_PTR(-ENOMEM);
 732
 733         data->key = kzalloc(key_len, GFP_KERNEL);
 734         if (!data->key) {
 735                 track_data_free(data);
 736                 return ERR_PTR(-ENOMEM);
 737         }
 738
 739         data->key_len = key_len;
 740         data->action_data = action_data;
 741         data->hist_data = hist_data;
 742
 743         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
 744         if (!elt_data) {
 745                 track_data_free(data);
 746                 return ERR_PTR(-ENOMEM);
 747         }
 748
 749         data->elt.private_data = elt_data;
 750
 751         elt_data->comm = kzalloc(TASK_COMM_LEN, GFP_KERNEL);
 752         if (!elt_data->comm) {
 753                 track_data_free(data);
 754                 return ERR_PTR(-ENOMEM);
 755         }
 756
 757         return data;
 758 }
 759
 760 #define HIST_PREFIX "hist:"
 761
 762 static char *last_cmd;
 763 static char last_cmd_loc[MAX_FILTER_STR_VAL];
 764
 765 static int errpos(char *str)
 766 {
 767         if (!str || !last_cmd)
 768                 return 0;
 769
 770         return err_pos(last_cmd, str);
 771 }
 772
 773 static void last_cmd_set(struct trace_event_file *file, char *str)
 774 {
 775         const char *system = NULL, *name = NULL;
 776         struct trace_event_call *call;
 777
 778         if (!str)
 779                 return;
 780
 781         kfree(last_cmd);
 782
 783         last_cmd = kasprintf(GFP_KERNEL, HIST_PREFIX "%s", str);
 784         if (!last_cmd)
 785                 return;
 786
 787         if (file) {
 788                 call = file->event_call;
 789                 system = call->class->system;
 790                 if (system) {
 791                         name = trace_event_name(call);
 792                         if (!name)
 793                                 system = NULL;
 794                 }
 795         }
 796
 797         if (system)
 798                 snprintf(last_cmd_loc, MAX_FILTER_STR_VAL, HIST_PREFIX "%s:%s", system, name);
 799 }
 800
 801 static void hist_err(struct trace_array *tr, u8 err_type, u16 err_pos)
 802 {
 803         if (!last_cmd)
 804                 return;
 805
 806         tracing_log_err(tr, last_cmd_loc, last_cmd, err_text,
 807                         err_type, err_pos);
 808 }
 809
 810 static void hist_err_clear(void)
 811 {
 812         if (last_cmd)
 813                 last_cmd[0] = '\0';
 814         last_cmd_loc[0] = '\0';
 815 }
 816
 817 typedef void (*synth_probe_func_t) (void *__data, u64 *var_ref_vals,
 818                                     unsigned int *var_ref_idx);
 819
 820 static inline void trace_synth(struct synth_event *event, u64 *var_ref_vals,
 821                                unsigned int *var_ref_idx)
 822 {
 823         struct tracepoint *tp = event->tp;
 824
 825         if (unlikely(atomic_read(&tp->key.enabled) > 0)) {
 826                 struct tracepoint_func *probe_func_ptr;
 827                 synth_probe_func_t probe_func;
 828                 void *__data;
 829
 830                 if (!(cpu_online(raw_smp_processor_id())))
 831                         return;
 832
 833                 probe_func_ptr = rcu_dereference_sched((tp)->funcs);
 834                 if (probe_func_ptr) {
 835                         do {
 836                                 probe_func = probe_func_ptr->func;
 837                                 __data = probe_func_ptr->data;
 838                                 probe_func(__data, var_ref_vals, var_ref_idx);
 839                         } while ((++probe_func_ptr)->func);
 840                 }
 841         }
 842 }
 843
 844 static void action_trace(struct hist_trigger_data *hist_data,
 845                          struct tracing_map_elt *elt,
 846                          struct trace_buffer *buffer, void *rec,
 847                          struct ring_buffer_event *rbe, void *key,
 848                          struct action_data *data, u64 *var_ref_vals)
 849 {
 850         struct synth_event *event = data->synth_event;
 851
 852         trace_synth(event, var_ref_vals, data->var_ref_idx);
 853 }
 854
 855 struct hist_var_data {
 856         struct list_head list;
 857         struct hist_trigger_data *hist_data;
 858 };
 859
 860 static u64 hist_field_timestamp(struct hist_field *hist_field,
 861                                 struct tracing_map_elt *elt,
 862                                 struct trace_buffer *buffer,
 863                                 struct ring_buffer_event *rbe,
 864                                 void *event)
 865 {
 866         struct hist_trigger_data *hist_data = hist_field->hist_data;
 867         struct trace_array *tr = hist_data->event_file->tr;
 868
 869         u64 ts = ring_buffer_event_time_stamp(buffer, rbe);
 870
 871         if (hist_data->attrs->ts_in_usecs && trace_clock_in_ns(tr))
 872                 ts = ns2usecs(ts);
 873
 874         return ts;
 875 }
 876
 877 static u64 hist_field_cpu(struct hist_field *hist_field,
 878                           struct tracing_map_elt *elt,
 879                           struct trace_buffer *buffer,
 880                           struct ring_buffer_event *rbe,
 881                           void *event)
 882 {
 883         int cpu = smp_processor_id();
 884
 885         return cpu;
 886 }
 887
 888 /**
 889  * check_field_for_var_ref - Check if a VAR_REF field references a variable
 890  * @hist_field: The VAR_REF field to check
 891  * @var_data: The hist trigger that owns the variable
 892  * @var_idx: The trigger variable identifier
 893  *
 894  * Check the given VAR_REF field to see whether or not it references
 895  * the given variable associated with the given trigger.
 896  *
 897  * Return: The VAR_REF field if it does reference the variable, NULL if not
 898  */
 899 static struct hist_field *
 900 check_field_for_var_ref(struct hist_field *hist_field,
 901                         struct hist_trigger_data *var_data,
 902                         unsigned int var_idx)
 903 {
 904         WARN_ON(!(hist_field && hist_field->flags & HIST_FIELD_FL_VAR_REF));
 905
 906         if (hist_field && hist_field->var.idx == var_idx &&
 907             hist_field->var.hist_data == var_data)
 908                 return hist_field;
 909
 910         return NULL;
 911 }
 912
 913 /**
 914  * find_var_ref - Check if a trigger has a reference to a trigger variable
 915  * @hist_data: The hist trigger that might have a reference to the variable
 916  * @var_data: The hist trigger that owns the variable
 917  * @var_idx: The trigger variable identifier
 918  *
 919  * Check the list of var_refs[] on the first hist trigger to see
 920  * whether any of them are references to the variable on the second
 921  * trigger.
 922  *
 923  * Return: The VAR_REF field referencing the variable if so, NULL if not
 924  */
 925 static struct hist_field *find_var_ref(struct hist_trigger_data *hist_data,
 926                                        struct hist_trigger_data *var_data,
 927                                        unsigned int var_idx)
 928 {
 929         struct hist_field *hist_field;
 930         unsigned int i;
 931
 932         for (i = 0; i < hist_data->n_var_refs; i++) {
 933                 hist_field = hist_data->var_refs[i];
 934                 if (check_field_for_var_ref(hist_field, var_data, var_idx))
 935                         return hist_field;
 936         }
 937
 938         return NULL;
 939 }
 940
 941 /**
 942  * find_any_var_ref - Check if there is a reference to a given trigger variable
 943  * @hist_data: The hist trigger
 944  * @var_idx: The trigger variable identifier
 945  *
 946  * Check to see whether the given variable is currently referenced by
 947  * any other trigger.
 948  *
 949  * The trigger the variable is defined on is explicitly excluded - the
 950  * assumption being that a self-reference doesn't prevent a trigger
 951  * from being removed.
 952  *
 953  * Return: The VAR_REF field referencing the variable if so, NULL if not
 954  */
 955 static struct hist_field *find_any_var_ref(struct hist_trigger_data *hist_data,
 956                                            unsigned int var_idx)
 957 {
 958         struct trace_array *tr = hist_data->event_file->tr;
 959         struct hist_field *found = NULL;
 960         struct hist_var_data *var_data;
 961
 962         list_for_each_entry(var_data, &tr->hist_vars, list) {
 963                 if (var_data->hist_data == hist_data)
 964                         continue;
 965                 found = find_var_ref(var_data->hist_data, hist_data, var_idx);
 966                 if (found)
 967                         break;
 968         }
 969
 970         return found;
 971 }
 972
 973 /**
 974  * check_var_refs - Check if there is a reference to any of trigger's variables
 975  * @hist_data: The hist trigger
 976  *
 977  * A trigger can define one or more variables.  If any one of them is
 978  * currently referenced by any other trigger, this function will
 979  * determine that.
 980  *
 981  * Typically used to determine whether or not a trigger can be removed
 982  * - if there are any references to a trigger's variables, it cannot.
 983  *
 984  * Return: True if there is a reference to any of trigger's variables
 985  */
 986 static bool check_var_refs(struct hist_trigger_data *hist_data)
 987 {
 988         struct hist_field *field;
 989         bool found = false;
 990         int i;
 991
 992         for_each_hist_field(i, hist_data) {
 993                 field = hist_data->fields[i];
 994                 if (field && field->flags & HIST_FIELD_FL_VAR) {
 995                         if (find_any_var_ref(hist_data, field->var.idx)) {
 996                                 found = true;
 997                                 break;
 998                         }
 999                 }
1000         }
1001
1002         return found;
1003 }
1004
1005 static struct hist_var_data *find_hist_vars(struct hist_trigger_data *hist_data)
1006 {
1007         struct trace_array *tr = hist_data->event_file->tr;
1008         struct hist_var_data *var_data, *found = NULL;
1009
1010         list_for_each_entry(var_data, &tr->hist_vars, list) {
1011                 if (var_data->hist_data == hist_data) {
1012                         found = var_data;
1013                         break;
1014                 }
1015         }
1016
1017         return found;
1018 }
1019
1020 static bool field_has_hist_vars(struct hist_field *hist_field,
1021                                 unsigned int level)
1022 {
1023         int i;
1024
1025         if (level > 3)
1026                 return false;
1027
1028         if (!hist_field)
1029                 return false;
1030
1031         if (hist_field->flags & HIST_FIELD_FL_VAR ||
1032             hist_field->flags & HIST_FIELD_FL_VAR_REF)
1033                 return true;
1034
1035         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++) {
1036                 struct hist_field *operand;
1037
1038                 operand = hist_field->operands[i];
1039                 if (field_has_hist_vars(operand, level + 1))
1040                         return true;
1041         }
1042
1043         return false;
1044 }
1045
1046 static bool has_hist_vars(struct hist_trigger_data *hist_data)
1047 {
1048         struct hist_field *hist_field;
1049         int i;
1050
1051         for_each_hist_field(i, hist_data) {
1052                 hist_field = hist_data->fields[i];
1053                 if (field_has_hist_vars(hist_field, 0))
1054                         return true;
1055         }
1056
1057         return false;
1058 }
1059
1060 static int save_hist_vars(struct hist_trigger_data *hist_data)
1061 {
1062         struct trace_array *tr = hist_data->event_file->tr;
1063         struct hist_var_data *var_data;
1064
1065         var_data = find_hist_vars(hist_data);
1066         if (var_data)
1067                 return 0;
1068
1069         if (tracing_check_open_get_tr(tr))
1070                 return -ENODEV;
1071
1072         var_data = kzalloc(sizeof(*var_data), GFP_KERNEL);
1073         if (!var_data) {
1074                 trace_array_put(tr);
1075                 return -ENOMEM;
1076         }
1077
1078         var_data->hist_data = hist_data;
1079         list_add(&var_data->list, &tr->hist_vars);
1080
1081         return 0;
1082 }
1083
1084 static void remove_hist_vars(struct hist_trigger_data *hist_data)
1085 {
1086         struct trace_array *tr = hist_data->event_file->tr;
1087         struct hist_var_data *var_data;
1088
1089         var_data = find_hist_vars(hist_data);
1090         if (!var_data)
1091                 return;
1092
1093         if (WARN_ON(check_var_refs(hist_data)))
1094                 return;
1095
1096         list_del(&var_data->list);
1097
1098         kfree(var_data);
1099
1100         trace_array_put(tr);
1101 }
1102
1103 static struct hist_field *find_var_field(struct hist_trigger_data *hist_data,
1104                                          const char *var_name)
1105 {
1106         struct hist_field *hist_field, *found = NULL;
1107         int i;
1108
1109         for_each_hist_field(i, hist_data) {
1110                 hist_field = hist_data->fields[i];
1111                 if (hist_field && hist_field->flags & HIST_FIELD_FL_VAR &&
1112                     strcmp(hist_field->var.name, var_name) == 0) {
1113                         found = hist_field;
1114                         break;
1115                 }
1116         }
1117
1118         return found;
1119 }
1120
1121 static struct hist_field *find_var(struct hist_trigger_data *hist_data,
1122                                    struct trace_event_file *file,
1123                                    const char *var_name)
1124 {
1125         struct hist_trigger_data *test_data;
1126         struct event_trigger_data *test;
1127         struct hist_field *hist_field;
1128
1129         lockdep_assert_held(&event_mutex);
1130
1131         hist_field = find_var_field(hist_data, var_name);
1132         if (hist_field)
1133                 return hist_field;
1134
1135         list_for_each_entry(test, &file->triggers, list) {
1136                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1137                         test_data = test->private_data;
1138                         hist_field = find_var_field(test_data, var_name);
1139                         if (hist_field)
1140                                 return hist_field;
1141                 }
1142         }
1143
1144         return NULL;
1145 }
1146
1147 static struct trace_event_file *find_var_file(struct trace_array *tr,
1148                                               char *system,
1149                                               char *event_name,
1150                                               char *var_name)
1151 {
1152         struct hist_trigger_data *var_hist_data;
1153         struct hist_var_data *var_data;
1154         struct trace_event_file *file, *found = NULL;
1155
1156         if (system)
1157                 return find_event_file(tr, system, event_name);
1158
1159         list_for_each_entry(var_data, &tr->hist_vars, list) {
1160                 var_hist_data = var_data->hist_data;
1161                 file = var_hist_data->event_file;
1162                 if (file == found)
1163                         continue;
1164
1165                 if (find_var_field(var_hist_data, var_name)) {
1166                         if (found) {
1167                                 hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE, errpos(var_name));
1168                                 return NULL;
1169                         }
1170
1171                         found = file;
1172                 }
1173         }
1174
1175         return found;
1176 }
1177
1178 static struct hist_field *find_file_var(struct trace_event_file *file,
1179                                         const char *var_name)
1180 {
1181         struct hist_trigger_data *test_data;
1182         struct event_trigger_data *test;
1183         struct hist_field *hist_field;
1184
1185         lockdep_assert_held(&event_mutex);
1186
1187         list_for_each_entry(test, &file->triggers, list) {
1188                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1189                         test_data = test->private_data;
1190                         hist_field = find_var_field(test_data, var_name);
1191                         if (hist_field)
1192                                 return hist_field;
1193                 }
1194         }
1195
1196         return NULL;
1197 }
1198
1199 static struct hist_field *
1200 find_match_var(struct hist_trigger_data *hist_data, char *var_name)
1201 {
1202         struct trace_array *tr = hist_data->event_file->tr;
1203         struct hist_field *hist_field, *found = NULL;
1204         struct trace_event_file *file;
1205         unsigned int i;
1206
1207         for (i = 0; i < hist_data->n_actions; i++) {
1208                 struct action_data *data = hist_data->actions[i];
1209
1210                 if (data->handler == HANDLER_ONMATCH) {
1211                         char *system = data->match_data.event_system;
1212                         char *event_name = data->match_data.event;
1213
1214                         file = find_var_file(tr, system, event_name, var_name);
1215                         if (!file)
1216                                 continue;
1217                         hist_field = find_file_var(file, var_name);
1218                         if (hist_field) {
1219                                 if (found) {
1220                                         hist_err(tr, HIST_ERR_VAR_NOT_UNIQUE,
1221                                                  errpos(var_name));
1222                                         return ERR_PTR(-EINVAL);
1223                                 }
1224
1225                                 found = hist_field;
1226                         }
1227                 }
1228         }
1229         return found;
1230 }
1231
1232 static struct hist_field *find_event_var(struct hist_trigger_data *hist_data,
1233                                          char *system,
1234                                          char *event_name,
1235                                          char *var_name)
1236 {
1237         struct trace_array *tr = hist_data->event_file->tr;
1238         struct hist_field *hist_field = NULL;
1239         struct trace_event_file *file;
1240
1241         if (!system || !event_name) {
1242                 hist_field = find_match_var(hist_data, var_name);
1243                 if (IS_ERR(hist_field))
1244                         return NULL;
1245                 if (hist_field)
1246                         return hist_field;
1247         }
1248
1249         file = find_var_file(tr, system, event_name, var_name);
1250         if (!file)
1251                 return NULL;
1252
1253         hist_field = find_file_var(file, var_name);
1254
1255         return hist_field;
1256 }
1257
1258 static u64 hist_field_var_ref(struct hist_field *hist_field,
1259                               struct tracing_map_elt *elt,
1260                               struct trace_buffer *buffer,
1261                               struct ring_buffer_event *rbe,
1262                               void *event)
1263 {
1264         struct hist_elt_data *elt_data;
1265         u64 var_val = 0;
1266
1267         if (WARN_ON_ONCE(!elt))
1268                 return var_val;
1269
1270         elt_data = elt->private_data;
1271         var_val = elt_data->var_ref_vals[hist_field->var_ref_idx];
1272
1273         return var_val;
1274 }
1275
1276 static bool resolve_var_refs(struct hist_trigger_data *hist_data, void *key,
1277                              u64 *var_ref_vals, bool self)
1278 {
1279         struct hist_trigger_data *var_data;
1280         struct tracing_map_elt *var_elt;
1281         struct hist_field *hist_field;
1282         unsigned int i, var_idx;
1283         bool resolved = true;
1284         u64 var_val = 0;
1285
1286         for (i = 0; i < hist_data->n_var_refs; i++) {
1287                 hist_field = hist_data->var_refs[i];
1288                 var_idx = hist_field->var.idx;
1289                 var_data = hist_field->var.hist_data;
1290
1291                 if (var_data == NULL) {
1292                         resolved = false;
1293                         break;
1294                 }
1295
1296                 if ((self && var_data != hist_data) ||
1297                     (!self && var_data == hist_data))
1298                         continue;
1299
1300                 var_elt = tracing_map_lookup(var_data->map, key);
1301                 if (!var_elt) {
1302                         resolved = false;
1303                         break;
1304                 }
1305
1306                 if (!tracing_map_var_set(var_elt, var_idx)) {
1307                         resolved = false;
1308                         break;
1309                 }
1310
1311                 if (self || !hist_field->read_once)
1312                         var_val = tracing_map_read_var(var_elt, var_idx);
1313                 else
1314                         var_val = tracing_map_read_var_once(var_elt, var_idx);
1315
1316                 var_ref_vals[i] = var_val;
1317         }
1318
1319         return resolved;
1320 }
1321
1322 static const char *hist_field_name(struct hist_field *field,
1323                                    unsigned int level)
1324 {
1325         const char *field_name = "";
1326
1327         if (WARN_ON_ONCE(!field))
1328                 return field_name;
1329
1330         if (level > 1)
1331                 return field_name;
1332
1333         if (field->field)
1334                 field_name = field->field->name;
1335         else if (field->flags & HIST_FIELD_FL_LOG2 ||
1336                  field->flags & HIST_FIELD_FL_ALIAS ||
1337                  field->flags & HIST_FIELD_FL_BUCKET)
1338                 field_name = hist_field_name(field->operands[0], ++level);
1339         else if (field->flags & HIST_FIELD_FL_CPU)
1340                 field_name = "common_cpu";
1341         else if (field->flags & HIST_FIELD_FL_EXPR ||
1342                  field->flags & HIST_FIELD_FL_VAR_REF) {
1343                 if (field->system) {
1344                         static char full_name[MAX_FILTER_STR_VAL];
1345
1346                         strcat(full_name, field->system);
1347                         strcat(full_name, ".");
1348                         strcat(full_name, field->event_name);
1349                         strcat(full_name, ".");
1350                         strcat(full_name, field->name);
1351                         field_name = full_name;
1352                 } else
1353                         field_name = field->name;
1354         } else if (field->flags & HIST_FIELD_FL_TIMESTAMP)
1355                 field_name = "common_timestamp";
1356         else if (field->flags & HIST_FIELD_FL_STACKTRACE) {
1357                 if (field->field)
1358                         field_name = field->field->name;
1359                 else
1360                         field_name = "common_stacktrace";
1361         } else if (field->flags & HIST_FIELD_FL_HITCOUNT)
1362                 field_name = "hitcount";
1363
1364         if (field_name == NULL)
1365                 field_name = "";
1366
1367         return field_name;
1368 }
1369
1370 static enum hist_field_fn select_value_fn(int field_size, int field_is_signed)
1371 {
1372         switch (field_size) {
1373         case 8:
1374                 if (field_is_signed)
1375                         return HIST_FIELD_FN_S64;
1376                 else
1377                         return HIST_FIELD_FN_U64;
1378         case 4:
1379                 if (field_is_signed)
1380                         return HIST_FIELD_FN_S32;
1381                 else
1382                         return HIST_FIELD_FN_U32;
1383         case 2:
1384                 if (field_is_signed)
1385                         return HIST_FIELD_FN_S16;
1386                 else
1387                         return HIST_FIELD_FN_U16;
1388         case 1:
1389                 if (field_is_signed)
1390                         return HIST_FIELD_FN_S8;
1391                 else
1392                         return HIST_FIELD_FN_U8;
1393         }
1394
1395         return HIST_FIELD_FN_NOP;
1396 }
1397
1398 static int parse_map_size(char *str)
1399 {
1400         unsigned long size, map_bits;
1401         int ret;
1402
1403         ret = kstrtoul(str, 0, &size);
1404         if (ret)
1405                 goto out;
1406
1407         map_bits = ilog2(roundup_pow_of_two(size));
1408         if (map_bits < TRACING_MAP_BITS_MIN ||
1409             map_bits > TRACING_MAP_BITS_MAX)
1410                 ret = -EINVAL;
1411         else
1412                 ret = map_bits;
1413  out:
1414         return ret;
1415 }
1416
1417 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
1418 {
1419         unsigned int i;
1420
1421         if (!attrs)
1422                 return;
1423
1424         for (i = 0; i < attrs->n_assignments; i++)
1425                 kfree(attrs->assignment_str[i]);
1426
1427         for (i = 0; i < attrs->n_actions; i++)
1428                 kfree(attrs->action_str[i]);
1429
1430         kfree(attrs->name);
1431         kfree(attrs->sort_key_str);
1432         kfree(attrs->keys_str);
1433         kfree(attrs->vals_str);
1434         kfree(attrs->clock);
1435         kfree(attrs);
1436 }
1437
1438 static int parse_action(char *str, struct hist_trigger_attrs *attrs)
1439 {
1440         int ret = -EINVAL;
1441
1442         if (attrs->n_actions >= HIST_ACTIONS_MAX)
1443                 return ret;
1444
1445         if ((str_has_prefix(str, "onmatch(")) ||
1446             (str_has_prefix(str, "onmax(")) ||
1447             (str_has_prefix(str, "onchange("))) {
1448                 attrs->action_str[attrs->n_actions] = kstrdup(str, GFP_KERNEL);
1449                 if (!attrs->action_str[attrs->n_actions]) {
1450                         ret = -ENOMEM;
1451                         return ret;
1452                 }
1453                 attrs->n_actions++;
1454                 ret = 0;
1455         }
1456         return ret;
1457 }
1458
1459 static int parse_assignment(struct trace_array *tr,
1460                             char *str, struct hist_trigger_attrs *attrs)
1461 {
1462         int len, ret = 0;
1463
1464         if ((len = str_has_prefix(str, "key=")) ||
1465             (len = str_has_prefix(str, "keys="))) {
1466                 attrs->keys_str = kstrdup(str + len, GFP_KERNEL);
1467                 if (!attrs->keys_str) {
1468                         ret = -ENOMEM;
1469                         goto out;
1470                 }
1471         } else if ((len = str_has_prefix(str, "val=")) ||
1472                    (len = str_has_prefix(str, "vals=")) ||
1473                    (len = str_has_prefix(str, "values="))) {
1474                 attrs->vals_str = kstrdup(str + len, GFP_KERNEL);
1475                 if (!attrs->vals_str) {
1476                         ret = -ENOMEM;
1477                         goto out;
1478                 }
1479         } else if ((len = str_has_prefix(str, "sort="))) {
1480                 attrs->sort_key_str = kstrdup(str + len, GFP_KERNEL);
1481                 if (!attrs->sort_key_str) {
1482                         ret = -ENOMEM;
1483                         goto out;
1484                 }
1485         } else if (str_has_prefix(str, "name=")) {
1486                 attrs->name = kstrdup(str, GFP_KERNEL);
1487                 if (!attrs->name) {
1488                         ret = -ENOMEM;
1489                         goto out;
1490                 }
1491         } else if ((len = str_has_prefix(str, "clock="))) {
1492                 str += len;
1493
1494                 str = strstrip(str);
1495                 attrs->clock = kstrdup(str, GFP_KERNEL);
1496                 if (!attrs->clock) {
1497                         ret = -ENOMEM;
1498                         goto out;
1499                 }
1500         } else if ((len = str_has_prefix(str, "size="))) {
1501                 int map_bits = parse_map_size(str + len);
1502
1503                 if (map_bits < 0) {
1504                         ret = map_bits;
1505                         goto out;
1506                 }
1507                 attrs->map_bits = map_bits;
1508         } else {
1509                 char *assignment;
1510
1511                 if (attrs->n_assignments == TRACING_MAP_VARS_MAX) {
1512                         hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(str));
1513                         ret = -EINVAL;
1514                         goto out;
1515                 }
1516
1517                 assignment = kstrdup(str, GFP_KERNEL);
1518                 if (!assignment) {
1519                         ret = -ENOMEM;
1520                         goto out;
1521                 }
1522
1523                 attrs->assignment_str[attrs->n_assignments++] = assignment;
1524         }
1525  out:
1526         return ret;
1527 }
1528
1529 static struct hist_trigger_attrs *
1530 parse_hist_trigger_attrs(struct trace_array *tr, char *trigger_str)
1531 {
1532         struct hist_trigger_attrs *attrs;
1533         int ret = 0;
1534
1535         attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
1536         if (!attrs)
1537                 return ERR_PTR(-ENOMEM);
1538
1539         while (trigger_str) {
1540                 char *str = strsep(&trigger_str, ":");
1541                 char *rhs;
1542
1543                 rhs = strchr(str, '=');
1544                 if (rhs) {
1545                         if (!strlen(++rhs)) {
1546                                 ret = -EINVAL;
1547                                 hist_err(tr, HIST_ERR_EMPTY_ASSIGNMENT, errpos(str));
1548                                 goto free;
1549                         }
1550                         ret = parse_assignment(tr, str, attrs);
1551                         if (ret)
1552                                 goto free;
1553                 } else if (strcmp(str, "nohitcount") == 0 ||
1554                            strcmp(str, "NOHC") == 0)
1555                         attrs->no_hitcount = true;
1556                 else if (strcmp(str, "pause") == 0)
1557                         attrs->pause = true;
1558                 else if ((strcmp(str, "cont") == 0) ||
1559                          (strcmp(str, "continue") == 0))
1560                         attrs->cont = true;
1561                 else if (strcmp(str, "clear") == 0)
1562                         attrs->clear = true;
1563                 else {
1564                         ret = parse_action(str, attrs);
1565                         if (ret)
1566                                 goto free;
1567                 }
1568         }
1569
1570         if (!attrs->keys_str) {
1571                 ret = -EINVAL;
1572                 goto free;
1573         }
1574
1575         if (!attrs->clock) {
1576                 attrs->clock = kstrdup("global", GFP_KERNEL);
1577                 if (!attrs->clock) {
1578                         ret = -ENOMEM;
1579                         goto free;
1580                 }
1581         }
1582
1583         return attrs;
1584  free:
1585         destroy_hist_trigger_attrs(attrs);
1586
1587         return ERR_PTR(ret);
1588 }
1589
1590 static inline void save_comm(char *comm, struct task_struct *task)
1591 {
1592         if (!task->pid) {
1593                 strcpy(comm, "<idle>");
1594                 return;
1595         }
1596
1597         if (WARN_ON_ONCE(task->pid < 0)) {
1598                 strcpy(comm, "<XXX>");
1599                 return;
1600         }
1601
1602         strncpy(comm, task->comm, TASK_COMM_LEN);
1603 }
1604
1605 static void hist_elt_data_free(struct hist_elt_data *elt_data)
1606 {
1607         unsigned int i;
1608
1609         for (i = 0; i < elt_data->n_field_var_str; i++)
1610                 kfree(elt_data->field_var_str[i]);
1611
1612         kfree(elt_data->field_var_str);
1613
1614         kfree(elt_data->comm);
1615         kfree(elt_data);
1616 }
1617
1618 static void hist_trigger_elt_data_free(struct tracing_map_elt *elt)
1619 {
1620         struct hist_elt_data *elt_data = elt->private_data;
1621
1622         hist_elt_data_free(elt_data);
1623 }
1624
1625 static int hist_trigger_elt_data_alloc(struct tracing_map_elt *elt)
1626 {
1627         struct hist_trigger_data *hist_data = elt->map->private_data;
1628         unsigned int size = TASK_COMM_LEN;
1629         struct hist_elt_data *elt_data;
1630         struct hist_field *hist_field;
1631         unsigned int i, n_str;
1632
1633         elt_data = kzalloc(sizeof(*elt_data), GFP_KERNEL);
1634         if (!elt_data)
1635                 return -ENOMEM;
1636
1637         for_each_hist_field(i, hist_data) {
1638                 hist_field = hist_data->fields[i];
1639
1640                 if (hist_field->flags & HIST_FIELD_FL_EXECNAME) {
1641                         elt_data->comm = kzalloc(size, GFP_KERNEL);
1642                         if (!elt_data->comm) {
1643                                 kfree(elt_data);
1644                                 return -ENOMEM;
1645                         }
1646                         break;
1647                 }
1648         }
1649
1650         n_str = hist_data->n_field_var_str + hist_data->n_save_var_str +
1651                 hist_data->n_var_str;
1652         if (n_str > SYNTH_FIELDS_MAX) {
1653                 hist_elt_data_free(elt_data);
1654                 return -EINVAL;
1655         }
1656
1657         BUILD_BUG_ON(STR_VAR_LEN_MAX & (sizeof(u64) - 1));
1658
1659         size = STR_VAR_LEN_MAX;
1660
1661         elt_data->field_var_str = kcalloc(n_str, sizeof(char *), GFP_KERNEL);
1662         if (!elt_data->field_var_str) {
1663                 hist_elt_data_free(elt_data);
1664                 return -EINVAL;
1665         }
1666         elt_data->n_field_var_str = n_str;
1667
1668         for (i = 0; i < n_str; i++) {
1669                 elt_data->field_var_str[i] = kzalloc(size, GFP_KERNEL);
1670                 if (!elt_data->field_var_str[i]) {
1671                         hist_elt_data_free(elt_data);
1672                         return -ENOMEM;
1673                 }
1674         }
1675
1676         elt->private_data = elt_data;
1677
1678         return 0;
1679 }
1680
1681 static void hist_trigger_elt_data_init(struct tracing_map_elt *elt)
1682 {
1683         struct hist_elt_data *elt_data = elt->private_data;
1684
1685         if (elt_data->comm)
1686                 save_comm(elt_data->comm, current);
1687 }
1688
1689 static const struct tracing_map_ops hist_trigger_elt_data_ops = {
1690         .elt_alloc      = hist_trigger_elt_data_alloc,
1691         .elt_free       = hist_trigger_elt_data_free,
1692         .elt_init       = hist_trigger_elt_data_init,
1693 };
1694
1695 static const char *get_hist_field_flags(struct hist_field *hist_field)
1696 {
1697         const char *flags_str = NULL;
1698
1699         if (hist_field->flags & HIST_FIELD_FL_HEX)
1700                 flags_str = "hex";
1701         else if (hist_field->flags & HIST_FIELD_FL_SYM)
1702                 flags_str = "sym";
1703         else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1704                 flags_str = "sym-offset";
1705         else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1706                 flags_str = "execname";
1707         else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1708                 flags_str = "syscall";
1709         else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1710                 flags_str = "log2";
1711         else if (hist_field->flags & HIST_FIELD_FL_BUCKET)
1712                 flags_str = "buckets";
1713         else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP_USECS)
1714                 flags_str = "usecs";
1715         else if (hist_field->flags & HIST_FIELD_FL_PERCENT)
1716                 flags_str = "percent";
1717         else if (hist_field->flags & HIST_FIELD_FL_GRAPH)
1718                 flags_str = "graph";
1719         else if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
1720                 flags_str = "stacktrace";
1721
1722         return flags_str;
1723 }
1724
1725 static void expr_field_str(struct hist_field *field, char *expr)
1726 {
1727         if (field->flags & HIST_FIELD_FL_VAR_REF)
1728                 strcat(expr, "$");
1729         else if (field->flags & HIST_FIELD_FL_CONST) {
1730                 char str[HIST_CONST_DIGITS_MAX];
1731
1732                 snprintf(str, HIST_CONST_DIGITS_MAX, "%llu", field->constant);
1733                 strcat(expr, str);
1734         }
1735
1736         strcat(expr, hist_field_name(field, 0));
1737
1738         if (field->flags && !(field->flags & HIST_FIELD_FL_VAR_REF)) {
1739                 const char *flags_str = get_hist_field_flags(field);
1740
1741                 if (flags_str) {
1742                         strcat(expr, ".");
1743                         strcat(expr, flags_str);
1744                 }
1745         }
1746 }
1747
1748 static char *expr_str(struct hist_field *field, unsigned int level)
1749 {
1750         char *expr;
1751
1752         if (level > 1)
1753                 return NULL;
1754
1755         expr = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
1756         if (!expr)
1757                 return NULL;
1758
1759         if (!field->operands[0]) {
1760                 expr_field_str(field, expr);
1761                 return expr;
1762         }
1763
1764         if (field->operator == FIELD_OP_UNARY_MINUS) {
1765                 char *subexpr;
1766
1767                 strcat(expr, "-(");
1768                 subexpr = expr_str(field->operands[0], ++level);
1769                 if (!subexpr) {
1770                         kfree(expr);
1771                         return NULL;
1772                 }
1773                 strcat(expr, subexpr);
1774                 strcat(expr, ")");
1775
1776                 kfree(subexpr);
1777
1778                 return expr;
1779         }
1780
1781         expr_field_str(field->operands[0], expr);
1782
1783         switch (field->operator) {
1784         case FIELD_OP_MINUS:
1785                 strcat(expr, "-");
1786                 break;
1787         case FIELD_OP_PLUS:
1788                 strcat(expr, "+");
1789                 break;
1790         case FIELD_OP_DIV:
1791                 strcat(expr, "/");
1792                 break;
1793         case FIELD_OP_MULT:
1794                 strcat(expr, "*");
1795                 break;
1796         default:
1797                 kfree(expr);
1798                 return NULL;
1799         }
1800
1801         expr_field_str(field->operands[1], expr);
1802
1803         return expr;
1804 }
1805
1806 /*
1807  * If field_op != FIELD_OP_NONE, *sep points to the root operator
1808  * of the expression tree to be evaluated.
1809  */
1810 static int contains_operator(char *str, char **sep)
1811 {
1812         enum field_op_id field_op = FIELD_OP_NONE;
1813         char *minus_op, *plus_op, *div_op, *mult_op;
1814
1815
1816         /*
1817          * Report the last occurrence of the operators first, so that the
1818          * expression is evaluated left to right. This is important since
1819          * subtraction and division are not associative.
1820          *
1821          *      e.g
1822          *              64/8/4/2 is 1, i.e 64/8/4/2 = ((64/8)/4)/2
1823          *              14-7-5-2 is 0, i.e 14-7-5-2 = ((14-7)-5)-2
1824          */
1825
1826         /*
1827          * First, find lower precedence addition and subtraction
1828          * since the expression will be evaluated recursively.
1829          */
1830         minus_op = strrchr(str, '-');
1831         if (minus_op) {
1832                 /*
1833                  * Unary minus is not supported in sub-expressions. If
1834                  * present, it is always the next root operator.
1835                  */
1836                 if (minus_op == str) {
1837                         field_op = FIELD_OP_UNARY_MINUS;
1838                         goto out;
1839                 }
1840
1841                 field_op = FIELD_OP_MINUS;
1842         }
1843
1844         plus_op = strrchr(str, '+');
1845         if (plus_op || minus_op) {
1846                 /*
1847                  * For operators of the same precedence use to rightmost as the
1848                  * root, so that the expression is evaluated left to right.
1849                  */
1850                 if (plus_op > minus_op)
1851                         field_op = FIELD_OP_PLUS;
1852                 goto out;
1853         }
1854
1855         /*
1856          * Multiplication and division have higher precedence than addition and
1857          * subtraction.
1858          */
1859         div_op = strrchr(str, '/');
1860         if (div_op)
1861                 field_op = FIELD_OP_DIV;
1862
1863         mult_op = strrchr(str, '*');
1864         /*
1865          * For operators of the same precedence use to rightmost as the
1866          * root, so that the expression is evaluated left to right.
1867          */
1868         if (mult_op > div_op)
1869                 field_op = FIELD_OP_MULT;
1870
1871 out:
1872         if (sep) {
1873                 switch (field_op) {
1874                 case FIELD_OP_UNARY_MINUS:
1875                 case FIELD_OP_MINUS:
1876                         *sep = minus_op;
1877                         break;
1878                 case FIELD_OP_PLUS:
1879                         *sep = plus_op;
1880                         break;
1881                 case FIELD_OP_DIV:
1882                         *sep = div_op;
1883                         break;
1884                 case FIELD_OP_MULT:
1885                         *sep = mult_op;
1886                         break;
1887                 case FIELD_OP_NONE:
1888                 default:
1889                         *sep = NULL;
1890                         break;
1891                 }
1892         }
1893
1894         return field_op;
1895 }
1896
1897 static void get_hist_field(struct hist_field *hist_field)
1898 {
1899         hist_field->ref++;
1900 }
1901
1902 static void __destroy_hist_field(struct hist_field *hist_field)
1903 {
1904         if (--hist_field->ref > 1)
1905                 return;
1906
1907         kfree(hist_field->var.name);
1908         kfree(hist_field->name);
1909
1910         /* Can likely be a const */
1911         kfree_const(hist_field->type);
1912
1913         kfree(hist_field->system);
1914         kfree(hist_field->event_name);
1915
1916         kfree(hist_field);
1917 }
1918
1919 static void destroy_hist_field(struct hist_field *hist_field,
1920                                unsigned int level)
1921 {
1922         unsigned int i;
1923
1924         if (level > 3)
1925                 return;
1926
1927         if (!hist_field)
1928                 return;
1929
1930         if (hist_field->flags & HIST_FIELD_FL_VAR_REF)
1931                 return; /* var refs will be destroyed separately */
1932
1933         for (i = 0; i < HIST_FIELD_OPERANDS_MAX; i++)
1934                 destroy_hist_field(hist_field->operands[i], level + 1);
1935
1936         __destroy_hist_field(hist_field);
1937 }
1938
1939 static struct hist_field *create_hist_field(struct hist_trigger_data *hist_data,
1940                                             struct ftrace_event_field *field,
1941                                             unsigned long flags,
1942                                             char *var_name)
1943 {
1944         struct hist_field *hist_field;
1945
1946         if (field && is_function_field(field))
1947                 return NULL;
1948
1949         hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
1950         if (!hist_field)
1951                 return NULL;
1952
1953         hist_field->ref = 1;
1954
1955         hist_field->hist_data = hist_data;
1956
1957         if (flags & HIST_FIELD_FL_EXPR || flags & HIST_FIELD_FL_ALIAS)
1958                 goto out; /* caller will populate */
1959
1960         if (flags & HIST_FIELD_FL_VAR_REF) {
1961                 hist_field->fn_num = HIST_FIELD_FN_VAR_REF;
1962                 goto out;
1963         }
1964
1965         if (flags & HIST_FIELD_FL_HITCOUNT) {
1966                 hist_field->fn_num = HIST_FIELD_FN_COUNTER;
1967                 hist_field->size = sizeof(u64);
1968                 hist_field->type = "u64";
1969                 goto out;
1970         }
1971
1972         if (flags & HIST_FIELD_FL_CONST) {
1973                 hist_field->fn_num = HIST_FIELD_FN_CONST;
1974                 hist_field->size = sizeof(u64);
1975                 hist_field->type = kstrdup("u64", GFP_KERNEL);
1976                 if (!hist_field->type)
1977                         goto free;
1978                 goto out;
1979         }
1980
1981         if (flags & HIST_FIELD_FL_STACKTRACE) {
1982                 if (field)
1983                         hist_field->fn_num = HIST_FIELD_FN_STACK;
1984                 else
1985                         hist_field->fn_num = HIST_FIELD_FN_NOP;
1986                 hist_field->size = HIST_STACKTRACE_SIZE;
1987                 hist_field->type = kstrdup_const("unsigned long[]", GFP_KERNEL);
1988                 if (!hist_field->type)
1989                         goto free;
1990                 goto out;
1991         }
1992
1993         if (flags & (HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET)) {
1994                 unsigned long fl = flags & ~(HIST_FIELD_FL_LOG2 | HIST_FIELD_FL_BUCKET);
1995                 hist_field->fn_num = flags & HIST_FIELD_FL_LOG2 ? HIST_FIELD_FN_LOG2 :
1996                         HIST_FIELD_FN_BUCKET;
1997                 hist_field->operands[0] = create_hist_field(hist_data, field, fl, NULL);
1998                 if (!hist_field->operands[0])
1999                         goto free;
2000                 hist_field->size = hist_field->operands[0]->size;
2001                 hist_field->type = kstrdup_const(hist_field->operands[0]->type, GFP_KERNEL);
2002                 if (!hist_field->type)
2003                         goto free;
2004                 goto out;
2005         }
2006
2007         if (flags & HIST_FIELD_FL_TIMESTAMP) {
2008                 hist_field->fn_num = HIST_FIELD_FN_TIMESTAMP;
2009                 hist_field->size = sizeof(u64);
2010                 hist_field->type = "u64";
2011                 goto out;
2012         }
2013
2014         if (flags & HIST_FIELD_FL_CPU) {
2015                 hist_field->fn_num = HIST_FIELD_FN_CPU;
2016                 hist_field->size = sizeof(int);
2017                 hist_field->type = "unsigned int";
2018                 goto out;
2019         }
2020
2021         if (WARN_ON_ONCE(!field))
2022                 goto out;
2023
2024         /* Pointers to strings are just pointers and dangerous to dereference */
2025         if (is_string_field(field) &&
2026             (field->filter_type != FILTER_PTR_STRING)) {
2027                 flags |= HIST_FIELD_FL_STRING;
2028
2029                 hist_field->size = MAX_FILTER_STR_VAL;
2030                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2031                 if (!hist_field->type)
2032                         goto free;
2033
2034                 if (field->filter_type == FILTER_STATIC_STRING) {
2035                         hist_field->fn_num = HIST_FIELD_FN_STRING;
2036                         hist_field->size = field->size;
2037                 } else if (field->filter_type == FILTER_DYN_STRING) {
2038                         hist_field->fn_num = HIST_FIELD_FN_DYNSTRING;
2039                 } else if (field->filter_type == FILTER_RDYN_STRING)
2040                         hist_field->fn_num = HIST_FIELD_FN_RELDYNSTRING;
2041                 else
2042                         hist_field->fn_num = HIST_FIELD_FN_PSTRING;
2043         } else {
2044                 hist_field->size = field->size;
2045                 hist_field->is_signed = field->is_signed;
2046                 hist_field->type = kstrdup_const(field->type, GFP_KERNEL);
2047                 if (!hist_field->type)
2048                         goto free;
2049
2050                 hist_field->fn_num = select_value_fn(field->size,
2051                                                      field->is_signed);
2052                 if (hist_field->fn_num == HIST_FIELD_FN_NOP) {
2053                         destroy_hist_field(hist_field, 0);
2054                         return NULL;
2055                 }
2056         }
2057  out:
2058         hist_field->field = field;
2059         hist_field->flags = flags;
2060
2061         if (var_name) {
2062                 hist_field->var.name = kstrdup(var_name, GFP_KERNEL);
2063                 if (!hist_field->var.name)
2064                         goto free;
2065         }
2066
2067         return hist_field;
2068  free:
2069         destroy_hist_field(hist_field, 0);
2070         return NULL;
2071 }
2072
2073 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
2074 {
2075         unsigned int i;
2076
2077         for (i = 0; i < HIST_FIELDS_MAX; i++) {
2078                 if (hist_data->fields[i]) {
2079                         destroy_hist_field(hist_data->fields[i], 0);
2080                         hist_data->fields[i] = NULL;
2081                 }
2082         }
2083
2084         for (i = 0; i < hist_data->n_var_refs; i++) {
2085                 WARN_ON(!(hist_data->var_refs[i]->flags & HIST_FIELD_FL_VAR_REF));
2086                 __destroy_hist_field(hist_data->var_refs[i]);
2087                 hist_data->var_refs[i] = NULL;
2088         }
2089 }
2090
2091 static int init_var_ref(struct hist_field *ref_field,
2092                         struct hist_field *var_field,
2093                         char *system, char *event_name)
2094 {
2095         int err = 0;
2096
2097         ref_field->var.idx = var_field->var.idx;
2098         ref_field->var.hist_data = var_field->hist_data;
2099         ref_field->size = var_field->size;
2100         ref_field->is_signed = var_field->is_signed;
2101         ref_field->flags |= var_field->flags &
2102                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2103
2104         if (system) {
2105                 ref_field->system = kstrdup(system, GFP_KERNEL);
2106                 if (!ref_field->system)
2107                         return -ENOMEM;
2108         }
2109
2110         if (event_name) {
2111                 ref_field->event_name = kstrdup(event_name, GFP_KERNEL);
2112                 if (!ref_field->event_name) {
2113                         err = -ENOMEM;
2114                         goto free;
2115                 }
2116         }
2117
2118         if (var_field->var.name) {
2119                 ref_field->name = kstrdup(var_field->var.name, GFP_KERNEL);
2120                 if (!ref_field->name) {
2121                         err = -ENOMEM;
2122                         goto free;
2123                 }
2124         } else if (var_field->name) {
2125                 ref_field->name = kstrdup(var_field->name, GFP_KERNEL);
2126                 if (!ref_field->name) {
2127                         err = -ENOMEM;
2128                         goto free;
2129                 }
2130         }
2131
2132         ref_field->type = kstrdup_const(var_field->type, GFP_KERNEL);
2133         if (!ref_field->type) {
2134                 err = -ENOMEM;
2135                 goto free;
2136         }
2137  out:
2138         return err;
2139  free:
2140         kfree(ref_field->system);
2141         ref_field->system = NULL;
2142         kfree(ref_field->event_name);
2143         ref_field->event_name = NULL;
2144         kfree(ref_field->name);
2145         ref_field->name = NULL;
2146
2147         goto out;
2148 }
2149
2150 static int find_var_ref_idx(struct hist_trigger_data *hist_data,
2151                             struct hist_field *var_field)
2152 {
2153         struct hist_field *ref_field;
2154         int i;
2155
2156         for (i = 0; i < hist_data->n_var_refs; i++) {
2157                 ref_field = hist_data->var_refs[i];
2158                 if (ref_field->var.idx == var_field->var.idx &&
2159                     ref_field->var.hist_data == var_field->hist_data)
2160                         return i;
2161         }
2162
2163         return -ENOENT;
2164 }
2165
2166 /**
2167  * create_var_ref - Create a variable reference and attach it to trigger
2168  * @hist_data: The trigger that will be referencing the variable
2169  * @var_field: The VAR field to create a reference to
2170  * @system: The optional system string
2171  * @event_name: The optional event_name string
2172  *
2173  * Given a variable hist_field, create a VAR_REF hist_field that
2174  * represents a reference to it.
2175  *
2176  * This function also adds the reference to the trigger that
2177  * now references the variable.
2178  *
2179  * Return: The VAR_REF field if successful, NULL if not
2180  */
2181 static struct hist_field *create_var_ref(struct hist_trigger_data *hist_data,
2182                                          struct hist_field *var_field,
2183                                          char *system, char *event_name)
2184 {
2185         unsigned long flags = HIST_FIELD_FL_VAR_REF;
2186         struct hist_field *ref_field;
2187         int i;
2188
2189         /* Check if the variable already exists */
2190         for (i = 0; i < hist_data->n_var_refs; i++) {
2191                 ref_field = hist_data->var_refs[i];
2192                 if (ref_field->var.idx == var_field->var.idx &&
2193                     ref_field->var.hist_data == var_field->hist_data) {
2194                         get_hist_field(ref_field);
2195                         return ref_field;
2196                 }
2197         }
2198         /* Sanity check to avoid out-of-bound write on 'hist_data->var_refs' */
2199         if (hist_data->n_var_refs >= TRACING_MAP_VARS_MAX)
2200                 return NULL;
2201         ref_field = create_hist_field(var_field->hist_data, NULL, flags, NULL);
2202         if (ref_field) {
2203                 if (init_var_ref(ref_field, var_field, system, event_name)) {
2204                         destroy_hist_field(ref_field, 0);
2205                         return NULL;
2206                 }
2207
2208                 hist_data->var_refs[hist_data->n_var_refs] = ref_field;
2209                 ref_field->var_ref_idx = hist_data->n_var_refs++;
2210         }
2211
2212         return ref_field;
2213 }
2214
2215 static bool is_var_ref(char *var_name)
2216 {
2217         if (!var_name || strlen(var_name) < 2 || var_name[0] != '$')
2218                 return false;
2219
2220         return true;
2221 }
2222
2223 static char *field_name_from_var(struct hist_trigger_data *hist_data,
2224                                  char *var_name)
2225 {
2226         char *name, *field;
2227         unsigned int i;
2228
2229         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
2230                 name = hist_data->attrs->var_defs.name[i];
2231
2232                 if (strcmp(var_name, name) == 0) {
2233                         field = hist_data->attrs->var_defs.expr[i];
2234                         if (contains_operator(field, NULL) || is_var_ref(field))
2235                                 continue;
2236                         return field;
2237                 }
2238         }
2239
2240         return NULL;
2241 }
2242
2243 static char *local_field_var_ref(struct hist_trigger_data *hist_data,
2244                                  char *system, char *event_name,
2245                                  char *var_name)
2246 {
2247         struct trace_event_call *call;
2248
2249         if (system && event_name) {
2250                 call = hist_data->event_file->event_call;
2251
2252                 if (strcmp(system, call->class->system) != 0)
2253                         return NULL;
2254
2255                 if (strcmp(event_name, trace_event_name(call)) != 0)
2256                         return NULL;
2257         }
2258
2259         if (!!system != !!event_name)
2260                 return NULL;
2261
2262         if (!is_var_ref(var_name))
2263                 return NULL;
2264
2265         var_name++;
2266
2267         return field_name_from_var(hist_data, var_name);
2268 }
2269
2270 static struct hist_field *parse_var_ref(struct hist_trigger_data *hist_data,
2271                                         char *system, char *event_name,
2272                                         char *var_name)
2273 {
2274         struct hist_field *var_field = NULL, *ref_field = NULL;
2275         struct trace_array *tr = hist_data->event_file->tr;
2276
2277         if (!is_var_ref(var_name))
2278                 return NULL;
2279
2280         var_name++;
2281
2282         var_field = find_event_var(hist_data, system, event_name, var_name);
2283         if (var_field)
2284                 ref_field = create_var_ref(hist_data, var_field,
2285                                            system, event_name);
2286
2287         if (!ref_field)
2288                 hist_err(tr, HIST_ERR_VAR_NOT_FOUND, errpos(var_name));
2289
2290         return ref_field;
2291 }
2292
2293 static struct ftrace_event_field *
2294 parse_field(struct hist_trigger_data *hist_data, struct trace_event_file *file,
2295             char *field_str, unsigned long *flags, unsigned long *buckets)
2296 {
2297         struct ftrace_event_field *field = NULL;
2298         char *field_name, *modifier, *str;
2299         struct trace_array *tr = file->tr;
2300
2301         modifier = str = kstrdup(field_str, GFP_KERNEL);
2302         if (!modifier)
2303                 return ERR_PTR(-ENOMEM);
2304
2305         field_name = strsep(&modifier, ".");
2306         if (modifier) {
2307                 if (strcmp(modifier, "hex") == 0)
2308                         *flags |= HIST_FIELD_FL_HEX;
2309                 else if (strcmp(modifier, "sym") == 0)
2310                         *flags |= HIST_FIELD_FL_SYM;
2311                 /*
2312                  * 'sym-offset' occurrences in the trigger string are modified
2313                  * to 'symXoffset' to simplify arithmetic expression parsing.
2314                  */
2315                 else if (strcmp(modifier, "symXoffset") == 0)
2316                         *flags |= HIST_FIELD_FL_SYM_OFFSET;
2317                 else if ((strcmp(modifier, "execname") == 0) &&
2318                          (strcmp(field_name, "common_pid") == 0))
2319                         *flags |= HIST_FIELD_FL_EXECNAME;
2320                 else if (strcmp(modifier, "syscall") == 0)
2321                         *flags |= HIST_FIELD_FL_SYSCALL;
2322                 else if (strcmp(modifier, "stacktrace") == 0)
2323                         *flags |= HIST_FIELD_FL_STACKTRACE;
2324                 else if (strcmp(modifier, "log2") == 0)
2325                         *flags |= HIST_FIELD_FL_LOG2;
2326                 else if (strcmp(modifier, "usecs") == 0)
2327                         *flags |= HIST_FIELD_FL_TIMESTAMP_USECS;
2328                 else if (strncmp(modifier, "bucket", 6) == 0) {
2329                         int ret;
2330
2331                         modifier += 6;
2332
2333                         if (*modifier == 's')
2334                                 modifier++;
2335                         if (*modifier != '=')
2336                                 goto error;
2337                         modifier++;
2338                         ret = kstrtoul(modifier, 0, buckets);
2339                         if (ret || !(*buckets))
2340                                 goto error;
2341                         *flags |= HIST_FIELD_FL_BUCKET;
2342                 } else if (strncmp(modifier, "percent", 7) == 0) {
2343                         if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2344                                 goto error;
2345                         *flags |= HIST_FIELD_FL_PERCENT;
2346                 } else if (strncmp(modifier, "graph", 5) == 0) {
2347                         if (*flags & (HIST_FIELD_FL_VAR | HIST_FIELD_FL_KEY))
2348                                 goto error;
2349                         *flags |= HIST_FIELD_FL_GRAPH;
2350                 } else {
2351  error:
2352                         hist_err(tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(modifier));
2353                         field = ERR_PTR(-EINVAL);
2354                         goto out;
2355                 }
2356         }
2357
2358         if (strcmp(field_name, "common_timestamp") == 0) {
2359                 *flags |= HIST_FIELD_FL_TIMESTAMP;
2360                 hist_data->enable_timestamps = true;
2361                 if (*flags & HIST_FIELD_FL_TIMESTAMP_USECS)
2362                         hist_data->attrs->ts_in_usecs = true;
2363         } else if (strcmp(field_name, "common_stacktrace") == 0) {
2364                 *flags |= HIST_FIELD_FL_STACKTRACE;
2365         } else if (strcmp(field_name, "common_cpu") == 0)
2366                 *flags |= HIST_FIELD_FL_CPU;
2367         else if (strcmp(field_name, "hitcount") == 0)
2368                 *flags |= HIST_FIELD_FL_HITCOUNT;
2369         else {
2370                 field = trace_find_event_field(file->event_call, field_name);
2371                 if (!field || !field->size) {
2372                         /*
2373                          * For backward compatibility, if field_name
2374                          * was "cpu" or "stacktrace", then we treat this
2375                          * the same as common_cpu and common_stacktrace
2376                          * respectively. This also works for "CPU", and
2377                          * "STACKTRACE".
2378                          */
2379                         if (field && field->filter_type == FILTER_CPU) {
2380                                 *flags |= HIST_FIELD_FL_CPU;
2381                         } else if (field && field->filter_type == FILTER_STACKTRACE) {
2382                                 *flags |= HIST_FIELD_FL_STACKTRACE;
2383                         } else {
2384                                 hist_err(tr, HIST_ERR_FIELD_NOT_FOUND,
2385                                          errpos(field_name));
2386                                 field = ERR_PTR(-EINVAL);
2387                                 goto out;
2388                         }
2389                 }
2390         }
2391  out:
2392         kfree(str);
2393
2394         return field;
2395 }
2396
2397 static struct hist_field *create_alias(struct hist_trigger_data *hist_data,
2398                                        struct hist_field *var_ref,
2399                                        char *var_name)
2400 {
2401         struct hist_field *alias = NULL;
2402         unsigned long flags = HIST_FIELD_FL_ALIAS | HIST_FIELD_FL_VAR;
2403
2404         alias = create_hist_field(hist_data, NULL, flags, var_name);
2405         if (!alias)
2406                 return NULL;
2407
2408         alias->fn_num = var_ref->fn_num;
2409         alias->operands[0] = var_ref;
2410
2411         if (init_var_ref(alias, var_ref, var_ref->system, var_ref->event_name)) {
2412                 destroy_hist_field(alias, 0);
2413                 return NULL;
2414         }
2415
2416         alias->var_ref_idx = var_ref->var_ref_idx;
2417
2418         return alias;
2419 }
2420
2421 static struct hist_field *parse_const(struct hist_trigger_data *hist_data,
2422                                       char *str, char *var_name,
2423                                       unsigned long *flags)
2424 {
2425         struct trace_array *tr = hist_data->event_file->tr;
2426         struct hist_field *field = NULL;
2427         u64 constant;
2428
2429         if (kstrtoull(str, 0, &constant)) {
2430                 hist_err(tr, HIST_ERR_EXPECT_NUMBER, errpos(str));
2431                 return NULL;
2432         }
2433
2434         *flags |= HIST_FIELD_FL_CONST;
2435         field = create_hist_field(hist_data, NULL, *flags, var_name);
2436         if (!field)
2437                 return NULL;
2438
2439         field->constant = constant;
2440
2441         return field;
2442 }
2443
2444 static struct hist_field *parse_atom(struct hist_trigger_data *hist_data,
2445                                      struct trace_event_file *file, char *str,
2446                                      unsigned long *flags, char *var_name)
2447 {
2448         char *s, *ref_system = NULL, *ref_event = NULL, *ref_var = str;
2449         struct ftrace_event_field *field = NULL;
2450         struct hist_field *hist_field = NULL;
2451         unsigned long buckets = 0;
2452         int ret = 0;
2453
2454         if (isdigit(str[0])) {
2455                 hist_field = parse_const(hist_data, str, var_name, flags);
2456                 if (!hist_field) {
2457                         ret = -EINVAL;
2458                         goto out;
2459                 }
2460                 return hist_field;
2461         }
2462
2463         s = strchr(str, '.');
2464         if (s) {
2465                 s = strchr(++s, '.');
2466                 if (s) {
2467                         ref_system = strsep(&str, ".");
2468                         if (!str) {
2469                                 ret = -EINVAL;
2470                                 goto out;
2471                         }
2472                         ref_event = strsep(&str, ".");
2473                         if (!str) {
2474                                 ret = -EINVAL;
2475                                 goto out;
2476                         }
2477                         ref_var = str;
2478                 }
2479         }
2480
2481         s = local_field_var_ref(hist_data, ref_system, ref_event, ref_var);
2482         if (!s) {
2483                 hist_field = parse_var_ref(hist_data, ref_system,
2484                                            ref_event, ref_var);
2485                 if (hist_field) {
2486                         if (var_name) {
2487                                 hist_field = create_alias(hist_data, hist_field, var_name);
2488                                 if (!hist_field) {
2489                                         ret = -ENOMEM;
2490                                         goto out;
2491                                 }
2492                         }
2493                         return hist_field;
2494                 }
2495         } else
2496                 str = s;
2497
2498         field = parse_field(hist_data, file, str, flags, &buckets);
2499         if (IS_ERR(field)) {
2500                 ret = PTR_ERR(field);
2501                 goto out;
2502         }
2503
2504         hist_field = create_hist_field(hist_data, field, *flags, var_name);
2505         if (!hist_field) {
2506                 ret = -ENOMEM;
2507                 goto out;
2508         }
2509         hist_field->buckets = buckets;
2510
2511         return hist_field;
2512  out:
2513         return ERR_PTR(ret);
2514 }
2515
2516 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2517                                      struct trace_event_file *file,
2518                                      char *str, unsigned long flags,
2519                                      char *var_name, unsigned int *n_subexprs);
2520
2521 static struct hist_field *parse_unary(struct hist_trigger_data *hist_data,
2522                                       struct trace_event_file *file,
2523                                       char *str, unsigned long flags,
2524                                       char *var_name, unsigned int *n_subexprs)
2525 {
2526         struct hist_field *operand1, *expr = NULL;
2527         unsigned long operand_flags;
2528         int ret = 0;
2529         char *s;
2530
2531         /* Unary minus operator, increment n_subexprs */
2532         ++*n_subexprs;
2533
2534         /* we support only -(xxx) i.e. explicit parens required */
2535
2536         if (*n_subexprs > 3) {
2537                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2538                 ret = -EINVAL;
2539                 goto free;
2540         }
2541
2542         str++; /* skip leading '-' */
2543
2544         s = strchr(str, '(');
2545         if (s)
2546                 str++;
2547         else {
2548                 ret = -EINVAL;
2549                 goto free;
2550         }
2551
2552         s = strrchr(str, ')');
2553         if (s) {
2554                  /* unary minus not supported in sub-expressions */
2555                 if (*(s+1) != '\0') {
2556                         hist_err(file->tr, HIST_ERR_UNARY_MINUS_SUBEXPR,
2557                                  errpos(str));
2558                         ret = -EINVAL;
2559                         goto free;
2560                 }
2561                 *s = '\0';
2562         }
2563         else {
2564                 ret = -EINVAL; /* no closing ')' */
2565                 goto free;
2566         }
2567
2568         flags |= HIST_FIELD_FL_EXPR;
2569         expr = create_hist_field(hist_data, NULL, flags, var_name);
2570         if (!expr) {
2571                 ret = -ENOMEM;
2572                 goto free;
2573         }
2574
2575         operand_flags = 0;
2576         operand1 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2577         if (IS_ERR(operand1)) {
2578                 ret = PTR_ERR(operand1);
2579                 goto free;
2580         }
2581         if (operand1->flags & HIST_FIELD_FL_STRING) {
2582                 /* String type can not be the operand of unary operator. */
2583                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2584                 destroy_hist_field(operand1, 0);
2585                 ret = -EINVAL;
2586                 goto free;
2587         }
2588
2589         expr->flags |= operand1->flags &
2590                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2591         expr->fn_num = HIST_FIELD_FN_UMINUS;
2592         expr->operands[0] = operand1;
2593         expr->size = operand1->size;
2594         expr->is_signed = operand1->is_signed;
2595         expr->operator = FIELD_OP_UNARY_MINUS;
2596         expr->name = expr_str(expr, 0);
2597         expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2598         if (!expr->type) {
2599                 ret = -ENOMEM;
2600                 goto free;
2601         }
2602
2603         return expr;
2604  free:
2605         destroy_hist_field(expr, 0);
2606         return ERR_PTR(ret);
2607 }
2608
2609 /*
2610  * If the operands are var refs, return pointers the
2611  * variable(s) referenced in var1 and var2, else NULL.
2612  */
2613 static int check_expr_operands(struct trace_array *tr,
2614                                struct hist_field *operand1,
2615                                struct hist_field *operand2,
2616                                struct hist_field **var1,
2617                                struct hist_field **var2)
2618 {
2619         unsigned long operand1_flags = operand1->flags;
2620         unsigned long operand2_flags = operand2->flags;
2621
2622         if ((operand1_flags & HIST_FIELD_FL_VAR_REF) ||
2623             (operand1_flags & HIST_FIELD_FL_ALIAS)) {
2624                 struct hist_field *var;
2625
2626                 var = find_var_field(operand1->var.hist_data, operand1->name);
2627                 if (!var)
2628                         return -EINVAL;
2629                 operand1_flags = var->flags;
2630                 *var1 = var;
2631         }
2632
2633         if ((operand2_flags & HIST_FIELD_FL_VAR_REF) ||
2634             (operand2_flags & HIST_FIELD_FL_ALIAS)) {
2635                 struct hist_field *var;
2636
2637                 var = find_var_field(operand2->var.hist_data, operand2->name);
2638                 if (!var)
2639                         return -EINVAL;
2640                 operand2_flags = var->flags;
2641                 *var2 = var;
2642         }
2643
2644         if ((operand1_flags & HIST_FIELD_FL_TIMESTAMP_USECS) !=
2645             (operand2_flags & HIST_FIELD_FL_TIMESTAMP_USECS)) {
2646                 hist_err(tr, HIST_ERR_TIMESTAMP_MISMATCH, 0);
2647                 return -EINVAL;
2648         }
2649
2650         return 0;
2651 }
2652
2653 static struct hist_field *parse_expr(struct hist_trigger_data *hist_data,
2654                                      struct trace_event_file *file,
2655                                      char *str, unsigned long flags,
2656                                      char *var_name, unsigned int *n_subexprs)
2657 {
2658         struct hist_field *operand1 = NULL, *operand2 = NULL, *expr = NULL;
2659         struct hist_field *var1 = NULL, *var2 = NULL;
2660         unsigned long operand_flags, operand2_flags;
2661         int field_op, ret = -EINVAL;
2662         char *sep, *operand1_str;
2663         enum hist_field_fn op_fn;
2664         bool combine_consts;
2665
2666         if (*n_subexprs > 3) {
2667                 hist_err(file->tr, HIST_ERR_TOO_MANY_SUBEXPR, errpos(str));
2668                 return ERR_PTR(-EINVAL);
2669         }
2670
2671         field_op = contains_operator(str, &sep);
2672
2673         if (field_op == FIELD_OP_NONE)
2674                 return parse_atom(hist_data, file, str, &flags, var_name);
2675
2676         if (field_op == FIELD_OP_UNARY_MINUS)
2677                 return parse_unary(hist_data, file, str, flags, var_name, n_subexprs);
2678
2679         /* Binary operator found, increment n_subexprs */
2680         ++*n_subexprs;
2681
2682         /* Split the expression string at the root operator */
2683         if (!sep)
2684                 return ERR_PTR(-EINVAL);
2685
2686         *sep = '\0';
2687         operand1_str = str;
2688         str = sep+1;
2689
2690         /* Binary operator requires both operands */
2691         if (*operand1_str == '\0' || *str == '\0')
2692                 return ERR_PTR(-EINVAL);
2693
2694         operand_flags = 0;
2695
2696         /* LHS of string is an expression e.g. a+b in a+b+c */
2697         operand1 = parse_expr(hist_data, file, operand1_str, operand_flags, NULL, n_subexprs);
2698         if (IS_ERR(operand1))
2699                 return ERR_CAST(operand1);
2700
2701         if (operand1->flags & HIST_FIELD_FL_STRING) {
2702                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(operand1_str));
2703                 ret = -EINVAL;
2704                 goto free_op1;
2705         }
2706
2707         /* RHS of string is another expression e.g. c in a+b+c */
2708         operand_flags = 0;
2709         operand2 = parse_expr(hist_data, file, str, operand_flags, NULL, n_subexprs);
2710         if (IS_ERR(operand2)) {
2711                 ret = PTR_ERR(operand2);
2712                 goto free_op1;
2713         }
2714         if (operand2->flags & HIST_FIELD_FL_STRING) {
2715                 hist_err(file->tr, HIST_ERR_INVALID_STR_OPERAND, errpos(str));
2716                 ret = -EINVAL;
2717                 goto free_operands;
2718         }
2719
2720         switch (field_op) {
2721         case FIELD_OP_MINUS:
2722                 op_fn = HIST_FIELD_FN_MINUS;
2723                 break;
2724         case FIELD_OP_PLUS:
2725                 op_fn = HIST_FIELD_FN_PLUS;
2726                 break;
2727         case FIELD_OP_DIV:
2728                 op_fn = HIST_FIELD_FN_DIV;
2729                 break;
2730         case FIELD_OP_MULT:
2731                 op_fn = HIST_FIELD_FN_MULT;
2732                 break;
2733         default:
2734                 ret = -EINVAL;
2735                 goto free_operands;
2736         }
2737
2738         ret = check_expr_operands(file->tr, operand1, operand2, &var1, &var2);
2739         if (ret)
2740                 goto free_operands;
2741
2742         operand_flags = var1 ? var1->flags : operand1->flags;
2743         operand2_flags = var2 ? var2->flags : operand2->flags;
2744
2745         /*
2746          * If both operands are constant, the expression can be
2747          * collapsed to a single constant.
2748          */
2749         combine_consts = operand_flags & operand2_flags & HIST_FIELD_FL_CONST;
2750
2751         flags |= combine_consts ? HIST_FIELD_FL_CONST : HIST_FIELD_FL_EXPR;
2752
2753         flags |= operand1->flags &
2754                 (HIST_FIELD_FL_TIMESTAMP | HIST_FIELD_FL_TIMESTAMP_USECS);
2755
2756         expr = create_hist_field(hist_data, NULL, flags, var_name);
2757         if (!expr) {
2758                 ret = -ENOMEM;
2759                 goto free_operands;
2760         }
2761
2762         operand1->read_once = true;
2763         operand2->read_once = true;
2764
2765         /* The operands are now owned and free'd by 'expr' */
2766         expr->operands[0] = operand1;
2767         expr->operands[1] = operand2;
2768
2769         if (field_op == FIELD_OP_DIV &&
2770                         operand2_flags & HIST_FIELD_FL_CONST) {
2771                 u64 divisor = var2 ? var2->constant : operand2->constant;
2772
2773                 if (!divisor) {
2774                         hist_err(file->tr, HIST_ERR_DIVISION_BY_ZERO, errpos(str));
2775                         ret = -EDOM;
2776                         goto free_expr;
2777                 }
2778
2779                 /*
2780                  * Copy the divisor here so we don't have to look it up
2781                  * later if this is a var ref
2782                  */
2783                 operand2->constant = divisor;
2784                 op_fn = hist_field_get_div_fn(operand2);
2785         }
2786
2787         expr->fn_num = op_fn;
2788
2789         if (combine_consts) {
2790                 if (var1)
2791                         expr->operands[0] = var1;
2792                 if (var2)
2793                         expr->operands[1] = var2;
2794
2795                 expr->constant = hist_fn_call(expr, NULL, NULL, NULL, NULL);
2796                 expr->fn_num = HIST_FIELD_FN_CONST;
2797
2798                 expr->operands[0] = NULL;
2799                 expr->operands[1] = NULL;
2800
2801                 /*
2802                  * var refs won't be destroyed immediately
2803                  * See: destroy_hist_field()
2804                  */
2805                 destroy_hist_field(operand2, 0);
2806                 destroy_hist_field(operand1, 0);
2807
2808                 expr->name = expr_str(expr, 0);
2809         } else {
2810                 /* The operand sizes should be the same, so just pick one */
2811                 expr->size = operand1->size;
2812                 expr->is_signed = operand1->is_signed;
2813
2814                 expr->operator = field_op;
2815                 expr->type = kstrdup_const(operand1->type, GFP_KERNEL);
2816                 if (!expr->type) {
2817                         ret = -ENOMEM;
2818                         goto free_expr;
2819                 }
2820
2821                 expr->name = expr_str(expr, 0);
2822         }
2823
2824         return expr;
2825
2826 free_operands:
2827         destroy_hist_field(operand2, 0);
2828 free_op1:
2829         destroy_hist_field(operand1, 0);
2830         return ERR_PTR(ret);
2831
2832 free_expr:
2833         destroy_hist_field(expr, 0);
2834         return ERR_PTR(ret);
2835 }
2836
2837 static char *find_trigger_filter(struct hist_trigger_data *hist_data,
2838                                  struct trace_event_file *file)
2839 {
2840         struct event_trigger_data *test;
2841
2842         lockdep_assert_held(&event_mutex);
2843
2844         list_for_each_entry(test, &file->triggers, list) {
2845                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2846                         if (test->private_data == hist_data)
2847                                 return test->filter_str;
2848                 }
2849         }
2850
2851         return NULL;
2852 }
2853
2854 static struct event_command trigger_hist_cmd;
2855 static int event_hist_trigger_parse(struct event_command *cmd_ops,
2856                                     struct trace_event_file *file,
2857                                     char *glob, char *cmd,
2858                                     char *param_and_filter);
2859
2860 static bool compatible_keys(struct hist_trigger_data *target_hist_data,
2861                             struct hist_trigger_data *hist_data,
2862                             unsigned int n_keys)
2863 {
2864         struct hist_field *target_hist_field, *hist_field;
2865         unsigned int n, i, j;
2866
2867         if (hist_data->n_fields - hist_data->n_vals != n_keys)
2868                 return false;
2869
2870         i = hist_data->n_vals;
2871         j = target_hist_data->n_vals;
2872
2873         for (n = 0; n < n_keys; n++) {
2874                 hist_field = hist_data->fields[i + n];
2875                 target_hist_field = target_hist_data->fields[j + n];
2876
2877                 if (strcmp(hist_field->type, target_hist_field->type) != 0)
2878                         return false;
2879                 if (hist_field->size != target_hist_field->size)
2880                         return false;
2881                 if (hist_field->is_signed != target_hist_field->is_signed)
2882                         return false;
2883         }
2884
2885         return true;
2886 }
2887
2888 static struct hist_trigger_data *
2889 find_compatible_hist(struct hist_trigger_data *target_hist_data,
2890                      struct trace_event_file *file)
2891 {
2892         struct hist_trigger_data *hist_data;
2893         struct event_trigger_data *test;
2894         unsigned int n_keys;
2895
2896         lockdep_assert_held(&event_mutex);
2897
2898         n_keys = target_hist_data->n_fields - target_hist_data->n_vals;
2899
2900         list_for_each_entry(test, &file->triggers, list) {
2901                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
2902                         hist_data = test->private_data;
2903
2904                         if (compatible_keys(target_hist_data, hist_data, n_keys))
2905                                 return hist_data;
2906                 }
2907         }
2908
2909         return NULL;
2910 }
2911
2912 static struct trace_event_file *event_file(struct trace_array *tr,
2913                                            char *system, char *event_name)
2914 {
2915         struct trace_event_file *file;
2916
2917         file = __find_event_file(tr, system, event_name);
2918         if (!file)
2919                 return ERR_PTR(-EINVAL);
2920
2921         return file;
2922 }
2923
2924 static struct hist_field *
2925 find_synthetic_field_var(struct hist_trigger_data *target_hist_data,
2926                          char *system, char *event_name, char *field_name)
2927 {
2928         struct hist_field *event_var;
2929         char *synthetic_name;
2930
2931         synthetic_name = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
2932         if (!synthetic_name)
2933                 return ERR_PTR(-ENOMEM);
2934
2935         strcpy(synthetic_name, "synthetic_");
2936         strcat(synthetic_name, field_name);
2937
2938         event_var = find_event_var(target_hist_data, system, event_name, synthetic_name);
2939
2940         kfree(synthetic_name);
2941
2942         return event_var;
2943 }
2944
2945 /**
2946  * create_field_var_hist - Automatically create a histogram and var for a field
2947  * @target_hist_data: The target hist trigger
2948  * @subsys_name: Optional subsystem name
2949  * @event_name: Optional event name
2950  * @field_name: The name of the field (and the resulting variable)
2951  *
2952  * Hist trigger actions fetch data from variables, not directly from
2953  * events.  However, for convenience, users are allowed to directly
2954  * specify an event field in an action, which will be automatically
2955  * converted into a variable on their behalf.
2956  *
2957  * If a user specifies a field on an event that isn't the event the
2958  * histogram currently being defined (the target event histogram), the
2959  * only way that can be accomplished is if a new hist trigger is
2960  * created and the field variable defined on that.
2961  *
2962  * This function creates a new histogram compatible with the target
2963  * event (meaning a histogram with the same key as the target
2964  * histogram), and creates a variable for the specified field, but
2965  * with 'synthetic_' prepended to the variable name in order to avoid
2966  * collision with normal field variables.
2967  *
2968  * Return: The variable created for the field.
2969  */
2970 static struct hist_field *
2971 create_field_var_hist(struct hist_trigger_data *target_hist_data,
2972                       char *subsys_name, char *event_name, char *field_name)
2973 {
2974         struct trace_array *tr = target_hist_data->event_file->tr;
2975         struct hist_trigger_data *hist_data;
2976         unsigned int i, n, first = true;
2977         struct field_var_hist *var_hist;
2978         struct trace_event_file *file;
2979         struct hist_field *key_field;
2980         struct hist_field *event_var;
2981         char *saved_filter;
2982         char *cmd;
2983         int ret;
2984
2985         if (target_hist_data->n_field_var_hists >= SYNTH_FIELDS_MAX) {
2986                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
2987                 return ERR_PTR(-EINVAL);
2988         }
2989
2990         file = event_file(tr, subsys_name, event_name);
2991
2992         if (IS_ERR(file)) {
2993                 hist_err(tr, HIST_ERR_EVENT_FILE_NOT_FOUND, errpos(field_name));
2994                 ret = PTR_ERR(file);
2995                 return ERR_PTR(ret);
2996         }
2997
2998         /*
2999          * Look for a histogram compatible with target.  We'll use the
3000          * found histogram specification to create a new matching
3001          * histogram with our variable on it.  target_hist_data is not
3002          * yet a registered histogram so we can't use that.
3003          */
3004         hist_data = find_compatible_hist(target_hist_data, file);
3005         if (!hist_data) {
3006                 hist_err(tr, HIST_ERR_HIST_NOT_FOUND, errpos(field_name));
3007                 return ERR_PTR(-EINVAL);
3008         }
3009
3010         /* See if a synthetic field variable has already been created */
3011         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3012                                              event_name, field_name);
3013         if (!IS_ERR_OR_NULL(event_var))
3014                 return event_var;
3015
3016         var_hist = kzalloc(sizeof(*var_hist), GFP_KERNEL);
3017         if (!var_hist)
3018                 return ERR_PTR(-ENOMEM);
3019
3020         cmd = kzalloc(MAX_FILTER_STR_VAL, GFP_KERNEL);
3021         if (!cmd) {
3022                 kfree(var_hist);
3023                 return ERR_PTR(-ENOMEM);
3024         }
3025
3026         /* Use the same keys as the compatible histogram */
3027         strcat(cmd, "keys=");
3028
3029         for_each_hist_key_field(i, hist_data) {
3030                 key_field = hist_data->fields[i];
3031                 if (!first)
3032                         strcat(cmd, ",");
3033                 strcat(cmd, key_field->field->name);
3034                 first = false;
3035         }
3036
3037         /* Create the synthetic field variable specification */
3038         strcat(cmd, ":synthetic_");
3039         strcat(cmd, field_name);
3040         strcat(cmd, "=");
3041         strcat(cmd, field_name);
3042
3043         /* Use the same filter as the compatible histogram */
3044         saved_filter = find_trigger_filter(hist_data, file);
3045         if (saved_filter) {
3046                 strcat(cmd, " if ");
3047                 strcat(cmd, saved_filter);
3048         }
3049
3050         var_hist->cmd = kstrdup(cmd, GFP_KERNEL);
3051         if (!var_hist->cmd) {
3052                 kfree(cmd);
3053                 kfree(var_hist);
3054                 return ERR_PTR(-ENOMEM);
3055         }
3056
3057         /* Save the compatible histogram information */
3058         var_hist->hist_data = hist_data;
3059
3060         /* Create the new histogram with our variable */
3061         ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
3062                                        "", "hist", cmd);
3063         if (ret) {
3064                 kfree(cmd);
3065                 kfree(var_hist->cmd);
3066                 kfree(var_hist);
3067                 hist_err(tr, HIST_ERR_HIST_CREATE_FAIL, errpos(field_name));
3068                 return ERR_PTR(ret);
3069         }
3070
3071         kfree(cmd);
3072
3073         /* If we can't find the variable, something went wrong */
3074         event_var = find_synthetic_field_var(target_hist_data, subsys_name,
3075                                              event_name, field_name);
3076         if (IS_ERR_OR_NULL(event_var)) {
3077                 kfree(var_hist->cmd);
3078                 kfree(var_hist);
3079                 hist_err(tr, HIST_ERR_SYNTH_VAR_NOT_FOUND, errpos(field_name));
3080                 return ERR_PTR(-EINVAL);
3081         }
3082
3083         n = target_hist_data->n_field_var_hists;
3084         target_hist_data->field_var_hists[n] = var_hist;
3085         target_hist_data->n_field_var_hists++;
3086
3087         return event_var;
3088 }
3089
3090 static struct hist_field *
3091 find_target_event_var(struct hist_trigger_data *hist_data,
3092                       char *subsys_name, char *event_name, char *var_name)
3093 {
3094         struct trace_event_file *file = hist_data->event_file;
3095         struct hist_field *hist_field = NULL;
3096
3097         if (subsys_name) {
3098                 struct trace_event_call *call;
3099
3100                 if (!event_name)
3101                         return NULL;
3102
3103                 call = file->event_call;
3104
3105                 if (strcmp(subsys_name, call->class->system) != 0)
3106                         return NULL;
3107
3108                 if (strcmp(event_name, trace_event_name(call)) != 0)
3109                         return NULL;
3110         }
3111
3112         hist_field = find_var_field(hist_data, var_name);
3113
3114         return hist_field;
3115 }
3116
3117 static inline void __update_field_vars(struct tracing_map_elt *elt,
3118                                        struct trace_buffer *buffer,
3119                                        struct ring_buffer_event *rbe,
3120                                        void *rec,
3121                                        struct field_var **field_vars,
3122                                        unsigned int n_field_vars,
3123                                        unsigned int field_var_str_start)
3124 {
3125         struct hist_elt_data *elt_data = elt->private_data;
3126         unsigned int i, j, var_idx;
3127         u64 var_val;
3128
3129         /* Make sure stacktrace can fit in the string variable length */
3130         BUILD_BUG_ON((HIST_STACKTRACE_DEPTH + 1) * sizeof(long) >= STR_VAR_LEN_MAX);
3131
3132         for (i = 0, j = field_var_str_start; i < n_field_vars; i++) {
3133                 struct field_var *field_var = field_vars[i];
3134                 struct hist_field *var = field_var->var;
3135                 struct hist_field *val = field_var->val;
3136
3137                 var_val = hist_fn_call(val, elt, buffer, rbe, rec);
3138                 var_idx = var->var.idx;
3139
3140                 if (val->flags & (HIST_FIELD_FL_STRING |
3141                                   HIST_FIELD_FL_STACKTRACE)) {
3142                         char *str = elt_data->field_var_str[j++];
3143                         char *val_str = (char *)(uintptr_t)var_val;
3144                         unsigned int size;
3145
3146                         if (val->flags & HIST_FIELD_FL_STRING) {
3147                                 size = min(val->size, STR_VAR_LEN_MAX);
3148                                 strscpy(str, val_str, size);
3149                         } else {
3150                                 char *stack_start = str + sizeof(unsigned long);
3151                                 int e;
3152
3153                                 e = stack_trace_save((void *)stack_start,
3154                                                      HIST_STACKTRACE_DEPTH,
3155                                                      HIST_STACKTRACE_SKIP);
3156                                 if (e < HIST_STACKTRACE_DEPTH - 1)
3157                                         ((unsigned long *)stack_start)[e] = 0;
3158                                 *((unsigned long *)str) = e;
3159                         }
3160                         var_val = (u64)(uintptr_t)str;
3161                 }
3162                 tracing_map_set_var(elt, var_idx, var_val);
3163         }
3164 }
3165
3166 static void update_field_vars(struct hist_trigger_data *hist_data,
3167                               struct tracing_map_elt *elt,
3168                               struct trace_buffer *buffer,
3169                               struct ring_buffer_event *rbe,
3170                               void *rec)
3171 {
3172         __update_field_vars(elt, buffer, rbe, rec, hist_data->field_vars,
3173                             hist_data->n_field_vars, 0);
3174 }
3175
3176 static void save_track_data_vars(struct hist_trigger_data *hist_data,
3177                                  struct tracing_map_elt *elt,
3178                                  struct trace_buffer *buffer,  void *rec,
3179                                  struct ring_buffer_event *rbe, void *key,
3180                                  struct action_data *data, u64 *var_ref_vals)
3181 {
3182         __update_field_vars(elt, buffer, rbe, rec, hist_data->save_vars,
3183                             hist_data->n_save_vars, hist_data->n_field_var_str);
3184 }
3185
3186 static struct hist_field *create_var(struct hist_trigger_data *hist_data,
3187                                      struct trace_event_file *file,
3188                                      char *name, int size, const char *type)
3189 {
3190         struct hist_field *var;
3191         int idx;
3192
3193         if (find_var(hist_data, file, name) && !hist_data->remove) {
3194                 var = ERR_PTR(-EINVAL);
3195                 goto out;
3196         }
3197
3198         var = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
3199         if (!var) {
3200                 var = ERR_PTR(-ENOMEM);
3201                 goto out;
3202         }
3203
3204         idx = tracing_map_add_var(hist_data->map);
3205         if (idx < 0) {
3206                 kfree(var);
3207                 var = ERR_PTR(-EINVAL);
3208                 goto out;
3209         }
3210
3211         var->ref = 1;
3212         var->flags = HIST_FIELD_FL_VAR;
3213         var->var.idx = idx;
3214         var->var.hist_data = var->hist_data = hist_data;
3215         var->size = size;
3216         var->var.name = kstrdup(name, GFP_KERNEL);
3217         var->type = kstrdup_const(type, GFP_KERNEL);
3218         if (!var->var.name || !var->type) {
3219                 kfree_const(var->type);
3220                 kfree(var->var.name);
3221                 kfree(var);
3222                 var = ERR_PTR(-ENOMEM);
3223         }
3224  out:
3225         return var;
3226 }
3227
3228 static struct field_var *create_field_var(struct hist_trigger_data *hist_data,
3229                                           struct trace_event_file *file,
3230                                           char *field_name)
3231 {
3232         struct hist_field *val = NULL, *var = NULL;
3233         unsigned long flags = HIST_FIELD_FL_VAR;
3234         struct trace_array *tr = file->tr;
3235         struct field_var *field_var;
3236         int ret = 0;
3237
3238         if (hist_data->n_field_vars >= SYNTH_FIELDS_MAX) {
3239                 hist_err(tr, HIST_ERR_TOO_MANY_FIELD_VARS, errpos(field_name));
3240                 ret = -EINVAL;
3241                 goto err;
3242         }
3243
3244         val = parse_atom(hist_data, file, field_name, &flags, NULL);
3245         if (IS_ERR(val)) {
3246                 hist_err(tr, HIST_ERR_FIELD_VAR_PARSE_FAIL, errpos(field_name));
3247                 ret = PTR_ERR(val);
3248                 goto err;
3249         }
3250
3251         var = create_var(hist_data, file, field_name, val->size, val->type);
3252         if (IS_ERR(var)) {
3253                 hist_err(tr, HIST_ERR_VAR_CREATE_FIND_FAIL, errpos(field_name));
3254                 kfree(val);
3255                 ret = PTR_ERR(var);
3256                 goto err;
3257         }
3258
3259         field_var = kzalloc(sizeof(struct field_var), GFP_KERNEL);
3260         if (!field_var) {
3261                 kfree(val);
3262                 kfree(var);
3263                 ret =  -ENOMEM;
3264                 goto err;
3265         }
3266
3267         field_var->var = var;
3268         field_var->val = val;
3269  out:
3270         return field_var;
3271  err:
3272         field_var = ERR_PTR(ret);
3273         goto out;
3274 }
3275
3276 /**
3277  * create_target_field_var - Automatically create a variable for a field
3278  * @target_hist_data: The target hist trigger
3279  * @subsys_name: Optional subsystem name
3280  * @event_name: Optional event name
3281  * @var_name: The name of the field (and the resulting variable)
3282  *
3283  * Hist trigger actions fetch data from variables, not directly from
3284  * events.  However, for convenience, users are allowed to directly
3285  * specify an event field in an action, which will be automatically
3286  * converted into a variable on their behalf.
3287  *
3288  * This function creates a field variable with the name var_name on
3289  * the hist trigger currently being defined on the target event.  If
3290  * subsys_name and event_name are specified, this function simply
3291  * verifies that they do in fact match the target event subsystem and
3292  * event name.
3293  *
3294  * Return: The variable created for the field.
3295  */
3296 static struct field_var *
3297 create_target_field_var(struct hist_trigger_data *target_hist_data,
3298                         char *subsys_name, char *event_name, char *var_name)
3299 {
3300         struct trace_event_file *file = target_hist_data->event_file;
3301
3302         if (subsys_name) {
3303                 struct trace_event_call *call;
3304
3305                 if (!event_name)
3306                         return NULL;
3307
3308                 call = file->event_call;
3309
3310                 if (strcmp(subsys_name, call->class->system) != 0)
3311                         return NULL;
3312
3313                 if (strcmp(event_name, trace_event_name(call)) != 0)
3314                         return NULL;
3315         }
3316
3317         return create_field_var(target_hist_data, file, var_name);
3318 }
3319
3320 static bool check_track_val_max(u64 track_val, u64 var_val)
3321 {
3322         if (var_val <= track_val)
3323                 return false;
3324
3325         return true;
3326 }
3327
3328 static bool check_track_val_changed(u64 track_val, u64 var_val)
3329 {
3330         if (var_val == track_val)
3331                 return false;
3332
3333         return true;
3334 }
3335
3336 static u64 get_track_val(struct hist_trigger_data *hist_data,
3337                          struct tracing_map_elt *elt,
3338                          struct action_data *data)
3339 {
3340         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3341         u64 track_val;
3342
3343         track_val = tracing_map_read_var(elt, track_var_idx);
3344
3345         return track_val;
3346 }
3347
3348 static void save_track_val(struct hist_trigger_data *hist_data,
3349                            struct tracing_map_elt *elt,
3350                            struct action_data *data, u64 var_val)
3351 {
3352         unsigned int track_var_idx = data->track_data.track_var->var.idx;
3353
3354         tracing_map_set_var(elt, track_var_idx, var_val);
3355 }
3356
3357 static void save_track_data(struct hist_trigger_data *hist_data,
3358                             struct tracing_map_elt *elt,
3359                             struct trace_buffer *buffer, void *rec,
3360                             struct ring_buffer_event *rbe, void *key,
3361                             struct action_data *data, u64 *var_ref_vals)
3362 {
3363         if (data->track_data.save_data)
3364                 data->track_data.save_data(hist_data, elt, buffer, rec, rbe,
3365                                            key, data, var_ref_vals);
3366 }
3367
3368 static bool check_track_val(struct tracing_map_elt *elt,
3369                             struct action_data *data,
3370                             u64 var_val)
3371 {
3372         struct hist_trigger_data *hist_data;
3373         u64 track_val;
3374
3375         hist_data = data->track_data.track_var->hist_data;
3376         track_val = get_track_val(hist_data, elt, data);
3377
3378         return data->track_data.check_val(track_val, var_val);
3379 }
3380
3381 #ifdef CONFIG_TRACER_SNAPSHOT
3382 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3383 {
3384         /* called with tr->max_lock held */
3385         struct track_data *track_data = tr->cond_snapshot->cond_data;
3386         struct hist_elt_data *elt_data, *track_elt_data;
3387         struct snapshot_context *context = cond_data;
3388         struct action_data *action;
3389         u64 track_val;
3390
3391         if (!track_data)
3392                 return false;
3393
3394         action = track_data->action_data;
3395
3396         track_val = get_track_val(track_data->hist_data, context->elt,
3397                                   track_data->action_data);
3398
3399         if (!action->track_data.check_val(track_data->track_val, track_val))
3400                 return false;
3401
3402         track_data->track_val = track_val;
3403         memcpy(track_data->key, context->key, track_data->key_len);
3404
3405         elt_data = context->elt->private_data;
3406         track_elt_data = track_data->elt.private_data;
3407         if (elt_data->comm)
3408                 strncpy(track_elt_data->comm, elt_data->comm, TASK_COMM_LEN);
3409
3410         track_data->updated = true;
3411
3412         return true;
3413 }
3414
3415 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3416                                      struct tracing_map_elt *elt,
3417                                      struct trace_buffer *buffer, void *rec,
3418                                      struct ring_buffer_event *rbe, void *key,
3419                                      struct action_data *data,
3420                                      u64 *var_ref_vals)
3421 {
3422         struct trace_event_file *file = hist_data->event_file;
3423         struct snapshot_context context;
3424
3425         context.elt = elt;
3426         context.key = key;
3427
3428         tracing_snapshot_cond(file->tr, &context);
3429 }
3430
3431 static void hist_trigger_print_key(struct seq_file *m,
3432                                    struct hist_trigger_data *hist_data,
3433                                    void *key,
3434                                    struct tracing_map_elt *elt);
3435
3436 static struct action_data *snapshot_action(struct hist_trigger_data *hist_data)
3437 {
3438         unsigned int i;
3439
3440         if (!hist_data->n_actions)
3441                 return NULL;
3442
3443         for (i = 0; i < hist_data->n_actions; i++) {
3444                 struct action_data *data = hist_data->actions[i];
3445
3446                 if (data->action == ACTION_SNAPSHOT)
3447                         return data;
3448         }
3449
3450         return NULL;
3451 }
3452
3453 static void track_data_snapshot_print(struct seq_file *m,
3454                                       struct hist_trigger_data *hist_data)
3455 {
3456         struct trace_event_file *file = hist_data->event_file;
3457         struct track_data *track_data;
3458         struct action_data *action;
3459
3460         track_data = tracing_cond_snapshot_data(file->tr);
3461         if (!track_data)
3462                 return;
3463
3464         if (!track_data->updated)
3465                 return;
3466
3467         action = snapshot_action(hist_data);
3468         if (!action)
3469                 return;
3470
3471         seq_puts(m, "\nSnapshot taken (see tracing/snapshot).  Details:\n");
3472         seq_printf(m, "\ttriggering value { %s(%s) }: %10llu",
3473                    action->handler == HANDLER_ONMAX ? "onmax" : "onchange",
3474                    action->track_data.var_str, track_data->track_val);
3475
3476         seq_puts(m, "\ttriggered by event with key: ");
3477         hist_trigger_print_key(m, hist_data, track_data->key, &track_data->elt);
3478         seq_putc(m, '\n');
3479 }
3480 #else
3481 static bool cond_snapshot_update(struct trace_array *tr, void *cond_data)
3482 {
3483         return false;
3484 }
3485 static void save_track_data_snapshot(struct hist_trigger_data *hist_data,
3486                                      struct tracing_map_elt *elt,
3487                                      struct trace_buffer *buffer, void *rec,
3488                                      struct ring_buffer_event *rbe, void *key,
3489                                      struct action_data *data,
3490                                      u64 *var_ref_vals) {}
3491 static void track_data_snapshot_print(struct seq_file *m,
3492                                       struct hist_trigger_data *hist_data) {}
3493 #endif /* CONFIG_TRACER_SNAPSHOT */
3494
3495 static void track_data_print(struct seq_file *m,
3496                              struct hist_trigger_data *hist_data,
3497                              struct tracing_map_elt *elt,
3498                              struct action_data *data)
3499 {
3500         u64 track_val = get_track_val(hist_data, elt, data);
3501         unsigned int i, save_var_idx;
3502
3503         if (data->handler == HANDLER_ONMAX)
3504                 seq_printf(m, "\n\tmax: %10llu", track_val);
3505         else if (data->handler == HANDLER_ONCHANGE)
3506                 seq_printf(m, "\n\tchanged: %10llu", track_val);
3507
3508         if (data->action == ACTION_SNAPSHOT)
3509                 return;
3510
3511         for (i = 0; i < hist_data->n_save_vars; i++) {
3512                 struct hist_field *save_val = hist_data->save_vars[i]->val;
3513                 struct hist_field *save_var = hist_data->save_vars[i]->var;
3514                 u64 val;
3515
3516                 save_var_idx = save_var->var.idx;
3517
3518                 val = tracing_map_read_var(elt, save_var_idx);
3519
3520                 if (save_val->flags & HIST_FIELD_FL_STRING) {
3521                         seq_printf(m, "  %s: %-32s", save_var->var.name,
3522                                    (char *)(uintptr_t)(val));
3523                 } else
3524                         seq_printf(m, "  %s: %10llu", save_var->var.name, val);
3525         }
3526 }
3527
3528 static void ontrack_action(struct hist_trigger_data *hist_data,
3529                            struct tracing_map_elt *elt,
3530                            struct trace_buffer *buffer, void *rec,
3531                            struct ring_buffer_event *rbe, void *key,
3532                            struct action_data *data, u64 *var_ref_vals)
3533 {
3534         u64 var_val = var_ref_vals[data->track_data.var_ref->var_ref_idx];
3535
3536         if (check_track_val(elt, data, var_val)) {
3537                 save_track_val(hist_data, elt, data, var_val);
3538                 save_track_data(hist_data, elt, buffer, rec, rbe,
3539                                 key, data, var_ref_vals);
3540         }
3541 }
3542
3543 static void action_data_destroy(struct action_data *data)
3544 {
3545         unsigned int i;
3546
3547         lockdep_assert_held(&event_mutex);
3548
3549         kfree(data->action_name);
3550
3551         for (i = 0; i < data->n_params; i++)
3552                 kfree(data->params[i]);
3553
3554         if (data->synth_event)
3555                 data->synth_event->ref--;
3556
3557         kfree(data->synth_event_name);
3558
3559         kfree(data);
3560 }
3561
3562 static void track_data_destroy(struct hist_trigger_data *hist_data,
3563                                struct action_data *data)
3564 {
3565         struct trace_event_file *file = hist_data->event_file;
3566
3567         destroy_hist_field(data->track_data.track_var, 0);
3568
3569         if (data->action == ACTION_SNAPSHOT) {
3570                 struct track_data *track_data;
3571
3572                 track_data = tracing_cond_snapshot_data(file->tr);
3573                 if (track_data && track_data->hist_data == hist_data) {
3574                         tracing_snapshot_cond_disable(file->tr);
3575                         track_data_free(track_data);
3576                 }
3577         }
3578
3579         kfree(data->track_data.var_str);
3580
3581         action_data_destroy(data);
3582 }
3583
3584 static int action_create(struct hist_trigger_data *hist_data,
3585                          struct action_data *data);
3586
3587 static int track_data_create(struct hist_trigger_data *hist_data,
3588                              struct action_data *data)
3589 {
3590         struct hist_field *var_field, *ref_field, *track_var = NULL;
3591         struct trace_event_file *file = hist_data->event_file;
3592         struct trace_array *tr = file->tr;
3593         char *track_data_var_str;
3594         int ret = 0;
3595
3596         track_data_var_str = data->track_data.var_str;
3597         if (track_data_var_str[0] != '$') {
3598                 hist_err(tr, HIST_ERR_ONX_NOT_VAR, errpos(track_data_var_str));
3599                 return -EINVAL;
3600         }
3601         track_data_var_str++;
3602
3603         var_field = find_target_event_var(hist_data, NULL, NULL, track_data_var_str);
3604         if (!var_field) {
3605                 hist_err(tr, HIST_ERR_ONX_VAR_NOT_FOUND, errpos(track_data_var_str));
3606                 return -EINVAL;
3607         }
3608
3609         ref_field = create_var_ref(hist_data, var_field, NULL, NULL);
3610         if (!ref_field)
3611                 return -ENOMEM;
3612
3613         data->track_data.var_ref = ref_field;
3614
3615         if (data->handler == HANDLER_ONMAX)
3616                 track_var = create_var(hist_data, file, "__max", sizeof(u64), "u64");
3617         if (IS_ERR(track_var)) {
3618                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3619                 ret = PTR_ERR(track_var);
3620                 goto out;
3621         }
3622
3623         if (data->handler == HANDLER_ONCHANGE)
3624                 track_var = create_var(hist_data, file, "__change", sizeof(u64), "u64");
3625         if (IS_ERR(track_var)) {
3626                 hist_err(tr, HIST_ERR_ONX_VAR_CREATE_FAIL, 0);
3627                 ret = PTR_ERR(track_var);
3628                 goto out;
3629         }
3630         data->track_data.track_var = track_var;
3631
3632         ret = action_create(hist_data, data);
3633  out:
3634         return ret;
3635 }
3636
3637 static int parse_action_params(struct trace_array *tr, char *params,
3638                                struct action_data *data)
3639 {
3640         char *param, *saved_param;
3641         bool first_param = true;
3642         int ret = 0;
3643
3644         while (params) {
3645                 if (data->n_params >= SYNTH_FIELDS_MAX) {
3646                         hist_err(tr, HIST_ERR_TOO_MANY_PARAMS, 0);
3647                         ret = -EINVAL;
3648                         goto out;
3649                 }
3650
3651                 param = strsep(&params, ",");
3652                 if (!param) {
3653                         hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, 0);
3654                         ret = -EINVAL;
3655                         goto out;
3656                 }
3657
3658                 param = strstrip(param);
3659                 if (strlen(param) < 2) {
3660                         hist_err(tr, HIST_ERR_INVALID_PARAM, errpos(param));
3661                         ret = -EINVAL;
3662                         goto out;
3663                 }
3664
3665                 saved_param = kstrdup(param, GFP_KERNEL);
3666                 if (!saved_param) {
3667                         ret = -ENOMEM;
3668                         goto out;
3669                 }
3670
3671                 if (first_param && data->use_trace_keyword) {
3672                         data->synth_event_name = saved_param;
3673                         first_param = false;
3674                         continue;
3675                 }
3676                 first_param = false;
3677
3678                 data->params[data->n_params++] = saved_param;
3679         }
3680  out:
3681         return ret;
3682 }
3683
3684 static int action_parse(struct trace_array *tr, char *str, struct action_data *data,
3685                         enum handler_id handler)
3686 {
3687         char *action_name;
3688         int ret = 0;
3689
3690         strsep(&str, ".");
3691         if (!str) {
3692                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3693                 ret = -EINVAL;
3694                 goto out;
3695         }
3696
3697         action_name = strsep(&str, "(");
3698         if (!action_name || !str) {
3699                 hist_err(tr, HIST_ERR_ACTION_NOT_FOUND, 0);
3700                 ret = -EINVAL;
3701                 goto out;
3702         }
3703
3704         if (str_has_prefix(action_name, "save")) {
3705                 char *params = strsep(&str, ")");
3706
3707                 if (!params) {
3708                         hist_err(tr, HIST_ERR_NO_SAVE_PARAMS, 0);
3709                         ret = -EINVAL;
3710                         goto out;
3711                 }
3712
3713                 ret = parse_action_params(tr, params, data);
3714                 if (ret)
3715                         goto out;
3716
3717                 if (handler == HANDLER_ONMAX)
3718                         data->track_data.check_val = check_track_val_max;
3719                 else if (handler == HANDLER_ONCHANGE)
3720                         data->track_data.check_val = check_track_val_changed;
3721                 else {
3722                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3723                         ret = -EINVAL;
3724                         goto out;
3725                 }
3726
3727                 data->track_data.save_data = save_track_data_vars;
3728                 data->fn = ontrack_action;
3729                 data->action = ACTION_SAVE;
3730         } else if (str_has_prefix(action_name, "snapshot")) {
3731                 char *params = strsep(&str, ")");
3732
3733                 if (!str) {
3734                         hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(params));
3735                         ret = -EINVAL;
3736                         goto out;
3737                 }
3738
3739                 if (handler == HANDLER_ONMAX)
3740                         data->track_data.check_val = check_track_val_max;
3741                 else if (handler == HANDLER_ONCHANGE)
3742                         data->track_data.check_val = check_track_val_changed;
3743                 else {
3744                         hist_err(tr, HIST_ERR_ACTION_MISMATCH, errpos(action_name));
3745                         ret = -EINVAL;
3746                         goto out;
3747                 }
3748
3749                 data->track_data.save_data = save_track_data_snapshot;
3750                 data->fn = ontrack_action;
3751                 data->action = ACTION_SNAPSHOT;
3752         } else {
3753                 char *params = strsep(&str, ")");
3754
3755                 if (str_has_prefix(action_name, "trace"))
3756                         data->use_trace_keyword = true;
3757
3758                 if (params) {
3759                         ret = parse_action_params(tr, params, data);
3760                         if (ret)
3761                                 goto out;
3762                 }
3763
3764                 if (handler == HANDLER_ONMAX)
3765                         data->track_data.check_val = check_track_val_max;
3766                 else if (handler == HANDLER_ONCHANGE)
3767                         data->track_data.check_val = check_track_val_changed;
3768
3769                 if (handler != HANDLER_ONMATCH) {
3770                         data->track_data.save_data = action_trace;
3771                         data->fn = ontrack_action;
3772                 } else
3773                         data->fn = action_trace;
3774
3775                 data->action = ACTION_TRACE;
3776         }
3777
3778         data->action_name = kstrdup(action_name, GFP_KERNEL);
3779         if (!data->action_name) {
3780                 ret = -ENOMEM;
3781                 goto out;
3782         }
3783
3784         data->handler = handler;
3785  out:
3786         return ret;
3787 }
3788
3789 static struct action_data *track_data_parse(struct hist_trigger_data *hist_data,
3790                                             char *str, enum handler_id handler)
3791 {
3792         struct action_data *data;
3793         int ret = -EINVAL;
3794         char *var_str;
3795
3796         data = kzalloc(sizeof(*data), GFP_KERNEL);
3797         if (!data)
3798                 return ERR_PTR(-ENOMEM);
3799
3800         var_str = strsep(&str, ")");
3801         if (!var_str || !str) {
3802                 ret = -EINVAL;
3803                 goto free;
3804         }
3805
3806         data->track_data.var_str = kstrdup(var_str, GFP_KERNEL);
3807         if (!data->track_data.var_str) {
3808                 ret = -ENOMEM;
3809                 goto free;
3810         }
3811
3812         ret = action_parse(hist_data->event_file->tr, str, data, handler);
3813         if (ret)
3814                 goto free;
3815  out:
3816         return data;
3817  free:
3818         track_data_destroy(hist_data, data);
3819         data = ERR_PTR(ret);
3820         goto out;
3821 }
3822
3823 static void onmatch_destroy(struct action_data *data)
3824 {
3825         kfree(data->match_data.event);
3826         kfree(data->match_data.event_system);
3827
3828         action_data_destroy(data);
3829 }
3830
3831 static void destroy_field_var(struct field_var *field_var)
3832 {
3833         if (!field_var)
3834                 return;
3835
3836         destroy_hist_field(field_var->var, 0);
3837         destroy_hist_field(field_var->val, 0);
3838
3839         kfree(field_var);
3840 }
3841
3842 static void destroy_field_vars(struct hist_trigger_data *hist_data)
3843 {
3844         unsigned int i;
3845
3846         for (i = 0; i < hist_data->n_field_vars; i++)
3847                 destroy_field_var(hist_data->field_vars[i]);
3848
3849         for (i = 0; i < hist_data->n_save_vars; i++)
3850                 destroy_field_var(hist_data->save_vars[i]);
3851 }
3852
3853 static void save_field_var(struct hist_trigger_data *hist_data,
3854                            struct field_var *field_var)
3855 {
3856         hist_data->field_vars[hist_data->n_field_vars++] = field_var;
3857
3858         /* Stack traces are saved in the string storage too */
3859         if (field_var->val->flags & (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
3860                 hist_data->n_field_var_str++;
3861 }
3862
3863
3864 static int check_synth_field(struct synth_event *event,
3865                              struct hist_field *hist_field,
3866                              unsigned int field_pos)
3867 {
3868         struct synth_field *field;
3869
3870         if (field_pos >= event->n_fields)
3871                 return -EINVAL;
3872
3873         field = event->fields[field_pos];
3874
3875         /*
3876          * A dynamic string synth field can accept static or
3877          * dynamic. A static string synth field can only accept a
3878          * same-sized static string, which is checked for later.
3879          */
3880         if (strstr(hist_field->type, "char[") && field->is_string
3881             && field->is_dynamic)
3882                 return 0;
3883
3884         if (strstr(hist_field->type, "long[") && field->is_stack)
3885                 return 0;
3886
3887         if (strcmp(field->type, hist_field->type) != 0) {
3888                 if (field->size != hist_field->size ||
3889                     (!field->is_string && field->is_signed != hist_field->is_signed))
3890                         return -EINVAL;
3891         }
3892
3893         return 0;
3894 }
3895
3896 static struct hist_field *
3897 trace_action_find_var(struct hist_trigger_data *hist_data,
3898                       struct action_data *data,
3899                       char *system, char *event, char *var)
3900 {
3901         struct trace_array *tr = hist_data->event_file->tr;
3902         struct hist_field *hist_field;
3903
3904         var++; /* skip '$' */
3905
3906         hist_field = find_target_event_var(hist_data, system, event, var);
3907         if (!hist_field) {
3908                 if (!system && data->handler == HANDLER_ONMATCH) {
3909                         system = data->match_data.event_system;
3910                         event = data->match_data.event;
3911                 }
3912
3913                 hist_field = find_event_var(hist_data, system, event, var);
3914         }
3915
3916         if (!hist_field)
3917                 hist_err(tr, HIST_ERR_PARAM_NOT_FOUND, errpos(var));
3918
3919         return hist_field;
3920 }
3921
3922 static struct hist_field *
3923 trace_action_create_field_var(struct hist_trigger_data *hist_data,
3924                               struct action_data *data, char *system,
3925                               char *event, char *var)
3926 {
3927         struct hist_field *hist_field = NULL;
3928         struct field_var *field_var;
3929
3930         /*
3931          * First try to create a field var on the target event (the
3932          * currently being defined).  This will create a variable for
3933          * unqualified fields on the target event, or if qualified,
3934          * target fields that have qualified names matching the target.
3935          */
3936         field_var = create_target_field_var(hist_data, system, event, var);
3937
3938         if (field_var && !IS_ERR(field_var)) {
3939                 save_field_var(hist_data, field_var);
3940                 hist_field = field_var->var;
3941         } else {
3942                 field_var = NULL;
3943                 /*
3944                  * If no explicit system.event is specified, default to
3945                  * looking for fields on the onmatch(system.event.xxx)
3946                  * event.
3947                  */
3948                 if (!system && data->handler == HANDLER_ONMATCH) {
3949                         system = data->match_data.event_system;
3950                         event = data->match_data.event;
3951                 }
3952
3953                 if (!event)
3954                         goto free;
3955                 /*
3956                  * At this point, we're looking at a field on another
3957                  * event.  Because we can't modify a hist trigger on
3958                  * another event to add a variable for a field, we need
3959                  * to create a new trigger on that event and create the
3960                  * variable at the same time.
3961                  */
3962                 hist_field = create_field_var_hist(hist_data, system, event, var);
3963                 if (IS_ERR(hist_field))
3964                         goto free;
3965         }
3966  out:
3967         return hist_field;
3968  free:
3969         destroy_field_var(field_var);
3970         hist_field = NULL;
3971         goto out;
3972 }
3973
3974 static int trace_action_create(struct hist_trigger_data *hist_data,
3975                                struct action_data *data)
3976 {
3977         struct trace_array *tr = hist_data->event_file->tr;
3978         char *event_name, *param, *system = NULL;
3979         struct hist_field *hist_field, *var_ref;
3980         unsigned int i;
3981         unsigned int field_pos = 0;
3982         struct synth_event *event;
3983         char *synth_event_name;
3984         int var_ref_idx, ret = 0;
3985
3986         lockdep_assert_held(&event_mutex);
3987
3988         /* Sanity check to avoid out-of-bound write on 'data->var_ref_idx' */
3989         if (data->n_params > SYNTH_FIELDS_MAX)
3990                 return -EINVAL;
3991
3992         if (data->use_trace_keyword)
3993                 synth_event_name = data->synth_event_name;
3994         else
3995                 synth_event_name = data->action_name;
3996
3997         event = find_synth_event(synth_event_name);
3998         if (!event) {
3999                 hist_err(tr, HIST_ERR_SYNTH_EVENT_NOT_FOUND, errpos(synth_event_name));
4000                 return -EINVAL;
4001         }
4002
4003         event->ref++;
4004
4005         for (i = 0; i < data->n_params; i++) {
4006                 char *p;
4007
4008                 p = param = kstrdup(data->params[i], GFP_KERNEL);
4009                 if (!param) {
4010                         ret = -ENOMEM;
4011                         goto err;
4012                 }
4013
4014                 system = strsep(&param, ".");
4015                 if (!param) {
4016                         param = (char *)system;
4017                         system = event_name = NULL;
4018                 } else {
4019                         event_name = strsep(&param, ".");
4020                         if (!param) {
4021                                 kfree(p);
4022                                 ret = -EINVAL;
4023                                 goto err;
4024                         }
4025                 }
4026
4027                 if (param[0] == '$')
4028                         hist_field = trace_action_find_var(hist_data, data,
4029                                                            system, event_name,
4030                                                            param);
4031                 else
4032                         hist_field = trace_action_create_field_var(hist_data,
4033                                                                    data,
4034                                                                    system,
4035                                                                    event_name,
4036                                                                    param);
4037
4038                 if (!hist_field) {
4039                         kfree(p);
4040                         ret = -EINVAL;
4041                         goto err;
4042                 }
4043
4044                 if (check_synth_field(event, hist_field, field_pos) == 0) {
4045                         var_ref = create_var_ref(hist_data, hist_field,
4046                                                  system, event_name);
4047                         if (!var_ref) {
4048                                 kfree(p);
4049                                 ret = -ENOMEM;
4050                                 goto err;
4051                         }
4052
4053                         var_ref_idx = find_var_ref_idx(hist_data, var_ref);
4054                         if (WARN_ON(var_ref_idx < 0)) {
4055                                 kfree(p);
4056                                 ret = var_ref_idx;
4057                                 goto err;
4058                         }
4059
4060                         data->var_ref_idx[i] = var_ref_idx;
4061
4062                         field_pos++;
4063                         kfree(p);
4064                         continue;
4065                 }
4066
4067                 hist_err(tr, HIST_ERR_SYNTH_TYPE_MISMATCH, errpos(param));
4068                 kfree(p);
4069                 ret = -EINVAL;
4070                 goto err;
4071         }
4072
4073         if (field_pos != event->n_fields) {
4074                 hist_err(tr, HIST_ERR_SYNTH_COUNT_MISMATCH, errpos(event->name));
4075                 ret = -EINVAL;
4076                 goto err;
4077         }
4078
4079         data->synth_event = event;
4080  out:
4081         return ret;
4082  err:
4083         event->ref--;
4084
4085         goto out;
4086 }
4087
4088 static int action_create(struct hist_trigger_data *hist_data,
4089                          struct action_data *data)
4090 {
4091         struct trace_event_file *file = hist_data->event_file;
4092         struct trace_array *tr = file->tr;
4093         struct track_data *track_data;
4094         struct field_var *field_var;
4095         unsigned int i;
4096         char *param;
4097         int ret = 0;
4098
4099         if (data->action == ACTION_TRACE)
4100                 return trace_action_create(hist_data, data);
4101
4102         if (data->action == ACTION_SNAPSHOT) {
4103                 track_data = track_data_alloc(hist_data->key_size, data, hist_data);
4104                 if (IS_ERR(track_data)) {
4105                         ret = PTR_ERR(track_data);
4106                         goto out;
4107                 }
4108
4109                 ret = tracing_snapshot_cond_enable(file->tr, track_data,
4110                                                    cond_snapshot_update);
4111                 if (ret)
4112                         track_data_free(track_data);
4113
4114                 goto out;
4115         }
4116
4117         if (data->action == ACTION_SAVE) {
4118                 if (hist_data->n_save_vars) {
4119                         ret = -EEXIST;
4120                         hist_err(tr, HIST_ERR_TOO_MANY_SAVE_ACTIONS, 0);
4121                         goto out;
4122                 }
4123
4124                 for (i = 0; i < data->n_params; i++) {
4125                         param = kstrdup(data->params[i], GFP_KERNEL);
4126                         if (!param) {
4127                                 ret = -ENOMEM;
4128                                 goto out;
4129                         }
4130
4131                         field_var = create_target_field_var(hist_data, NULL, NULL, param);
4132                         if (IS_ERR(field_var)) {
4133                                 hist_err(tr, HIST_ERR_FIELD_VAR_CREATE_FAIL,
4134                                          errpos(param));
4135                                 ret = PTR_ERR(field_var);
4136                                 kfree(param);
4137                                 goto out;
4138                         }
4139
4140                         hist_data->save_vars[hist_data->n_save_vars++] = field_var;
4141                         if (field_var->val->flags &
4142                             (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
4143                                 hist_data->n_save_var_str++;
4144                         kfree(param);
4145                 }
4146         }
4147  out:
4148         return ret;
4149 }
4150
4151 static int onmatch_create(struct hist_trigger_data *hist_data,
4152                           struct action_data *data)
4153 {
4154         return action_create(hist_data, data);
4155 }
4156
4157 static struct action_data *onmatch_parse(struct trace_array *tr, char *str)
4158 {
4159         char *match_event, *match_event_system;
4160         struct action_data *data;
4161         int ret = -EINVAL;
4162
4163         data = kzalloc(sizeof(*data), GFP_KERNEL);
4164         if (!data)
4165                 return ERR_PTR(-ENOMEM);
4166
4167         match_event = strsep(&str, ")");
4168         if (!match_event || !str) {
4169                 hist_err(tr, HIST_ERR_NO_CLOSING_PAREN, errpos(match_event));
4170                 goto free;
4171         }
4172
4173         match_event_system = strsep(&match_event, ".");
4174         if (!match_event) {
4175                 hist_err(tr, HIST_ERR_SUBSYS_NOT_FOUND, errpos(match_event_system));
4176                 goto free;
4177         }
4178
4179         if (IS_ERR(event_file(tr, match_event_system, match_event))) {
4180                 hist_err(tr, HIST_ERR_INVALID_SUBSYS_EVENT, errpos(match_event));
4181                 goto free;
4182         }
4183
4184         data->match_data.event = kstrdup(match_event, GFP_KERNEL);
4185         if (!data->match_data.event) {
4186                 ret = -ENOMEM;
4187                 goto free;
4188         }
4189
4190         data->match_data.event_system = kstrdup(match_event_system, GFP_KERNEL);
4191         if (!data->match_data.event_system) {
4192                 ret = -ENOMEM;
4193                 goto free;
4194         }
4195
4196         ret = action_parse(tr, str, data, HANDLER_ONMATCH);
4197         if (ret)
4198                 goto free;
4199  out:
4200         return data;
4201  free:
4202         onmatch_destroy(data);
4203         data = ERR_PTR(ret);
4204         goto out;
4205 }
4206
4207 static int create_hitcount_val(struct hist_trigger_data *hist_data)
4208 {
4209         hist_data->fields[HITCOUNT_IDX] =
4210                 create_hist_field(hist_data, NULL, HIST_FIELD_FL_HITCOUNT, NULL);
4211         if (!hist_data->fields[HITCOUNT_IDX])
4212                 return -ENOMEM;
4213
4214         hist_data->n_vals++;
4215         hist_data->n_fields++;
4216
4217         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
4218                 return -EINVAL;
4219
4220         return 0;
4221 }
4222
4223 static int __create_val_field(struct hist_trigger_data *hist_data,
4224                               unsigned int val_idx,
4225                               struct trace_event_file *file,
4226                               char *var_name, char *field_str,
4227                               unsigned long flags)
4228 {
4229         struct hist_field *hist_field;
4230         int ret = 0, n_subexprs = 0;
4231
4232         hist_field = parse_expr(hist_data, file, field_str, flags, var_name, &n_subexprs);
4233         if (IS_ERR(hist_field)) {
4234                 ret = PTR_ERR(hist_field);
4235                 goto out;
4236         }
4237
4238         /* values and variables should not have some modifiers */
4239         if (hist_field->flags & HIST_FIELD_FL_VAR) {
4240                 /* Variable */
4241                 if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT |
4242                                          HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2))
4243                         goto err;
4244         } else {
4245                 /* Value */
4246                 if (hist_field->flags & (HIST_FIELD_FL_GRAPH | HIST_FIELD_FL_PERCENT |
4247                                          HIST_FIELD_FL_BUCKET | HIST_FIELD_FL_LOG2 |
4248                                          HIST_FIELD_FL_SYM | HIST_FIELD_FL_SYM_OFFSET |
4249                                          HIST_FIELD_FL_SYSCALL | HIST_FIELD_FL_STACKTRACE))
4250                         goto err;
4251         }
4252
4253         hist_data->fields[val_idx] = hist_field;
4254
4255         ++hist_data->n_vals;
4256         ++hist_data->n_fields;
4257
4258         if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4259                 ret = -EINVAL;
4260  out:
4261         return ret;
4262  err:
4263         hist_err(file->tr, HIST_ERR_BAD_FIELD_MODIFIER, errpos(field_str));
4264         return -EINVAL;
4265 }
4266
4267 static int create_val_field(struct hist_trigger_data *hist_data,
4268                             unsigned int val_idx,
4269                             struct trace_event_file *file,
4270                             char *field_str)
4271 {
4272         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
4273                 return -EINVAL;
4274
4275         return __create_val_field(hist_data, val_idx, file, NULL, field_str, 0);
4276 }
4277
4278 static const char no_comm[] = "(no comm)";
4279
4280 static u64 hist_field_execname(struct hist_field *hist_field,
4281                                struct tracing_map_elt *elt,
4282                                struct trace_buffer *buffer,
4283                                struct ring_buffer_event *rbe,
4284                                void *event)
4285 {
4286         struct hist_elt_data *elt_data;
4287
4288         if (WARN_ON_ONCE(!elt))
4289                 return (u64)(unsigned long)no_comm;
4290
4291         elt_data = elt->private_data;
4292
4293         if (WARN_ON_ONCE(!elt_data->comm))
4294                 return (u64)(unsigned long)no_comm;
4295
4296         return (u64)(unsigned long)(elt_data->comm);
4297 }
4298
4299 static u64 hist_field_stack(struct hist_field *hist_field,
4300                             struct tracing_map_elt *elt,
4301                             struct trace_buffer *buffer,
4302                             struct ring_buffer_event *rbe,
4303                             void *event)
4304 {
4305         u32 str_item = *(u32 *)(event + hist_field->field->offset);
4306         int str_loc = str_item & 0xffff;
4307         char *addr = (char *)(event + str_loc);
4308
4309         return (u64)(unsigned long)addr;
4310 }
4311
4312 static u64 hist_fn_call(struct hist_field *hist_field,
4313                         struct tracing_map_elt *elt,
4314                         struct trace_buffer *buffer,
4315                         struct ring_buffer_event *rbe,
4316                         void *event)
4317 {
4318         switch (hist_field->fn_num) {
4319         case HIST_FIELD_FN_VAR_REF:
4320                 return hist_field_var_ref(hist_field, elt, buffer, rbe, event);
4321         case HIST_FIELD_FN_COUNTER:
4322                 return hist_field_counter(hist_field, elt, buffer, rbe, event);
4323         case HIST_FIELD_FN_CONST:
4324                 return hist_field_const(hist_field, elt, buffer, rbe, event);
4325         case HIST_FIELD_FN_LOG2:
4326                 return hist_field_log2(hist_field, elt, buffer, rbe, event);
4327         case HIST_FIELD_FN_BUCKET:
4328                 return hist_field_bucket(hist_field, elt, buffer, rbe, event);
4329         case HIST_FIELD_FN_TIMESTAMP:
4330                 return hist_field_timestamp(hist_field, elt, buffer, rbe, event);
4331         case HIST_FIELD_FN_CPU:
4332                 return hist_field_cpu(hist_field, elt, buffer, rbe, event);
4333         case HIST_FIELD_FN_STRING:
4334                 return hist_field_string(hist_field, elt, buffer, rbe, event);
4335         case HIST_FIELD_FN_DYNSTRING:
4336                 return hist_field_dynstring(hist_field, elt, buffer, rbe, event);
4337         case HIST_FIELD_FN_RELDYNSTRING:
4338                 return hist_field_reldynstring(hist_field, elt, buffer, rbe, event);
4339         case HIST_FIELD_FN_PSTRING:
4340                 return hist_field_pstring(hist_field, elt, buffer, rbe, event);
4341         case HIST_FIELD_FN_S64:
4342                 return hist_field_s64(hist_field, elt, buffer, rbe, event);
4343         case HIST_FIELD_FN_U64:
4344                 return hist_field_u64(hist_field, elt, buffer, rbe, event);
4345         case HIST_FIELD_FN_S32:
4346                 return hist_field_s32(hist_field, elt, buffer, rbe, event);
4347         case HIST_FIELD_FN_U32:
4348                 return hist_field_u32(hist_field, elt, buffer, rbe, event);
4349         case HIST_FIELD_FN_S16:
4350                 return hist_field_s16(hist_field, elt, buffer, rbe, event);
4351         case HIST_FIELD_FN_U16:
4352                 return hist_field_u16(hist_field, elt, buffer, rbe, event);
4353         case HIST_FIELD_FN_S8:
4354                 return hist_field_s8(hist_field, elt, buffer, rbe, event);
4355         case HIST_FIELD_FN_U8:
4356                 return hist_field_u8(hist_field, elt, buffer, rbe, event);
4357         case HIST_FIELD_FN_UMINUS:
4358                 return hist_field_unary_minus(hist_field, elt, buffer, rbe, event);
4359         case HIST_FIELD_FN_MINUS:
4360                 return hist_field_minus(hist_field, elt, buffer, rbe, event);
4361         case HIST_FIELD_FN_PLUS:
4362                 return hist_field_plus(hist_field, elt, buffer, rbe, event);
4363         case HIST_FIELD_FN_DIV:
4364                 return hist_field_div(hist_field, elt, buffer, rbe, event);
4365         case HIST_FIELD_FN_MULT:
4366                 return hist_field_mult(hist_field, elt, buffer, rbe, event);
4367         case HIST_FIELD_FN_DIV_POWER2:
4368                 return div_by_power_of_two(hist_field, elt, buffer, rbe, event);
4369         case HIST_FIELD_FN_DIV_NOT_POWER2:
4370                 return div_by_not_power_of_two(hist_field, elt, buffer, rbe, event);
4371         case HIST_FIELD_FN_DIV_MULT_SHIFT:
4372                 return div_by_mult_and_shift(hist_field, elt, buffer, rbe, event);
4373         case HIST_FIELD_FN_EXECNAME:
4374                 return hist_field_execname(hist_field, elt, buffer, rbe, event);
4375         case HIST_FIELD_FN_STACK:
4376                 return hist_field_stack(hist_field, elt, buffer, rbe, event);
4377         default:
4378                 return 0;
4379         }
4380 }
4381
4382 /* Convert a var that points to common_pid.execname to a string */
4383 static void update_var_execname(struct hist_field *hist_field)
4384 {
4385         hist_field->flags = HIST_FIELD_FL_STRING | HIST_FIELD_FL_VAR |
4386                 HIST_FIELD_FL_EXECNAME;
4387         hist_field->size = MAX_FILTER_STR_VAL;
4388         hist_field->is_signed = 0;
4389
4390         kfree_const(hist_field->type);
4391         hist_field->type = "char[]";
4392
4393         hist_field->fn_num = HIST_FIELD_FN_EXECNAME;
4394 }
4395
4396 static int create_var_field(struct hist_trigger_data *hist_data,
4397                             unsigned int val_idx,
4398                             struct trace_event_file *file,
4399                             char *var_name, char *expr_str)
4400 {
4401         struct trace_array *tr = hist_data->event_file->tr;
4402         unsigned long flags = 0;
4403         int ret;
4404
4405         if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX + TRACING_MAP_VARS_MAX))
4406                 return -EINVAL;
4407
4408         if (find_var(hist_data, file, var_name) && !hist_data->remove) {
4409                 hist_err(tr, HIST_ERR_DUPLICATE_VAR, errpos(var_name));
4410                 return -EINVAL;
4411         }
4412
4413         flags |= HIST_FIELD_FL_VAR;
4414         hist_data->n_vars++;
4415         if (WARN_ON(hist_data->n_vars > TRACING_MAP_VARS_MAX))
4416                 return -EINVAL;
4417
4418         ret = __create_val_field(hist_data, val_idx, file, var_name, expr_str, flags);
4419
4420         if (!ret && hist_data->fields[val_idx]->flags & HIST_FIELD_FL_EXECNAME)
4421                 update_var_execname(hist_data->fields[val_idx]);
4422
4423         if (!ret && hist_data->fields[val_idx]->flags &
4424             (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE))
4425                 hist_data->fields[val_idx]->var_str_idx = hist_data->n_var_str++;
4426
4427         return ret;
4428 }
4429
4430 static int create_val_fields(struct hist_trigger_data *hist_data,
4431                              struct trace_event_file *file)
4432 {
4433         unsigned int i, j = 1, n_hitcount = 0;
4434         char *fields_str, *field_str;
4435         int ret;
4436
4437         ret = create_hitcount_val(hist_data);
4438         if (ret)
4439                 goto out;
4440
4441         fields_str = hist_data->attrs->vals_str;
4442         if (!fields_str)
4443                 goto out;
4444
4445         for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
4446                      j < TRACING_MAP_VALS_MAX; i++) {
4447                 field_str = strsep(&fields_str, ",");
4448                 if (!field_str)
4449                         break;
4450
4451                 if (strcmp(field_str, "hitcount") == 0) {
4452                         if (!n_hitcount++)
4453                                 continue;
4454                 }
4455
4456                 ret = create_val_field(hist_data, j++, file, field_str);
4457                 if (ret)
4458                         goto out;
4459         }
4460
4461         if (fields_str && (strcmp(fields_str, "hitcount") != 0))
4462                 ret = -EINVAL;
4463  out:
4464         /* There is only raw hitcount but nohitcount suppresses it. */
4465         if (j == 1 && hist_data->attrs->no_hitcount) {
4466                 hist_err(hist_data->event_file->tr, HIST_ERR_NEED_NOHC_VAL, 0);
4467                 ret = -ENOENT;
4468         }
4469
4470         return ret;
4471 }
4472
4473 static int create_key_field(struct hist_trigger_data *hist_data,
4474                             unsigned int key_idx,
4475                             unsigned int key_offset,
4476                             struct trace_event_file *file,
4477                             char *field_str)
4478 {
4479         struct trace_array *tr = hist_data->event_file->tr;
4480         struct hist_field *hist_field = NULL;
4481         unsigned long flags = 0;
4482         unsigned int key_size;
4483         int ret = 0, n_subexprs = 0;
4484
4485         if (WARN_ON(key_idx >= HIST_FIELDS_MAX))
4486                 return -EINVAL;
4487
4488         flags |= HIST_FIELD_FL_KEY;
4489
4490         if (strcmp(field_str, "stacktrace") == 0) {
4491                 flags |= HIST_FIELD_FL_STACKTRACE;
4492                 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
4493                 hist_field = create_hist_field(hist_data, NULL, flags, NULL);
4494         } else {
4495                 hist_field = parse_expr(hist_data, file, field_str, flags,
4496                                         NULL, &n_subexprs);
4497                 if (IS_ERR(hist_field)) {
4498                         ret = PTR_ERR(hist_field);
4499                         goto out;
4500                 }
4501
4502                 if (field_has_hist_vars(hist_field, 0)) {
4503                         hist_err(tr, HIST_ERR_INVALID_REF_KEY, errpos(field_str));
4504                         destroy_hist_field(hist_field, 0);
4505                         ret = -EINVAL;
4506                         goto out;
4507                 }
4508
4509                 key_size = hist_field->size;
4510         }
4511
4512         hist_data->fields[key_idx] = hist_field;
4513
4514         key_size = ALIGN(key_size, sizeof(u64));
4515         hist_data->fields[key_idx]->size = key_size;
4516         hist_data->fields[key_idx]->offset = key_offset;
4517
4518         hist_data->key_size += key_size;
4519
4520         if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
4521                 ret = -EINVAL;
4522                 goto out;
4523         }
4524
4525         hist_data->n_keys++;
4526         hist_data->n_fields++;
4527
4528         if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
4529                 return -EINVAL;
4530
4531         ret = key_size;
4532  out:
4533         return ret;
4534 }
4535
4536 static int create_key_fields(struct hist_trigger_data *hist_data,
4537                              struct trace_event_file *file)
4538 {
4539         unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
4540         char *fields_str, *field_str;
4541         int ret = -EINVAL;
4542
4543         fields_str = hist_data->attrs->keys_str;
4544         if (!fields_str)
4545                 goto out;
4546
4547         for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
4548                 field_str = strsep(&fields_str, ",");
4549                 if (!field_str)
4550                         break;
4551                 ret = create_key_field(hist_data, i, key_offset,
4552                                        file, field_str);
4553                 if (ret < 0)
4554                         goto out;
4555                 key_offset += ret;
4556         }
4557         if (fields_str) {
4558                 ret = -EINVAL;
4559                 goto out;
4560         }
4561         ret = 0;
4562  out:
4563         return ret;
4564 }
4565
4566 static int create_var_fields(struct hist_trigger_data *hist_data,
4567                              struct trace_event_file *file)
4568 {
4569         unsigned int i, j = hist_data->n_vals;
4570         int ret = 0;
4571
4572         unsigned int n_vars = hist_data->attrs->var_defs.n_vars;
4573
4574         for (i = 0; i < n_vars; i++) {
4575                 char *var_name = hist_data->attrs->var_defs.name[i];
4576                 char *expr = hist_data->attrs->var_defs.expr[i];
4577
4578                 ret = create_var_field(hist_data, j++, file, var_name, expr);
4579                 if (ret)
4580                         goto out;
4581         }
4582  out:
4583         return ret;
4584 }
4585
4586 static void free_var_defs(struct hist_trigger_data *hist_data)
4587 {
4588         unsigned int i;
4589
4590         for (i = 0; i < hist_data->attrs->var_defs.n_vars; i++) {
4591                 kfree(hist_data->attrs->var_defs.name[i]);
4592                 kfree(hist_data->attrs->var_defs.expr[i]);
4593         }
4594
4595         hist_data->attrs->var_defs.n_vars = 0;
4596 }
4597
4598 static int parse_var_defs(struct hist_trigger_data *hist_data)
4599 {
4600         struct trace_array *tr = hist_data->event_file->tr;
4601         char *s, *str, *var_name, *field_str;
4602         unsigned int i, j, n_vars = 0;
4603         int ret = 0;
4604
4605         for (i = 0; i < hist_data->attrs->n_assignments; i++) {
4606                 str = hist_data->attrs->assignment_str[i];
4607                 for (j = 0; j < TRACING_MAP_VARS_MAX; j++) {
4608                         field_str = strsep(&str, ",");
4609                         if (!field_str)
4610                                 break;
4611
4612                         var_name = strsep(&field_str, "=");
4613                         if (!var_name || !field_str) {
4614                                 hist_err(tr, HIST_ERR_MALFORMED_ASSIGNMENT,
4615                                          errpos(var_name));
4616                                 ret = -EINVAL;
4617                                 goto free;
4618                         }
4619
4620                         if (n_vars == TRACING_MAP_VARS_MAX) {
4621                                 hist_err(tr, HIST_ERR_TOO_MANY_VARS, errpos(var_name));
4622                                 ret = -EINVAL;
4623                                 goto free;
4624                         }
4625
4626                         s = kstrdup(var_name, GFP_KERNEL);
4627                         if (!s) {
4628                                 ret = -ENOMEM;
4629                                 goto free;
4630                         }
4631                         hist_data->attrs->var_defs.name[n_vars] = s;
4632
4633                         s = kstrdup(field_str, GFP_KERNEL);
4634                         if (!s) {
4635                                 kfree(hist_data->attrs->var_defs.name[n_vars]);
4636                                 hist_data->attrs->var_defs.name[n_vars] = NULL;
4637                                 ret = -ENOMEM;
4638                                 goto free;
4639                         }
4640                         hist_data->attrs->var_defs.expr[n_vars++] = s;
4641
4642                         hist_data->attrs->var_defs.n_vars = n_vars;
4643                 }
4644         }
4645
4646         return ret;
4647  free:
4648         free_var_defs(hist_data);
4649
4650         return ret;
4651 }
4652
4653 static int create_hist_fields(struct hist_trigger_data *hist_data,
4654                               struct trace_event_file *file)
4655 {
4656         int ret;
4657
4658         ret = parse_var_defs(hist_data);
4659         if (ret)
4660                 return ret;
4661
4662         ret = create_val_fields(hist_data, file);
4663         if (ret)
4664                 goto out;
4665
4666         ret = create_var_fields(hist_data, file);
4667         if (ret)
4668                 goto out;
4669
4670         ret = create_key_fields(hist_data, file);
4671
4672  out:
4673         free_var_defs(hist_data);
4674
4675         return ret;
4676 }
4677
4678 static int is_descending(struct trace_array *tr, const char *str)
4679 {
4680         if (!str)
4681                 return 0;
4682
4683         if (strcmp(str, "descending") == 0)
4684                 return 1;
4685
4686         if (strcmp(str, "ascending") == 0)
4687                 return 0;
4688
4689         hist_err(tr, HIST_ERR_INVALID_SORT_MODIFIER, errpos((char *)str));
4690
4691         return -EINVAL;
4692 }
4693
4694 static int create_sort_keys(struct hist_trigger_data *hist_data)
4695 {
4696         struct trace_array *tr = hist_data->event_file->tr;
4697         char *fields_str = hist_data->attrs->sort_key_str;
4698         struct tracing_map_sort_key *sort_key;
4699         int descending, ret = 0;
4700         unsigned int i, j, k;
4701
4702         hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
4703
4704         if (!fields_str)
4705                 goto out;
4706
4707         for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
4708                 struct hist_field *hist_field;
4709                 char *field_str, *field_name;
4710                 const char *test_name;
4711
4712                 sort_key = &hist_data->sort_keys[i];
4713
4714                 field_str = strsep(&fields_str, ",");
4715                 if (!field_str)
4716                         break;
4717
4718                 if (!*field_str) {
4719                         ret = -EINVAL;
4720                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4721                         break;
4722                 }
4723
4724                 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
4725                         hist_err(tr, HIST_ERR_TOO_MANY_SORT_FIELDS, errpos("sort="));
4726                         ret = -EINVAL;
4727                         break;
4728                 }
4729
4730                 field_name = strsep(&field_str, ".");
4731                 if (!field_name || !*field_name) {
4732                         ret = -EINVAL;
4733                         hist_err(tr, HIST_ERR_EMPTY_SORT_FIELD, errpos("sort="));
4734                         break;
4735                 }
4736
4737                 if (strcmp(field_name, "hitcount") == 0) {
4738                         descending = is_descending(tr, field_str);
4739                         if (descending < 0) {
4740                                 ret = descending;
4741                                 break;
4742                         }
4743                         sort_key->descending = descending;
4744                         continue;
4745                 }
4746
4747                 for (j = 1, k = 1; j < hist_data->n_fields; j++) {
4748                         unsigned int idx;
4749
4750                         hist_field = hist_data->fields[j];
4751                         if (hist_field->flags & HIST_FIELD_FL_VAR)
4752                                 continue;
4753
4754                         idx = k++;
4755
4756                         test_name = hist_field_name(hist_field, 0);
4757
4758                         if (strcmp(field_name, test_name) == 0) {
4759                                 sort_key->field_idx = idx;
4760                                 descending = is_descending(tr, field_str);
4761                                 if (descending < 0) {
4762                                         ret = descending;
4763                                         goto out;
4764                                 }
4765                                 sort_key->descending = descending;
4766                                 break;
4767                         }
4768                 }
4769                 if (j == hist_data->n_fields) {
4770                         ret = -EINVAL;
4771                         hist_err(tr, HIST_ERR_INVALID_SORT_FIELD, errpos(field_name));
4772                         break;
4773                 }
4774         }
4775
4776         hist_data->n_sort_keys = i;
4777  out:
4778         return ret;
4779 }
4780
4781 static void destroy_actions(struct hist_trigger_data *hist_data)
4782 {
4783         unsigned int i;
4784
4785         for (i = 0; i < hist_data->n_actions; i++) {
4786                 struct action_data *data = hist_data->actions[i];
4787
4788                 if (data->handler == HANDLER_ONMATCH)
4789                         onmatch_destroy(data);
4790                 else if (data->handler == HANDLER_ONMAX ||
4791                          data->handler == HANDLER_ONCHANGE)
4792                         track_data_destroy(hist_data, data);
4793                 else
4794                         kfree(data);
4795         }
4796 }
4797
4798 static int parse_actions(struct hist_trigger_data *hist_data)
4799 {
4800         struct trace_array *tr = hist_data->event_file->tr;
4801         struct action_data *data;
4802         unsigned int i;
4803         int ret = 0;
4804         char *str;
4805         int len;
4806
4807         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4808                 enum handler_id hid = 0;
4809                 char *action_str;
4810
4811                 str = hist_data->attrs->action_str[i];
4812
4813                 if ((len = str_has_prefix(str, "onmatch(")))
4814                         hid = HANDLER_ONMATCH;
4815                 else if ((len = str_has_prefix(str, "onmax(")))
4816                         hid = HANDLER_ONMAX;
4817                 else if ((len = str_has_prefix(str, "onchange(")))
4818                         hid = HANDLER_ONCHANGE;
4819
4820                 action_str = str + len;
4821
4822                 switch (hid) {
4823                 case HANDLER_ONMATCH:
4824                         data = onmatch_parse(tr, action_str);
4825                         break;
4826                 case HANDLER_ONMAX:
4827                 case HANDLER_ONCHANGE:
4828                         data = track_data_parse(hist_data, action_str, hid);
4829                         break;
4830                 default:
4831                         data = ERR_PTR(-EINVAL);
4832                         break;
4833                 }
4834
4835                 if (IS_ERR(data)) {
4836                         ret = PTR_ERR(data);
4837                         break;
4838                 }
4839
4840                 hist_data->actions[hist_data->n_actions++] = data;
4841         }
4842
4843         return ret;
4844 }
4845
4846 static int create_actions(struct hist_trigger_data *hist_data)
4847 {
4848         struct action_data *data;
4849         unsigned int i;
4850         int ret = 0;
4851
4852         for (i = 0; i < hist_data->attrs->n_actions; i++) {
4853                 data = hist_data->actions[i];
4854
4855                 if (data->handler == HANDLER_ONMATCH) {
4856                         ret = onmatch_create(hist_data, data);
4857                         if (ret)
4858                                 break;
4859                 } else if (data->handler == HANDLER_ONMAX ||
4860                            data->handler == HANDLER_ONCHANGE) {
4861                         ret = track_data_create(hist_data, data);
4862                         if (ret)
4863                                 break;
4864                 } else {
4865                         ret = -EINVAL;
4866                         break;
4867                 }
4868         }
4869
4870         return ret;
4871 }
4872
4873 static void print_actions(struct seq_file *m,
4874                           struct hist_trigger_data *hist_data,
4875                           struct tracing_map_elt *elt)
4876 {
4877         unsigned int i;
4878
4879         for (i = 0; i < hist_data->n_actions; i++) {
4880                 struct action_data *data = hist_data->actions[i];
4881
4882                 if (data->action == ACTION_SNAPSHOT)
4883                         continue;
4884
4885                 if (data->handler == HANDLER_ONMAX ||
4886                     data->handler == HANDLER_ONCHANGE)
4887                         track_data_print(m, hist_data, elt, data);
4888         }
4889 }
4890
4891 static void print_action_spec(struct seq_file *m,
4892                               struct hist_trigger_data *hist_data,
4893                               struct action_data *data)
4894 {
4895         unsigned int i;
4896
4897         if (data->action == ACTION_SAVE) {
4898                 for (i = 0; i < hist_data->n_save_vars; i++) {
4899                         seq_printf(m, "%s", hist_data->save_vars[i]->var->var.name);
4900                         if (i < hist_data->n_save_vars - 1)
4901                                 seq_puts(m, ",");
4902                 }
4903         } else if (data->action == ACTION_TRACE) {
4904                 if (data->use_trace_keyword)
4905                         seq_printf(m, "%s", data->synth_event_name);
4906                 for (i = 0; i < data->n_params; i++) {
4907                         if (i || data->use_trace_keyword)
4908                                 seq_puts(m, ",");
4909                         seq_printf(m, "%s", data->params[i]);
4910                 }
4911         }
4912 }
4913
4914 static void print_track_data_spec(struct seq_file *m,
4915                                   struct hist_trigger_data *hist_data,
4916                                   struct action_data *data)
4917 {
4918         if (data->handler == HANDLER_ONMAX)
4919                 seq_puts(m, ":onmax(");
4920         else if (data->handler == HANDLER_ONCHANGE)
4921                 seq_puts(m, ":onchange(");
4922         seq_printf(m, "%s", data->track_data.var_str);
4923         seq_printf(m, ").%s(", data->action_name);
4924
4925         print_action_spec(m, hist_data, data);
4926
4927         seq_puts(m, ")");
4928 }
4929
4930 static void print_onmatch_spec(struct seq_file *m,
4931                                struct hist_trigger_data *hist_data,
4932                                struct action_data *data)
4933 {
4934         seq_printf(m, ":onmatch(%s.%s).", data->match_data.event_system,
4935                    data->match_data.event);
4936
4937         seq_printf(m, "%s(", data->action_name);
4938
4939         print_action_spec(m, hist_data, data);
4940
4941         seq_puts(m, ")");
4942 }
4943
4944 static bool actions_match(struct hist_trigger_data *hist_data,
4945                           struct hist_trigger_data *hist_data_test)
4946 {
4947         unsigned int i, j;
4948
4949         if (hist_data->n_actions != hist_data_test->n_actions)
4950                 return false;
4951
4952         for (i = 0; i < hist_data->n_actions; i++) {
4953                 struct action_data *data = hist_data->actions[i];
4954                 struct action_data *data_test = hist_data_test->actions[i];
4955                 char *action_name, *action_name_test;
4956
4957                 if (data->handler != data_test->handler)
4958                         return false;
4959                 if (data->action != data_test->action)
4960                         return false;
4961
4962                 if (data->n_params != data_test->n_params)
4963                         return false;
4964
4965                 for (j = 0; j < data->n_params; j++) {
4966                         if (strcmp(data->params[j], data_test->params[j]) != 0)
4967                                 return false;
4968                 }
4969
4970                 if (data->use_trace_keyword)
4971                         action_name = data->synth_event_name;
4972                 else
4973                         action_name = data->action_name;
4974
4975                 if (data_test->use_trace_keyword)
4976                         action_name_test = data_test->synth_event_name;
4977                 else
4978                         action_name_test = data_test->action_name;
4979
4980                 if (strcmp(action_name, action_name_test) != 0)
4981                         return false;
4982
4983                 if (data->handler == HANDLER_ONMATCH) {
4984                         if (strcmp(data->match_data.event_system,
4985                                    data_test->match_data.event_system) != 0)
4986                                 return false;
4987                         if (strcmp(data->match_data.event,
4988                                    data_test->match_data.event) != 0)
4989                                 return false;
4990                 } else if (data->handler == HANDLER_ONMAX ||
4991                            data->handler == HANDLER_ONCHANGE) {
4992                         if (strcmp(data->track_data.var_str,
4993                                    data_test->track_data.var_str) != 0)
4994                                 return false;
4995                 }
4996         }
4997
4998         return true;
4999 }
5000
5001
5002 static void print_actions_spec(struct seq_file *m,
5003                                struct hist_trigger_data *hist_data)
5004 {
5005         unsigned int i;
5006
5007         for (i = 0; i < hist_data->n_actions; i++) {
5008                 struct action_data *data = hist_data->actions[i];
5009
5010                 if (data->handler == HANDLER_ONMATCH)
5011                         print_onmatch_spec(m, hist_data, data);
5012                 else if (data->handler == HANDLER_ONMAX ||
5013                          data->handler == HANDLER_ONCHANGE)
5014                         print_track_data_spec(m, hist_data, data);
5015         }
5016 }
5017
5018 static void destroy_field_var_hists(struct hist_trigger_data *hist_data)
5019 {
5020         unsigned int i;
5021
5022         for (i = 0; i < hist_data->n_field_var_hists; i++) {
5023                 kfree(hist_data->field_var_hists[i]->cmd);
5024                 kfree(hist_data->field_var_hists[i]);
5025         }
5026 }
5027
5028 static void destroy_hist_data(struct hist_trigger_data *hist_data)
5029 {
5030         if (!hist_data)
5031                 return;
5032
5033         destroy_hist_trigger_attrs(hist_data->attrs);
5034         destroy_hist_fields(hist_data);
5035         tracing_map_destroy(hist_data->map);
5036
5037         destroy_actions(hist_data);
5038         destroy_field_vars(hist_data);
5039         destroy_field_var_hists(hist_data);
5040
5041         kfree(hist_data);
5042 }
5043
5044 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
5045 {
5046         struct tracing_map *map = hist_data->map;
5047         struct ftrace_event_field *field;
5048         struct hist_field *hist_field;
5049         int i, idx = 0;
5050
5051         for_each_hist_field(i, hist_data) {
5052                 hist_field = hist_data->fields[i];
5053                 if (hist_field->flags & HIST_FIELD_FL_KEY) {
5054                         tracing_map_cmp_fn_t cmp_fn;
5055
5056                         field = hist_field->field;
5057
5058                         if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
5059                                 cmp_fn = tracing_map_cmp_none;
5060                         else if (!field || hist_field->flags & HIST_FIELD_FL_CPU)
5061                                 cmp_fn = tracing_map_cmp_num(hist_field->size,
5062                                                              hist_field->is_signed);
5063                         else if (is_string_field(field))
5064                                 cmp_fn = tracing_map_cmp_string;
5065                         else
5066                                 cmp_fn = tracing_map_cmp_num(field->size,
5067                                                              field->is_signed);
5068                         idx = tracing_map_add_key_field(map,
5069                                                         hist_field->offset,
5070                                                         cmp_fn);
5071                 } else if (!(hist_field->flags & HIST_FIELD_FL_VAR))
5072                         idx = tracing_map_add_sum_field(map);
5073
5074                 if (idx < 0)
5075                         return idx;
5076
5077                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5078                         idx = tracing_map_add_var(map);
5079                         if (idx < 0)
5080                                 return idx;
5081                         hist_field->var.idx = idx;
5082                         hist_field->var.hist_data = hist_data;
5083                 }
5084         }
5085
5086         return 0;
5087 }
5088
5089 static struct hist_trigger_data *
5090 create_hist_data(unsigned int map_bits,
5091                  struct hist_trigger_attrs *attrs,
5092                  struct trace_event_file *file,
5093                  bool remove)
5094 {
5095         const struct tracing_map_ops *map_ops = NULL;
5096         struct hist_trigger_data *hist_data;
5097         int ret = 0;
5098
5099         hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
5100         if (!hist_data)
5101                 return ERR_PTR(-ENOMEM);
5102
5103         hist_data->attrs = attrs;
5104         hist_data->remove = remove;
5105         hist_data->event_file = file;
5106
5107         ret = parse_actions(hist_data);
5108         if (ret)
5109                 goto free;
5110
5111         ret = create_hist_fields(hist_data, file);
5112         if (ret)
5113                 goto free;
5114
5115         ret = create_sort_keys(hist_data);
5116         if (ret)
5117                 goto free;
5118
5119         map_ops = &hist_trigger_elt_data_ops;
5120
5121         hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
5122                                             map_ops, hist_data);
5123         if (IS_ERR(hist_data->map)) {
5124                 ret = PTR_ERR(hist_data->map);
5125                 hist_data->map = NULL;
5126                 goto free;
5127         }
5128
5129         ret = create_tracing_map_fields(hist_data);
5130         if (ret)
5131                 goto free;
5132  out:
5133         return hist_data;
5134  free:
5135         hist_data->attrs = NULL;
5136
5137         destroy_hist_data(hist_data);
5138
5139         hist_data = ERR_PTR(ret);
5140
5141         goto out;
5142 }
5143
5144 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
5145                                     struct tracing_map_elt *elt,
5146                                     struct trace_buffer *buffer, void *rec,
5147                                     struct ring_buffer_event *rbe,
5148                                     u64 *var_ref_vals)
5149 {
5150         struct hist_elt_data *elt_data;
5151         struct hist_field *hist_field;
5152         unsigned int i, var_idx;
5153         u64 hist_val;
5154
5155         elt_data = elt->private_data;
5156         elt_data->var_ref_vals = var_ref_vals;
5157
5158         for_each_hist_val_field(i, hist_data) {
5159                 hist_field = hist_data->fields[i];
5160                 hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5161                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5162                         var_idx = hist_field->var.idx;
5163
5164                         if (hist_field->flags &
5165                             (HIST_FIELD_FL_STRING | HIST_FIELD_FL_STACKTRACE)) {
5166                                 unsigned int str_start, var_str_idx, idx;
5167                                 char *str, *val_str;
5168                                 unsigned int size;
5169
5170                                 str_start = hist_data->n_field_var_str +
5171                                         hist_data->n_save_var_str;
5172                                 var_str_idx = hist_field->var_str_idx;
5173                                 idx = str_start + var_str_idx;
5174
5175                                 str = elt_data->field_var_str[idx];
5176                                 val_str = (char *)(uintptr_t)hist_val;
5177
5178                                 if (hist_field->flags & HIST_FIELD_FL_STRING) {
5179                                         size = min(hist_field->size, STR_VAR_LEN_MAX);
5180                                         strscpy(str, val_str, size);
5181                                 } else {
5182                                         char *stack_start = str + sizeof(unsigned long);
5183                                         int e;
5184
5185                                         e = stack_trace_save((void *)stack_start,
5186                                                              HIST_STACKTRACE_DEPTH,
5187                                                              HIST_STACKTRACE_SKIP);
5188                                         if (e < HIST_STACKTRACE_DEPTH - 1)
5189                                                 ((unsigned long *)stack_start)[e] = 0;
5190                                         *((unsigned long *)str) = e;
5191                                 }
5192                                 hist_val = (u64)(uintptr_t)str;
5193                         }
5194                         tracing_map_set_var(elt, var_idx, hist_val);
5195                         continue;
5196                 }
5197                 tracing_map_update_sum(elt, i, hist_val);
5198         }
5199
5200         for_each_hist_key_field(i, hist_data) {
5201                 hist_field = hist_data->fields[i];
5202                 if (hist_field->flags & HIST_FIELD_FL_VAR) {
5203                         hist_val = hist_fn_call(hist_field, elt, buffer, rbe, rec);
5204                         var_idx = hist_field->var.idx;
5205                         tracing_map_set_var(elt, var_idx, hist_val);
5206                 }
5207         }
5208
5209         update_field_vars(hist_data, elt, buffer, rbe, rec);
5210 }
5211
5212 static inline void add_to_key(char *compound_key, void *key,
5213                               struct hist_field *key_field, void *rec)
5214 {
5215         size_t size = key_field->size;
5216
5217         if (key_field->flags & HIST_FIELD_FL_STRING) {
5218                 struct ftrace_event_field *field;
5219
5220                 field = key_field->field;
5221                 if (field->filter_type == FILTER_DYN_STRING ||
5222                     field->filter_type == FILTER_RDYN_STRING)
5223                         size = *(u32 *)(rec + field->offset) >> 16;
5224                 else if (field->filter_type == FILTER_STATIC_STRING)
5225                         size = field->size;
5226
5227                 /* ensure NULL-termination */
5228                 if (size > key_field->size - 1)
5229                         size = key_field->size - 1;
5230
5231                 strncpy(compound_key + key_field->offset, (char *)key, size);
5232         } else
5233                 memcpy(compound_key + key_field->offset, key, size);
5234 }
5235
5236 static void
5237 hist_trigger_actions(struct hist_trigger_data *hist_data,
5238                      struct tracing_map_elt *elt,
5239                      struct trace_buffer *buffer, void *rec,
5240                      struct ring_buffer_event *rbe, void *key,
5241                      u64 *var_ref_vals)
5242 {
5243         struct action_data *data;
5244         unsigned int i;
5245
5246         for (i = 0; i < hist_data->n_actions; i++) {
5247                 data = hist_data->actions[i];
5248                 data->fn(hist_data, elt, buffer, rec, rbe, key, data, var_ref_vals);
5249         }
5250 }
5251
5252 static void event_hist_trigger(struct event_trigger_data *data,
5253                                struct trace_buffer *buffer, void *rec,
5254                                struct ring_buffer_event *rbe)
5255 {
5256         struct hist_trigger_data *hist_data = data->private_data;
5257         bool use_compound_key = (hist_data->n_keys > 1);
5258         unsigned long entries[HIST_STACKTRACE_DEPTH];
5259         u64 var_ref_vals[TRACING_MAP_VARS_MAX];
5260         char compound_key[HIST_KEY_SIZE_MAX];
5261         struct tracing_map_elt *elt = NULL;
5262         struct hist_field *key_field;
5263         u64 field_contents;
5264         void *key = NULL;
5265         unsigned int i;
5266
5267         if (unlikely(!rbe))
5268                 return;
5269
5270         memset(compound_key, 0, hist_data->key_size);
5271
5272         for_each_hist_key_field(i, hist_data) {
5273                 key_field = hist_data->fields[i];
5274
5275                 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5276                         memset(entries, 0, HIST_STACKTRACE_SIZE);
5277                         if (key_field->field) {
5278                                 unsigned long *stack, n_entries;
5279
5280                                 field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5281                                 stack = (unsigned long *)(long)field_contents;
5282                                 n_entries = *stack;
5283                                 memcpy(entries, ++stack, n_entries * sizeof(unsigned long));
5284                         } else {
5285                                 stack_trace_save(entries, HIST_STACKTRACE_DEPTH,
5286                                                  HIST_STACKTRACE_SKIP);
5287                         }
5288                         key = entries;
5289                 } else {
5290                         field_contents = hist_fn_call(key_field, elt, buffer, rbe, rec);
5291                         if (key_field->flags & HIST_FIELD_FL_STRING) {
5292                                 key = (void *)(unsigned long)field_contents;
5293                                 use_compound_key = true;
5294                         } else
5295                                 key = (void *)&field_contents;
5296                 }
5297
5298                 if (use_compound_key)
5299                         add_to_key(compound_key, key, key_field, rec);
5300         }
5301
5302         if (use_compound_key)
5303                 key = compound_key;
5304
5305         if (hist_data->n_var_refs &&
5306             !resolve_var_refs(hist_data, key, var_ref_vals, false))
5307                 return;
5308
5309         elt = tracing_map_insert(hist_data->map, key);
5310         if (!elt)
5311                 return;
5312
5313         hist_trigger_elt_update(hist_data, elt, buffer, rec, rbe, var_ref_vals);
5314
5315         if (resolve_var_refs(hist_data, key, var_ref_vals, true))
5316                 hist_trigger_actions(hist_data, elt, buffer, rec, rbe, key, var_ref_vals);
5317 }
5318
5319 static void hist_trigger_stacktrace_print(struct seq_file *m,
5320                                           unsigned long *stacktrace_entries,
5321                                           unsigned int max_entries)
5322 {
5323         unsigned int spaces = 8;
5324         unsigned int i;
5325
5326         for (i = 0; i < max_entries; i++) {
5327                 if (!stacktrace_entries[i])
5328                         return;
5329
5330                 seq_printf(m, "%*c", 1 + spaces, ' ');
5331                 seq_printf(m, "%pS\n", (void*)stacktrace_entries[i]);
5332         }
5333 }
5334
5335 static void hist_trigger_print_key(struct seq_file *m,
5336                                    struct hist_trigger_data *hist_data,
5337                                    void *key,
5338                                    struct tracing_map_elt *elt)
5339 {
5340         struct hist_field *key_field;
5341         bool multiline = false;
5342         const char *field_name;
5343         unsigned int i;
5344         u64 uval;
5345
5346         seq_puts(m, "{ ");
5347
5348         for_each_hist_key_field(i, hist_data) {
5349                 key_field = hist_data->fields[i];
5350
5351                 if (i > hist_data->n_vals)
5352                         seq_puts(m, ", ");
5353
5354                 field_name = hist_field_name(key_field, 0);
5355
5356                 if (key_field->flags & HIST_FIELD_FL_HEX) {
5357                         uval = *(u64 *)(key + key_field->offset);
5358                         seq_printf(m, "%s: %llx", field_name, uval);
5359                 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
5360                         uval = *(u64 *)(key + key_field->offset);
5361                         seq_printf(m, "%s: [%llx] %-45ps", field_name,
5362                                    uval, (void *)(uintptr_t)uval);
5363                 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
5364                         uval = *(u64 *)(key + key_field->offset);
5365                         seq_printf(m, "%s: [%llx] %-55pS", field_name,
5366                                    uval, (void *)(uintptr_t)uval);
5367                 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
5368                         struct hist_elt_data *elt_data = elt->private_data;
5369                         char *comm;
5370
5371                         if (WARN_ON_ONCE(!elt_data))
5372                                 return;
5373
5374                         comm = elt_data->comm;
5375
5376                         uval = *(u64 *)(key + key_field->offset);
5377                         seq_printf(m, "%s: %-16s[%10llu]", field_name,
5378                                    comm, uval);
5379                 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
5380                         const char *syscall_name;
5381
5382                         uval = *(u64 *)(key + key_field->offset);
5383                         syscall_name = get_syscall_name(uval);
5384                         if (!syscall_name)
5385                                 syscall_name = "unknown_syscall";
5386
5387                         seq_printf(m, "%s: %-30s[%3llu]", field_name,
5388                                    syscall_name, uval);
5389                 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
5390                         if (key_field->field)
5391                                 seq_printf(m, "%s.stacktrace", key_field->field->name);
5392                         else
5393                                 seq_puts(m, "common_stacktrace:\n");
5394                         hist_trigger_stacktrace_print(m,
5395                                                       key + key_field->offset,
5396                                                       HIST_STACKTRACE_DEPTH);
5397                         multiline = true;
5398                 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
5399                         seq_printf(m, "%s: ~ 2^%-2llu", field_name,
5400                                    *(u64 *)(key + key_field->offset));
5401                 } else if (key_field->flags & HIST_FIELD_FL_BUCKET) {
5402                         unsigned long buckets = key_field->buckets;
5403                         uval = *(u64 *)(key + key_field->offset);
5404                         seq_printf(m, "%s: ~ %llu-%llu", field_name,
5405                                    uval, uval + buckets -1);
5406                 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
5407                         seq_printf(m, "%s: %-50s", field_name,
5408                                    (char *)(key + key_field->offset));
5409                 } else {
5410                         uval = *(u64 *)(key + key_field->offset);
5411                         seq_printf(m, "%s: %10llu", field_name, uval);
5412                 }
5413         }
5414
5415         if (!multiline)
5416                 seq_puts(m, " ");
5417
5418         seq_puts(m, "}");
5419 }
5420
5421 /* Get the 100 times of the percentage of @val in @total */
5422 static inline unsigned int __get_percentage(u64 val, u64 total)
5423 {
5424         if (!total)
5425                 goto div0;
5426
5427         if (val < (U64_MAX / 10000))
5428                 return (unsigned int)div64_ul(val * 10000, total);
5429
5430         total = div64_u64(total, 10000);
5431         if (!total)
5432                 goto div0;
5433
5434         return (unsigned int)div64_ul(val, total);
5435 div0:
5436         return val ? UINT_MAX : 0;
5437 }
5438
5439 #define BAR_CHAR '#'
5440
5441 static inline const char *__fill_bar_str(char *buf, int size, u64 val, u64 max)
5442 {
5443         unsigned int len = __get_percentage(val, max);
5444         int i;
5445
5446         if (len == UINT_MAX) {
5447                 snprintf(buf, size, "[ERROR]");
5448                 return buf;
5449         }
5450
5451         len = len * size / 10000;
5452         for (i = 0; i < len && i < size; i++)
5453                 buf[i] = BAR_CHAR;
5454         while (i < size)
5455                 buf[i++] = ' ';
5456         buf[size] = '\0';
5457
5458         return buf;
5459 }
5460
5461 struct hist_val_stat {
5462         u64 max;
5463         u64 total;
5464 };
5465
5466 static void hist_trigger_print_val(struct seq_file *m, unsigned int idx,
5467                                    const char *field_name, unsigned long flags,
5468                                    struct hist_val_stat *stats,
5469                                    struct tracing_map_elt *elt)
5470 {
5471         u64 val = tracing_map_read_sum(elt, idx);
5472         unsigned int pc;
5473         char bar[21];
5474
5475         if (flags & HIST_FIELD_FL_PERCENT) {
5476                 pc = __get_percentage(val, stats[idx].total);
5477                 if (pc == UINT_MAX)
5478                         seq_printf(m, " %s (%%):[ERROR]", field_name);
5479                 else
5480                         seq_printf(m, " %s (%%): %3u.%02u", field_name,
5481                                         pc / 100, pc % 100);
5482         } else if (flags & HIST_FIELD_FL_GRAPH) {
5483                 seq_printf(m, " %s: %20s", field_name,
5484                            __fill_bar_str(bar, 20, val, stats[idx].max));
5485         } else if (flags & HIST_FIELD_FL_HEX) {
5486                 seq_printf(m, " %s: %10llx", field_name, val);
5487         } else {
5488                 seq_printf(m, " %s: %10llu", field_name, val);
5489         }
5490 }
5491
5492 static void hist_trigger_entry_print(struct seq_file *m,
5493                                      struct hist_trigger_data *hist_data,
5494                                      struct hist_val_stat *stats,
5495                                      void *key,
5496                                      struct tracing_map_elt *elt)
5497 {
5498         const char *field_name;
5499         unsigned int i = HITCOUNT_IDX;
5500         unsigned long flags;
5501
5502         hist_trigger_print_key(m, hist_data, key, elt);
5503
5504         /* At first, show the raw hitcount if !nohitcount */
5505         if (!hist_data->attrs->no_hitcount)
5506                 hist_trigger_print_val(m, i, "hitcount", 0, stats, elt);
5507
5508         for (i = 1; i < hist_data->n_vals; i++) {
5509                 field_name = hist_field_name(hist_data->fields[i], 0);
5510                 flags = hist_data->fields[i]->flags;
5511                 if (flags & HIST_FIELD_FL_VAR || flags & HIST_FIELD_FL_EXPR)
5512                         continue;
5513
5514                 seq_puts(m, " ");
5515                 hist_trigger_print_val(m, i, field_name, flags, stats, elt);
5516         }
5517
5518         print_actions(m, hist_data, elt);
5519
5520         seq_puts(m, "\n");
5521 }
5522
5523 static int print_entries(struct seq_file *m,
5524                          struct hist_trigger_data *hist_data)
5525 {
5526         struct tracing_map_sort_entry **sort_entries = NULL;
5527         struct tracing_map *map = hist_data->map;
5528         int i, j, n_entries;
5529         struct hist_val_stat *stats = NULL;
5530         u64 val;
5531
5532         n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
5533                                              hist_data->n_sort_keys,
5534                                              &sort_entries);
5535         if (n_entries < 0)
5536                 return n_entries;
5537
5538         /* Calculate the max and the total for each field if needed. */
5539         for (j = 0; j < hist_data->n_vals; j++) {
5540                 if (!(hist_data->fields[j]->flags &
5541                         (HIST_FIELD_FL_PERCENT | HIST_FIELD_FL_GRAPH)))
5542                         continue;
5543                 if (!stats) {
5544                         stats = kcalloc(hist_data->n_vals, sizeof(*stats),
5545                                        GFP_KERNEL);
5546                         if (!stats) {
5547                                 n_entries = -ENOMEM;
5548                                 goto out;
5549                         }
5550                 }
5551                 for (i = 0; i < n_entries; i++) {
5552                         val = tracing_map_read_sum(sort_entries[i]->elt, j);
5553                         stats[j].total += val;
5554                         if (stats[j].max < val)
5555                                 stats[j].max = val;
5556                 }
5557         }
5558
5559         for (i = 0; i < n_entries; i++)
5560                 hist_trigger_entry_print(m, hist_data, stats,
5561                                          sort_entries[i]->key,
5562                                          sort_entries[i]->elt);
5563
5564         kfree(stats);
5565 out:
5566         tracing_map_destroy_sort_entries(sort_entries, n_entries);
5567
5568         return n_entries;
5569 }
5570
5571 static void hist_trigger_show(struct seq_file *m,
5572                               struct event_trigger_data *data, int n)
5573 {
5574         struct hist_trigger_data *hist_data;
5575         int n_entries;
5576
5577         if (n > 0)
5578                 seq_puts(m, "\n\n");
5579
5580         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5581         data->ops->print(m, data);
5582         seq_puts(m, "#\n\n");
5583
5584         hist_data = data->private_data;
5585         n_entries = print_entries(m, hist_data);
5586         if (n_entries < 0)
5587                 n_entries = 0;
5588
5589         track_data_snapshot_print(m, hist_data);
5590
5591         seq_printf(m, "\nTotals:\n    Hits: %llu\n    Entries: %u\n    Dropped: %llu\n",
5592                    (u64)atomic64_read(&hist_data->map->hits),
5593                    n_entries, (u64)atomic64_read(&hist_data->map->drops));
5594 }
5595
5596 static int hist_show(struct seq_file *m, void *v)
5597 {
5598         struct event_trigger_data *data;
5599         struct trace_event_file *event_file;
5600         int n = 0, ret = 0;
5601
5602         mutex_lock(&event_mutex);
5603
5604         event_file = event_file_data(m->private);
5605         if (unlikely(!event_file)) {
5606                 ret = -ENODEV;
5607                 goto out_unlock;
5608         }
5609
5610         list_for_each_entry(data, &event_file->triggers, list) {
5611                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5612                         hist_trigger_show(m, data, n++);
5613         }
5614
5615  out_unlock:
5616         mutex_unlock(&event_mutex);
5617
5618         return ret;
5619 }
5620
5621 static int event_hist_open(struct inode *inode, struct file *file)
5622 {
5623         int ret;
5624
5625         ret = tracing_open_file_tr(inode, file);
5626         if (ret)
5627                 return ret;
5628
5629         /* Clear private_data to avoid warning in single_open() */
5630         file->private_data = NULL;
5631         return single_open(file, hist_show, file);
5632 }
5633
5634 const struct file_operations event_hist_fops = {
5635         .open = event_hist_open,
5636         .read = seq_read,
5637         .llseek = seq_lseek,
5638         .release = tracing_single_release_file_tr,
5639 };
5640
5641 #ifdef CONFIG_HIST_TRIGGERS_DEBUG
5642 static void hist_field_debug_show_flags(struct seq_file *m,
5643                                         unsigned long flags)
5644 {
5645         seq_puts(m, "      flags:\n");
5646
5647         if (flags & HIST_FIELD_FL_KEY)
5648                 seq_puts(m, "        HIST_FIELD_FL_KEY\n");
5649         else if (flags & HIST_FIELD_FL_HITCOUNT)
5650                 seq_puts(m, "        VAL: HIST_FIELD_FL_HITCOUNT\n");
5651         else if (flags & HIST_FIELD_FL_VAR)
5652                 seq_puts(m, "        HIST_FIELD_FL_VAR\n");
5653         else if (flags & HIST_FIELD_FL_VAR_REF)
5654                 seq_puts(m, "        HIST_FIELD_FL_VAR_REF\n");
5655         else
5656                 seq_puts(m, "        VAL: normal u64 value\n");
5657
5658         if (flags & HIST_FIELD_FL_ALIAS)
5659                 seq_puts(m, "        HIST_FIELD_FL_ALIAS\n");
5660         else if (flags & HIST_FIELD_FL_CONST)
5661                 seq_puts(m, "        HIST_FIELD_FL_CONST\n");
5662 }
5663
5664 static int hist_field_debug_show(struct seq_file *m,
5665                                  struct hist_field *field, unsigned long flags)
5666 {
5667         if ((field->flags & flags) != flags) {
5668                 seq_printf(m, "ERROR: bad flags - %lx\n", flags);
5669                 return -EINVAL;
5670         }
5671
5672         hist_field_debug_show_flags(m, field->flags);
5673         if (field->field)
5674                 seq_printf(m, "      ftrace_event_field name: %s\n",
5675                            field->field->name);
5676
5677         if (field->flags & HIST_FIELD_FL_VAR) {
5678                 seq_printf(m, "      var.name: %s\n", field->var.name);
5679                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5680                            field->var.idx);
5681         }
5682
5683         if (field->flags & HIST_FIELD_FL_CONST)
5684                 seq_printf(m, "      constant: %llu\n", field->constant);
5685
5686         if (field->flags & HIST_FIELD_FL_ALIAS)
5687                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5688                            field->var_ref_idx);
5689
5690         if (field->flags & HIST_FIELD_FL_VAR_REF) {
5691                 seq_printf(m, "      name: %s\n", field->name);
5692                 seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5693                            field->var.idx);
5694                 seq_printf(m, "      var.hist_data: %p\n", field->var.hist_data);
5695                 seq_printf(m, "      var_ref_idx (into hist_data->var_refs[]): %u\n",
5696                            field->var_ref_idx);
5697                 if (field->system)
5698                         seq_printf(m, "      system: %s\n", field->system);
5699                 if (field->event_name)
5700                         seq_printf(m, "      event_name: %s\n", field->event_name);
5701         }
5702
5703         seq_printf(m, "      type: %s\n", field->type);
5704         seq_printf(m, "      size: %u\n", field->size);
5705         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5706
5707         return 0;
5708 }
5709
5710 static int field_var_debug_show(struct seq_file *m,
5711                                 struct field_var *field_var, unsigned int i,
5712                                 bool save_vars)
5713 {
5714         const char *vars_name = save_vars ? "save_vars" : "field_vars";
5715         struct hist_field *field;
5716         int ret = 0;
5717
5718         seq_printf(m, "\n    hist_data->%s[%d]:\n", vars_name, i);
5719
5720         field = field_var->var;
5721
5722         seq_printf(m, "\n      %s[%d].var:\n", vars_name, i);
5723
5724         hist_field_debug_show_flags(m, field->flags);
5725         seq_printf(m, "      var.name: %s\n", field->var.name);
5726         seq_printf(m, "      var.idx (into tracing_map_elt.vars[]): %u\n",
5727                    field->var.idx);
5728
5729         field = field_var->val;
5730
5731         seq_printf(m, "\n      %s[%d].val:\n", vars_name, i);
5732         if (field->field)
5733                 seq_printf(m, "      ftrace_event_field name: %s\n",
5734                            field->field->name);
5735         else {
5736                 ret = -EINVAL;
5737                 goto out;
5738         }
5739
5740         seq_printf(m, "      type: %s\n", field->type);
5741         seq_printf(m, "      size: %u\n", field->size);
5742         seq_printf(m, "      is_signed: %u\n", field->is_signed);
5743 out:
5744         return ret;
5745 }
5746
5747 static int hist_action_debug_show(struct seq_file *m,
5748                                   struct action_data *data, int i)
5749 {
5750         int ret = 0;
5751
5752         if (data->handler == HANDLER_ONMAX ||
5753             data->handler == HANDLER_ONCHANGE) {
5754                 seq_printf(m, "\n    hist_data->actions[%d].track_data.var_ref:\n", i);
5755                 ret = hist_field_debug_show(m, data->track_data.var_ref,
5756                                             HIST_FIELD_FL_VAR_REF);
5757                 if (ret)
5758                         goto out;
5759
5760                 seq_printf(m, "\n    hist_data->actions[%d].track_data.track_var:\n", i);
5761                 ret = hist_field_debug_show(m, data->track_data.track_var,
5762                                             HIST_FIELD_FL_VAR);
5763                 if (ret)
5764                         goto out;
5765         }
5766
5767         if (data->handler == HANDLER_ONMATCH) {
5768                 seq_printf(m, "\n    hist_data->actions[%d].match_data.event_system: %s\n",
5769                            i, data->match_data.event_system);
5770                 seq_printf(m, "    hist_data->actions[%d].match_data.event: %s\n",
5771                            i, data->match_data.event);
5772         }
5773 out:
5774         return ret;
5775 }
5776
5777 static int hist_actions_debug_show(struct seq_file *m,
5778                                    struct hist_trigger_data *hist_data)
5779 {
5780         int i, ret = 0;
5781
5782         if (hist_data->n_actions)
5783                 seq_puts(m, "\n  action tracking variables (for onmax()/onchange()/onmatch()):\n");
5784
5785         for (i = 0; i < hist_data->n_actions; i++) {
5786                 struct action_data *action = hist_data->actions[i];
5787
5788                 ret = hist_action_debug_show(m, action, i);
5789                 if (ret)
5790                         goto out;
5791         }
5792
5793         if (hist_data->n_save_vars)
5794                 seq_puts(m, "\n  save action variables (save() params):\n");
5795
5796         for (i = 0; i < hist_data->n_save_vars; i++) {
5797                 ret = field_var_debug_show(m, hist_data->save_vars[i], i, true);
5798                 if (ret)
5799                         goto out;
5800         }
5801 out:
5802         return ret;
5803 }
5804
5805 static void hist_trigger_debug_show(struct seq_file *m,
5806                                     struct event_trigger_data *data, int n)
5807 {
5808         struct hist_trigger_data *hist_data;
5809         int i, ret;
5810
5811         if (n > 0)
5812                 seq_puts(m, "\n\n");
5813
5814         seq_puts(m, "# event histogram\n#\n# trigger info: ");
5815         data->ops->print(m, data);
5816         seq_puts(m, "#\n\n");
5817
5818         hist_data = data->private_data;
5819
5820         seq_printf(m, "hist_data: %p\n\n", hist_data);
5821         seq_printf(m, "  n_vals: %u\n", hist_data->n_vals);
5822         seq_printf(m, "  n_keys: %u\n", hist_data->n_keys);
5823         seq_printf(m, "  n_fields: %u\n", hist_data->n_fields);
5824
5825         seq_puts(m, "\n  val fields:\n\n");
5826
5827         seq_puts(m, "    hist_data->fields[0]:\n");
5828         ret = hist_field_debug_show(m, hist_data->fields[0],
5829                                     HIST_FIELD_FL_HITCOUNT);
5830         if (ret)
5831                 return;
5832
5833         for (i = 1; i < hist_data->n_vals; i++) {
5834                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5835                 ret = hist_field_debug_show(m, hist_data->fields[i], 0);
5836                 if (ret)
5837                         return;
5838         }
5839
5840         seq_puts(m, "\n  key fields:\n");
5841
5842         for (i = hist_data->n_vals; i < hist_data->n_fields; i++) {
5843                 seq_printf(m, "\n    hist_data->fields[%d]:\n", i);
5844                 ret = hist_field_debug_show(m, hist_data->fields[i],
5845                                             HIST_FIELD_FL_KEY);
5846                 if (ret)
5847                         return;
5848         }
5849
5850         if (hist_data->n_var_refs)
5851                 seq_puts(m, "\n  variable reference fields:\n");
5852
5853         for (i = 0; i < hist_data->n_var_refs; i++) {
5854                 seq_printf(m, "\n    hist_data->var_refs[%d]:\n", i);
5855                 ret = hist_field_debug_show(m, hist_data->var_refs[i],
5856                                             HIST_FIELD_FL_VAR_REF);
5857                 if (ret)
5858                         return;
5859         }
5860
5861         if (hist_data->n_field_vars)
5862                 seq_puts(m, "\n  field variables:\n");
5863
5864         for (i = 0; i < hist_data->n_field_vars; i++) {
5865                 ret = field_var_debug_show(m, hist_data->field_vars[i], i, false);
5866                 if (ret)
5867                         return;
5868         }
5869
5870         ret = hist_actions_debug_show(m, hist_data);
5871         if (ret)
5872                 return;
5873 }
5874
5875 static int hist_debug_show(struct seq_file *m, void *v)
5876 {
5877         struct event_trigger_data *data;
5878         struct trace_event_file *event_file;
5879         int n = 0, ret = 0;
5880
5881         mutex_lock(&event_mutex);
5882
5883         event_file = event_file_data(m->private);
5884         if (unlikely(!event_file)) {
5885                 ret = -ENODEV;
5886                 goto out_unlock;
5887         }
5888
5889         list_for_each_entry(data, &event_file->triggers, list) {
5890                 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
5891                         hist_trigger_debug_show(m, data, n++);
5892         }
5893
5894  out_unlock:
5895         mutex_unlock(&event_mutex);
5896
5897         return ret;
5898 }
5899
5900 static int event_hist_debug_open(struct inode *inode, struct file *file)
5901 {
5902         int ret;
5903
5904         ret = tracing_open_file_tr(inode, file);
5905         if (ret)
5906                 return ret;
5907
5908         /* Clear private_data to avoid warning in single_open() */
5909         file->private_data = NULL;
5910         return single_open(file, hist_debug_show, file);
5911 }
5912
5913 const struct file_operations event_hist_debug_fops = {
5914         .open = event_hist_debug_open,
5915         .read = seq_read,
5916         .llseek = seq_lseek,
5917         .release = tracing_single_release_file_tr,
5918 };
5919 #endif
5920
5921 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
5922 {
5923         const char *field_name = hist_field_name(hist_field, 0);
5924
5925         if (hist_field->var.name)
5926                 seq_printf(m, "%s=", hist_field->var.name);
5927
5928         if (hist_field->flags & HIST_FIELD_FL_CPU)
5929                 seq_puts(m, "common_cpu");
5930         else if (hist_field->flags & HIST_FIELD_FL_CONST)
5931                 seq_printf(m, "%llu", hist_field->constant);
5932         else if (field_name) {
5933                 if (hist_field->flags & HIST_FIELD_FL_VAR_REF ||
5934                     hist_field->flags & HIST_FIELD_FL_ALIAS)
5935                         seq_putc(m, '$');
5936                 seq_printf(m, "%s", field_name);
5937         } else if (hist_field->flags & HIST_FIELD_FL_TIMESTAMP)
5938                 seq_puts(m, "common_timestamp");
5939
5940         if (hist_field->flags) {
5941                 if (!(hist_field->flags & HIST_FIELD_FL_VAR_REF) &&
5942                     !(hist_field->flags & HIST_FIELD_FL_EXPR) &&
5943                     !(hist_field->flags & HIST_FIELD_FL_STACKTRACE)) {
5944                         const char *flags = get_hist_field_flags(hist_field);
5945
5946                         if (flags)
5947                                 seq_printf(m, ".%s", flags);
5948                 }
5949         }
5950         if (hist_field->buckets)
5951                 seq_printf(m, "=%ld", hist_field->buckets);
5952 }
5953
5954 static int event_hist_trigger_print(struct seq_file *m,
5955                                     struct event_trigger_data *data)
5956 {
5957         struct hist_trigger_data *hist_data = data->private_data;
5958         struct hist_field *field;
5959         bool have_var = false;
5960         bool show_val = false;
5961         unsigned int i;
5962
5963         seq_puts(m, HIST_PREFIX);
5964
5965         if (data->name)
5966                 seq_printf(m, "%s:", data->name);
5967
5968         seq_puts(m, "keys=");
5969
5970         for_each_hist_key_field(i, hist_data) {
5971                 field = hist_data->fields[i];
5972
5973                 if (i > hist_data->n_vals)
5974                         seq_puts(m, ",");
5975
5976                 if (field->flags & HIST_FIELD_FL_STACKTRACE) {
5977                         if (field->field)
5978                                 seq_printf(m, "%s.stacktrace", field->field->name);
5979                         else
5980                                 seq_puts(m, "common_stacktrace");
5981                 } else
5982                         hist_field_print(m, field);
5983         }
5984
5985         seq_puts(m, ":vals=");
5986
5987         for_each_hist_val_field(i, hist_data) {
5988                 field = hist_data->fields[i];
5989                 if (field->flags & HIST_FIELD_FL_VAR) {
5990                         have_var = true;
5991                         continue;
5992                 }
5993
5994                 if (i == HITCOUNT_IDX) {
5995                         if (hist_data->attrs->no_hitcount)
5996                                 continue;
5997                         seq_puts(m, "hitcount");
5998                 } else {
5999                         if (show_val)
6000                                 seq_puts(m, ",");
6001                         hist_field_print(m, field);
6002                 }
6003                 show_val = true;
6004         }
6005
6006         if (have_var) {
6007                 unsigned int n = 0;
6008
6009                 seq_puts(m, ":");
6010
6011                 for_each_hist_val_field(i, hist_data) {
6012                         field = hist_data->fields[i];
6013
6014                         if (field->flags & HIST_FIELD_FL_VAR) {
6015                                 if (n++)
6016                                         seq_puts(m, ",");
6017                                 hist_field_print(m, field);
6018                         }
6019                 }
6020         }
6021
6022         seq_puts(m, ":sort=");
6023
6024         for (i = 0; i < hist_data->n_sort_keys; i++) {
6025                 struct tracing_map_sort_key *sort_key;
6026                 unsigned int idx, first_key_idx;
6027
6028                 /* skip VAR vals */
6029                 first_key_idx = hist_data->n_vals - hist_data->n_vars;
6030
6031                 sort_key = &hist_data->sort_keys[i];
6032                 idx = sort_key->field_idx;
6033
6034                 if (WARN_ON(idx >= HIST_FIELDS_MAX))
6035                         return -EINVAL;
6036
6037                 if (i > 0)
6038                         seq_puts(m, ",");
6039
6040                 if (idx == HITCOUNT_IDX)
6041                         seq_puts(m, "hitcount");
6042                 else {
6043                         if (idx >= first_key_idx)
6044                                 idx += hist_data->n_vars;
6045                         hist_field_print(m, hist_data->fields[idx]);
6046                 }
6047
6048                 if (sort_key->descending)
6049                         seq_puts(m, ".descending");
6050         }
6051         seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
6052         if (hist_data->enable_timestamps)
6053                 seq_printf(m, ":clock=%s", hist_data->attrs->clock);
6054         if (hist_data->attrs->no_hitcount)
6055                 seq_puts(m, ":nohitcount");
6056
6057         print_actions_spec(m, hist_data);
6058
6059         if (data->filter_str)
6060                 seq_printf(m, " if %s", data->filter_str);
6061
6062         if (data->paused)
6063                 seq_puts(m, " [paused]");
6064         else
6065                 seq_puts(m, " [active]");
6066
6067         seq_putc(m, '\n');
6068
6069         return 0;
6070 }
6071
6072 static int event_hist_trigger_init(struct event_trigger_data *data)
6073 {
6074         struct hist_trigger_data *hist_data = data->private_data;
6075
6076         if (!data->ref && hist_data->attrs->name)
6077                 save_named_trigger(hist_data->attrs->name, data);
6078
6079         data->ref++;
6080
6081         return 0;
6082 }
6083
6084 static void unregister_field_var_hists(struct hist_trigger_data *hist_data)
6085 {
6086         struct trace_event_file *file;
6087         unsigned int i;
6088         char *cmd;
6089         int ret;
6090
6091         for (i = 0; i < hist_data->n_field_var_hists; i++) {
6092                 file = hist_data->field_var_hists[i]->hist_data->event_file;
6093                 cmd = hist_data->field_var_hists[i]->cmd;
6094                 ret = event_hist_trigger_parse(&trigger_hist_cmd, file,
6095                                                "!hist", "hist", cmd);
6096                 WARN_ON_ONCE(ret < 0);
6097         }
6098 }
6099
6100 static void event_hist_trigger_free(struct event_trigger_data *data)
6101 {
6102         struct hist_trigger_data *hist_data = data->private_data;
6103
6104         if (WARN_ON_ONCE(data->ref <= 0))
6105                 return;
6106
6107         data->ref--;
6108         if (!data->ref) {
6109                 if (data->name)
6110                         del_named_trigger(data);
6111
6112                 trigger_data_free(data);
6113
6114                 remove_hist_vars(hist_data);
6115
6116                 unregister_field_var_hists(hist_data);
6117
6118                 destroy_hist_data(hist_data);
6119         }
6120 }
6121
6122 static struct event_trigger_ops event_hist_trigger_ops = {
6123         .trigger                = event_hist_trigger,
6124         .print                  = event_hist_trigger_print,
6125         .init                   = event_hist_trigger_init,
6126         .free                   = event_hist_trigger_free,
6127 };
6128
6129 static int event_hist_trigger_named_init(struct event_trigger_data *data)
6130 {
6131         data->ref++;
6132
6133         save_named_trigger(data->named_data->name, data);
6134
6135         event_hist_trigger_init(data->named_data);
6136
6137         return 0;
6138 }
6139
6140 static void event_hist_trigger_named_free(struct event_trigger_data *data)
6141 {
6142         if (WARN_ON_ONCE(data->ref <= 0))
6143                 return;
6144
6145         event_hist_trigger_free(data->named_data);
6146
6147         data->ref--;
6148         if (!data->ref) {
6149                 del_named_trigger(data);
6150                 trigger_data_free(data);
6151         }
6152 }
6153
6154 static struct event_trigger_ops event_hist_trigger_named_ops = {
6155         .trigger                = event_hist_trigger,
6156         .print                  = event_hist_trigger_print,
6157         .init                   = event_hist_trigger_named_init,
6158         .free                   = event_hist_trigger_named_free,
6159 };
6160
6161 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
6162                                                             char *param)
6163 {
6164         return &event_hist_trigger_ops;
6165 }
6166
6167 static void hist_clear(struct event_trigger_data *data)
6168 {
6169         struct hist_trigger_data *hist_data = data->private_data;
6170
6171         if (data->name)
6172                 pause_named_trigger(data);
6173
6174         tracepoint_synchronize_unregister();
6175
6176         tracing_map_clear(hist_data->map);
6177
6178         if (data->name)
6179                 unpause_named_trigger(data);
6180 }
6181
6182 static bool compatible_field(struct ftrace_event_field *field,
6183                              struct ftrace_event_field *test_field)
6184 {
6185         if (field == test_field)
6186                 return true;
6187         if (field == NULL || test_field == NULL)
6188                 return false;
6189         if (strcmp(field->name, test_field->name) != 0)
6190                 return false;
6191         if (strcmp(field->type, test_field->type) != 0)
6192                 return false;
6193         if (field->size != test_field->size)
6194                 return false;
6195         if (field->is_signed != test_field->is_signed)
6196                 return false;
6197
6198         return true;
6199 }
6200
6201 static bool hist_trigger_match(struct event_trigger_data *data,
6202                                struct event_trigger_data *data_test,
6203                                struct event_trigger_data *named_data,
6204                                bool ignore_filter)
6205 {
6206         struct tracing_map_sort_key *sort_key, *sort_key_test;
6207         struct hist_trigger_data *hist_data, *hist_data_test;
6208         struct hist_field *key_field, *key_field_test;
6209         unsigned int i;
6210
6211         if (named_data && (named_data != data_test) &&
6212             (named_data != data_test->named_data))
6213                 return false;
6214
6215         if (!named_data && is_named_trigger(data_test))
6216                 return false;
6217
6218         hist_data = data->private_data;
6219         hist_data_test = data_test->private_data;
6220
6221         if (hist_data->n_vals != hist_data_test->n_vals ||
6222             hist_data->n_fields != hist_data_test->n_fields ||
6223             hist_data->n_sort_keys != hist_data_test->n_sort_keys)
6224                 return false;
6225
6226         if (!ignore_filter) {
6227                 if ((data->filter_str && !data_test->filter_str) ||
6228                    (!data->filter_str && data_test->filter_str))
6229                         return false;
6230         }
6231
6232         for_each_hist_field(i, hist_data) {
6233                 key_field = hist_data->fields[i];
6234                 key_field_test = hist_data_test->fields[i];
6235
6236                 if (key_field->flags != key_field_test->flags)
6237                         return false;
6238                 if (!compatible_field(key_field->field, key_field_test->field))
6239                         return false;
6240                 if (key_field->offset != key_field_test->offset)
6241                         return false;
6242                 if (key_field->size != key_field_test->size)
6243                         return false;
6244                 if (key_field->is_signed != key_field_test->is_signed)
6245                         return false;
6246                 if (!!key_field->var.name != !!key_field_test->var.name)
6247                         return false;
6248                 if (key_field->var.name &&
6249                     strcmp(key_field->var.name, key_field_test->var.name) != 0)
6250                         return false;
6251         }
6252
6253         for (i = 0; i < hist_data->n_sort_keys; i++) {
6254                 sort_key = &hist_data->sort_keys[i];
6255                 sort_key_test = &hist_data_test->sort_keys[i];
6256
6257                 if (sort_key->field_idx != sort_key_test->field_idx ||
6258                     sort_key->descending != sort_key_test->descending)
6259                         return false;
6260         }
6261
6262         if (!ignore_filter && data->filter_str &&
6263             (strcmp(data->filter_str, data_test->filter_str) != 0))
6264                 return false;
6265
6266         if (!actions_match(hist_data, hist_data_test))
6267                 return false;
6268
6269         return true;
6270 }
6271
6272 static bool existing_hist_update_only(char *glob,
6273                                       struct event_trigger_data *data,
6274                                       struct trace_event_file *file)
6275 {
6276         struct hist_trigger_data *hist_data = data->private_data;
6277         struct event_trigger_data *test, *named_data = NULL;
6278         bool updated = false;
6279
6280         if (!hist_data->attrs->pause && !hist_data->attrs->cont &&
6281             !hist_data->attrs->clear)
6282                 goto out;
6283
6284         if (hist_data->attrs->name) {
6285                 named_data = find_named_trigger(hist_data->attrs->name);
6286                 if (named_data) {
6287                         if (!hist_trigger_match(data, named_data, named_data,
6288                                                 true))
6289                                 goto out;
6290                 }
6291         }
6292
6293         if (hist_data->attrs->name && !named_data)
6294                 goto out;
6295
6296         list_for_each_entry(test, &file->triggers, list) {
6297                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6298                         if (!hist_trigger_match(data, test, named_data, false))
6299                                 continue;
6300                         if (hist_data->attrs->pause)
6301                                 test->paused = true;
6302                         else if (hist_data->attrs->cont)
6303                                 test->paused = false;
6304                         else if (hist_data->attrs->clear)
6305                                 hist_clear(test);
6306                         updated = true;
6307                         goto out;
6308                 }
6309         }
6310  out:
6311         return updated;
6312 }
6313
6314 static int hist_register_trigger(char *glob,
6315                                  struct event_trigger_data *data,
6316                                  struct trace_event_file *file)
6317 {
6318         struct hist_trigger_data *hist_data = data->private_data;
6319         struct event_trigger_data *test, *named_data = NULL;
6320         struct trace_array *tr = file->tr;
6321         int ret = 0;
6322
6323         if (hist_data->attrs->name) {
6324                 named_data = find_named_trigger(hist_data->attrs->name);
6325                 if (named_data) {
6326                         if (!hist_trigger_match(data, named_data, named_data,
6327                                                 true)) {
6328                                 hist_err(tr, HIST_ERR_NAMED_MISMATCH, errpos(hist_data->attrs->name));
6329                                 ret = -EINVAL;
6330                                 goto out;
6331                         }
6332                 }
6333         }
6334
6335         if (hist_data->attrs->name && !named_data)
6336                 goto new;
6337
6338         lockdep_assert_held(&event_mutex);
6339
6340         list_for_each_entry(test, &file->triggers, list) {
6341                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6342                         if (hist_trigger_match(data, test, named_data, false)) {
6343                                 hist_err(tr, HIST_ERR_TRIGGER_EEXIST, 0);
6344                                 ret = -EEXIST;
6345                                 goto out;
6346                         }
6347                 }
6348         }
6349  new:
6350         if (hist_data->attrs->cont || hist_data->attrs->clear) {
6351                 hist_err(tr, HIST_ERR_TRIGGER_ENOENT_CLEAR, 0);
6352                 ret = -ENOENT;
6353                 goto out;
6354         }
6355
6356         if (hist_data->attrs->pause)
6357                 data->paused = true;
6358
6359         if (named_data) {
6360                 data->private_data = named_data->private_data;
6361                 set_named_trigger_data(data, named_data);
6362                 data->ops = &event_hist_trigger_named_ops;
6363         }
6364
6365         if (data->ops->init) {
6366                 ret = data->ops->init(data);
6367                 if (ret < 0)
6368                         goto out;
6369         }
6370
6371         if (hist_data->enable_timestamps) {
6372                 char *clock = hist_data->attrs->clock;
6373
6374                 ret = tracing_set_clock(file->tr, hist_data->attrs->clock);
6375                 if (ret) {
6376                         hist_err(tr, HIST_ERR_SET_CLOCK_FAIL, errpos(clock));
6377                         goto out;
6378                 }
6379
6380                 tracing_set_filter_buffering(file->tr, true);
6381         }
6382
6383         if (named_data)
6384                 destroy_hist_data(hist_data);
6385  out:
6386         return ret;
6387 }
6388
6389 static int hist_trigger_enable(struct event_trigger_data *data,
6390                                struct trace_event_file *file)
6391 {
6392         int ret = 0;
6393
6394         list_add_tail_rcu(&data->list, &file->triggers);
6395
6396         update_cond_flag(file);
6397
6398         if (trace_event_trigger_enable_disable(file, 1) < 0) {
6399                 list_del_rcu(&data->list);
6400                 update_cond_flag(file);
6401                 ret--;
6402         }
6403
6404         return ret;
6405 }
6406
6407 static bool have_hist_trigger_match(struct event_trigger_data *data,
6408                                     struct trace_event_file *file)
6409 {
6410         struct hist_trigger_data *hist_data = data->private_data;
6411         struct event_trigger_data *test, *named_data = NULL;
6412         bool match = false;
6413
6414         lockdep_assert_held(&event_mutex);
6415
6416         if (hist_data->attrs->name)
6417                 named_data = find_named_trigger(hist_data->attrs->name);
6418
6419         list_for_each_entry(test, &file->triggers, list) {
6420                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6421                         if (hist_trigger_match(data, test, named_data, false)) {
6422                                 match = true;
6423                                 break;
6424                         }
6425                 }
6426         }
6427
6428         return match;
6429 }
6430
6431 static bool hist_trigger_check_refs(struct event_trigger_data *data,
6432                                     struct trace_event_file *file)
6433 {
6434         struct hist_trigger_data *hist_data = data->private_data;
6435         struct event_trigger_data *test, *named_data = NULL;
6436
6437         lockdep_assert_held(&event_mutex);
6438
6439         if (hist_data->attrs->name)
6440                 named_data = find_named_trigger(hist_data->attrs->name);
6441
6442         list_for_each_entry(test, &file->triggers, list) {
6443                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6444                         if (!hist_trigger_match(data, test, named_data, false))
6445                                 continue;
6446                         hist_data = test->private_data;
6447                         if (check_var_refs(hist_data))
6448                                 return true;
6449                         break;
6450                 }
6451         }
6452
6453         return false;
6454 }
6455
6456 static void hist_unregister_trigger(char *glob,
6457                                     struct event_trigger_data *data,
6458                                     struct trace_event_file *file)
6459 {
6460         struct event_trigger_data *test = NULL, *iter, *named_data = NULL;
6461         struct hist_trigger_data *hist_data = data->private_data;
6462
6463         lockdep_assert_held(&event_mutex);
6464
6465         if (hist_data->attrs->name)
6466                 named_data = find_named_trigger(hist_data->attrs->name);
6467
6468         list_for_each_entry(iter, &file->triggers, list) {
6469                 if (iter->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6470                         if (!hist_trigger_match(data, iter, named_data, false))
6471                                 continue;
6472                         test = iter;
6473                         list_del_rcu(&test->list);
6474                         trace_event_trigger_enable_disable(file, 0);
6475                         update_cond_flag(file);
6476                         break;
6477                 }
6478         }
6479
6480         if (test && test->ops->free)
6481                 test->ops->free(test);
6482
6483         if (hist_data->enable_timestamps) {
6484                 if (!hist_data->remove || test)
6485                         tracing_set_filter_buffering(file->tr, false);
6486         }
6487 }
6488
6489 static bool hist_file_check_refs(struct trace_event_file *file)
6490 {
6491         struct hist_trigger_data *hist_data;
6492         struct event_trigger_data *test;
6493
6494         lockdep_assert_held(&event_mutex);
6495
6496         list_for_each_entry(test, &file->triggers, list) {
6497                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6498                         hist_data = test->private_data;
6499                         if (check_var_refs(hist_data))
6500                                 return true;
6501                 }
6502         }
6503
6504         return false;
6505 }
6506
6507 static void hist_unreg_all(struct trace_event_file *file)
6508 {
6509         struct event_trigger_data *test, *n;
6510         struct hist_trigger_data *hist_data;
6511         struct synth_event *se;
6512         const char *se_name;
6513
6514         lockdep_assert_held(&event_mutex);
6515
6516         if (hist_file_check_refs(file))
6517                 return;
6518
6519         list_for_each_entry_safe(test, n, &file->triggers, list) {
6520                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6521                         hist_data = test->private_data;
6522                         list_del_rcu(&test->list);
6523                         trace_event_trigger_enable_disable(file, 0);
6524
6525                         se_name = trace_event_name(file->event_call);
6526                         se = find_synth_event(se_name);
6527                         if (se)
6528                                 se->ref--;
6529
6530                         update_cond_flag(file);
6531                         if (hist_data->enable_timestamps)
6532                                 tracing_set_filter_buffering(file->tr, false);
6533                         if (test->ops->free)
6534                                 test->ops->free(test);
6535                 }
6536         }
6537 }
6538
6539 static int event_hist_trigger_parse(struct event_command *cmd_ops,
6540                                     struct trace_event_file *file,
6541                                     char *glob, char *cmd,
6542                                     char *param_and_filter)
6543 {
6544         unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
6545         struct event_trigger_data *trigger_data;
6546         struct hist_trigger_attrs *attrs;
6547         struct hist_trigger_data *hist_data;
6548         char *param, *filter, *p, *start;
6549         struct synth_event *se;
6550         const char *se_name;
6551         bool remove;
6552         int ret = 0;
6553
6554         lockdep_assert_held(&event_mutex);
6555
6556         if (WARN_ON(!glob))
6557                 return -EINVAL;
6558
6559         if (glob[0]) {
6560                 hist_err_clear();
6561                 last_cmd_set(file, param_and_filter);
6562         }
6563
6564         remove = event_trigger_check_remove(glob);
6565
6566         if (event_trigger_empty_param(param_and_filter))
6567                 return -EINVAL;
6568
6569         /*
6570          * separate the trigger from the filter (k:v [if filter])
6571          * allowing for whitespace in the trigger
6572          */
6573         p = param = param_and_filter;
6574         do {
6575                 p = strstr(p, "if");
6576                 if (!p)
6577                         break;
6578                 if (p == param_and_filter)
6579                         return -EINVAL;
6580                 if (*(p - 1) != ' ' && *(p - 1) != '\t') {
6581                         p++;
6582                         continue;
6583                 }
6584                 if (p >= param_and_filter + strlen(param_and_filter) - (sizeof("if") - 1) - 1)
6585                         return -EINVAL;
6586                 if (*(p + sizeof("if") - 1) != ' ' && *(p + sizeof("if") - 1) != '\t') {
6587                         p++;
6588                         continue;
6589                 }
6590                 break;
6591         } while (1);
6592
6593         if (!p)
6594                 filter = NULL;
6595         else {
6596                 *(p - 1) = '\0';
6597                 filter = strstrip(p);
6598                 param = strstrip(param);
6599         }
6600
6601         /*
6602          * To simplify arithmetic expression parsing, replace occurrences of
6603          * '.sym-offset' modifier with '.symXoffset'
6604          */
6605         start = strstr(param, ".sym-offset");
6606         while (start) {
6607                 *(start + 4) = 'X';
6608                 start = strstr(start + 11, ".sym-offset");
6609         }
6610
6611         attrs = parse_hist_trigger_attrs(file->tr, param);
6612         if (IS_ERR(attrs))
6613                 return PTR_ERR(attrs);
6614
6615         if (attrs->map_bits)
6616                 hist_trigger_bits = attrs->map_bits;
6617
6618         hist_data = create_hist_data(hist_trigger_bits, attrs, file, remove);
6619         if (IS_ERR(hist_data)) {
6620                 destroy_hist_trigger_attrs(attrs);
6621                 return PTR_ERR(hist_data);
6622         }
6623
6624         trigger_data = event_trigger_alloc(cmd_ops, cmd, param, hist_data);
6625         if (!trigger_data) {
6626                 ret = -ENOMEM;
6627                 goto out_free;
6628         }
6629
6630         ret = event_trigger_set_filter(cmd_ops, file, filter, trigger_data);
6631         if (ret < 0)
6632                 goto out_free;
6633
6634         if (remove) {
6635                 if (!have_hist_trigger_match(trigger_data, file))
6636                         goto out_free;
6637
6638                 if (hist_trigger_check_refs(trigger_data, file)) {
6639                         ret = -EBUSY;
6640                         goto out_free;
6641                 }
6642
6643                 event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6644                 se_name = trace_event_name(file->event_call);
6645                 se = find_synth_event(se_name);
6646                 if (se)
6647                         se->ref--;
6648                 ret = 0;
6649                 goto out_free;
6650         }
6651
6652         if (existing_hist_update_only(glob, trigger_data, file))
6653                 goto out_free;
6654
6655         ret = event_trigger_register(cmd_ops, file, glob, trigger_data);
6656         if (ret < 0)
6657                 goto out_free;
6658
6659         if (get_named_trigger_data(trigger_data))
6660                 goto enable;
6661
6662         ret = create_actions(hist_data);
6663         if (ret)
6664                 goto out_unreg;
6665
6666         if (has_hist_vars(hist_data) || hist_data->n_var_refs) {
6667                 ret = save_hist_vars(hist_data);
6668                 if (ret)
6669                         goto out_unreg;
6670         }
6671
6672         ret = tracing_map_init(hist_data->map);
6673         if (ret)
6674                 goto out_unreg;
6675 enable:
6676         ret = hist_trigger_enable(trigger_data, file);
6677         if (ret)
6678                 goto out_unreg;
6679
6680         se_name = trace_event_name(file->event_call);
6681         se = find_synth_event(se_name);
6682         if (se)
6683                 se->ref++;
6684  out:
6685         if (ret == 0 && glob[0])
6686                 hist_err_clear();
6687
6688         return ret;
6689  out_unreg:
6690         event_trigger_unregister(cmd_ops, file, glob+1, trigger_data);
6691  out_free:
6692         event_trigger_reset_filter(cmd_ops, trigger_data);
6693
6694         remove_hist_vars(hist_data);
6695
6696         kfree(trigger_data);
6697
6698         destroy_hist_data(hist_data);
6699         goto out;
6700 }
6701
6702 static struct event_command trigger_hist_cmd = {
6703         .name                   = "hist",
6704         .trigger_type           = ETT_EVENT_HIST,
6705         .flags                  = EVENT_CMD_FL_NEEDS_REC,
6706         .parse                  = event_hist_trigger_parse,
6707         .reg                    = hist_register_trigger,
6708         .unreg                  = hist_unregister_trigger,
6709         .unreg_all              = hist_unreg_all,
6710         .get_trigger_ops        = event_hist_get_trigger_ops,
6711         .set_filter             = set_trigger_filter,
6712 };
6713
6714 __init int register_trigger_hist_cmd(void)
6715 {
6716         int ret;
6717
6718         ret = register_event_command(&trigger_hist_cmd);
6719         WARN_ON(ret < 0);
6720
6721         return ret;
6722 }
6723
6724 static void
6725 hist_enable_trigger(struct event_trigger_data *data,
6726                     struct trace_buffer *buffer,  void *rec,
6727                     struct ring_buffer_event *event)
6728 {
6729         struct enable_trigger_data *enable_data = data->private_data;
6730         struct event_trigger_data *test;
6731
6732         list_for_each_entry_rcu(test, &enable_data->file->triggers, list,
6733                                 lockdep_is_held(&event_mutex)) {
6734                 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
6735                         if (enable_data->enable)
6736                                 test->paused = false;
6737                         else
6738                                 test->paused = true;
6739                 }
6740         }
6741 }
6742
6743 static void
6744 hist_enable_count_trigger(struct event_trigger_data *data,
6745                           struct trace_buffer *buffer,  void *rec,
6746                           struct ring_buffer_event *event)
6747 {
6748         if (!data->count)
6749                 return;
6750
6751         if (data->count != -1)
6752                 (data->count)--;
6753
6754         hist_enable_trigger(data, buffer, rec, event);
6755 }
6756
6757 static struct event_trigger_ops hist_enable_trigger_ops = {
6758         .trigger                = hist_enable_trigger,
6759         .print                  = event_enable_trigger_print,
6760         .init                   = event_trigger_init,
6761         .free                   = event_enable_trigger_free,
6762 };
6763
6764 static struct event_trigger_ops hist_enable_count_trigger_ops = {
6765         .trigger                = hist_enable_count_trigger,
6766         .print                  = event_enable_trigger_print,
6767         .init                   = event_trigger_init,
6768         .free                   = event_enable_trigger_free,
6769 };
6770
6771 static struct event_trigger_ops hist_disable_trigger_ops = {
6772         .trigger                = hist_enable_trigger,
6773         .print                  = event_enable_trigger_print,
6774         .init                   = event_trigger_init,
6775         .free                   = event_enable_trigger_free,
6776 };
6777
6778 static struct event_trigger_ops hist_disable_count_trigger_ops = {
6779         .trigger                = hist_enable_count_trigger,
6780         .print                  = event_enable_trigger_print,
6781         .init                   = event_trigger_init,
6782         .free                   = event_enable_trigger_free,
6783 };
6784
6785 static struct event_trigger_ops *
6786 hist_enable_get_trigger_ops(char *cmd, char *param)
6787 {
6788         struct event_trigger_ops *ops;
6789         bool enable;
6790
6791         enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
6792
6793         if (enable)
6794                 ops = param ? &hist_enable_count_trigger_ops :
6795                         &hist_enable_trigger_ops;
6796         else
6797                 ops = param ? &hist_disable_count_trigger_ops :
6798                         &hist_disable_trigger_ops;
6799
6800         return ops;
6801 }
6802
6803 static void hist_enable_unreg_all(struct trace_event_file *file)
6804 {
6805         struct event_trigger_data *test, *n;
6806
6807         list_for_each_entry_safe(test, n, &file->triggers, list) {
6808                 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
6809                         list_del_rcu(&test->list);
6810                         update_cond_flag(file);
6811                         trace_event_trigger_enable_disable(file, 0);
6812                         if (test->ops->free)
6813                                 test->ops->free(test);
6814                 }
6815         }
6816 }
6817
6818 static struct event_command trigger_hist_enable_cmd = {
6819         .name                   = ENABLE_HIST_STR,
6820         .trigger_type           = ETT_HIST_ENABLE,
6821         .parse                  = event_enable_trigger_parse,
6822         .reg                    = event_enable_register_trigger,
6823         .unreg                  = event_enable_unregister_trigger,
6824         .unreg_all              = hist_enable_unreg_all,
6825         .get_trigger_ops        = hist_enable_get_trigger_ops,
6826         .set_filter             = set_trigger_filter,
6827 };
6828
6829 static struct event_command trigger_hist_disable_cmd = {
6830         .name                   = DISABLE_HIST_STR,
6831         .trigger_type           = ETT_HIST_ENABLE,
6832         .parse                  = event_enable_trigger_parse,
6833         .reg                    = event_enable_register_trigger,
6834         .unreg                  = event_enable_unregister_trigger,
6835         .unreg_all              = hist_enable_unreg_all,
6836         .get_trigger_ops        = hist_enable_get_trigger_ops,
6837         .set_filter             = set_trigger_filter,
6838 };
6839
6840 static __init void unregister_trigger_hist_enable_disable_cmds(void)
6841 {
6842         unregister_event_command(&trigger_hist_enable_cmd);
6843         unregister_event_command(&trigger_hist_disable_cmd);
6844 }
6845
6846 __init int register_trigger_hist_enable_disable_cmds(void)
6847 {
6848         int ret;
6849
6850         ret = register_event_command(&trigger_hist_enable_cmd);
6851         if (WARN_ON(ret < 0))
6852                 return ret;
6853         ret = register_event_command(&trigger_hist_disable_cmd);
6854         if (WARN_ON(ret < 0))
6855                 unregister_trigger_hist_enable_disable_cmds();
6856
6857         return ret;
6858 }