2 * trace_events_hist - trace event hist triggers
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; either version 2 of the License, or
7 * (at your option) any later version.
9 * This program is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 * GNU General Public License for more details.
14 * Copyright (C) 2015 Tom Zanussi <tom.zanussi@linux.intel.com>
17 #include <linux/module.h>
18 #include <linux/kallsyms.h>
19 #include <linux/mutex.h>
20 #include <linux/slab.h>
21 #include <linux/stacktrace.h>
22 #include <linux/rculist.h>
24 #include "tracing_map.h"
29 typedef u64 (*hist_field_fn_t) (struct hist_field *field, void *event);
32 struct ftrace_event_field *field;
39 static u64 hist_field_none(struct hist_field *field, void *event)
44 static u64 hist_field_counter(struct hist_field *field, void *event)
49 static u64 hist_field_string(struct hist_field *hist_field, void *event)
51 char *addr = (char *)(event + hist_field->field->offset);
53 return (u64)(unsigned long)addr;
56 static u64 hist_field_dynstring(struct hist_field *hist_field, void *event)
58 u32 str_item = *(u32 *)(event + hist_field->field->offset);
59 int str_loc = str_item & 0xffff;
60 char *addr = (char *)(event + str_loc);
62 return (u64)(unsigned long)addr;
65 static u64 hist_field_pstring(struct hist_field *hist_field, void *event)
67 char **addr = (char **)(event + hist_field->field->offset);
69 return (u64)(unsigned long)*addr;
72 static u64 hist_field_log2(struct hist_field *hist_field, void *event)
74 u64 val = *(u64 *)(event + hist_field->field->offset);
76 return (u64) ilog2(roundup_pow_of_two(val));
79 #define DEFINE_HIST_FIELD_FN(type) \
80 static u64 hist_field_##type(struct hist_field *hist_field, void *event)\
82 type *addr = (type *)(event + hist_field->field->offset); \
84 return (u64)(unsigned long)*addr; \
87 DEFINE_HIST_FIELD_FN(s64);
88 DEFINE_HIST_FIELD_FN(u64);
89 DEFINE_HIST_FIELD_FN(s32);
90 DEFINE_HIST_FIELD_FN(u32);
91 DEFINE_HIST_FIELD_FN(s16);
92 DEFINE_HIST_FIELD_FN(u16);
93 DEFINE_HIST_FIELD_FN(s8);
94 DEFINE_HIST_FIELD_FN(u8);
96 #define for_each_hist_field(i, hist_data) \
97 for ((i) = 0; (i) < (hist_data)->n_fields; (i)++)
99 #define for_each_hist_val_field(i, hist_data) \
100 for ((i) = 0; (i) < (hist_data)->n_vals; (i)++)
102 #define for_each_hist_key_field(i, hist_data) \
103 for ((i) = (hist_data)->n_vals; (i) < (hist_data)->n_fields; (i)++)
105 #define HIST_STACKTRACE_DEPTH 16
106 #define HIST_STACKTRACE_SIZE (HIST_STACKTRACE_DEPTH * sizeof(unsigned long))
107 #define HIST_STACKTRACE_SKIP 5
109 #define HITCOUNT_IDX 0
110 #define HIST_KEY_SIZE_MAX (MAX_FILTER_STR_VAL + HIST_STACKTRACE_SIZE)
112 enum hist_field_flags {
113 HIST_FIELD_FL_HITCOUNT = 1,
114 HIST_FIELD_FL_KEY = 2,
115 HIST_FIELD_FL_STRING = 4,
116 HIST_FIELD_FL_HEX = 8,
117 HIST_FIELD_FL_SYM = 16,
118 HIST_FIELD_FL_SYM_OFFSET = 32,
119 HIST_FIELD_FL_EXECNAME = 64,
120 HIST_FIELD_FL_SYSCALL = 128,
121 HIST_FIELD_FL_STACKTRACE = 256,
122 HIST_FIELD_FL_LOG2 = 512,
125 struct hist_trigger_attrs {
133 unsigned int map_bits;
136 struct hist_trigger_data {
137 struct hist_field *fields[TRACING_MAP_FIELDS_MAX];
140 unsigned int n_fields;
141 unsigned int key_size;
142 struct tracing_map_sort_key sort_keys[TRACING_MAP_SORT_KEYS_MAX];
143 unsigned int n_sort_keys;
144 struct trace_event_file *event_file;
145 struct hist_trigger_attrs *attrs;
146 struct tracing_map *map;
149 static hist_field_fn_t select_value_fn(int field_size, int field_is_signed)
151 hist_field_fn_t fn = NULL;
153 switch (field_size) {
183 static int parse_map_size(char *str)
185 unsigned long size, map_bits;
194 ret = kstrtoul(str, 0, &size);
198 map_bits = ilog2(roundup_pow_of_two(size));
199 if (map_bits < TRACING_MAP_BITS_MIN ||
200 map_bits > TRACING_MAP_BITS_MAX)
208 static void destroy_hist_trigger_attrs(struct hist_trigger_attrs *attrs)
214 kfree(attrs->sort_key_str);
215 kfree(attrs->keys_str);
216 kfree(attrs->vals_str);
220 static struct hist_trigger_attrs *parse_hist_trigger_attrs(char *trigger_str)
222 struct hist_trigger_attrs *attrs;
225 attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
227 return ERR_PTR(-ENOMEM);
229 while (trigger_str) {
230 char *str = strsep(&trigger_str, ":");
232 if ((strncmp(str, "key=", strlen("key=")) == 0) ||
233 (strncmp(str, "keys=", strlen("keys=")) == 0))
234 attrs->keys_str = kstrdup(str, GFP_KERNEL);
235 else if ((strncmp(str, "val=", strlen("val=")) == 0) ||
236 (strncmp(str, "vals=", strlen("vals=")) == 0) ||
237 (strncmp(str, "values=", strlen("values=")) == 0))
238 attrs->vals_str = kstrdup(str, GFP_KERNEL);
239 else if (strncmp(str, "sort=", strlen("sort=")) == 0)
240 attrs->sort_key_str = kstrdup(str, GFP_KERNEL);
241 else if (strncmp(str, "name=", strlen("name=")) == 0)
242 attrs->name = kstrdup(str, GFP_KERNEL);
243 else if (strcmp(str, "pause") == 0)
245 else if ((strcmp(str, "cont") == 0) ||
246 (strcmp(str, "continue") == 0))
248 else if (strcmp(str, "clear") == 0)
250 else if (strncmp(str, "size=", strlen("size=")) == 0) {
251 int map_bits = parse_map_size(str);
257 attrs->map_bits = map_bits;
264 if (!attrs->keys_str) {
271 destroy_hist_trigger_attrs(attrs);
276 static inline void save_comm(char *comm, struct task_struct *task)
279 strcpy(comm, "<idle>");
283 if (WARN_ON_ONCE(task->pid < 0)) {
284 strcpy(comm, "<XXX>");
288 memcpy(comm, task->comm, TASK_COMM_LEN);
291 static void hist_trigger_elt_comm_free(struct tracing_map_elt *elt)
293 kfree((char *)elt->private_data);
296 static int hist_trigger_elt_comm_alloc(struct tracing_map_elt *elt)
298 struct hist_trigger_data *hist_data = elt->map->private_data;
299 struct hist_field *key_field;
302 for_each_hist_key_field(i, hist_data) {
303 key_field = hist_data->fields[i];
305 if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
306 unsigned int size = TASK_COMM_LEN + 1;
308 elt->private_data = kzalloc(size, GFP_KERNEL);
309 if (!elt->private_data)
318 static void hist_trigger_elt_comm_copy(struct tracing_map_elt *to,
319 struct tracing_map_elt *from)
321 char *comm_from = from->private_data;
322 char *comm_to = to->private_data;
325 memcpy(comm_to, comm_from, TASK_COMM_LEN + 1);
328 static void hist_trigger_elt_comm_init(struct tracing_map_elt *elt)
330 char *comm = elt->private_data;
333 save_comm(comm, current);
336 static const struct tracing_map_ops hist_trigger_elt_comm_ops = {
337 .elt_alloc = hist_trigger_elt_comm_alloc,
338 .elt_copy = hist_trigger_elt_comm_copy,
339 .elt_free = hist_trigger_elt_comm_free,
340 .elt_init = hist_trigger_elt_comm_init,
343 static void destroy_hist_field(struct hist_field *hist_field)
348 static struct hist_field *create_hist_field(struct ftrace_event_field *field,
351 struct hist_field *hist_field;
353 if (field && is_function_field(field))
356 hist_field = kzalloc(sizeof(struct hist_field), GFP_KERNEL);
360 if (flags & HIST_FIELD_FL_HITCOUNT) {
361 hist_field->fn = hist_field_counter;
365 if (flags & HIST_FIELD_FL_STACKTRACE) {
366 hist_field->fn = hist_field_none;
370 if (flags & HIST_FIELD_FL_LOG2) {
371 hist_field->fn = hist_field_log2;
375 if (WARN_ON_ONCE(!field))
378 /* Pointers to strings are just pointers and dangerous to dereference */
379 if (is_string_field(field) &&
380 (field->filter_type != FILTER_PTR_STRING)) {
381 flags |= HIST_FIELD_FL_STRING;
383 if (field->filter_type == FILTER_STATIC_STRING)
384 hist_field->fn = hist_field_string;
385 else if (field->filter_type == FILTER_DYN_STRING)
386 hist_field->fn = hist_field_dynstring;
388 hist_field->fn = hist_field_pstring;
390 hist_field->fn = select_value_fn(field->size,
392 if (!hist_field->fn) {
393 destroy_hist_field(hist_field);
398 hist_field->field = field;
399 hist_field->flags = flags;
404 static void destroy_hist_fields(struct hist_trigger_data *hist_data)
408 for (i = 0; i < TRACING_MAP_FIELDS_MAX; i++) {
409 if (hist_data->fields[i]) {
410 destroy_hist_field(hist_data->fields[i]);
411 hist_data->fields[i] = NULL;
416 static int create_hitcount_val(struct hist_trigger_data *hist_data)
418 hist_data->fields[HITCOUNT_IDX] =
419 create_hist_field(NULL, HIST_FIELD_FL_HITCOUNT);
420 if (!hist_data->fields[HITCOUNT_IDX])
425 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
431 static int create_val_field(struct hist_trigger_data *hist_data,
432 unsigned int val_idx,
433 struct trace_event_file *file,
436 struct ftrace_event_field *field = NULL;
437 unsigned long flags = 0;
441 if (WARN_ON(val_idx >= TRACING_MAP_VALS_MAX))
444 field_name = strsep(&field_str, ".");
446 if (strcmp(field_str, "hex") == 0)
447 flags |= HIST_FIELD_FL_HEX;
454 field = trace_find_event_field(file->event_call, field_name);
455 if (!field || !field->size) {
460 hist_data->fields[val_idx] = create_hist_field(field, flags);
461 if (!hist_data->fields[val_idx]) {
468 if (WARN_ON(hist_data->n_vals > TRACING_MAP_VALS_MAX))
474 static int create_val_fields(struct hist_trigger_data *hist_data,
475 struct trace_event_file *file)
477 char *fields_str, *field_str;
481 ret = create_hitcount_val(hist_data);
485 fields_str = hist_data->attrs->vals_str;
489 strsep(&fields_str, "=");
493 for (i = 0, j = 1; i < TRACING_MAP_VALS_MAX &&
494 j < TRACING_MAP_VALS_MAX; i++) {
495 field_str = strsep(&fields_str, ",");
498 if (strcmp(field_str, "hitcount") == 0)
500 ret = create_val_field(hist_data, j++, file, field_str);
504 if (fields_str && (strcmp(fields_str, "hitcount") != 0))
510 static int create_key_field(struct hist_trigger_data *hist_data,
511 unsigned int key_idx,
512 unsigned int key_offset,
513 struct trace_event_file *file,
516 struct ftrace_event_field *field = NULL;
517 unsigned long flags = 0;
518 unsigned int key_size;
521 if (WARN_ON(key_idx >= TRACING_MAP_FIELDS_MAX))
524 flags |= HIST_FIELD_FL_KEY;
526 if (strcmp(field_str, "stacktrace") == 0) {
527 flags |= HIST_FIELD_FL_STACKTRACE;
528 key_size = sizeof(unsigned long) * HIST_STACKTRACE_DEPTH;
530 char *field_name = strsep(&field_str, ".");
533 if (strcmp(field_str, "hex") == 0)
534 flags |= HIST_FIELD_FL_HEX;
535 else if (strcmp(field_str, "sym") == 0)
536 flags |= HIST_FIELD_FL_SYM;
537 else if (strcmp(field_str, "sym-offset") == 0)
538 flags |= HIST_FIELD_FL_SYM_OFFSET;
539 else if ((strcmp(field_str, "execname") == 0) &&
540 (strcmp(field_name, "common_pid") == 0))
541 flags |= HIST_FIELD_FL_EXECNAME;
542 else if (strcmp(field_str, "syscall") == 0)
543 flags |= HIST_FIELD_FL_SYSCALL;
544 else if (strcmp(field_str, "log2") == 0)
545 flags |= HIST_FIELD_FL_LOG2;
552 field = trace_find_event_field(file->event_call, field_name);
553 if (!field || !field->size) {
558 if (is_string_field(field))
559 key_size = MAX_FILTER_STR_VAL;
561 key_size = field->size;
564 hist_data->fields[key_idx] = create_hist_field(field, flags);
565 if (!hist_data->fields[key_idx]) {
570 key_size = ALIGN(key_size, sizeof(u64));
571 hist_data->fields[key_idx]->size = key_size;
572 hist_data->fields[key_idx]->offset = key_offset;
573 hist_data->key_size += key_size;
574 if (hist_data->key_size > HIST_KEY_SIZE_MAX) {
581 if (WARN_ON(hist_data->n_keys > TRACING_MAP_KEYS_MAX))
589 static int create_key_fields(struct hist_trigger_data *hist_data,
590 struct trace_event_file *file)
592 unsigned int i, key_offset = 0, n_vals = hist_data->n_vals;
593 char *fields_str, *field_str;
596 fields_str = hist_data->attrs->keys_str;
600 strsep(&fields_str, "=");
604 for (i = n_vals; i < n_vals + TRACING_MAP_KEYS_MAX; i++) {
605 field_str = strsep(&fields_str, ",");
608 ret = create_key_field(hist_data, i, key_offset,
623 static int create_hist_fields(struct hist_trigger_data *hist_data,
624 struct trace_event_file *file)
628 ret = create_val_fields(hist_data, file);
632 ret = create_key_fields(hist_data, file);
636 hist_data->n_fields = hist_data->n_vals + hist_data->n_keys;
641 static int is_descending(const char *str)
646 if (strcmp(str, "descending") == 0)
649 if (strcmp(str, "ascending") == 0)
655 static int create_sort_keys(struct hist_trigger_data *hist_data)
657 char *fields_str = hist_data->attrs->sort_key_str;
658 struct ftrace_event_field *field = NULL;
659 struct tracing_map_sort_key *sort_key;
660 int descending, ret = 0;
663 hist_data->n_sort_keys = 1; /* we always have at least one, hitcount */
668 strsep(&fields_str, "=");
674 for (i = 0; i < TRACING_MAP_SORT_KEYS_MAX; i++) {
675 char *field_str, *field_name;
677 sort_key = &hist_data->sort_keys[i];
679 field_str = strsep(&fields_str, ",");
686 if ((i == TRACING_MAP_SORT_KEYS_MAX - 1) && fields_str) {
691 field_name = strsep(&field_str, ".");
697 if (strcmp(field_name, "hitcount") == 0) {
698 descending = is_descending(field_str);
699 if (descending < 0) {
703 sort_key->descending = descending;
707 for (j = 1; j < hist_data->n_fields; j++) {
708 field = hist_data->fields[j]->field;
709 if (field && (strcmp(field_name, field->name) == 0)) {
710 sort_key->field_idx = j;
711 descending = is_descending(field_str);
712 if (descending < 0) {
716 sort_key->descending = descending;
720 if (j == hist_data->n_fields) {
725 hist_data->n_sort_keys = i;
730 static void destroy_hist_data(struct hist_trigger_data *hist_data)
732 destroy_hist_trigger_attrs(hist_data->attrs);
733 destroy_hist_fields(hist_data);
734 tracing_map_destroy(hist_data->map);
738 static int create_tracing_map_fields(struct hist_trigger_data *hist_data)
740 struct tracing_map *map = hist_data->map;
741 struct ftrace_event_field *field;
742 struct hist_field *hist_field;
745 for_each_hist_field(i, hist_data) {
746 hist_field = hist_data->fields[i];
747 if (hist_field->flags & HIST_FIELD_FL_KEY) {
748 tracing_map_cmp_fn_t cmp_fn;
750 field = hist_field->field;
752 if (hist_field->flags & HIST_FIELD_FL_STACKTRACE)
753 cmp_fn = tracing_map_cmp_none;
754 else if (is_string_field(field))
755 cmp_fn = tracing_map_cmp_string;
757 cmp_fn = tracing_map_cmp_num(field->size,
759 idx = tracing_map_add_key_field(map,
764 idx = tracing_map_add_sum_field(map);
773 static bool need_tracing_map_ops(struct hist_trigger_data *hist_data)
775 struct hist_field *key_field;
778 for_each_hist_key_field(i, hist_data) {
779 key_field = hist_data->fields[i];
781 if (key_field->flags & HIST_FIELD_FL_EXECNAME)
788 static struct hist_trigger_data *
789 create_hist_data(unsigned int map_bits,
790 struct hist_trigger_attrs *attrs,
791 struct trace_event_file *file)
793 const struct tracing_map_ops *map_ops = NULL;
794 struct hist_trigger_data *hist_data;
797 hist_data = kzalloc(sizeof(*hist_data), GFP_KERNEL);
799 return ERR_PTR(-ENOMEM);
801 hist_data->attrs = attrs;
803 ret = create_hist_fields(hist_data, file);
807 ret = create_sort_keys(hist_data);
811 if (need_tracing_map_ops(hist_data))
812 map_ops = &hist_trigger_elt_comm_ops;
814 hist_data->map = tracing_map_create(map_bits, hist_data->key_size,
816 if (IS_ERR(hist_data->map)) {
817 ret = PTR_ERR(hist_data->map);
818 hist_data->map = NULL;
822 ret = create_tracing_map_fields(hist_data);
826 ret = tracing_map_init(hist_data->map);
830 hist_data->event_file = file;
834 hist_data->attrs = NULL;
836 destroy_hist_data(hist_data);
838 hist_data = ERR_PTR(ret);
843 static void hist_trigger_elt_update(struct hist_trigger_data *hist_data,
844 struct tracing_map_elt *elt,
847 struct hist_field *hist_field;
851 for_each_hist_val_field(i, hist_data) {
852 hist_field = hist_data->fields[i];
853 hist_val = hist_field->fn(hist_field, rec);
854 tracing_map_update_sum(elt, i, hist_val);
858 static inline void add_to_key(char *compound_key, void *key,
859 struct hist_field *key_field, void *rec)
861 size_t size = key_field->size;
863 if (key_field->flags & HIST_FIELD_FL_STRING) {
864 struct ftrace_event_field *field;
866 field = key_field->field;
867 if (field->filter_type == FILTER_DYN_STRING)
868 size = *(u32 *)(rec + field->offset) >> 16;
869 else if (field->filter_type == FILTER_STATIC_STRING)
872 /* ensure NULL-termination */
873 if (size > key_field->size - 1)
874 size = key_field->size - 1;
876 strncpy(compound_key + key_field->offset, (char *)key, size);
878 memcpy(compound_key + key_field->offset, key, size);
881 static void event_hist_trigger(struct event_trigger_data *data, void *rec)
883 struct hist_trigger_data *hist_data = data->private_data;
884 bool use_compound_key = (hist_data->n_keys > 1);
885 unsigned long entries[HIST_STACKTRACE_DEPTH];
886 char compound_key[HIST_KEY_SIZE_MAX];
887 struct stack_trace stacktrace;
888 struct hist_field *key_field;
889 struct tracing_map_elt *elt;
894 memset(compound_key, 0, hist_data->key_size);
896 for_each_hist_key_field(i, hist_data) {
897 key_field = hist_data->fields[i];
899 if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
900 stacktrace.max_entries = HIST_STACKTRACE_DEPTH;
901 stacktrace.entries = entries;
902 stacktrace.nr_entries = 0;
903 stacktrace.skip = HIST_STACKTRACE_SKIP;
905 memset(stacktrace.entries, 0, HIST_STACKTRACE_SIZE);
906 save_stack_trace(&stacktrace);
910 field_contents = key_field->fn(key_field, rec);
911 if (key_field->flags & HIST_FIELD_FL_STRING) {
912 key = (void *)(unsigned long)field_contents;
913 use_compound_key = true;
915 key = (void *)&field_contents;
918 if (use_compound_key)
919 add_to_key(compound_key, key, key_field, rec);
922 if (use_compound_key)
925 elt = tracing_map_insert(hist_data->map, key);
927 hist_trigger_elt_update(hist_data, elt, rec);
930 static void hist_trigger_stacktrace_print(struct seq_file *m,
931 unsigned long *stacktrace_entries,
932 unsigned int max_entries)
934 char str[KSYM_SYMBOL_LEN];
935 unsigned int spaces = 8;
938 for (i = 0; i < max_entries; i++) {
939 if (stacktrace_entries[i] == ULONG_MAX)
942 seq_printf(m, "%*c", 1 + spaces, ' ');
943 sprint_symbol(str, stacktrace_entries[i]);
944 seq_printf(m, "%s\n", str);
949 hist_trigger_entry_print(struct seq_file *m,
950 struct hist_trigger_data *hist_data, void *key,
951 struct tracing_map_elt *elt)
953 struct hist_field *key_field;
954 char str[KSYM_SYMBOL_LEN];
955 bool multiline = false;
961 for_each_hist_key_field(i, hist_data) {
962 key_field = hist_data->fields[i];
964 if (i > hist_data->n_vals)
967 if (key_field->flags & HIST_FIELD_FL_HEX) {
968 uval = *(u64 *)(key + key_field->offset);
969 seq_printf(m, "%s: %llx",
970 key_field->field->name, uval);
971 } else if (key_field->flags & HIST_FIELD_FL_SYM) {
972 uval = *(u64 *)(key + key_field->offset);
973 sprint_symbol_no_offset(str, uval);
974 seq_printf(m, "%s: [%llx] %-45s",
975 key_field->field->name, uval, str);
976 } else if (key_field->flags & HIST_FIELD_FL_SYM_OFFSET) {
977 uval = *(u64 *)(key + key_field->offset);
978 sprint_symbol(str, uval);
979 seq_printf(m, "%s: [%llx] %-55s",
980 key_field->field->name, uval, str);
981 } else if (key_field->flags & HIST_FIELD_FL_EXECNAME) {
982 char *comm = elt->private_data;
984 uval = *(u64 *)(key + key_field->offset);
985 seq_printf(m, "%s: %-16s[%10llu]",
986 key_field->field->name, comm, uval);
987 } else if (key_field->flags & HIST_FIELD_FL_SYSCALL) {
988 const char *syscall_name;
990 uval = *(u64 *)(key + key_field->offset);
991 syscall_name = get_syscall_name(uval);
993 syscall_name = "unknown_syscall";
995 seq_printf(m, "%s: %-30s[%3llu]",
996 key_field->field->name, syscall_name, uval);
997 } else if (key_field->flags & HIST_FIELD_FL_STACKTRACE) {
998 seq_puts(m, "stacktrace:\n");
999 hist_trigger_stacktrace_print(m,
1000 key + key_field->offset,
1001 HIST_STACKTRACE_DEPTH);
1003 } else if (key_field->flags & HIST_FIELD_FL_LOG2) {
1004 seq_printf(m, "%s: ~ 2^%-2llu", key_field->field->name,
1005 *(u64 *)(key + key_field->offset));
1006 } else if (key_field->flags & HIST_FIELD_FL_STRING) {
1007 seq_printf(m, "%s: %-50s", key_field->field->name,
1008 (char *)(key + key_field->offset));
1010 uval = *(u64 *)(key + key_field->offset);
1011 seq_printf(m, "%s: %10llu", key_field->field->name,
1021 seq_printf(m, " hitcount: %10llu",
1022 tracing_map_read_sum(elt, HITCOUNT_IDX));
1024 for (i = 1; i < hist_data->n_vals; i++) {
1025 if (hist_data->fields[i]->flags & HIST_FIELD_FL_HEX) {
1026 seq_printf(m, " %s: %10llx",
1027 hist_data->fields[i]->field->name,
1028 tracing_map_read_sum(elt, i));
1030 seq_printf(m, " %s: %10llu",
1031 hist_data->fields[i]->field->name,
1032 tracing_map_read_sum(elt, i));
1039 static int print_entries(struct seq_file *m,
1040 struct hist_trigger_data *hist_data)
1042 struct tracing_map_sort_entry **sort_entries = NULL;
1043 struct tracing_map *map = hist_data->map;
1046 n_entries = tracing_map_sort_entries(map, hist_data->sort_keys,
1047 hist_data->n_sort_keys,
1052 for (i = 0; i < n_entries; i++)
1053 hist_trigger_entry_print(m, hist_data,
1054 sort_entries[i]->key,
1055 sort_entries[i]->elt);
1057 tracing_map_destroy_sort_entries(sort_entries, n_entries);
1062 static void hist_trigger_show(struct seq_file *m,
1063 struct event_trigger_data *data, int n)
1065 struct hist_trigger_data *hist_data;
1066 int n_entries, ret = 0;
1069 seq_puts(m, "\n\n");
1071 seq_puts(m, "# event histogram\n#\n# trigger info: ");
1072 data->ops->print(m, data->ops, data);
1073 seq_puts(m, "#\n\n");
1075 hist_data = data->private_data;
1076 n_entries = print_entries(m, hist_data);
1077 if (n_entries < 0) {
1082 seq_printf(m, "\nTotals:\n Hits: %llu\n Entries: %u\n Dropped: %llu\n",
1083 (u64)atomic64_read(&hist_data->map->hits),
1084 n_entries, (u64)atomic64_read(&hist_data->map->drops));
1087 static int hist_show(struct seq_file *m, void *v)
1089 struct event_trigger_data *data;
1090 struct trace_event_file *event_file;
1093 mutex_lock(&event_mutex);
1095 event_file = event_file_data(m->private);
1096 if (unlikely(!event_file)) {
1101 list_for_each_entry_rcu(data, &event_file->triggers, list) {
1102 if (data->cmd_ops->trigger_type == ETT_EVENT_HIST)
1103 hist_trigger_show(m, data, n++);
1107 mutex_unlock(&event_mutex);
1112 static int event_hist_open(struct inode *inode, struct file *file)
1114 return single_open(file, hist_show, file);
1117 const struct file_operations event_hist_fops = {
1118 .open = event_hist_open,
1120 .llseek = seq_lseek,
1121 .release = single_release,
1124 static const char *get_hist_field_flags(struct hist_field *hist_field)
1126 const char *flags_str = NULL;
1128 if (hist_field->flags & HIST_FIELD_FL_HEX)
1130 else if (hist_field->flags & HIST_FIELD_FL_SYM)
1132 else if (hist_field->flags & HIST_FIELD_FL_SYM_OFFSET)
1133 flags_str = "sym-offset";
1134 else if (hist_field->flags & HIST_FIELD_FL_EXECNAME)
1135 flags_str = "execname";
1136 else if (hist_field->flags & HIST_FIELD_FL_SYSCALL)
1137 flags_str = "syscall";
1138 else if (hist_field->flags & HIST_FIELD_FL_LOG2)
1144 static void hist_field_print(struct seq_file *m, struct hist_field *hist_field)
1146 seq_printf(m, "%s", hist_field->field->name);
1147 if (hist_field->flags) {
1148 const char *flags_str = get_hist_field_flags(hist_field);
1151 seq_printf(m, ".%s", flags_str);
1155 static int event_hist_trigger_print(struct seq_file *m,
1156 struct event_trigger_ops *ops,
1157 struct event_trigger_data *data)
1159 struct hist_trigger_data *hist_data = data->private_data;
1160 struct hist_field *key_field;
1163 seq_puts(m, "hist:");
1166 seq_printf(m, "%s:", data->name);
1168 seq_puts(m, "keys=");
1170 for_each_hist_key_field(i, hist_data) {
1171 key_field = hist_data->fields[i];
1173 if (i > hist_data->n_vals)
1176 if (key_field->flags & HIST_FIELD_FL_STACKTRACE)
1177 seq_puts(m, "stacktrace");
1179 hist_field_print(m, key_field);
1182 seq_puts(m, ":vals=");
1184 for_each_hist_val_field(i, hist_data) {
1185 if (i == HITCOUNT_IDX)
1186 seq_puts(m, "hitcount");
1189 hist_field_print(m, hist_data->fields[i]);
1193 seq_puts(m, ":sort=");
1195 for (i = 0; i < hist_data->n_sort_keys; i++) {
1196 struct tracing_map_sort_key *sort_key;
1198 sort_key = &hist_data->sort_keys[i];
1203 if (sort_key->field_idx == HITCOUNT_IDX)
1204 seq_puts(m, "hitcount");
1206 unsigned int idx = sort_key->field_idx;
1208 if (WARN_ON(idx >= TRACING_MAP_FIELDS_MAX))
1211 hist_field_print(m, hist_data->fields[idx]);
1214 if (sort_key->descending)
1215 seq_puts(m, ".descending");
1218 seq_printf(m, ":size=%u", (1 << hist_data->map->map_bits));
1220 if (data->filter_str)
1221 seq_printf(m, " if %s", data->filter_str);
1224 seq_puts(m, " [paused]");
1226 seq_puts(m, " [active]");
1233 static int event_hist_trigger_init(struct event_trigger_ops *ops,
1234 struct event_trigger_data *data)
1236 struct hist_trigger_data *hist_data = data->private_data;
1238 if (!data->ref && hist_data->attrs->name)
1239 save_named_trigger(hist_data->attrs->name, data);
1246 static void event_hist_trigger_free(struct event_trigger_ops *ops,
1247 struct event_trigger_data *data)
1249 struct hist_trigger_data *hist_data = data->private_data;
1251 if (WARN_ON_ONCE(data->ref <= 0))
1257 del_named_trigger(data);
1258 trigger_data_free(data);
1259 destroy_hist_data(hist_data);
1263 static struct event_trigger_ops event_hist_trigger_ops = {
1264 .func = event_hist_trigger,
1265 .print = event_hist_trigger_print,
1266 .init = event_hist_trigger_init,
1267 .free = event_hist_trigger_free,
1270 static int event_hist_trigger_named_init(struct event_trigger_ops *ops,
1271 struct event_trigger_data *data)
1275 save_named_trigger(data->named_data->name, data);
1277 event_hist_trigger_init(ops, data->named_data);
1282 static void event_hist_trigger_named_free(struct event_trigger_ops *ops,
1283 struct event_trigger_data *data)
1285 if (WARN_ON_ONCE(data->ref <= 0))
1288 event_hist_trigger_free(ops, data->named_data);
1292 del_named_trigger(data);
1293 trigger_data_free(data);
1297 static struct event_trigger_ops event_hist_trigger_named_ops = {
1298 .func = event_hist_trigger,
1299 .print = event_hist_trigger_print,
1300 .init = event_hist_trigger_named_init,
1301 .free = event_hist_trigger_named_free,
1304 static struct event_trigger_ops *event_hist_get_trigger_ops(char *cmd,
1307 return &event_hist_trigger_ops;
1310 static void hist_clear(struct event_trigger_data *data)
1312 struct hist_trigger_data *hist_data = data->private_data;
1315 pause_named_trigger(data);
1317 synchronize_sched();
1319 tracing_map_clear(hist_data->map);
1322 unpause_named_trigger(data);
1325 static bool compatible_field(struct ftrace_event_field *field,
1326 struct ftrace_event_field *test_field)
1328 if (field == test_field)
1330 if (field == NULL || test_field == NULL)
1332 if (strcmp(field->name, test_field->name) != 0)
1334 if (strcmp(field->type, test_field->type) != 0)
1336 if (field->size != test_field->size)
1338 if (field->is_signed != test_field->is_signed)
1344 static bool hist_trigger_match(struct event_trigger_data *data,
1345 struct event_trigger_data *data_test,
1346 struct event_trigger_data *named_data,
1349 struct tracing_map_sort_key *sort_key, *sort_key_test;
1350 struct hist_trigger_data *hist_data, *hist_data_test;
1351 struct hist_field *key_field, *key_field_test;
1354 if (named_data && (named_data != data_test) &&
1355 (named_data != data_test->named_data))
1358 if (!named_data && is_named_trigger(data_test))
1361 hist_data = data->private_data;
1362 hist_data_test = data_test->private_data;
1364 if (hist_data->n_vals != hist_data_test->n_vals ||
1365 hist_data->n_fields != hist_data_test->n_fields ||
1366 hist_data->n_sort_keys != hist_data_test->n_sort_keys)
1369 if (!ignore_filter) {
1370 if ((data->filter_str && !data_test->filter_str) ||
1371 (!data->filter_str && data_test->filter_str))
1375 for_each_hist_field(i, hist_data) {
1376 key_field = hist_data->fields[i];
1377 key_field_test = hist_data_test->fields[i];
1379 if (key_field->flags != key_field_test->flags)
1381 if (!compatible_field(key_field->field, key_field_test->field))
1383 if (key_field->offset != key_field_test->offset)
1387 for (i = 0; i < hist_data->n_sort_keys; i++) {
1388 sort_key = &hist_data->sort_keys[i];
1389 sort_key_test = &hist_data_test->sort_keys[i];
1391 if (sort_key->field_idx != sort_key_test->field_idx ||
1392 sort_key->descending != sort_key_test->descending)
1396 if (!ignore_filter && data->filter_str &&
1397 (strcmp(data->filter_str, data_test->filter_str) != 0))
1403 static int hist_register_trigger(char *glob, struct event_trigger_ops *ops,
1404 struct event_trigger_data *data,
1405 struct trace_event_file *file)
1407 struct hist_trigger_data *hist_data = data->private_data;
1408 struct event_trigger_data *test, *named_data = NULL;
1411 if (hist_data->attrs->name) {
1412 named_data = find_named_trigger(hist_data->attrs->name);
1414 if (!hist_trigger_match(data, named_data, named_data,
1422 if (hist_data->attrs->name && !named_data)
1425 list_for_each_entry_rcu(test, &file->triggers, list) {
1426 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1427 if (!hist_trigger_match(data, test, named_data, false))
1429 if (hist_data->attrs->pause)
1430 test->paused = true;
1431 else if (hist_data->attrs->cont)
1432 test->paused = false;
1433 else if (hist_data->attrs->clear)
1441 if (hist_data->attrs->cont || hist_data->attrs->clear) {
1446 if (hist_data->attrs->pause)
1447 data->paused = true;
1450 destroy_hist_data(data->private_data);
1451 data->private_data = named_data->private_data;
1452 set_named_trigger_data(data, named_data);
1453 data->ops = &event_hist_trigger_named_ops;
1456 if (data->ops->init) {
1457 ret = data->ops->init(data->ops, data);
1462 list_add_rcu(&data->list, &file->triggers);
1465 update_cond_flag(file);
1467 if (trace_event_trigger_enable_disable(file, 1) < 0) {
1468 list_del_rcu(&data->list);
1469 update_cond_flag(file);
1476 static void hist_unregister_trigger(char *glob, struct event_trigger_ops *ops,
1477 struct event_trigger_data *data,
1478 struct trace_event_file *file)
1480 struct hist_trigger_data *hist_data = data->private_data;
1481 struct event_trigger_data *test, *named_data = NULL;
1482 bool unregistered = false;
1484 if (hist_data->attrs->name)
1485 named_data = find_named_trigger(hist_data->attrs->name);
1487 list_for_each_entry_rcu(test, &file->triggers, list) {
1488 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1489 if (!hist_trigger_match(data, test, named_data, false))
1491 unregistered = true;
1492 list_del_rcu(&test->list);
1493 trace_event_trigger_enable_disable(file, 0);
1494 update_cond_flag(file);
1499 if (unregistered && test->ops->free)
1500 test->ops->free(test->ops, test);
1503 static void hist_unreg_all(struct trace_event_file *file)
1505 struct event_trigger_data *test, *n;
1507 list_for_each_entry_safe(test, n, &file->triggers, list) {
1508 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1509 list_del_rcu(&test->list);
1510 trace_event_trigger_enable_disable(file, 0);
1511 update_cond_flag(file);
1512 if (test->ops->free)
1513 test->ops->free(test->ops, test);
1518 static int event_hist_trigger_func(struct event_command *cmd_ops,
1519 struct trace_event_file *file,
1520 char *glob, char *cmd, char *param)
1522 unsigned int hist_trigger_bits = TRACING_MAP_BITS_DEFAULT;
1523 struct event_trigger_data *trigger_data;
1524 struct hist_trigger_attrs *attrs;
1525 struct event_trigger_ops *trigger_ops;
1526 struct hist_trigger_data *hist_data;
1533 /* separate the trigger from the filter (k:v [if filter]) */
1534 trigger = strsep(¶m, " \t");
1538 attrs = parse_hist_trigger_attrs(trigger);
1540 return PTR_ERR(attrs);
1542 if (attrs->map_bits)
1543 hist_trigger_bits = attrs->map_bits;
1545 hist_data = create_hist_data(hist_trigger_bits, attrs, file);
1546 if (IS_ERR(hist_data)) {
1547 destroy_hist_trigger_attrs(attrs);
1548 return PTR_ERR(hist_data);
1551 trigger_ops = cmd_ops->get_trigger_ops(cmd, trigger);
1554 trigger_data = kzalloc(sizeof(*trigger_data), GFP_KERNEL);
1558 trigger_data->count = -1;
1559 trigger_data->ops = trigger_ops;
1560 trigger_data->cmd_ops = cmd_ops;
1562 INIT_LIST_HEAD(&trigger_data->list);
1563 RCU_INIT_POINTER(trigger_data->filter, NULL);
1565 trigger_data->private_data = hist_data;
1567 /* if param is non-empty, it's supposed to be a filter */
1568 if (param && cmd_ops->set_filter) {
1569 ret = cmd_ops->set_filter(param, trigger_data, file);
1574 if (glob[0] == '!') {
1575 cmd_ops->unreg(glob+1, trigger_ops, trigger_data, file);
1580 ret = cmd_ops->reg(glob, trigger_ops, trigger_data, file);
1582 * The above returns on success the # of triggers registered,
1583 * but if it didn't register any it returns zero. Consider no
1584 * triggers registered a failure too.
1587 if (!(attrs->pause || attrs->cont || attrs->clear))
1592 /* Just return zero, not the number of registered triggers */
1597 if (cmd_ops->set_filter)
1598 cmd_ops->set_filter(NULL, trigger_data, NULL);
1600 kfree(trigger_data);
1602 destroy_hist_data(hist_data);
1606 static struct event_command trigger_hist_cmd = {
1608 .trigger_type = ETT_EVENT_HIST,
1609 .flags = EVENT_CMD_FL_NEEDS_REC,
1610 .func = event_hist_trigger_func,
1611 .reg = hist_register_trigger,
1612 .unreg = hist_unregister_trigger,
1613 .unreg_all = hist_unreg_all,
1614 .get_trigger_ops = event_hist_get_trigger_ops,
1615 .set_filter = set_trigger_filter,
1618 __init int register_trigger_hist_cmd(void)
1622 ret = register_event_command(&trigger_hist_cmd);
1629 hist_enable_trigger(struct event_trigger_data *data, void *rec)
1631 struct enable_trigger_data *enable_data = data->private_data;
1632 struct event_trigger_data *test;
1634 list_for_each_entry_rcu(test, &enable_data->file->triggers, list) {
1635 if (test->cmd_ops->trigger_type == ETT_EVENT_HIST) {
1636 if (enable_data->enable)
1637 test->paused = false;
1639 test->paused = true;
1645 hist_enable_count_trigger(struct event_trigger_data *data, void *rec)
1650 if (data->count != -1)
1653 hist_enable_trigger(data, rec);
1656 static struct event_trigger_ops hist_enable_trigger_ops = {
1657 .func = hist_enable_trigger,
1658 .print = event_enable_trigger_print,
1659 .init = event_trigger_init,
1660 .free = event_enable_trigger_free,
1663 static struct event_trigger_ops hist_enable_count_trigger_ops = {
1664 .func = hist_enable_count_trigger,
1665 .print = event_enable_trigger_print,
1666 .init = event_trigger_init,
1667 .free = event_enable_trigger_free,
1670 static struct event_trigger_ops hist_disable_trigger_ops = {
1671 .func = hist_enable_trigger,
1672 .print = event_enable_trigger_print,
1673 .init = event_trigger_init,
1674 .free = event_enable_trigger_free,
1677 static struct event_trigger_ops hist_disable_count_trigger_ops = {
1678 .func = hist_enable_count_trigger,
1679 .print = event_enable_trigger_print,
1680 .init = event_trigger_init,
1681 .free = event_enable_trigger_free,
1684 static struct event_trigger_ops *
1685 hist_enable_get_trigger_ops(char *cmd, char *param)
1687 struct event_trigger_ops *ops;
1690 enable = (strcmp(cmd, ENABLE_HIST_STR) == 0);
1693 ops = param ? &hist_enable_count_trigger_ops :
1694 &hist_enable_trigger_ops;
1696 ops = param ? &hist_disable_count_trigger_ops :
1697 &hist_disable_trigger_ops;
1702 static void hist_enable_unreg_all(struct trace_event_file *file)
1704 struct event_trigger_data *test, *n;
1706 list_for_each_entry_safe(test, n, &file->triggers, list) {
1707 if (test->cmd_ops->trigger_type == ETT_HIST_ENABLE) {
1708 list_del_rcu(&test->list);
1709 update_cond_flag(file);
1710 trace_event_trigger_enable_disable(file, 0);
1711 if (test->ops->free)
1712 test->ops->free(test->ops, test);
1717 static struct event_command trigger_hist_enable_cmd = {
1718 .name = ENABLE_HIST_STR,
1719 .trigger_type = ETT_HIST_ENABLE,
1720 .func = event_enable_trigger_func,
1721 .reg = event_enable_register_trigger,
1722 .unreg = event_enable_unregister_trigger,
1723 .unreg_all = hist_enable_unreg_all,
1724 .get_trigger_ops = hist_enable_get_trigger_ops,
1725 .set_filter = set_trigger_filter,
1728 static struct event_command trigger_hist_disable_cmd = {
1729 .name = DISABLE_HIST_STR,
1730 .trigger_type = ETT_HIST_ENABLE,
1731 .func = event_enable_trigger_func,
1732 .reg = event_enable_register_trigger,
1733 .unreg = event_enable_unregister_trigger,
1734 .unreg_all = hist_enable_unreg_all,
1735 .get_trigger_ops = hist_enable_get_trigger_ops,
1736 .set_filter = set_trigger_filter,
1739 static __init void unregister_trigger_hist_enable_disable_cmds(void)
1741 unregister_event_command(&trigger_hist_enable_cmd);
1742 unregister_event_command(&trigger_hist_disable_cmd);
1745 __init int register_trigger_hist_enable_disable_cmds(void)
1749 ret = register_event_command(&trigger_hist_enable_cmd);
1750 if (WARN_ON(ret < 0))
1752 ret = register_event_command(&trigger_hist_disable_cmd);
1753 if (WARN_ON(ret < 0))
1754 unregister_trigger_hist_enable_disable_cmds();