1 // SPDX-License-Identifier: GPL-2.0
3 * Kprobes-based tracing events
5 * Created by Masami Hiramatsu <mhiramat@redhat.com>
8 #define pr_fmt(fmt) "trace_kprobe: " fmt
10 #include <linux/module.h>
11 #include <linux/uaccess.h>
12 #include <linux/rculist.h>
13 #include <linux/error-injection.h>
15 #include "trace_kprobe_selftest.h"
16 #include "trace_probe.h"
18 #define KPROBE_EVENT_SYSTEM "kprobes"
19 #define KRETPROBE_MAXACTIVE_MAX 4096
22 * Kprobe event core functions
25 struct list_head list;
26 struct kretprobe rp; /* Use rp.kp for kprobe use */
27 unsigned long __percpu *nhit;
28 const char *symbol; /* symbol name */
29 struct trace_probe tp;
32 #define SIZEOF_TRACE_KPROBE(n) \
33 (offsetof(struct trace_kprobe, tp.args) + \
34 (sizeof(struct probe_arg) * (n)))
36 static nokprobe_inline bool trace_kprobe_is_return(struct trace_kprobe *tk)
38 return tk->rp.handler != NULL;
41 static nokprobe_inline const char *trace_kprobe_symbol(struct trace_kprobe *tk)
43 return tk->symbol ? tk->symbol : "unknown";
46 static nokprobe_inline unsigned long trace_kprobe_offset(struct trace_kprobe *tk)
48 return tk->rp.kp.offset;
51 static nokprobe_inline bool trace_kprobe_has_gone(struct trace_kprobe *tk)
53 return !!(kprobe_gone(&tk->rp.kp));
56 static nokprobe_inline bool trace_kprobe_within_module(struct trace_kprobe *tk,
59 int len = strlen(mod->name);
60 const char *name = trace_kprobe_symbol(tk);
61 return strncmp(mod->name, name, len) == 0 && name[len] == ':';
64 static nokprobe_inline bool trace_kprobe_module_exist(struct trace_kprobe *tk)
71 p = strchr(tk->symbol, ':');
75 mutex_lock(&module_mutex);
76 ret = !!find_module(tk->symbol);
77 mutex_unlock(&module_mutex);
83 static nokprobe_inline unsigned long trace_kprobe_nhit(struct trace_kprobe *tk)
85 unsigned long nhit = 0;
88 for_each_possible_cpu(cpu)
89 nhit += *per_cpu_ptr(tk->nhit, cpu);
94 /* Return 0 if it fails to find the symbol address */
95 static nokprobe_inline
96 unsigned long trace_kprobe_address(struct trace_kprobe *tk)
101 addr = (unsigned long)
102 kallsyms_lookup_name(trace_kprobe_symbol(tk));
104 addr += tk->rp.kp.offset;
106 addr = (unsigned long)tk->rp.kp.addr;
111 bool trace_kprobe_on_func_entry(struct trace_event_call *call)
113 struct trace_kprobe *tk = (struct trace_kprobe *)call->data;
115 return (kprobe_on_func_entry(tk->rp.kp.addr,
116 tk->rp.kp.addr ? NULL : tk->rp.kp.symbol_name,
117 tk->rp.kp.addr ? 0 : tk->rp.kp.offset) == 0);
120 bool trace_kprobe_error_injectable(struct trace_event_call *call)
122 struct trace_kprobe *tk = (struct trace_kprobe *)call->data;
124 return within_error_injection_list(trace_kprobe_address(tk));
127 static int register_kprobe_event(struct trace_kprobe *tk);
128 static int unregister_kprobe_event(struct trace_kprobe *tk);
130 static DEFINE_MUTEX(probe_lock);
131 static LIST_HEAD(probe_list);
133 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs);
134 static int kretprobe_dispatcher(struct kretprobe_instance *ri,
135 struct pt_regs *regs);
137 /* Memory fetching by symbol */
138 struct symbol_cache {
144 unsigned long update_symbol_cache(struct symbol_cache *sc)
146 sc->addr = (unsigned long)kallsyms_lookup_name(sc->symbol);
149 sc->addr += sc->offset;
154 void free_symbol_cache(struct symbol_cache *sc)
160 struct symbol_cache *alloc_symbol_cache(const char *sym, long offset)
162 struct symbol_cache *sc;
164 if (!sym || strlen(sym) == 0)
167 sc = kzalloc(sizeof(struct symbol_cache), GFP_KERNEL);
171 sc->symbol = kstrdup(sym, GFP_KERNEL);
177 update_symbol_cache(sc);
183 * Kprobes-specific fetch functions
185 #define DEFINE_FETCH_stack(type) \
186 static void FETCH_FUNC_NAME(stack, type)(struct pt_regs *regs, \
187 void *offset, void *dest) \
189 *(type *)dest = (type)regs_get_kernel_stack_nth(regs, \
190 (unsigned int)((unsigned long)offset)); \
192 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(stack, type));
194 DEFINE_BASIC_FETCH_FUNCS(stack)
195 /* No string on the stack entry */
196 #define fetch_stack_string NULL
197 #define fetch_stack_string_size NULL
199 #define DEFINE_FETCH_memory(type) \
200 static void FETCH_FUNC_NAME(memory, type)(struct pt_regs *regs, \
201 void *addr, void *dest) \
204 if (probe_kernel_address(addr, retval)) \
207 *(type *)dest = retval; \
209 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, type));
211 DEFINE_BASIC_FETCH_FUNCS(memory)
213 * Fetch a null-terminated string. Caller MUST set *(u32 *)dest with max
214 * length and relative data location.
216 static void FETCH_FUNC_NAME(memory, string)(struct pt_regs *regs,
217 void *addr, void *dest)
219 int maxlen = get_rloc_len(*(u32 *)dest);
220 u8 *dst = get_rloc_data(dest);
227 * Try to get string again, since the string can be changed while
230 ret = strncpy_from_unsafe(dst, addr, maxlen);
232 if (ret < 0) { /* Failed to fetch string */
234 *(u32 *)dest = make_data_rloc(0, get_rloc_offs(*(u32 *)dest));
236 *(u32 *)dest = make_data_rloc(ret, get_rloc_offs(*(u32 *)dest));
239 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string));
241 /* Return the length of string -- including null terminal byte */
242 static void FETCH_FUNC_NAME(memory, string_size)(struct pt_regs *regs,
243 void *addr, void *dest)
254 ret = __copy_from_user_inatomic(&c, (u8 *)addr + len, 1);
256 } while (c && ret == 0 && len < MAX_STRING_SIZE);
261 if (ret < 0) /* Failed to check the length */
266 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(memory, string_size));
268 #define DEFINE_FETCH_symbol(type) \
269 void FETCH_FUNC_NAME(symbol, type)(struct pt_regs *regs, void *data, void *dest)\
271 struct symbol_cache *sc = data; \
273 fetch_memory_##type(regs, (void *)sc->addr, dest); \
277 NOKPROBE_SYMBOL(FETCH_FUNC_NAME(symbol, type));
279 DEFINE_BASIC_FETCH_FUNCS(symbol)
280 DEFINE_FETCH_symbol(string)
281 DEFINE_FETCH_symbol(string_size)
283 /* kprobes don't support file_offset fetch methods */
284 #define fetch_file_offset_u8 NULL
285 #define fetch_file_offset_u16 NULL
286 #define fetch_file_offset_u32 NULL
287 #define fetch_file_offset_u64 NULL
288 #define fetch_file_offset_string NULL
289 #define fetch_file_offset_string_size NULL
291 /* Fetch type information table */
292 static const struct fetch_type kprobes_fetch_type_table[] = {
294 [FETCH_TYPE_STRING] = __ASSIGN_FETCH_TYPE("string", string, string,
295 sizeof(u32), 1, "__data_loc char[]"),
296 [FETCH_TYPE_STRSIZE] = __ASSIGN_FETCH_TYPE("string_size", u32,
297 string_size, sizeof(u32), 0, "u32"),
299 ASSIGN_FETCH_TYPE(u8, u8, 0),
300 ASSIGN_FETCH_TYPE(u16, u16, 0),
301 ASSIGN_FETCH_TYPE(u32, u32, 0),
302 ASSIGN_FETCH_TYPE(u64, u64, 0),
303 ASSIGN_FETCH_TYPE(s8, u8, 1),
304 ASSIGN_FETCH_TYPE(s16, u16, 1),
305 ASSIGN_FETCH_TYPE(s32, u32, 1),
306 ASSIGN_FETCH_TYPE(s64, u64, 1),
307 ASSIGN_FETCH_TYPE_ALIAS(x8, u8, u8, 0),
308 ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
309 ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
310 ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
312 ASSIGN_FETCH_TYPE_END
316 * Allocate new trace_probe and initialize it (including kprobes).
318 static struct trace_kprobe *alloc_trace_kprobe(const char *group,
324 int nargs, bool is_return)
326 struct trace_kprobe *tk;
329 tk = kzalloc(SIZEOF_TRACE_KPROBE(nargs), GFP_KERNEL);
333 tk->nhit = alloc_percpu(unsigned long);
338 tk->symbol = kstrdup(symbol, GFP_KERNEL);
341 tk->rp.kp.symbol_name = tk->symbol;
342 tk->rp.kp.offset = offs;
344 tk->rp.kp.addr = addr;
347 tk->rp.handler = kretprobe_dispatcher;
349 tk->rp.kp.pre_handler = kprobe_dispatcher;
351 tk->rp.maxactive = maxactive;
353 if (!event || !is_good_name(event)) {
358 tk->tp.call.class = &tk->tp.class;
359 tk->tp.call.name = kstrdup(event, GFP_KERNEL);
360 if (!tk->tp.call.name)
363 if (!group || !is_good_name(group)) {
368 tk->tp.class.system = kstrdup(group, GFP_KERNEL);
369 if (!tk->tp.class.system)
372 INIT_LIST_HEAD(&tk->list);
373 INIT_LIST_HEAD(&tk->tp.files);
376 kfree(tk->tp.call.name);
378 free_percpu(tk->nhit);
383 static void free_trace_kprobe(struct trace_kprobe *tk)
387 for (i = 0; i < tk->tp.nr_args; i++)
388 traceprobe_free_probe_arg(&tk->tp.args[i]);
390 kfree(tk->tp.call.class->system);
391 kfree(tk->tp.call.name);
393 free_percpu(tk->nhit);
397 static struct trace_kprobe *find_trace_kprobe(const char *event,
400 struct trace_kprobe *tk;
402 list_for_each_entry(tk, &probe_list, list)
403 if (strcmp(trace_event_name(&tk->tp.call), event) == 0 &&
404 strcmp(tk->tp.call.class->system, group) == 0)
409 static inline int __enable_trace_kprobe(struct trace_kprobe *tk)
413 if (trace_probe_is_registered(&tk->tp) && !trace_kprobe_has_gone(tk)) {
414 if (trace_kprobe_is_return(tk))
415 ret = enable_kretprobe(&tk->rp);
417 ret = enable_kprobe(&tk->rp.kp);
425 * if the file is NULL, enable "perf" handler, or enable "trace" handler.
428 enable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file)
430 struct event_file_link *link;
434 link = kmalloc(sizeof(*link), GFP_KERNEL);
441 list_add_tail_rcu(&link->list, &tk->tp.files);
443 tk->tp.flags |= TP_FLAG_TRACE;
444 ret = __enable_trace_kprobe(tk);
446 list_del_rcu(&link->list);
448 tk->tp.flags &= ~TP_FLAG_TRACE;
452 tk->tp.flags |= TP_FLAG_PROFILE;
453 ret = __enable_trace_kprobe(tk);
455 tk->tp.flags &= ~TP_FLAG_PROFILE;
462 * Disable trace_probe
463 * if the file is NULL, disable "perf" handler, or disable "trace" handler.
466 disable_trace_kprobe(struct trace_kprobe *tk, struct trace_event_file *file)
468 struct event_file_link *link = NULL;
473 link = find_event_file_link(&tk->tp, file);
479 list_del_rcu(&link->list);
481 if (!list_empty(&tk->tp.files))
484 tk->tp.flags &= ~TP_FLAG_TRACE;
486 tk->tp.flags &= ~TP_FLAG_PROFILE;
488 if (!trace_probe_is_enabled(&tk->tp) && trace_probe_is_registered(&tk->tp)) {
489 if (trace_kprobe_is_return(tk))
490 disable_kretprobe(&tk->rp);
492 disable_kprobe(&tk->rp.kp);
497 * if tk is not added to any list, it must be a local trace_kprobe
498 * created with perf_event_open. We don't need to wait for these
501 if (list_empty(&tk->list))
506 * Synchronize with kprobe_trace_func/kretprobe_trace_func
507 * to ensure disabled (all running handlers are finished).
508 * This is not only for kfree(), but also the caller,
509 * trace_remove_event_call() supposes it for releasing
510 * event_call related objects, which will be accessed in
511 * the kprobe_trace_func/kretprobe_trace_func.
514 kfree(link); /* Ignored if link == NULL */
520 #if defined(CONFIG_DYNAMIC_FTRACE) && \
521 !defined(CONFIG_KPROBE_EVENTS_ON_NOTRACE)
522 static bool __within_notrace_func(unsigned long addr)
524 unsigned long offset, size;
526 if (!addr || !kallsyms_lookup_size_offset(addr, &size, &offset))
529 /* Get the entry address of the target function */
533 * Since ftrace_location_range() does inclusive range check, we need
534 * to subtract 1 byte from the end address.
536 return !ftrace_location_range(addr, addr + size - 1);
539 static bool within_notrace_func(struct trace_kprobe *tk)
541 unsigned long addr = trace_kprobe_address(tk);
542 char symname[KSYM_NAME_LEN], *p;
544 if (!__within_notrace_func(addr))
547 /* Check if the address is on a suffixed-symbol */
548 if (!lookup_symbol_name(addr, symname)) {
549 p = strchr(symname, '.');
553 addr = (unsigned long)kprobe_lookup_name(symname, 0);
555 return __within_notrace_func(addr);
561 #define within_notrace_func(tk) (false)
564 /* Internal register function - just handle k*probes and flags */
565 static int __register_trace_kprobe(struct trace_kprobe *tk)
569 if (trace_probe_is_registered(&tk->tp))
572 if (within_notrace_func(tk)) {
573 pr_warn("Could not probe notrace function %s\n",
574 trace_kprobe_symbol(tk));
578 for (i = 0; i < tk->tp.nr_args; i++)
579 traceprobe_update_arg(&tk->tp.args[i]);
581 /* Set/clear disabled flag according to tp->flag */
582 if (trace_probe_is_enabled(&tk->tp))
583 tk->rp.kp.flags &= ~KPROBE_FLAG_DISABLED;
585 tk->rp.kp.flags |= KPROBE_FLAG_DISABLED;
587 if (trace_kprobe_is_return(tk))
588 ret = register_kretprobe(&tk->rp);
590 ret = register_kprobe(&tk->rp.kp);
593 tk->tp.flags |= TP_FLAG_REGISTERED;
594 } else if (ret == -EILSEQ) {
595 pr_warn("Probing address(0x%p) is not an instruction boundary.\n",
602 /* Internal unregister function - just handle k*probes and flags */
603 static void __unregister_trace_kprobe(struct trace_kprobe *tk)
605 if (trace_probe_is_registered(&tk->tp)) {
606 if (trace_kprobe_is_return(tk))
607 unregister_kretprobe(&tk->rp);
609 unregister_kprobe(&tk->rp.kp);
610 tk->tp.flags &= ~TP_FLAG_REGISTERED;
611 /* Cleanup kprobe for reuse */
612 if (tk->rp.kp.symbol_name)
613 tk->rp.kp.addr = NULL;
617 /* Unregister a trace_probe and probe_event: call with locking probe_lock */
618 static int unregister_trace_kprobe(struct trace_kprobe *tk)
620 /* Enabled event can not be unregistered */
621 if (trace_probe_is_enabled(&tk->tp))
624 /* Will fail if probe is being used by ftrace or perf */
625 if (unregister_kprobe_event(tk))
628 __unregister_trace_kprobe(tk);
634 /* Register a trace_probe and probe_event */
635 static int register_trace_kprobe(struct trace_kprobe *tk)
637 struct trace_kprobe *old_tk;
640 mutex_lock(&probe_lock);
642 /* Delete old (same name) event if exist */
643 old_tk = find_trace_kprobe(trace_event_name(&tk->tp.call),
644 tk->tp.call.class->system);
646 ret = unregister_trace_kprobe(old_tk);
649 free_trace_kprobe(old_tk);
652 /* Register new event */
653 ret = register_kprobe_event(tk);
655 pr_warn("Failed to register probe event(%d)\n", ret);
659 /* Register k*probe */
660 ret = __register_trace_kprobe(tk);
661 if (ret == -ENOENT && !trace_kprobe_module_exist(tk)) {
662 pr_warn("This probe might be able to register after target module is loaded. Continue.\n");
667 unregister_kprobe_event(tk);
669 list_add_tail(&tk->list, &probe_list);
672 mutex_unlock(&probe_lock);
676 /* Module notifier call back, checking event on the module */
677 static int trace_kprobe_module_callback(struct notifier_block *nb,
678 unsigned long val, void *data)
680 struct module *mod = data;
681 struct trace_kprobe *tk;
684 if (val != MODULE_STATE_COMING)
687 /* Update probes on coming module */
688 mutex_lock(&probe_lock);
689 list_for_each_entry(tk, &probe_list, list) {
690 if (trace_kprobe_within_module(tk, mod)) {
691 /* Don't need to check busy - this should have gone. */
692 __unregister_trace_kprobe(tk);
693 ret = __register_trace_kprobe(tk);
695 pr_warn("Failed to re-register probe %s on %s: %d\n",
696 trace_event_name(&tk->tp.call),
700 mutex_unlock(&probe_lock);
705 static struct notifier_block trace_kprobe_module_nb = {
706 .notifier_call = trace_kprobe_module_callback,
707 .priority = 1 /* Invoked after kprobe module callback */
710 /* Convert certain expected symbols into '_' when generating event names */
711 static inline void sanitize_event_name(char *name)
713 while (*name++ != '\0')
714 if (*name == ':' || *name == '.')
718 static int create_trace_kprobe(int argc, char **argv)
723 * p[:[GRP/]EVENT] [MOD:]KSYM[+OFFS]|KADDR [FETCHARGS]
725 * r[MAXACTIVE][:[GRP/]EVENT] [MOD:]KSYM[+0] [FETCHARGS]
727 * $retval : fetch return value
728 * $stack : fetch stack address
729 * $stackN : fetch Nth of stack (N:0-)
730 * $comm : fetch current task comm
731 * @ADDR : fetch memory at ADDR (ADDR should be in kernel)
732 * @SYM[+|-offs] : fetch memory at SYM +|- offs (SYM is a data symbol)
733 * %REG : fetch register REG
734 * Dereferencing memory fetch:
735 * +|-offs(ARG) : fetch memory at ARG +|- offs address.
736 * Alias name of args:
737 * NAME=FETCHARG : set NAME as alias of FETCHARG.
739 * FETCHARG:TYPE : use TYPE instead of unsigned long.
741 struct trace_kprobe *tk;
743 bool is_return = false, is_delete = false;
744 char *symbol = NULL, *event = NULL, *group = NULL;
749 char buf[MAX_EVENT_NAME_LEN];
751 /* argc must be >= 1 */
752 if (argv[0][0] == 'p')
754 else if (argv[0][0] == 'r')
756 else if (argv[0][0] == '-')
759 pr_info("Probe definition must be started with 'p', 'r' or"
764 event = strchr(&argv[0][1], ':');
769 if (is_return && isdigit(argv[0][1])) {
770 ret = kstrtouint(&argv[0][1], 0, &maxactive);
772 pr_info("Failed to parse maxactive.\n");
775 /* kretprobes instances are iterated over via a list. The
776 * maximum should stay reasonable.
778 if (maxactive > KRETPROBE_MAXACTIVE_MAX) {
779 pr_info("Maxactive is too big (%d > %d).\n",
780 maxactive, KRETPROBE_MAXACTIVE_MAX);
786 if (strchr(event, '/')) {
788 event = strchr(group, '/') + 1;
790 if (strlen(group) == 0) {
791 pr_info("Group name is not specified\n");
795 if (strlen(event) == 0) {
796 pr_info("Event name is not specified\n");
801 group = KPROBE_EVENT_SYSTEM;
805 pr_info("Delete command needs an event name.\n");
808 mutex_lock(&probe_lock);
809 tk = find_trace_kprobe(event, group);
811 mutex_unlock(&probe_lock);
812 pr_info("Event %s/%s doesn't exist.\n", group, event);
815 /* delete an event */
816 ret = unregister_trace_kprobe(tk);
818 free_trace_kprobe(tk);
819 mutex_unlock(&probe_lock);
824 pr_info("Probe point is not specified.\n");
828 /* try to parse an address. if that fails, try to read the
829 * input as a symbol. */
830 if (kstrtoul(argv[1], 0, (unsigned long *)&addr)) {
831 /* a symbol specified */
833 /* TODO: support .init module functions */
834 ret = traceprobe_split_symbol_offset(symbol, &offset);
835 if (ret || offset < 0 || offset > UINT_MAX) {
836 pr_info("Failed to parse either an address or a symbol.\n");
839 /* Defer the ENOENT case until register kprobe */
840 if (offset && is_return &&
841 kprobe_on_func_entry(NULL, symbol, offset) == -EINVAL) {
842 pr_info("Given offset is not valid for return probe.\n");
846 argc -= 2; argv += 2;
850 /* Make a new event name */
852 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_%s_%ld",
853 is_return ? 'r' : 'p', symbol, offset);
855 snprintf(buf, MAX_EVENT_NAME_LEN, "%c_0x%p",
856 is_return ? 'r' : 'p', addr);
857 sanitize_event_name(buf);
860 tk = alloc_trace_kprobe(group, event, addr, symbol, offset, maxactive,
863 pr_info("Failed to allocate trace_probe.(%d)\n",
868 /* parse arguments */
870 for (i = 0; i < argc && i < MAX_TRACE_ARGS; i++) {
871 struct probe_arg *parg = &tk->tp.args[i];
873 /* Increment count for freeing args in error case */
876 /* Parse argument name */
877 arg = strchr(argv[i], '=');
880 parg->name = kstrdup(argv[i], GFP_KERNEL);
883 /* If argument name is omitted, set "argN" */
884 snprintf(buf, MAX_EVENT_NAME_LEN, "arg%d", i + 1);
885 parg->name = kstrdup(buf, GFP_KERNEL);
889 pr_info("Failed to allocate argument[%d] name.\n", i);
894 if (!is_good_name(parg->name)) {
895 pr_info("Invalid argument[%d] name: %s\n",
901 if (traceprobe_conflict_field_name(parg->name,
903 pr_info("Argument[%d] name '%s' conflicts with "
904 "another field.\n", i, argv[i]);
909 /* Parse fetch argument */
910 ret = traceprobe_parse_probe_arg(arg, &tk->tp.size, parg,
912 kprobes_fetch_type_table);
914 pr_info("Parse error at argument[%d]. (%d)\n", i, ret);
919 ret = register_trace_kprobe(tk);
925 free_trace_kprobe(tk);
929 static int release_all_trace_kprobes(void)
931 struct trace_kprobe *tk;
934 mutex_lock(&probe_lock);
935 /* Ensure no probe is in use. */
936 list_for_each_entry(tk, &probe_list, list)
937 if (trace_probe_is_enabled(&tk->tp)) {
941 /* TODO: Use batch unregistration */
942 while (!list_empty(&probe_list)) {
943 tk = list_entry(probe_list.next, struct trace_kprobe, list);
944 ret = unregister_trace_kprobe(tk);
947 free_trace_kprobe(tk);
951 mutex_unlock(&probe_lock);
956 /* Probes listing interfaces */
957 static void *probes_seq_start(struct seq_file *m, loff_t *pos)
959 mutex_lock(&probe_lock);
960 return seq_list_start(&probe_list, *pos);
963 static void *probes_seq_next(struct seq_file *m, void *v, loff_t *pos)
965 return seq_list_next(v, &probe_list, pos);
968 static void probes_seq_stop(struct seq_file *m, void *v)
970 mutex_unlock(&probe_lock);
973 static int probes_seq_show(struct seq_file *m, void *v)
975 struct trace_kprobe *tk = v;
978 seq_putc(m, trace_kprobe_is_return(tk) ? 'r' : 'p');
979 if (trace_kprobe_is_return(tk) && tk->rp.maxactive)
980 seq_printf(m, "%d", tk->rp.maxactive);
981 seq_printf(m, ":%s/%s", tk->tp.call.class->system,
982 trace_event_name(&tk->tp.call));
985 seq_printf(m, " 0x%p", tk->rp.kp.addr);
986 else if (tk->rp.kp.offset)
987 seq_printf(m, " %s+%u", trace_kprobe_symbol(tk),
990 seq_printf(m, " %s", trace_kprobe_symbol(tk));
992 for (i = 0; i < tk->tp.nr_args; i++)
993 seq_printf(m, " %s=%s", tk->tp.args[i].name, tk->tp.args[i].comm);
999 static const struct seq_operations probes_seq_op = {
1000 .start = probes_seq_start,
1001 .next = probes_seq_next,
1002 .stop = probes_seq_stop,
1003 .show = probes_seq_show
1006 static int probes_open(struct inode *inode, struct file *file)
1010 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
1011 ret = release_all_trace_kprobes();
1016 return seq_open(file, &probes_seq_op);
1019 static ssize_t probes_write(struct file *file, const char __user *buffer,
1020 size_t count, loff_t *ppos)
1022 return trace_parse_run_command(file, buffer, count, ppos,
1023 create_trace_kprobe);
1026 static const struct file_operations kprobe_events_ops = {
1027 .owner = THIS_MODULE,
1028 .open = probes_open,
1030 .llseek = seq_lseek,
1031 .release = seq_release,
1032 .write = probes_write,
1035 /* Probes profiling interfaces */
1036 static int probes_profile_seq_show(struct seq_file *m, void *v)
1038 struct trace_kprobe *tk = v;
1040 seq_printf(m, " %-44s %15lu %15lu\n",
1041 trace_event_name(&tk->tp.call),
1042 trace_kprobe_nhit(tk),
1048 static const struct seq_operations profile_seq_op = {
1049 .start = probes_seq_start,
1050 .next = probes_seq_next,
1051 .stop = probes_seq_stop,
1052 .show = probes_profile_seq_show
1055 static int profile_open(struct inode *inode, struct file *file)
1057 return seq_open(file, &profile_seq_op);
1060 static const struct file_operations kprobe_profile_ops = {
1061 .owner = THIS_MODULE,
1062 .open = profile_open,
1064 .llseek = seq_lseek,
1065 .release = seq_release,
1068 /* Kprobe handler */
1069 static nokprobe_inline void
1070 __kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs,
1071 struct trace_event_file *trace_file)
1073 struct kprobe_trace_entry_head *entry;
1074 struct ring_buffer_event *event;
1075 struct ring_buffer *buffer;
1076 int size, dsize, pc;
1077 unsigned long irq_flags;
1078 struct trace_event_call *call = &tk->tp.call;
1080 WARN_ON(call != trace_file->event_call);
1082 if (trace_trigger_soft_disabled(trace_file))
1085 local_save_flags(irq_flags);
1086 pc = preempt_count();
1088 dsize = __get_data_size(&tk->tp, regs);
1089 size = sizeof(*entry) + tk->tp.size + dsize;
1091 event = trace_event_buffer_lock_reserve(&buffer, trace_file,
1093 size, irq_flags, pc);
1097 entry = ring_buffer_event_data(event);
1098 entry->ip = (unsigned long)tk->rp.kp.addr;
1099 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
1101 event_trigger_unlock_commit_regs(trace_file, buffer, event,
1102 entry, irq_flags, pc, regs);
1106 kprobe_trace_func(struct trace_kprobe *tk, struct pt_regs *regs)
1108 struct event_file_link *link;
1110 list_for_each_entry_rcu(link, &tk->tp.files, list)
1111 __kprobe_trace_func(tk, regs, link->file);
1113 NOKPROBE_SYMBOL(kprobe_trace_func);
1115 /* Kretprobe handler */
1116 static nokprobe_inline void
1117 __kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1118 struct pt_regs *regs,
1119 struct trace_event_file *trace_file)
1121 struct kretprobe_trace_entry_head *entry;
1122 struct ring_buffer_event *event;
1123 struct ring_buffer *buffer;
1124 int size, pc, dsize;
1125 unsigned long irq_flags;
1126 struct trace_event_call *call = &tk->tp.call;
1128 WARN_ON(call != trace_file->event_call);
1130 if (trace_trigger_soft_disabled(trace_file))
1133 local_save_flags(irq_flags);
1134 pc = preempt_count();
1136 dsize = __get_data_size(&tk->tp, regs);
1137 size = sizeof(*entry) + tk->tp.size + dsize;
1139 event = trace_event_buffer_lock_reserve(&buffer, trace_file,
1141 size, irq_flags, pc);
1145 entry = ring_buffer_event_data(event);
1146 entry->func = (unsigned long)tk->rp.kp.addr;
1147 entry->ret_ip = (unsigned long)ri->ret_addr;
1148 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
1150 event_trigger_unlock_commit_regs(trace_file, buffer, event,
1151 entry, irq_flags, pc, regs);
1155 kretprobe_trace_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1156 struct pt_regs *regs)
1158 struct event_file_link *link;
1160 list_for_each_entry_rcu(link, &tk->tp.files, list)
1161 __kretprobe_trace_func(tk, ri, regs, link->file);
1163 NOKPROBE_SYMBOL(kretprobe_trace_func);
1165 /* Event entry printers */
1166 static enum print_line_t
1167 print_kprobe_event(struct trace_iterator *iter, int flags,
1168 struct trace_event *event)
1170 struct kprobe_trace_entry_head *field;
1171 struct trace_seq *s = &iter->seq;
1172 struct trace_probe *tp;
1176 field = (struct kprobe_trace_entry_head *)iter->ent;
1177 tp = container_of(event, struct trace_probe, call.event);
1179 trace_seq_printf(s, "%s: (", trace_event_name(&tp->call));
1181 if (!seq_print_ip_sym(s, field->ip, flags | TRACE_ITER_SYM_OFFSET))
1184 trace_seq_putc(s, ')');
1186 data = (u8 *)&field[1];
1187 for (i = 0; i < tp->nr_args; i++)
1188 if (!tp->args[i].type->print(s, tp->args[i].name,
1189 data + tp->args[i].offset, field))
1192 trace_seq_putc(s, '\n');
1194 return trace_handle_return(s);
1197 static enum print_line_t
1198 print_kretprobe_event(struct trace_iterator *iter, int flags,
1199 struct trace_event *event)
1201 struct kretprobe_trace_entry_head *field;
1202 struct trace_seq *s = &iter->seq;
1203 struct trace_probe *tp;
1207 field = (struct kretprobe_trace_entry_head *)iter->ent;
1208 tp = container_of(event, struct trace_probe, call.event);
1210 trace_seq_printf(s, "%s: (", trace_event_name(&tp->call));
1212 if (!seq_print_ip_sym(s, field->ret_ip, flags | TRACE_ITER_SYM_OFFSET))
1215 trace_seq_puts(s, " <- ");
1217 if (!seq_print_ip_sym(s, field->func, flags & ~TRACE_ITER_SYM_OFFSET))
1220 trace_seq_putc(s, ')');
1222 data = (u8 *)&field[1];
1223 for (i = 0; i < tp->nr_args; i++)
1224 if (!tp->args[i].type->print(s, tp->args[i].name,
1225 data + tp->args[i].offset, field))
1228 trace_seq_putc(s, '\n');
1231 return trace_handle_return(s);
1235 static int kprobe_event_define_fields(struct trace_event_call *event_call)
1238 struct kprobe_trace_entry_head field;
1239 struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;
1241 DEFINE_FIELD(unsigned long, ip, FIELD_STRING_IP, 0);
1242 /* Set argument names as fields */
1243 for (i = 0; i < tk->tp.nr_args; i++) {
1244 struct probe_arg *parg = &tk->tp.args[i];
1246 ret = trace_define_field(event_call, parg->type->fmttype,
1248 sizeof(field) + parg->offset,
1250 parg->type->is_signed,
1258 static int kretprobe_event_define_fields(struct trace_event_call *event_call)
1261 struct kretprobe_trace_entry_head field;
1262 struct trace_kprobe *tk = (struct trace_kprobe *)event_call->data;
1264 DEFINE_FIELD(unsigned long, func, FIELD_STRING_FUNC, 0);
1265 DEFINE_FIELD(unsigned long, ret_ip, FIELD_STRING_RETIP, 0);
1266 /* Set argument names as fields */
1267 for (i = 0; i < tk->tp.nr_args; i++) {
1268 struct probe_arg *parg = &tk->tp.args[i];
1270 ret = trace_define_field(event_call, parg->type->fmttype,
1272 sizeof(field) + parg->offset,
1274 parg->type->is_signed,
1282 #ifdef CONFIG_PERF_EVENTS
1284 /* Kprobe profile handler */
1286 kprobe_perf_func(struct trace_kprobe *tk, struct pt_regs *regs)
1288 struct trace_event_call *call = &tk->tp.call;
1289 struct kprobe_trace_entry_head *entry;
1290 struct hlist_head *head;
1291 int size, __size, dsize;
1294 if (bpf_prog_array_valid(call)) {
1295 unsigned long orig_ip = instruction_pointer(regs);
1298 ret = trace_call_bpf(call, regs);
1301 * We need to check and see if we modified the pc of the
1302 * pt_regs, and if so return 1 so that we don't do the
1305 if (orig_ip != instruction_pointer(regs))
1311 head = this_cpu_ptr(call->perf_events);
1312 if (hlist_empty(head))
1315 dsize = __get_data_size(&tk->tp, regs);
1316 __size = sizeof(*entry) + tk->tp.size + dsize;
1317 size = ALIGN(__size + sizeof(u32), sizeof(u64));
1318 size -= sizeof(u32);
1320 entry = perf_trace_buf_alloc(size, NULL, &rctx);
1324 entry->ip = (unsigned long)tk->rp.kp.addr;
1325 memset(&entry[1], 0, dsize);
1326 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
1327 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1331 NOKPROBE_SYMBOL(kprobe_perf_func);
1333 /* Kretprobe profile handler */
1335 kretprobe_perf_func(struct trace_kprobe *tk, struct kretprobe_instance *ri,
1336 struct pt_regs *regs)
1338 struct trace_event_call *call = &tk->tp.call;
1339 struct kretprobe_trace_entry_head *entry;
1340 struct hlist_head *head;
1341 int size, __size, dsize;
1344 if (bpf_prog_array_valid(call) && !trace_call_bpf(call, regs))
1347 head = this_cpu_ptr(call->perf_events);
1348 if (hlist_empty(head))
1351 dsize = __get_data_size(&tk->tp, regs);
1352 __size = sizeof(*entry) + tk->tp.size + dsize;
1353 size = ALIGN(__size + sizeof(u32), sizeof(u64));
1354 size -= sizeof(u32);
1356 entry = perf_trace_buf_alloc(size, NULL, &rctx);
1360 entry->func = (unsigned long)tk->rp.kp.addr;
1361 entry->ret_ip = (unsigned long)ri->ret_addr;
1362 store_trace_args(sizeof(*entry), &tk->tp, regs, (u8 *)&entry[1], dsize);
1363 perf_trace_buf_submit(entry, size, rctx, call->event.type, 1, regs,
1366 NOKPROBE_SYMBOL(kretprobe_perf_func);
1368 int bpf_get_kprobe_info(const struct perf_event *event, u32 *fd_type,
1369 const char **symbol, u64 *probe_offset,
1370 u64 *probe_addr, bool perf_type_tracepoint)
1372 const char *pevent = trace_event_name(event->tp_event);
1373 const char *group = event->tp_event->class->system;
1374 struct trace_kprobe *tk;
1376 if (perf_type_tracepoint)
1377 tk = find_trace_kprobe(pevent, group);
1379 tk = event->tp_event->data;
1383 *fd_type = trace_kprobe_is_return(tk) ? BPF_FD_TYPE_KRETPROBE
1384 : BPF_FD_TYPE_KPROBE;
1386 *symbol = tk->symbol;
1387 *probe_offset = tk->rp.kp.offset;
1392 *probe_addr = (unsigned long)tk->rp.kp.addr;
1396 #endif /* CONFIG_PERF_EVENTS */
1399 * called by perf_trace_init() or __ftrace_set_clr_event() under event_mutex.
1401 * kprobe_trace_self_tests_init() does enable_trace_probe/disable_trace_probe
1402 * lockless, but we can't race with this __init function.
1404 static int kprobe_register(struct trace_event_call *event,
1405 enum trace_reg type, void *data)
1407 struct trace_kprobe *tk = (struct trace_kprobe *)event->data;
1408 struct trace_event_file *file = data;
1411 case TRACE_REG_REGISTER:
1412 return enable_trace_kprobe(tk, file);
1413 case TRACE_REG_UNREGISTER:
1414 return disable_trace_kprobe(tk, file);
1416 #ifdef CONFIG_PERF_EVENTS
1417 case TRACE_REG_PERF_REGISTER:
1418 return enable_trace_kprobe(tk, NULL);
1419 case TRACE_REG_PERF_UNREGISTER:
1420 return disable_trace_kprobe(tk, NULL);
1421 case TRACE_REG_PERF_OPEN:
1422 case TRACE_REG_PERF_CLOSE:
1423 case TRACE_REG_PERF_ADD:
1424 case TRACE_REG_PERF_DEL:
1431 static int kprobe_dispatcher(struct kprobe *kp, struct pt_regs *regs)
1433 struct trace_kprobe *tk = container_of(kp, struct trace_kprobe, rp.kp);
1436 raw_cpu_inc(*tk->nhit);
1438 if (tk->tp.flags & TP_FLAG_TRACE)
1439 kprobe_trace_func(tk, regs);
1440 #ifdef CONFIG_PERF_EVENTS
1441 if (tk->tp.flags & TP_FLAG_PROFILE)
1442 ret = kprobe_perf_func(tk, regs);
1446 NOKPROBE_SYMBOL(kprobe_dispatcher);
1449 kretprobe_dispatcher(struct kretprobe_instance *ri, struct pt_regs *regs)
1451 struct trace_kprobe *tk = container_of(ri->rp, struct trace_kprobe, rp);
1453 raw_cpu_inc(*tk->nhit);
1455 if (tk->tp.flags & TP_FLAG_TRACE)
1456 kretprobe_trace_func(tk, ri, regs);
1457 #ifdef CONFIG_PERF_EVENTS
1458 if (tk->tp.flags & TP_FLAG_PROFILE)
1459 kretprobe_perf_func(tk, ri, regs);
1461 return 0; /* We don't tweek kernel, so just return 0 */
1463 NOKPROBE_SYMBOL(kretprobe_dispatcher);
1465 static struct trace_event_functions kretprobe_funcs = {
1466 .trace = print_kretprobe_event
1469 static struct trace_event_functions kprobe_funcs = {
1470 .trace = print_kprobe_event
1473 static inline void init_trace_event_call(struct trace_kprobe *tk,
1474 struct trace_event_call *call)
1476 INIT_LIST_HEAD(&call->class->fields);
1477 if (trace_kprobe_is_return(tk)) {
1478 call->event.funcs = &kretprobe_funcs;
1479 call->class->define_fields = kretprobe_event_define_fields;
1481 call->event.funcs = &kprobe_funcs;
1482 call->class->define_fields = kprobe_event_define_fields;
1485 call->flags = TRACE_EVENT_FL_KPROBE;
1486 call->class->reg = kprobe_register;
1490 static int register_kprobe_event(struct trace_kprobe *tk)
1492 struct trace_event_call *call = &tk->tp.call;
1495 init_trace_event_call(tk, call);
1497 if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0)
1499 ret = register_trace_event(&call->event);
1501 kfree(call->print_fmt);
1504 ret = trace_add_event_call(call);
1506 pr_info("Failed to register kprobe event: %s\n",
1507 trace_event_name(call));
1508 kfree(call->print_fmt);
1509 unregister_trace_event(&call->event);
1514 static int unregister_kprobe_event(struct trace_kprobe *tk)
1518 /* tp->event is unregistered in trace_remove_event_call() */
1519 ret = trace_remove_event_call(&tk->tp.call);
1521 kfree(tk->tp.call.print_fmt);
1525 #ifdef CONFIG_PERF_EVENTS
1526 /* create a trace_kprobe, but don't add it to global lists */
1527 struct trace_event_call *
1528 create_local_trace_kprobe(char *func, void *addr, unsigned long offs,
1531 struct trace_kprobe *tk;
1536 * local trace_kprobes are not added to probe_list, so they are never
1537 * searched in find_trace_kprobe(). Therefore, there is no concern of
1538 * duplicated name here.
1540 event = func ? func : "DUMMY_EVENT";
1542 tk = alloc_trace_kprobe(KPROBE_EVENT_SYSTEM, event, (void *)addr, func,
1543 offs, 0 /* maxactive */, 0 /* nargs */,
1547 pr_info("Failed to allocate trace_probe.(%d)\n",
1549 return ERR_CAST(tk);
1552 init_trace_event_call(tk, &tk->tp.call);
1554 if (set_print_fmt(&tk->tp, trace_kprobe_is_return(tk)) < 0) {
1559 ret = __register_trace_kprobe(tk);
1561 kfree(tk->tp.call.print_fmt);
1565 return &tk->tp.call;
1567 free_trace_kprobe(tk);
1568 return ERR_PTR(ret);
1571 void destroy_local_trace_kprobe(struct trace_event_call *event_call)
1573 struct trace_kprobe *tk;
1575 tk = container_of(event_call, struct trace_kprobe, tp.call);
1577 if (trace_probe_is_enabled(&tk->tp)) {
1582 __unregister_trace_kprobe(tk);
1584 kfree(tk->tp.call.print_fmt);
1585 free_trace_kprobe(tk);
1587 #endif /* CONFIG_PERF_EVENTS */
1589 /* Make a tracefs interface for controlling probe points */
1590 static __init int init_kprobe_trace(void)
1592 struct dentry *d_tracer;
1593 struct dentry *entry;
1595 if (register_module_notifier(&trace_kprobe_module_nb))
1598 d_tracer = tracing_init_dentry();
1599 if (IS_ERR(d_tracer))
1602 entry = tracefs_create_file("kprobe_events", 0644, d_tracer,
1603 NULL, &kprobe_events_ops);
1605 /* Event list interface */
1607 pr_warn("Could not create tracefs 'kprobe_events' entry\n");
1609 /* Profile interface */
1610 entry = tracefs_create_file("kprobe_profile", 0444, d_tracer,
1611 NULL, &kprobe_profile_ops);
1614 pr_warn("Could not create tracefs 'kprobe_profile' entry\n");
1617 fs_initcall(init_kprobe_trace);
1620 #ifdef CONFIG_FTRACE_STARTUP_TEST
1621 static __init struct trace_event_file *
1622 find_trace_probe_file(struct trace_kprobe *tk, struct trace_array *tr)
1624 struct trace_event_file *file;
1626 list_for_each_entry(file, &tr->events, list)
1627 if (file->event_call == &tk->tp.call)
1634 * Nobody but us can call enable_trace_kprobe/disable_trace_kprobe at this
1635 * stage, we can do this lockless.
1637 static __init int kprobe_trace_self_tests_init(void)
1640 int (*target)(int, int, int, int, int, int);
1641 struct trace_kprobe *tk;
1642 struct trace_event_file *file;
1644 if (tracing_is_disabled())
1647 target = kprobe_trace_selftest_target;
1649 pr_info("Testing kprobe tracing: ");
1651 ret = trace_run_command("p:testprobe kprobe_trace_selftest_target "
1652 "$stack $stack0 +0($stack)",
1653 create_trace_kprobe);
1654 if (WARN_ON_ONCE(ret)) {
1655 pr_warn("error on probing function entry.\n");
1658 /* Enable trace point */
1659 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
1660 if (WARN_ON_ONCE(tk == NULL)) {
1661 pr_warn("error on getting new probe.\n");
1664 file = find_trace_probe_file(tk, top_trace_array());
1665 if (WARN_ON_ONCE(file == NULL)) {
1666 pr_warn("error on getting probe file.\n");
1669 enable_trace_kprobe(tk, file);
1673 ret = trace_run_command("r:testprobe2 kprobe_trace_selftest_target "
1674 "$retval", create_trace_kprobe);
1675 if (WARN_ON_ONCE(ret)) {
1676 pr_warn("error on probing function return.\n");
1679 /* Enable trace point */
1680 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
1681 if (WARN_ON_ONCE(tk == NULL)) {
1682 pr_warn("error on getting 2nd new probe.\n");
1685 file = find_trace_probe_file(tk, top_trace_array());
1686 if (WARN_ON_ONCE(file == NULL)) {
1687 pr_warn("error on getting probe file.\n");
1690 enable_trace_kprobe(tk, file);
1697 ret = target(1, 2, 3, 4, 5, 6);
1700 * Not expecting an error here, the check is only to prevent the
1701 * optimizer from removing the call to target() as otherwise there
1702 * are no side-effects and the call is never performed.
1707 /* Disable trace points before removing it */
1708 tk = find_trace_kprobe("testprobe", KPROBE_EVENT_SYSTEM);
1709 if (WARN_ON_ONCE(tk == NULL)) {
1710 pr_warn("error on getting test probe.\n");
1713 if (trace_kprobe_nhit(tk) != 1) {
1714 pr_warn("incorrect number of testprobe hits\n");
1718 file = find_trace_probe_file(tk, top_trace_array());
1719 if (WARN_ON_ONCE(file == NULL)) {
1720 pr_warn("error on getting probe file.\n");
1723 disable_trace_kprobe(tk, file);
1726 tk = find_trace_kprobe("testprobe2", KPROBE_EVENT_SYSTEM);
1727 if (WARN_ON_ONCE(tk == NULL)) {
1728 pr_warn("error on getting 2nd test probe.\n");
1731 if (trace_kprobe_nhit(tk) != 1) {
1732 pr_warn("incorrect number of testprobe2 hits\n");
1736 file = find_trace_probe_file(tk, top_trace_array());
1737 if (WARN_ON_ONCE(file == NULL)) {
1738 pr_warn("error on getting probe file.\n");
1741 disable_trace_kprobe(tk, file);
1744 ret = trace_run_command("-:testprobe", create_trace_kprobe);
1745 if (WARN_ON_ONCE(ret)) {
1746 pr_warn("error on deleting a probe.\n");
1750 ret = trace_run_command("-:testprobe2", create_trace_kprobe);
1751 if (WARN_ON_ONCE(ret)) {
1752 pr_warn("error on deleting a probe.\n");
1757 release_all_trace_kprobes();
1759 * Wait for the optimizer work to finish. Otherwise it might fiddle
1760 * with probes in already freed __init text.
1762 wait_for_kprobe_optimizer();
1764 pr_cont("NG: Some tests are failed. Please check them.\n");
1770 late_initcall(kprobe_trace_self_tests_init);