2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/tracefs.h>
24 #include <linux/pagemap.h>
25 #include <linux/hardirq.h>
26 #include <linux/linkage.h>
27 #include <linux/uaccess.h>
28 #include <linux/vmalloc.h>
29 #include <linux/ftrace.h>
30 #include <linux/module.h>
31 #include <linux/percpu.h>
32 #include <linux/splice.h>
33 #include <linux/kdebug.h>
34 #include <linux/string.h>
35 #include <linux/mount.h>
36 #include <linux/rwsem.h>
37 #include <linux/slab.h>
38 #include <linux/ctype.h>
39 #include <linux/init.h>
40 #include <linux/poll.h>
41 #include <linux/nmi.h>
43 #include <linux/trace.h>
44 #include <linux/sched/rt.h>
47 #include "trace_output.h"
50 * On boot up, the ring buffer is set to the minimum size, so that
51 * we do not waste memory on systems that are not using tracing.
53 bool ring_buffer_expanded;
56 * We need to change this state when a selftest is running.
57 * A selftest will lurk into the ring-buffer to count the
58 * entries inserted during the selftest although some concurrent
59 * insertions into the ring-buffer such as trace_printk could occurred
60 * at the same time, giving false positive or negative results.
62 static bool __read_mostly tracing_selftest_running;
65 * If a tracer is running, we do not want to run SELFTEST.
67 bool __read_mostly tracing_selftest_disabled;
69 /* Pipe tracepoints to printk */
70 struct trace_iterator *tracepoint_print_iter;
71 int tracepoint_printk;
72 static DEFINE_STATIC_KEY_FALSE(tracepoint_printk_key);
74 /* For tracers that don't implement custom flags */
75 static struct tracer_opt dummy_tracer_opt[] = {
80 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
86 * To prevent the comm cache from being overwritten when no
87 * tracing is active, only save the comm when a trace event
90 static DEFINE_PER_CPU(bool, trace_taskinfo_save);
93 * Kill all tracing for good (never come back).
94 * It is initialized to 1 but will turn to zero if the initialization
95 * of the tracer is successful. But that is the only place that sets
98 static int tracing_disabled = 1;
100 cpumask_var_t __read_mostly tracing_buffer_mask;
103 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
105 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
106 * is set, then ftrace_dump is called. This will output the contents
107 * of the ftrace buffers to the console. This is very useful for
108 * capturing traces that lead to crashes and outputing it to a
111 * It is default off, but you can enable it with either specifying
112 * "ftrace_dump_on_oops" in the kernel command line, or setting
113 * /proc/sys/kernel/ftrace_dump_on_oops
114 * Set 1 if you want to dump buffers of all CPUs
115 * Set 2 if you want to dump the buffer of the CPU that triggered oops
118 enum ftrace_dump_mode ftrace_dump_on_oops;
120 /* When set, tracing will stop when a WARN*() is hit */
121 int __disable_trace_on_warning;
123 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
124 /* Map of enums to their values, for "eval_map" file */
125 struct trace_eval_map_head {
127 unsigned long length;
130 union trace_eval_map_item;
132 struct trace_eval_map_tail {
134 * "end" is first and points to NULL as it must be different
135 * than "mod" or "eval_string"
137 union trace_eval_map_item *next;
138 const char *end; /* points to NULL */
141 static DEFINE_MUTEX(trace_eval_mutex);
144 * The trace_eval_maps are saved in an array with two extra elements,
145 * one at the beginning, and one at the end. The beginning item contains
146 * the count of the saved maps (head.length), and the module they
147 * belong to if not built in (head.mod). The ending item contains a
148 * pointer to the next array of saved eval_map items.
150 union trace_eval_map_item {
151 struct trace_eval_map map;
152 struct trace_eval_map_head head;
153 struct trace_eval_map_tail tail;
156 static union trace_eval_map_item *trace_eval_maps;
157 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
159 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
161 #define MAX_TRACER_SIZE 100
162 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
163 static char *default_bootup_tracer;
165 static bool allocate_snapshot;
167 static int __init set_cmdline_ftrace(char *str)
169 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
170 default_bootup_tracer = bootup_tracer_buf;
171 /* We are using ftrace early, expand it */
172 ring_buffer_expanded = true;
175 __setup("ftrace=", set_cmdline_ftrace);
177 static int __init set_ftrace_dump_on_oops(char *str)
179 if (*str++ != '=' || !*str) {
180 ftrace_dump_on_oops = DUMP_ALL;
184 if (!strcmp("orig_cpu", str)) {
185 ftrace_dump_on_oops = DUMP_ORIG;
191 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
193 static int __init stop_trace_on_warning(char *str)
195 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
196 __disable_trace_on_warning = 1;
199 __setup("traceoff_on_warning", stop_trace_on_warning);
201 static int __init boot_alloc_snapshot(char *str)
203 allocate_snapshot = true;
204 /* We also need the main ring buffer expanded */
205 ring_buffer_expanded = true;
208 __setup("alloc_snapshot", boot_alloc_snapshot);
211 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
213 static int __init set_trace_boot_options(char *str)
215 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
218 __setup("trace_options=", set_trace_boot_options);
220 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
221 static char *trace_boot_clock __initdata;
223 static int __init set_trace_boot_clock(char *str)
225 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
226 trace_boot_clock = trace_boot_clock_buf;
229 __setup("trace_clock=", set_trace_boot_clock);
231 static int __init set_tracepoint_printk(char *str)
233 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
234 tracepoint_printk = 1;
237 __setup("tp_printk", set_tracepoint_printk);
239 unsigned long long ns2usecs(u64 nsec)
246 /* trace_flags holds trace_options default values */
247 #define TRACE_DEFAULT_FLAGS \
248 (FUNCTION_DEFAULT_FLAGS | \
249 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
250 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
251 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
252 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
254 /* trace_options that are only supported by global_trace */
255 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
256 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
258 /* trace_flags that are default zero for instances */
259 #define ZEROED_TRACE_FLAGS \
260 (TRACE_ITER_EVENT_FORK | TRACE_ITER_FUNC_FORK)
263 * The global_trace is the descriptor that holds the top-level tracing
264 * buffers for the live tracing.
266 static struct trace_array global_trace = {
267 .trace_flags = TRACE_DEFAULT_FLAGS,
270 LIST_HEAD(ftrace_trace_arrays);
272 int trace_array_get(struct trace_array *this_tr)
274 struct trace_array *tr;
277 mutex_lock(&trace_types_lock);
278 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
285 mutex_unlock(&trace_types_lock);
290 static void __trace_array_put(struct trace_array *this_tr)
292 WARN_ON(!this_tr->ref);
296 void trace_array_put(struct trace_array *this_tr)
298 mutex_lock(&trace_types_lock);
299 __trace_array_put(this_tr);
300 mutex_unlock(&trace_types_lock);
303 int call_filter_check_discard(struct trace_event_call *call, void *rec,
304 struct ring_buffer *buffer,
305 struct ring_buffer_event *event)
307 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
308 !filter_match_preds(call->filter, rec)) {
309 __trace_event_discard_commit(buffer, event);
316 void trace_free_pid_list(struct trace_pid_list *pid_list)
318 vfree(pid_list->pids);
323 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
324 * @filtered_pids: The list of pids to check
325 * @search_pid: The PID to find in @filtered_pids
327 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis.
330 trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
333 * If pid_max changed after filtered_pids was created, we
334 * by default ignore all pids greater than the previous pid_max.
336 if (search_pid >= filtered_pids->pid_max)
339 return test_bit(search_pid, filtered_pids->pids);
343 * trace_ignore_this_task - should a task be ignored for tracing
344 * @filtered_pids: The list of pids to check
345 * @task: The task that should be ignored if not filtered
347 * Checks if @task should be traced or not from @filtered_pids.
348 * Returns true if @task should *NOT* be traced.
349 * Returns false if @task should be traced.
352 trace_ignore_this_task(struct trace_pid_list *filtered_pids, struct task_struct *task)
355 * Return false, because if filtered_pids does not exist,
356 * all pids are good to trace.
361 return !trace_find_filtered_pid(filtered_pids, task->pid);
365 * trace_pid_filter_add_remove - Add or remove a task from a pid_list
366 * @pid_list: The list to modify
367 * @self: The current task for fork or NULL for exit
368 * @task: The task to add or remove
370 * If adding a task, if @self is defined, the task is only added if @self
371 * is also included in @pid_list. This happens on fork and tasks should
372 * only be added when the parent is listed. If @self is NULL, then the
373 * @task pid will be removed from the list, which would happen on exit
376 void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
377 struct task_struct *self,
378 struct task_struct *task)
383 /* For forks, we only add if the forking task is listed */
385 if (!trace_find_filtered_pid(pid_list, self->pid))
389 /* Sorry, but we don't support pid_max changing after setting */
390 if (task->pid >= pid_list->pid_max)
393 /* "self" is set for forks, and NULL for exits */
395 set_bit(task->pid, pid_list->pids);
397 clear_bit(task->pid, pid_list->pids);
401 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
402 * @pid_list: The pid list to show
403 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
404 * @pos: The position of the file
406 * This is used by the seq_file "next" operation to iterate the pids
407 * listed in a trace_pid_list structure.
409 * Returns the pid+1 as we want to display pid of zero, but NULL would
410 * stop the iteration.
412 void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
414 unsigned long pid = (unsigned long)v;
418 /* pid already is +1 of the actual prevous bit */
419 pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);
421 /* Return pid + 1 to allow zero to be represented */
422 if (pid < pid_list->pid_max)
423 return (void *)(pid + 1);
429 * trace_pid_start - Used for seq_file to start reading pid lists
430 * @pid_list: The pid list to show
431 * @pos: The position of the file
433 * This is used by seq_file "start" operation to start the iteration
436 * Returns the pid+1 as we want to display pid of zero, but NULL would
437 * stop the iteration.
439 void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
444 pid = find_first_bit(pid_list->pids, pid_list->pid_max);
445 if (pid >= pid_list->pid_max)
448 /* Return pid + 1 so that zero can be the exit value */
449 for (pid++; pid && l < *pos;
450 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
456 * trace_pid_show - show the current pid in seq_file processing
457 * @m: The seq_file structure to write into
458 * @v: A void pointer of the pid (+1) value to display
460 * Can be directly used by seq_file operations to display the current
463 int trace_pid_show(struct seq_file *m, void *v)
465 unsigned long pid = (unsigned long)v - 1;
467 seq_printf(m, "%lu\n", pid);
471 /* 128 should be much more than enough */
472 #define PID_BUF_SIZE 127
474 int trace_pid_write(struct trace_pid_list *filtered_pids,
475 struct trace_pid_list **new_pid_list,
476 const char __user *ubuf, size_t cnt)
478 struct trace_pid_list *pid_list;
479 struct trace_parser parser;
487 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
491 * Always recreate a new array. The write is an all or nothing
492 * operation. Always create a new array when adding new pids by
493 * the user. If the operation fails, then the current list is
496 pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
498 trace_parser_put(&parser);
502 pid_list->pid_max = READ_ONCE(pid_max);
504 /* Only truncating will shrink pid_max */
505 if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max)
506 pid_list->pid_max = filtered_pids->pid_max;
508 pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
509 if (!pid_list->pids) {
510 trace_parser_put(&parser);
516 /* copy the current bits to the new max */
517 for_each_set_bit(pid, filtered_pids->pids,
518 filtered_pids->pid_max) {
519 set_bit(pid, pid_list->pids);
528 ret = trace_get_user(&parser, ubuf, cnt, &pos);
529 if (ret < 0 || !trace_parser_loaded(&parser))
536 parser.buffer[parser.idx] = 0;
539 if (kstrtoul(parser.buffer, 0, &val))
541 if (val >= pid_list->pid_max)
546 set_bit(pid, pid_list->pids);
549 trace_parser_clear(&parser);
552 trace_parser_put(&parser);
555 trace_free_pid_list(pid_list);
560 /* Cleared the list of pids */
561 trace_free_pid_list(pid_list);
566 *new_pid_list = pid_list;
571 static u64 buffer_ftrace_now(struct trace_buffer *buf, int cpu)
575 /* Early boot up does not have a buffer yet */
577 return trace_clock_local();
579 ts = ring_buffer_time_stamp(buf->buffer, cpu);
580 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
585 u64 ftrace_now(int cpu)
587 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
591 * tracing_is_enabled - Show if global_trace has been disabled
593 * Shows if the global trace has been enabled or not. It uses the
594 * mirror flag "buffer_disabled" to be used in fast paths such as for
595 * the irqsoff tracer. But it may be inaccurate due to races. If you
596 * need to know the accurate state, use tracing_is_on() which is a little
597 * slower, but accurate.
599 int tracing_is_enabled(void)
602 * For quick access (irqsoff uses this in fast path), just
603 * return the mirror variable of the state of the ring buffer.
604 * It's a little racy, but we don't really care.
607 return !global_trace.buffer_disabled;
611 * trace_buf_size is the size in bytes that is allocated
612 * for a buffer. Note, the number of bytes is always rounded
615 * This number is purposely set to a low number of 16384.
616 * If the dump on oops happens, it will be much appreciated
617 * to not have to wait for all that output. Anyway this can be
618 * boot time and run time configurable.
620 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
622 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
624 /* trace_types holds a link list of available tracers. */
625 static struct tracer *trace_types __read_mostly;
628 * trace_types_lock is used to protect the trace_types list.
630 DEFINE_MUTEX(trace_types_lock);
633 * serialize the access of the ring buffer
635 * ring buffer serializes readers, but it is low level protection.
636 * The validity of the events (which returns by ring_buffer_peek() ..etc)
637 * are not protected by ring buffer.
639 * The content of events may become garbage if we allow other process consumes
640 * these events concurrently:
641 * A) the page of the consumed events may become a normal page
642 * (not reader page) in ring buffer, and this page will be rewrited
643 * by events producer.
644 * B) The page of the consumed events may become a page for splice_read,
645 * and this page will be returned to system.
647 * These primitives allow multi process access to different cpu ring buffer
650 * These primitives don't distinguish read-only and read-consume access.
651 * Multi read-only access are also serialized.
655 static DECLARE_RWSEM(all_cpu_access_lock);
656 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
658 static inline void trace_access_lock(int cpu)
660 if (cpu == RING_BUFFER_ALL_CPUS) {
661 /* gain it for accessing the whole ring buffer. */
662 down_write(&all_cpu_access_lock);
664 /* gain it for accessing a cpu ring buffer. */
666 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
667 down_read(&all_cpu_access_lock);
669 /* Secondly block other access to this @cpu ring buffer. */
670 mutex_lock(&per_cpu(cpu_access_lock, cpu));
674 static inline void trace_access_unlock(int cpu)
676 if (cpu == RING_BUFFER_ALL_CPUS) {
677 up_write(&all_cpu_access_lock);
679 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
680 up_read(&all_cpu_access_lock);
684 static inline void trace_access_lock_init(void)
688 for_each_possible_cpu(cpu)
689 mutex_init(&per_cpu(cpu_access_lock, cpu));
694 static DEFINE_MUTEX(access_lock);
696 static inline void trace_access_lock(int cpu)
699 mutex_lock(&access_lock);
702 static inline void trace_access_unlock(int cpu)
705 mutex_unlock(&access_lock);
708 static inline void trace_access_lock_init(void)
714 #ifdef CONFIG_STACKTRACE
715 static void __ftrace_trace_stack(struct ring_buffer *buffer,
717 int skip, int pc, struct pt_regs *regs);
718 static inline void ftrace_trace_stack(struct trace_array *tr,
719 struct ring_buffer *buffer,
721 int skip, int pc, struct pt_regs *regs);
724 static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
726 int skip, int pc, struct pt_regs *regs)
729 static inline void ftrace_trace_stack(struct trace_array *tr,
730 struct ring_buffer *buffer,
732 int skip, int pc, struct pt_regs *regs)
738 static __always_inline void
739 trace_event_setup(struct ring_buffer_event *event,
740 int type, unsigned long flags, int pc)
742 struct trace_entry *ent = ring_buffer_event_data(event);
744 tracing_generic_entry_update(ent, flags, pc);
748 static __always_inline struct ring_buffer_event *
749 __trace_buffer_lock_reserve(struct ring_buffer *buffer,
752 unsigned long flags, int pc)
754 struct ring_buffer_event *event;
756 event = ring_buffer_lock_reserve(buffer, len);
758 trace_event_setup(event, type, flags, pc);
763 void tracer_tracing_on(struct trace_array *tr)
765 if (tr->trace_buffer.buffer)
766 ring_buffer_record_on(tr->trace_buffer.buffer);
768 * This flag is looked at when buffers haven't been allocated
769 * yet, or by some tracers (like irqsoff), that just want to
770 * know if the ring buffer has been disabled, but it can handle
771 * races of where it gets disabled but we still do a record.
772 * As the check is in the fast path of the tracers, it is more
773 * important to be fast than accurate.
775 tr->buffer_disabled = 0;
776 /* Make the flag seen by readers */
781 * tracing_on - enable tracing buffers
783 * This function enables tracing buffers that may have been
784 * disabled with tracing_off.
786 void tracing_on(void)
788 tracer_tracing_on(&global_trace);
790 EXPORT_SYMBOL_GPL(tracing_on);
793 static __always_inline void
794 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
796 __this_cpu_write(trace_taskinfo_save, true);
798 /* If this is the temp buffer, we need to commit fully */
799 if (this_cpu_read(trace_buffered_event) == event) {
800 /* Length is in event->array[0] */
801 ring_buffer_write(buffer, event->array[0], &event->array[1]);
802 /* Release the temp buffer */
803 this_cpu_dec(trace_buffered_event_cnt);
805 ring_buffer_unlock_commit(buffer, event);
809 * __trace_puts - write a constant string into the trace buffer.
810 * @ip: The address of the caller
811 * @str: The constant string to write
812 * @size: The size of the string.
814 int __trace_puts(unsigned long ip, const char *str, int size)
816 struct ring_buffer_event *event;
817 struct ring_buffer *buffer;
818 struct print_entry *entry;
819 unsigned long irq_flags;
823 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
826 pc = preempt_count();
828 if (unlikely(tracing_selftest_running || tracing_disabled))
831 alloc = sizeof(*entry) + size + 2; /* possible \n added */
833 local_save_flags(irq_flags);
834 buffer = global_trace.trace_buffer.buffer;
835 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
840 entry = ring_buffer_event_data(event);
843 memcpy(&entry->buf, str, size);
845 /* Add a newline if necessary */
846 if (entry->buf[size - 1] != '\n') {
847 entry->buf[size] = '\n';
848 entry->buf[size + 1] = '\0';
850 entry->buf[size] = '\0';
852 __buffer_unlock_commit(buffer, event);
853 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
857 EXPORT_SYMBOL_GPL(__trace_puts);
860 * __trace_bputs - write the pointer to a constant string into trace buffer
861 * @ip: The address of the caller
862 * @str: The constant string to write to the buffer to
864 int __trace_bputs(unsigned long ip, const char *str)
866 struct ring_buffer_event *event;
867 struct ring_buffer *buffer;
868 struct bputs_entry *entry;
869 unsigned long irq_flags;
870 int size = sizeof(struct bputs_entry);
873 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
876 pc = preempt_count();
878 if (unlikely(tracing_selftest_running || tracing_disabled))
881 local_save_flags(irq_flags);
882 buffer = global_trace.trace_buffer.buffer;
883 event = __trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
888 entry = ring_buffer_event_data(event);
892 __buffer_unlock_commit(buffer, event);
893 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
897 EXPORT_SYMBOL_GPL(__trace_bputs);
899 #ifdef CONFIG_TRACER_SNAPSHOT
900 void tracing_snapshot_instance(struct trace_array *tr)
902 struct tracer *tracer = tr->current_trace;
906 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
907 internal_trace_puts("*** snapshot is being ignored ***\n");
911 if (!tr->allocated_snapshot) {
912 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
913 internal_trace_puts("*** stopping trace here! ***\n");
918 /* Note, snapshot can not be used when the tracer uses it */
919 if (tracer->use_max_tr) {
920 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
921 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
925 local_irq_save(flags);
926 update_max_tr(tr, current, smp_processor_id());
927 local_irq_restore(flags);
931 * trace_snapshot - take a snapshot of the current buffer.
933 * This causes a swap between the snapshot buffer and the current live
934 * tracing buffer. You can use this to take snapshots of the live
935 * trace when some condition is triggered, but continue to trace.
937 * Note, make sure to allocate the snapshot with either
938 * a tracing_snapshot_alloc(), or by doing it manually
939 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
941 * If the snapshot buffer is not allocated, it will stop tracing.
942 * Basically making a permanent snapshot.
944 void tracing_snapshot(void)
946 struct trace_array *tr = &global_trace;
948 tracing_snapshot_instance(tr);
950 EXPORT_SYMBOL_GPL(tracing_snapshot);
952 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
953 struct trace_buffer *size_buf, int cpu_id);
954 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
956 int tracing_alloc_snapshot_instance(struct trace_array *tr)
960 if (!tr->allocated_snapshot) {
962 /* allocate spare buffer */
963 ret = resize_buffer_duplicate_size(&tr->max_buffer,
964 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
968 tr->allocated_snapshot = true;
974 static void free_snapshot(struct trace_array *tr)
977 * We don't free the ring buffer. instead, resize it because
978 * The max_tr ring buffer has some state (e.g. ring->clock) and
979 * we want preserve it.
981 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
982 set_buffer_entries(&tr->max_buffer, 1);
983 tracing_reset_online_cpus(&tr->max_buffer);
984 tr->allocated_snapshot = false;
988 * tracing_alloc_snapshot - allocate snapshot buffer.
990 * This only allocates the snapshot buffer if it isn't already
991 * allocated - it doesn't also take a snapshot.
993 * This is meant to be used in cases where the snapshot buffer needs
994 * to be set up for events that can't sleep but need to be able to
995 * trigger a snapshot.
997 int tracing_alloc_snapshot(void)
999 struct trace_array *tr = &global_trace;
1002 ret = tracing_alloc_snapshot_instance(tr);
1007 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1010 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
1012 * This is similar to trace_snapshot(), but it will allocate the
1013 * snapshot buffer if it isn't already allocated. Use this only
1014 * where it is safe to sleep, as the allocation may sleep.
1016 * This causes a swap between the snapshot buffer and the current live
1017 * tracing buffer. You can use this to take snapshots of the live
1018 * trace when some condition is triggered, but continue to trace.
1020 void tracing_snapshot_alloc(void)
1024 ret = tracing_alloc_snapshot();
1030 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1032 void tracing_snapshot(void)
1034 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
1036 EXPORT_SYMBOL_GPL(tracing_snapshot);
1037 int tracing_alloc_snapshot(void)
1039 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1042 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1043 void tracing_snapshot_alloc(void)
1048 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1049 #endif /* CONFIG_TRACER_SNAPSHOT */
1051 void tracer_tracing_off(struct trace_array *tr)
1053 if (tr->trace_buffer.buffer)
1054 ring_buffer_record_off(tr->trace_buffer.buffer);
1056 * This flag is looked at when buffers haven't been allocated
1057 * yet, or by some tracers (like irqsoff), that just want to
1058 * know if the ring buffer has been disabled, but it can handle
1059 * races of where it gets disabled but we still do a record.
1060 * As the check is in the fast path of the tracers, it is more
1061 * important to be fast than accurate.
1063 tr->buffer_disabled = 1;
1064 /* Make the flag seen by readers */
1069 * tracing_off - turn off tracing buffers
1071 * This function stops the tracing buffers from recording data.
1072 * It does not disable any overhead the tracers themselves may
1073 * be causing. This function simply causes all recording to
1074 * the ring buffers to fail.
1076 void tracing_off(void)
1078 tracer_tracing_off(&global_trace);
1080 EXPORT_SYMBOL_GPL(tracing_off);
1082 void disable_trace_on_warning(void)
1084 if (__disable_trace_on_warning)
1089 * tracer_tracing_is_on - show real state of ring buffer enabled
1090 * @tr : the trace array to know if ring buffer is enabled
1092 * Shows real state of the ring buffer if it is enabled or not.
1094 int tracer_tracing_is_on(struct trace_array *tr)
1096 if (tr->trace_buffer.buffer)
1097 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
1098 return !tr->buffer_disabled;
1102 * tracing_is_on - show state of ring buffers enabled
1104 int tracing_is_on(void)
1106 return tracer_tracing_is_on(&global_trace);
1108 EXPORT_SYMBOL_GPL(tracing_is_on);
1110 static int __init set_buf_size(char *str)
1112 unsigned long buf_size;
1116 buf_size = memparse(str, &str);
1117 /* nr_entries can not be zero */
1120 trace_buf_size = buf_size;
1123 __setup("trace_buf_size=", set_buf_size);
1125 static int __init set_tracing_thresh(char *str)
1127 unsigned long threshold;
1132 ret = kstrtoul(str, 0, &threshold);
1135 tracing_thresh = threshold * 1000;
1138 __setup("tracing_thresh=", set_tracing_thresh);
1140 unsigned long nsecs_to_usecs(unsigned long nsecs)
1142 return nsecs / 1000;
1146 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1147 * It uses C(a, b) where 'a' is the eval (enum) name and 'b' is the string that
1148 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1149 * of strings in the order that the evals (enum) were defined.
1154 /* These must match the bit postions in trace_iterator_flags */
1155 static const char *trace_options[] = {
1163 int in_ns; /* is this clock in nanoseconds? */
1164 } trace_clocks[] = {
1165 { trace_clock_local, "local", 1 },
1166 { trace_clock_global, "global", 1 },
1167 { trace_clock_counter, "counter", 0 },
1168 { trace_clock_jiffies, "uptime", 0 },
1169 { trace_clock, "perf", 1 },
1170 { ktime_get_mono_fast_ns, "mono", 1 },
1171 { ktime_get_raw_fast_ns, "mono_raw", 1 },
1172 { ktime_get_boot_fast_ns, "boot", 1 },
1177 * trace_parser_get_init - gets the buffer for trace parser
1179 int trace_parser_get_init(struct trace_parser *parser, int size)
1181 memset(parser, 0, sizeof(*parser));
1183 parser->buffer = kmalloc(size, GFP_KERNEL);
1184 if (!parser->buffer)
1187 parser->size = size;
1192 * trace_parser_put - frees the buffer for trace parser
1194 void trace_parser_put(struct trace_parser *parser)
1196 kfree(parser->buffer);
1197 parser->buffer = NULL;
1201 * trace_get_user - reads the user input string separated by space
1202 * (matched by isspace(ch))
1204 * For each string found the 'struct trace_parser' is updated,
1205 * and the function returns.
1207 * Returns number of bytes read.
1209 * See kernel/trace/trace.h for 'struct trace_parser' details.
1211 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1212 size_t cnt, loff_t *ppos)
1219 trace_parser_clear(parser);
1221 ret = get_user(ch, ubuf++);
1229 * The parser is not finished with the last write,
1230 * continue reading the user input without skipping spaces.
1232 if (!parser->cont) {
1233 /* skip white space */
1234 while (cnt && isspace(ch)) {
1235 ret = get_user(ch, ubuf++);
1242 /* only spaces were written */
1252 /* read the non-space input */
1253 while (cnt && !isspace(ch)) {
1254 if (parser->idx < parser->size - 1)
1255 parser->buffer[parser->idx++] = ch;
1260 ret = get_user(ch, ubuf++);
1267 /* We either got finished input or we have to wait for another call. */
1269 parser->buffer[parser->idx] = 0;
1270 parser->cont = false;
1271 } else if (parser->idx < parser->size - 1) {
1272 parser->cont = true;
1273 parser->buffer[parser->idx++] = ch;
1286 /* TODO add a seq_buf_to_buffer() */
1287 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1291 if (trace_seq_used(s) <= s->seq.readpos)
1294 len = trace_seq_used(s) - s->seq.readpos;
1297 memcpy(buf, s->buffer + s->seq.readpos, cnt);
1299 s->seq.readpos += cnt;
1303 unsigned long __read_mostly tracing_thresh;
1305 #ifdef CONFIG_TRACER_MAX_TRACE
1307 * Copy the new maximum trace into the separate maximum-trace
1308 * structure. (this way the maximum trace is permanently saved,
1309 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
1312 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1314 struct trace_buffer *trace_buf = &tr->trace_buffer;
1315 struct trace_buffer *max_buf = &tr->max_buffer;
1316 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1317 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1320 max_buf->time_start = data->preempt_timestamp;
1322 max_data->saved_latency = tr->max_latency;
1323 max_data->critical_start = data->critical_start;
1324 max_data->critical_end = data->critical_end;
1326 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1327 max_data->pid = tsk->pid;
1329 * If tsk == current, then use current_uid(), as that does not use
1330 * RCU. The irq tracer can be called out of RCU scope.
1333 max_data->uid = current_uid();
1335 max_data->uid = task_uid(tsk);
1337 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1338 max_data->policy = tsk->policy;
1339 max_data->rt_priority = tsk->rt_priority;
1341 /* record this tasks comm */
1342 tracing_record_cmdline(tsk);
1346 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1348 * @tsk: the task with the latency
1349 * @cpu: The cpu that initiated the trace.
1351 * Flip the buffers between the @tr and the max_tr and record information
1352 * about which task was the cause of this latency.
1355 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1357 struct ring_buffer *buf;
1362 WARN_ON_ONCE(!irqs_disabled());
1364 if (!tr->allocated_snapshot) {
1365 /* Only the nop tracer should hit this when disabling */
1366 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1370 arch_spin_lock(&tr->max_lock);
1372 /* Inherit the recordable setting from trace_buffer */
1373 if (ring_buffer_record_is_set_on(tr->trace_buffer.buffer))
1374 ring_buffer_record_on(tr->max_buffer.buffer);
1376 ring_buffer_record_off(tr->max_buffer.buffer);
1378 buf = tr->trace_buffer.buffer;
1379 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1380 tr->max_buffer.buffer = buf;
1382 __update_max_tr(tr, tsk, cpu);
1383 arch_spin_unlock(&tr->max_lock);
1387 * update_max_tr_single - only copy one trace over, and reset the rest
1389 * @tsk - task with the latency
1390 * @cpu - the cpu of the buffer to copy.
1392 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1395 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1402 WARN_ON_ONCE(!irqs_disabled());
1403 if (!tr->allocated_snapshot) {
1404 /* Only the nop tracer should hit this when disabling */
1405 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1409 arch_spin_lock(&tr->max_lock);
1411 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1413 if (ret == -EBUSY) {
1415 * We failed to swap the buffer due to a commit taking
1416 * place on this CPU. We fail to record, but we reset
1417 * the max trace buffer (no one writes directly to it)
1418 * and flag that it failed.
1420 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1421 "Failed to swap buffers due to commit in progress\n");
1424 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1426 __update_max_tr(tr, tsk, cpu);
1427 arch_spin_unlock(&tr->max_lock);
1429 #endif /* CONFIG_TRACER_MAX_TRACE */
1431 static int wait_on_pipe(struct trace_iterator *iter, bool full)
1433 /* Iterators are static, they should be filled or empty */
1434 if (trace_buffer_iter(iter, iter->cpu_file))
1437 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1441 #ifdef CONFIG_FTRACE_STARTUP_TEST
1442 static bool selftests_can_run;
1444 struct trace_selftests {
1445 struct list_head list;
1446 struct tracer *type;
1449 static LIST_HEAD(postponed_selftests);
1451 static int save_selftest(struct tracer *type)
1453 struct trace_selftests *selftest;
1455 selftest = kmalloc(sizeof(*selftest), GFP_KERNEL);
1459 selftest->type = type;
1460 list_add(&selftest->list, &postponed_selftests);
1464 static int run_tracer_selftest(struct tracer *type)
1466 struct trace_array *tr = &global_trace;
1467 struct tracer *saved_tracer = tr->current_trace;
1470 if (!type->selftest || tracing_selftest_disabled)
1474 * If a tracer registers early in boot up (before scheduling is
1475 * initialized and such), then do not run its selftests yet.
1476 * Instead, run it a little later in the boot process.
1478 if (!selftests_can_run)
1479 return save_selftest(type);
1482 * Run a selftest on this tracer.
1483 * Here we reset the trace buffer, and set the current
1484 * tracer to be this tracer. The tracer can then run some
1485 * internal tracing to verify that everything is in order.
1486 * If we fail, we do not register this tracer.
1488 tracing_reset_online_cpus(&tr->trace_buffer);
1490 tr->current_trace = type;
1492 #ifdef CONFIG_TRACER_MAX_TRACE
1493 if (type->use_max_tr) {
1494 /* If we expanded the buffers, make sure the max is expanded too */
1495 if (ring_buffer_expanded)
1496 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1497 RING_BUFFER_ALL_CPUS);
1498 tr->allocated_snapshot = true;
1502 /* the test is responsible for initializing and enabling */
1503 pr_info("Testing tracer %s: ", type->name);
1504 ret = type->selftest(type, tr);
1505 /* the test is responsible for resetting too */
1506 tr->current_trace = saved_tracer;
1508 printk(KERN_CONT "FAILED!\n");
1509 /* Add the warning after printing 'FAILED' */
1513 /* Only reset on passing, to avoid touching corrupted buffers */
1514 tracing_reset_online_cpus(&tr->trace_buffer);
1516 #ifdef CONFIG_TRACER_MAX_TRACE
1517 if (type->use_max_tr) {
1518 tr->allocated_snapshot = false;
1520 /* Shrink the max buffer again */
1521 if (ring_buffer_expanded)
1522 ring_buffer_resize(tr->max_buffer.buffer, 1,
1523 RING_BUFFER_ALL_CPUS);
1527 printk(KERN_CONT "PASSED\n");
1531 static __init int init_trace_selftests(void)
1533 struct trace_selftests *p, *n;
1534 struct tracer *t, **last;
1537 selftests_can_run = true;
1539 mutex_lock(&trace_types_lock);
1541 if (list_empty(&postponed_selftests))
1544 pr_info("Running postponed tracer tests:\n");
1546 tracing_selftest_running = true;
1547 list_for_each_entry_safe(p, n, &postponed_selftests, list) {
1548 ret = run_tracer_selftest(p->type);
1549 /* If the test fails, then warn and remove from available_tracers */
1551 WARN(1, "tracer: %s failed selftest, disabling\n",
1553 last = &trace_types;
1554 for (t = trace_types; t; t = t->next) {
1565 tracing_selftest_running = false;
1568 mutex_unlock(&trace_types_lock);
1572 core_initcall(init_trace_selftests);
1574 static inline int run_tracer_selftest(struct tracer *type)
1578 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1580 static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1582 static void __init apply_trace_boot_options(void);
1585 * register_tracer - register a tracer with the ftrace system.
1586 * @type - the plugin for the tracer
1588 * Register a new plugin tracer.
1590 int __init register_tracer(struct tracer *type)
1596 pr_info("Tracer must have a name\n");
1600 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1601 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1605 mutex_lock(&trace_types_lock);
1607 tracing_selftest_running = true;
1609 for (t = trace_types; t; t = t->next) {
1610 if (strcmp(type->name, t->name) == 0) {
1612 pr_info("Tracer %s already registered\n",
1619 if (!type->set_flag)
1620 type->set_flag = &dummy_set_flag;
1622 /*allocate a dummy tracer_flags*/
1623 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
1628 type->flags->val = 0;
1629 type->flags->opts = dummy_tracer_opt;
1631 if (!type->flags->opts)
1632 type->flags->opts = dummy_tracer_opt;
1634 /* store the tracer for __set_tracer_option */
1635 type->flags->trace = type;
1637 ret = run_tracer_selftest(type);
1641 type->next = trace_types;
1643 add_tracer_options(&global_trace, type);
1646 tracing_selftest_running = false;
1647 mutex_unlock(&trace_types_lock);
1649 if (ret || !default_bootup_tracer)
1652 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1655 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1656 /* Do we want this tracer to start on bootup? */
1657 tracing_set_tracer(&global_trace, type->name);
1658 default_bootup_tracer = NULL;
1660 apply_trace_boot_options();
1662 /* disable other selftests, since this will break it. */
1663 tracing_selftest_disabled = true;
1664 #ifdef CONFIG_FTRACE_STARTUP_TEST
1665 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1673 void tracing_reset(struct trace_buffer *buf, int cpu)
1675 struct ring_buffer *buffer = buf->buffer;
1680 ring_buffer_record_disable(buffer);
1682 /* Make sure all commits have finished */
1683 synchronize_sched();
1684 ring_buffer_reset_cpu(buffer, cpu);
1686 ring_buffer_record_enable(buffer);
1689 void tracing_reset_online_cpus(struct trace_buffer *buf)
1691 struct ring_buffer *buffer = buf->buffer;
1697 ring_buffer_record_disable(buffer);
1699 /* Make sure all commits have finished */
1700 synchronize_sched();
1702 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1704 for_each_online_cpu(cpu)
1705 ring_buffer_reset_cpu(buffer, cpu);
1707 ring_buffer_record_enable(buffer);
1710 /* Must have trace_types_lock held */
1711 void tracing_reset_all_online_cpus(void)
1713 struct trace_array *tr;
1715 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1716 if (!tr->clear_trace)
1718 tr->clear_trace = false;
1719 tracing_reset_online_cpus(&tr->trace_buffer);
1720 #ifdef CONFIG_TRACER_MAX_TRACE
1721 tracing_reset_online_cpus(&tr->max_buffer);
1727 * The tgid_map array maps from pid to tgid; i.e. the value stored at index i
1728 * is the tgid last observed corresponding to pid=i.
1730 static int *tgid_map;
1732 /* The maximum valid index into tgid_map. */
1733 static size_t tgid_map_max;
1735 #define SAVED_CMDLINES_DEFAULT 128
1736 #define NO_CMDLINE_MAP UINT_MAX
1737 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1738 struct saved_cmdlines_buffer {
1739 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1740 unsigned *map_cmdline_to_pid;
1741 unsigned cmdline_num;
1743 char *saved_cmdlines;
1745 static struct saved_cmdlines_buffer *savedcmd;
1747 static inline char *get_saved_cmdlines(int idx)
1749 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1752 static inline void set_cmdline(int idx, const char *cmdline)
1754 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1757 static int allocate_cmdlines_buffer(unsigned int val,
1758 struct saved_cmdlines_buffer *s)
1760 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1762 if (!s->map_cmdline_to_pid)
1765 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1766 if (!s->saved_cmdlines) {
1767 kfree(s->map_cmdline_to_pid);
1772 s->cmdline_num = val;
1773 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1774 sizeof(s->map_pid_to_cmdline));
1775 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1776 val * sizeof(*s->map_cmdline_to_pid));
1781 static int trace_create_savedcmd(void)
1785 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1789 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1799 int is_tracing_stopped(void)
1801 return global_trace.stop_count;
1805 * tracing_start - quick start of the tracer
1807 * If tracing is enabled but was stopped by tracing_stop,
1808 * this will start the tracer back up.
1810 void tracing_start(void)
1812 struct ring_buffer *buffer;
1813 unsigned long flags;
1815 if (tracing_disabled)
1818 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1819 if (--global_trace.stop_count) {
1820 if (global_trace.stop_count < 0) {
1821 /* Someone screwed up their debugging */
1823 global_trace.stop_count = 0;
1828 /* Prevent the buffers from switching */
1829 arch_spin_lock(&global_trace.max_lock);
1831 buffer = global_trace.trace_buffer.buffer;
1833 ring_buffer_record_enable(buffer);
1835 #ifdef CONFIG_TRACER_MAX_TRACE
1836 buffer = global_trace.max_buffer.buffer;
1838 ring_buffer_record_enable(buffer);
1841 arch_spin_unlock(&global_trace.max_lock);
1844 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1847 static void tracing_start_tr(struct trace_array *tr)
1849 struct ring_buffer *buffer;
1850 unsigned long flags;
1852 if (tracing_disabled)
1855 /* If global, we need to also start the max tracer */
1856 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1857 return tracing_start();
1859 raw_spin_lock_irqsave(&tr->start_lock, flags);
1861 if (--tr->stop_count) {
1862 if (tr->stop_count < 0) {
1863 /* Someone screwed up their debugging */
1870 buffer = tr->trace_buffer.buffer;
1872 ring_buffer_record_enable(buffer);
1875 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1879 * tracing_stop - quick stop of the tracer
1881 * Light weight way to stop tracing. Use in conjunction with
1884 void tracing_stop(void)
1886 struct ring_buffer *buffer;
1887 unsigned long flags;
1889 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1890 if (global_trace.stop_count++)
1893 /* Prevent the buffers from switching */
1894 arch_spin_lock(&global_trace.max_lock);
1896 buffer = global_trace.trace_buffer.buffer;
1898 ring_buffer_record_disable(buffer);
1900 #ifdef CONFIG_TRACER_MAX_TRACE
1901 buffer = global_trace.max_buffer.buffer;
1903 ring_buffer_record_disable(buffer);
1906 arch_spin_unlock(&global_trace.max_lock);
1909 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1912 static void tracing_stop_tr(struct trace_array *tr)
1914 struct ring_buffer *buffer;
1915 unsigned long flags;
1917 /* If global, we need to also stop the max tracer */
1918 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1919 return tracing_stop();
1921 raw_spin_lock_irqsave(&tr->start_lock, flags);
1922 if (tr->stop_count++)
1925 buffer = tr->trace_buffer.buffer;
1927 ring_buffer_record_disable(buffer);
1930 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1933 static int trace_save_cmdline(struct task_struct *tsk)
1937 /* treat recording of idle task as a success */
1941 tpid = tsk->pid & (PID_MAX_DEFAULT - 1);
1944 * It's not the end of the world if we don't get
1945 * the lock, but we also don't want to spin
1946 * nor do we want to disable interrupts,
1947 * so if we miss here, then better luck next time.
1949 if (!arch_spin_trylock(&trace_cmdline_lock))
1952 idx = savedcmd->map_pid_to_cmdline[tpid];
1953 if (idx == NO_CMDLINE_MAP) {
1954 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1956 savedcmd->map_pid_to_cmdline[tpid] = idx;
1957 savedcmd->cmdline_idx = idx;
1960 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1961 set_cmdline(idx, tsk->comm);
1963 arch_spin_unlock(&trace_cmdline_lock);
1968 static void __trace_find_cmdline(int pid, char comm[])
1974 strcpy(comm, "<idle>");
1978 if (WARN_ON_ONCE(pid < 0)) {
1979 strcpy(comm, "<XXX>");
1983 tpid = pid & (PID_MAX_DEFAULT - 1);
1984 map = savedcmd->map_pid_to_cmdline[tpid];
1985 if (map != NO_CMDLINE_MAP) {
1986 tpid = savedcmd->map_cmdline_to_pid[map];
1988 strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN);
1992 strcpy(comm, "<...>");
1995 void trace_find_cmdline(int pid, char comm[])
1998 arch_spin_lock(&trace_cmdline_lock);
2000 __trace_find_cmdline(pid, comm);
2002 arch_spin_unlock(&trace_cmdline_lock);
2006 static int *trace_find_tgid_ptr(int pid)
2009 * Pairs with the smp_store_release in set_tracer_flag() to ensure that
2010 * if we observe a non-NULL tgid_map then we also observe the correct
2013 int *map = smp_load_acquire(&tgid_map);
2015 if (unlikely(!map || pid > tgid_map_max))
2021 int trace_find_tgid(int pid)
2023 int *ptr = trace_find_tgid_ptr(pid);
2025 return ptr ? *ptr : 0;
2028 static int trace_save_tgid(struct task_struct *tsk)
2032 /* treat recording of idle task as a success */
2036 ptr = trace_find_tgid_ptr(tsk->pid);
2044 static bool tracing_record_taskinfo_skip(int flags)
2046 if (unlikely(!(flags & (TRACE_RECORD_CMDLINE | TRACE_RECORD_TGID))))
2048 if (!__this_cpu_read(trace_taskinfo_save))
2054 * tracing_record_taskinfo - record the task info of a task
2056 * @task - task to record
2057 * @flags - TRACE_RECORD_CMDLINE for recording comm
2058 * - TRACE_RECORD_TGID for recording tgid
2060 void tracing_record_taskinfo(struct task_struct *task, int flags)
2064 if (tracing_record_taskinfo_skip(flags))
2068 * Record as much task information as possible. If some fail, continue
2069 * to try to record the others.
2071 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(task);
2072 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(task);
2074 /* If recording any information failed, retry again soon. */
2078 __this_cpu_write(trace_taskinfo_save, false);
2082 * tracing_record_taskinfo_sched_switch - record task info for sched_switch
2084 * @prev - previous task during sched_switch
2085 * @next - next task during sched_switch
2086 * @flags - TRACE_RECORD_CMDLINE for recording comm
2087 * TRACE_RECORD_TGID for recording tgid
2089 void tracing_record_taskinfo_sched_switch(struct task_struct *prev,
2090 struct task_struct *next, int flags)
2094 if (tracing_record_taskinfo_skip(flags))
2098 * Record as much task information as possible. If some fail, continue
2099 * to try to record the others.
2101 done = !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(prev);
2102 done &= !(flags & TRACE_RECORD_CMDLINE) || trace_save_cmdline(next);
2103 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(prev);
2104 done &= !(flags & TRACE_RECORD_TGID) || trace_save_tgid(next);
2106 /* If recording any information failed, retry again soon. */
2110 __this_cpu_write(trace_taskinfo_save, false);
2113 /* Helpers to record a specific task information */
2114 void tracing_record_cmdline(struct task_struct *task)
2116 tracing_record_taskinfo(task, TRACE_RECORD_CMDLINE);
2119 void tracing_record_tgid(struct task_struct *task)
2121 tracing_record_taskinfo(task, TRACE_RECORD_TGID);
2125 * Several functions return TRACE_TYPE_PARTIAL_LINE if the trace_seq
2126 * overflowed, and TRACE_TYPE_HANDLED otherwise. This helper function
2127 * simplifies those functions and keeps them in sync.
2129 enum print_line_t trace_handle_return(struct trace_seq *s)
2131 return trace_seq_has_overflowed(s) ?
2132 TRACE_TYPE_PARTIAL_LINE : TRACE_TYPE_HANDLED;
2134 EXPORT_SYMBOL_GPL(trace_handle_return);
2137 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
2140 struct task_struct *tsk = current;
2142 entry->preempt_count = pc & 0xff;
2143 entry->pid = (tsk) ? tsk->pid : 0;
2145 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
2146 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
2148 TRACE_FLAG_IRQS_NOSUPPORT |
2150 ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) |
2151 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
2152 ((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) |
2153 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
2154 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
2156 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
2158 struct ring_buffer_event *
2159 trace_buffer_lock_reserve(struct ring_buffer *buffer,
2162 unsigned long flags, int pc)
2164 return __trace_buffer_lock_reserve(buffer, type, len, flags, pc);
2167 DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
2168 DEFINE_PER_CPU(int, trace_buffered_event_cnt);
2169 static int trace_buffered_event_ref;
2172 * trace_buffered_event_enable - enable buffering events
2174 * When events are being filtered, it is quicker to use a temporary
2175 * buffer to write the event data into if there's a likely chance
2176 * that it will not be committed. The discard of the ring buffer
2177 * is not as fast as committing, and is much slower than copying
2180 * When an event is to be filtered, allocate per cpu buffers to
2181 * write the event data into, and if the event is filtered and discarded
2182 * it is simply dropped, otherwise, the entire data is to be committed
2185 void trace_buffered_event_enable(void)
2187 struct ring_buffer_event *event;
2191 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2193 if (trace_buffered_event_ref++)
2196 for_each_tracing_cpu(cpu) {
2197 page = alloc_pages_node(cpu_to_node(cpu),
2198 GFP_KERNEL | __GFP_NORETRY, 0);
2202 event = page_address(page);
2203 memset(event, 0, sizeof(*event));
2205 per_cpu(trace_buffered_event, cpu) = event;
2208 if (cpu == smp_processor_id() &&
2209 this_cpu_read(trace_buffered_event) !=
2210 per_cpu(trace_buffered_event, cpu))
2217 trace_buffered_event_disable();
2220 static void enable_trace_buffered_event(void *data)
2222 /* Probably not needed, but do it anyway */
2224 this_cpu_dec(trace_buffered_event_cnt);
2227 static void disable_trace_buffered_event(void *data)
2229 this_cpu_inc(trace_buffered_event_cnt);
2233 * trace_buffered_event_disable - disable buffering events
2235 * When a filter is removed, it is faster to not use the buffered
2236 * events, and to commit directly into the ring buffer. Free up
2237 * the temp buffers when there are no more users. This requires
2238 * special synchronization with current events.
2240 void trace_buffered_event_disable(void)
2244 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2246 if (WARN_ON_ONCE(!trace_buffered_event_ref))
2249 if (--trace_buffered_event_ref)
2253 /* For each CPU, set the buffer as used. */
2254 smp_call_function_many(tracing_buffer_mask,
2255 disable_trace_buffered_event, NULL, 1);
2258 /* Wait for all current users to finish */
2259 synchronize_sched();
2261 for_each_tracing_cpu(cpu) {
2262 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2263 per_cpu(trace_buffered_event, cpu) = NULL;
2266 * Make sure trace_buffered_event is NULL before clearing
2267 * trace_buffered_event_cnt.
2272 /* Do the work on each cpu */
2273 smp_call_function_many(tracing_buffer_mask,
2274 enable_trace_buffered_event, NULL, 1);
2278 static struct ring_buffer *temp_buffer;
2280 struct ring_buffer_event *
2281 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
2282 struct trace_event_file *trace_file,
2283 int type, unsigned long len,
2284 unsigned long flags, int pc)
2286 struct ring_buffer_event *entry;
2289 *current_rb = trace_file->tr->trace_buffer.buffer;
2291 if ((trace_file->flags &
2292 (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) &&
2293 (entry = this_cpu_read(trace_buffered_event))) {
2294 /* Try to use the per cpu buffer first */
2295 val = this_cpu_inc_return(trace_buffered_event_cnt);
2296 if ((len < (PAGE_SIZE - sizeof(*entry) - sizeof(entry->array[0]))) && val == 1) {
2297 trace_event_setup(entry, type, flags, pc);
2298 entry->array[0] = len;
2301 this_cpu_dec(trace_buffered_event_cnt);
2304 entry = __trace_buffer_lock_reserve(*current_rb,
2305 type, len, flags, pc);
2307 * If tracing is off, but we have triggers enabled
2308 * we still need to look at the event data. Use the temp_buffer
2309 * to store the trace event for the tigger to use. It's recusive
2310 * safe and will not be recorded anywhere.
2312 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2313 *current_rb = temp_buffer;
2314 entry = __trace_buffer_lock_reserve(*current_rb,
2315 type, len, flags, pc);
2319 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2321 static DEFINE_SPINLOCK(tracepoint_iter_lock);
2322 static DEFINE_MUTEX(tracepoint_printk_mutex);
2324 static void output_printk(struct trace_event_buffer *fbuffer)
2326 struct trace_event_call *event_call;
2327 struct trace_event *event;
2328 unsigned long flags;
2329 struct trace_iterator *iter = tracepoint_print_iter;
2331 /* We should never get here if iter is NULL */
2332 if (WARN_ON_ONCE(!iter))
2335 event_call = fbuffer->trace_file->event_call;
2336 if (!event_call || !event_call->event.funcs ||
2337 !event_call->event.funcs->trace)
2340 event = &fbuffer->trace_file->event_call->event;
2342 spin_lock_irqsave(&tracepoint_iter_lock, flags);
2343 trace_seq_init(&iter->seq);
2344 iter->ent = fbuffer->entry;
2345 event_call->event.funcs->trace(iter, 0, event);
2346 trace_seq_putc(&iter->seq, 0);
2347 printk("%s", iter->seq.buffer);
2349 spin_unlock_irqrestore(&tracepoint_iter_lock, flags);
2352 int tracepoint_printk_sysctl(struct ctl_table *table, int write,
2353 void __user *buffer, size_t *lenp,
2356 int save_tracepoint_printk;
2359 mutex_lock(&tracepoint_printk_mutex);
2360 save_tracepoint_printk = tracepoint_printk;
2362 ret = proc_dointvec(table, write, buffer, lenp, ppos);
2365 * This will force exiting early, as tracepoint_printk
2366 * is always zero when tracepoint_printk_iter is not allocated
2368 if (!tracepoint_print_iter)
2369 tracepoint_printk = 0;
2371 if (save_tracepoint_printk == tracepoint_printk)
2374 if (tracepoint_printk)
2375 static_key_enable(&tracepoint_printk_key.key);
2377 static_key_disable(&tracepoint_printk_key.key);
2380 mutex_unlock(&tracepoint_printk_mutex);
2385 void trace_event_buffer_commit(struct trace_event_buffer *fbuffer)
2387 if (static_key_false(&tracepoint_printk_key.key))
2388 output_printk(fbuffer);
2390 event_trigger_unlock_commit(fbuffer->trace_file, fbuffer->buffer,
2391 fbuffer->event, fbuffer->entry,
2392 fbuffer->flags, fbuffer->pc);
2394 EXPORT_SYMBOL_GPL(trace_event_buffer_commit);
2396 void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2397 struct ring_buffer *buffer,
2398 struct ring_buffer_event *event,
2399 unsigned long flags, int pc,
2400 struct pt_regs *regs)
2402 __buffer_unlock_commit(buffer, event);
2405 * If regs is not set, then skip the following callers:
2406 * trace_buffer_unlock_commit_regs
2407 * event_trigger_unlock_commit
2408 * trace_event_buffer_commit
2409 * trace_event_raw_event_sched_switch
2410 * Note, we can still get here via blktrace, wakeup tracer
2411 * and mmiotrace, but that's ok if they lose a function or
2412 * two. They are that meaningful.
2414 ftrace_trace_stack(tr, buffer, flags, regs ? 0 : 4, pc, regs);
2415 ftrace_trace_userstack(tr, buffer, flags, pc);
2419 * Similar to trace_buffer_unlock_commit_regs() but do not dump stack.
2422 trace_buffer_unlock_commit_nostack(struct ring_buffer *buffer,
2423 struct ring_buffer_event *event)
2425 __buffer_unlock_commit(buffer, event);
2429 trace_process_export(struct trace_export *export,
2430 struct ring_buffer_event *event)
2432 struct trace_entry *entry;
2433 unsigned int size = 0;
2435 entry = ring_buffer_event_data(event);
2436 size = ring_buffer_event_length(event);
2437 export->write(entry, size);
2440 static DEFINE_MUTEX(ftrace_export_lock);
2442 static struct trace_export __rcu *ftrace_exports_list __read_mostly;
2444 static DEFINE_STATIC_KEY_FALSE(ftrace_exports_enabled);
2446 static inline void ftrace_exports_enable(void)
2448 static_branch_enable(&ftrace_exports_enabled);
2451 static inline void ftrace_exports_disable(void)
2453 static_branch_disable(&ftrace_exports_enabled);
2456 void ftrace_exports(struct ring_buffer_event *event)
2458 struct trace_export *export;
2460 preempt_disable_notrace();
2462 export = rcu_dereference_raw_notrace(ftrace_exports_list);
2464 trace_process_export(export, event);
2465 export = rcu_dereference_raw_notrace(export->next);
2468 preempt_enable_notrace();
2472 add_trace_export(struct trace_export **list, struct trace_export *export)
2474 rcu_assign_pointer(export->next, *list);
2476 * We are entering export into the list but another
2477 * CPU might be walking that list. We need to make sure
2478 * the export->next pointer is valid before another CPU sees
2479 * the export pointer included into the list.
2481 rcu_assign_pointer(*list, export);
2485 rm_trace_export(struct trace_export **list, struct trace_export *export)
2487 struct trace_export **p;
2489 for (p = list; *p != NULL; p = &(*p)->next)
2496 rcu_assign_pointer(*p, (*p)->next);
2502 add_ftrace_export(struct trace_export **list, struct trace_export *export)
2505 ftrace_exports_enable();
2507 add_trace_export(list, export);
2511 rm_ftrace_export(struct trace_export **list, struct trace_export *export)
2515 ret = rm_trace_export(list, export);
2517 ftrace_exports_disable();
2522 int register_ftrace_export(struct trace_export *export)
2524 if (WARN_ON_ONCE(!export->write))
2527 mutex_lock(&ftrace_export_lock);
2529 add_ftrace_export(&ftrace_exports_list, export);
2531 mutex_unlock(&ftrace_export_lock);
2535 EXPORT_SYMBOL_GPL(register_ftrace_export);
2537 int unregister_ftrace_export(struct trace_export *export)
2541 mutex_lock(&ftrace_export_lock);
2543 ret = rm_ftrace_export(&ftrace_exports_list, export);
2545 mutex_unlock(&ftrace_export_lock);
2549 EXPORT_SYMBOL_GPL(unregister_ftrace_export);
2552 trace_function(struct trace_array *tr,
2553 unsigned long ip, unsigned long parent_ip, unsigned long flags,
2556 struct trace_event_call *call = &event_function;
2557 struct ring_buffer *buffer = tr->trace_buffer.buffer;
2558 struct ring_buffer_event *event;
2559 struct ftrace_entry *entry;
2561 event = __trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
2565 entry = ring_buffer_event_data(event);
2567 entry->parent_ip = parent_ip;
2569 if (!call_filter_check_discard(call, entry, buffer, event)) {
2570 if (static_branch_unlikely(&ftrace_exports_enabled))
2571 ftrace_exports(event);
2572 __buffer_unlock_commit(buffer, event);
2576 #ifdef CONFIG_STACKTRACE
2578 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
2579 struct ftrace_stack {
2580 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
2583 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
2584 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2586 static void __ftrace_trace_stack(struct ring_buffer *buffer,
2587 unsigned long flags,
2588 int skip, int pc, struct pt_regs *regs)
2590 struct trace_event_call *call = &event_kernel_stack;
2591 struct ring_buffer_event *event;
2592 struct stack_entry *entry;
2593 struct stack_trace trace;
2595 int size = FTRACE_STACK_ENTRIES;
2597 trace.nr_entries = 0;
2601 * Add two, for this function and the call to save_stack_trace()
2602 * If regs is set, then these functions will not be in the way.
2608 * Since events can happen in NMIs there's no safe way to
2609 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
2610 * or NMI comes in, it will just have to use the default
2611 * FTRACE_STACK_SIZE.
2613 preempt_disable_notrace();
2615 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
2617 * We don't need any atomic variables, just a barrier.
2618 * If an interrupt comes in, we don't care, because it would
2619 * have exited and put the counter back to what we want.
2620 * We just need a barrier to keep gcc from moving things
2624 if (use_stack == 1) {
2625 trace.entries = this_cpu_ptr(ftrace_stack.calls);
2626 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
2629 save_stack_trace_regs(regs, &trace);
2631 save_stack_trace(&trace);
2633 if (trace.nr_entries > size)
2634 size = trace.nr_entries;
2636 /* From now on, use_stack is a boolean */
2639 size *= sizeof(unsigned long);
2641 event = __trace_buffer_lock_reserve(buffer, TRACE_STACK,
2642 (sizeof(*entry) - sizeof(entry->caller)) + size,
2646 entry = ring_buffer_event_data(event);
2648 memset(&entry->caller, 0, size);
2651 memcpy(&entry->caller, trace.entries,
2652 trace.nr_entries * sizeof(unsigned long));
2654 trace.max_entries = FTRACE_STACK_ENTRIES;
2655 trace.entries = entry->caller;
2657 save_stack_trace_regs(regs, &trace);
2659 save_stack_trace(&trace);
2662 entry->size = trace.nr_entries;
2664 if (!call_filter_check_discard(call, entry, buffer, event))
2665 __buffer_unlock_commit(buffer, event);
2668 /* Again, don't let gcc optimize things here */
2670 __this_cpu_dec(ftrace_stack_reserve);
2671 preempt_enable_notrace();
2675 static inline void ftrace_trace_stack(struct trace_array *tr,
2676 struct ring_buffer *buffer,
2677 unsigned long flags,
2678 int skip, int pc, struct pt_regs *regs)
2680 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
2683 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
2686 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
2689 struct ring_buffer *buffer = tr->trace_buffer.buffer;
2691 if (rcu_is_watching()) {
2692 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
2697 * When an NMI triggers, RCU is enabled via rcu_nmi_enter(),
2698 * but if the above rcu_is_watching() failed, then the NMI
2699 * triggered someplace critical, and rcu_irq_enter() should
2700 * not be called from NMI.
2702 if (unlikely(in_nmi()))
2706 * It is possible that a function is being traced in a
2707 * location that RCU is not watching. A call to
2708 * rcu_irq_enter() will make sure that it is, but there's
2709 * a few internal rcu functions that could be traced
2710 * where that wont work either. In those cases, we just
2713 if (unlikely(rcu_irq_enter_disabled()))
2716 rcu_irq_enter_irqson();
2717 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
2718 rcu_irq_exit_irqson();
2722 * trace_dump_stack - record a stack back trace in the trace buffer
2723 * @skip: Number of functions to skip (helper handlers)
2725 void trace_dump_stack(int skip)
2727 unsigned long flags;
2729 if (tracing_disabled || tracing_selftest_running)
2732 local_save_flags(flags);
2735 * Skip 3 more, seems to get us at the caller of
2739 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
2740 flags, skip, preempt_count(), NULL);
2743 static DEFINE_PER_CPU(int, user_stack_count);
2746 ftrace_trace_userstack(struct trace_array *tr,
2747 struct ring_buffer *buffer, unsigned long flags, int pc)
2749 struct trace_event_call *call = &event_user_stack;
2750 struct ring_buffer_event *event;
2751 struct userstack_entry *entry;
2752 struct stack_trace trace;
2754 if (!(tr->trace_flags & TRACE_ITER_USERSTACKTRACE))
2758 * NMIs can not handle page faults, even with fix ups.
2759 * The save user stack can (and often does) fault.
2761 if (unlikely(in_nmi()))
2765 * prevent recursion, since the user stack tracing may
2766 * trigger other kernel events.
2769 if (__this_cpu_read(user_stack_count))
2772 __this_cpu_inc(user_stack_count);
2774 event = __trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
2775 sizeof(*entry), flags, pc);
2777 goto out_drop_count;
2778 entry = ring_buffer_event_data(event);
2780 entry->tgid = current->tgid;
2781 memset(&entry->caller, 0, sizeof(entry->caller));
2783 trace.nr_entries = 0;
2784 trace.max_entries = FTRACE_STACK_ENTRIES;
2786 trace.entries = entry->caller;
2788 save_stack_trace_user(&trace);
2789 if (!call_filter_check_discard(call, entry, buffer, event))
2790 __buffer_unlock_commit(buffer, event);
2793 __this_cpu_dec(user_stack_count);
2799 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
2801 ftrace_trace_userstack(tr, flags, preempt_count());
2805 #endif /* CONFIG_STACKTRACE */
2807 /* created for use with alloc_percpu */
2808 struct trace_buffer_struct {
2810 char buffer[4][TRACE_BUF_SIZE];
2813 static struct trace_buffer_struct *trace_percpu_buffer;
2816 * Thise allows for lockless recording. If we're nested too deeply, then
2817 * this returns NULL.
2819 static char *get_trace_buf(void)
2821 struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
2823 if (!buffer || buffer->nesting >= 4)
2828 /* Interrupts must see nesting incremented before we use the buffer */
2830 return &buffer->buffer[buffer->nesting - 1][0];
2833 static void put_trace_buf(void)
2835 /* Don't let the decrement of nesting leak before this */
2837 this_cpu_dec(trace_percpu_buffer->nesting);
2840 static int alloc_percpu_trace_buffer(void)
2842 struct trace_buffer_struct *buffers;
2844 buffers = alloc_percpu(struct trace_buffer_struct);
2845 if (WARN(!buffers, "Could not allocate percpu trace_printk buffer"))
2848 trace_percpu_buffer = buffers;
2852 static int buffers_allocated;
2854 void trace_printk_init_buffers(void)
2856 if (buffers_allocated)
2859 if (alloc_percpu_trace_buffer())
2862 /* trace_printk() is for debug use only. Don't use it in production. */
2865 pr_warn("**********************************************************\n");
2866 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2868 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
2870 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
2871 pr_warn("** unsafe for production use. **\n");
2873 pr_warn("** If you see this message and you are not debugging **\n");
2874 pr_warn("** the kernel, report this immediately to your vendor! **\n");
2876 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2877 pr_warn("**********************************************************\n");
2879 /* Expand the buffers to set size */
2880 tracing_update_buffers();
2882 buffers_allocated = 1;
2885 * trace_printk_init_buffers() can be called by modules.
2886 * If that happens, then we need to start cmdline recording
2887 * directly here. If the global_trace.buffer is already
2888 * allocated here, then this was called by module code.
2890 if (global_trace.trace_buffer.buffer)
2891 tracing_start_cmdline_record();
2894 void trace_printk_start_comm(void)
2896 /* Start tracing comms if trace printk is set */
2897 if (!buffers_allocated)
2899 tracing_start_cmdline_record();
2902 static void trace_printk_start_stop_comm(int enabled)
2904 if (!buffers_allocated)
2908 tracing_start_cmdline_record();
2910 tracing_stop_cmdline_record();
2914 * trace_vbprintk - write binary msg to tracing buffer
2917 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2919 struct trace_event_call *call = &event_bprint;
2920 struct ring_buffer_event *event;
2921 struct ring_buffer *buffer;
2922 struct trace_array *tr = &global_trace;
2923 struct bprint_entry *entry;
2924 unsigned long flags;
2926 int len = 0, size, pc;
2928 if (unlikely(tracing_selftest_running || tracing_disabled))
2931 /* Don't pollute graph traces with trace_vprintk internals */
2932 pause_graph_tracing();
2934 pc = preempt_count();
2935 preempt_disable_notrace();
2937 tbuffer = get_trace_buf();
2943 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2945 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2948 local_save_flags(flags);
2949 size = sizeof(*entry) + sizeof(u32) * len;
2950 buffer = tr->trace_buffer.buffer;
2951 event = __trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2955 entry = ring_buffer_event_data(event);
2959 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2960 if (!call_filter_check_discard(call, entry, buffer, event)) {
2961 __buffer_unlock_commit(buffer, event);
2962 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
2969 preempt_enable_notrace();
2970 unpause_graph_tracing();
2974 EXPORT_SYMBOL_GPL(trace_vbprintk);
2978 __trace_array_vprintk(struct ring_buffer *buffer,
2979 unsigned long ip, const char *fmt, va_list args)
2981 struct trace_event_call *call = &event_print;
2982 struct ring_buffer_event *event;
2983 int len = 0, size, pc;
2984 struct print_entry *entry;
2985 unsigned long flags;
2988 if (tracing_disabled || tracing_selftest_running)
2991 /* Don't pollute graph traces with trace_vprintk internals */
2992 pause_graph_tracing();
2994 pc = preempt_count();
2995 preempt_disable_notrace();
2998 tbuffer = get_trace_buf();
3004 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
3006 local_save_flags(flags);
3007 size = sizeof(*entry) + len + 1;
3008 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
3012 entry = ring_buffer_event_data(event);
3015 memcpy(&entry->buf, tbuffer, len + 1);
3016 if (!call_filter_check_discard(call, entry, buffer, event)) {
3017 __buffer_unlock_commit(buffer, event);
3018 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
3025 preempt_enable_notrace();
3026 unpause_graph_tracing();
3032 int trace_array_vprintk(struct trace_array *tr,
3033 unsigned long ip, const char *fmt, va_list args)
3035 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
3039 int trace_array_printk(struct trace_array *tr,
3040 unsigned long ip, const char *fmt, ...)
3045 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3052 ret = trace_array_vprintk(tr, ip, fmt, ap);
3058 int trace_array_printk_buf(struct ring_buffer *buffer,
3059 unsigned long ip, const char *fmt, ...)
3064 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
3068 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
3074 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
3076 return trace_array_vprintk(&global_trace, ip, fmt, args);
3078 EXPORT_SYMBOL_GPL(trace_vprintk);
3080 static void trace_iterator_increment(struct trace_iterator *iter)
3082 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
3086 ring_buffer_read(buf_iter, NULL);
3089 static struct trace_entry *
3090 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
3091 unsigned long *lost_events)
3093 struct ring_buffer_event *event;
3094 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
3097 event = ring_buffer_iter_peek(buf_iter, ts);
3099 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
3103 iter->ent_size = ring_buffer_event_length(event);
3104 return ring_buffer_event_data(event);
3110 static struct trace_entry *
3111 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
3112 unsigned long *missing_events, u64 *ent_ts)
3114 struct ring_buffer *buffer = iter->trace_buffer->buffer;
3115 struct trace_entry *ent, *next = NULL;
3116 unsigned long lost_events = 0, next_lost = 0;
3117 int cpu_file = iter->cpu_file;
3118 u64 next_ts = 0, ts;
3124 * If we are in a per_cpu trace file, don't bother by iterating over
3125 * all cpu and peek directly.
3127 if (cpu_file > RING_BUFFER_ALL_CPUS) {
3128 if (ring_buffer_empty_cpu(buffer, cpu_file))
3130 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
3132 *ent_cpu = cpu_file;
3137 for_each_tracing_cpu(cpu) {
3139 if (ring_buffer_empty_cpu(buffer, cpu))
3142 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
3145 * Pick the entry with the smallest timestamp:
3147 if (ent && (!next || ts < next_ts)) {
3151 next_lost = lost_events;
3152 next_size = iter->ent_size;
3156 iter->ent_size = next_size;
3159 *ent_cpu = next_cpu;
3165 *missing_events = next_lost;
3170 /* Find the next real entry, without updating the iterator itself */
3171 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
3172 int *ent_cpu, u64 *ent_ts)
3174 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
3177 /* Find the next real entry, and increment the iterator to the next entry */
3178 void *trace_find_next_entry_inc(struct trace_iterator *iter)
3180 iter->ent = __find_next_entry(iter, &iter->cpu,
3181 &iter->lost_events, &iter->ts);
3184 trace_iterator_increment(iter);
3186 return iter->ent ? iter : NULL;
3189 static void trace_consume(struct trace_iterator *iter)
3191 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
3192 &iter->lost_events);
3195 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
3197 struct trace_iterator *iter = m->private;
3201 WARN_ON_ONCE(iter->leftover);
3205 /* can't go backwards */
3210 ent = trace_find_next_entry_inc(iter);
3214 while (ent && iter->idx < i)
3215 ent = trace_find_next_entry_inc(iter);
3222 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
3224 struct ring_buffer_event *event;
3225 struct ring_buffer_iter *buf_iter;
3226 unsigned long entries = 0;
3229 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
3231 buf_iter = trace_buffer_iter(iter, cpu);
3235 ring_buffer_iter_reset(buf_iter);
3238 * We could have the case with the max latency tracers
3239 * that a reset never took place on a cpu. This is evident
3240 * by the timestamp being before the start of the buffer.
3242 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
3243 if (ts >= iter->trace_buffer->time_start)
3246 ring_buffer_read(buf_iter, NULL);
3249 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
3253 * The current tracer is copied to avoid a global locking
3256 static void *s_start(struct seq_file *m, loff_t *pos)
3258 struct trace_iterator *iter = m->private;
3259 struct trace_array *tr = iter->tr;
3260 int cpu_file = iter->cpu_file;
3266 * copy the tracer to avoid using a global lock all around.
3267 * iter->trace is a copy of current_trace, the pointer to the
3268 * name may be used instead of a strcmp(), as iter->trace->name
3269 * will point to the same string as current_trace->name.
3271 mutex_lock(&trace_types_lock);
3272 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
3273 *iter->trace = *tr->current_trace;
3274 mutex_unlock(&trace_types_lock);
3276 #ifdef CONFIG_TRACER_MAX_TRACE
3277 if (iter->snapshot && iter->trace->use_max_tr)
3278 return ERR_PTR(-EBUSY);
3281 if (*pos != iter->pos) {
3286 if (cpu_file == RING_BUFFER_ALL_CPUS) {
3287 for_each_tracing_cpu(cpu)
3288 tracing_iter_reset(iter, cpu);
3290 tracing_iter_reset(iter, cpu_file);
3293 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
3298 * If we overflowed the seq_file before, then we want
3299 * to just reuse the trace_seq buffer again.
3305 p = s_next(m, p, &l);
3309 trace_event_read_lock();
3310 trace_access_lock(cpu_file);
3314 static void s_stop(struct seq_file *m, void *p)
3316 struct trace_iterator *iter = m->private;
3318 #ifdef CONFIG_TRACER_MAX_TRACE
3319 if (iter->snapshot && iter->trace->use_max_tr)
3323 trace_access_unlock(iter->cpu_file);
3324 trace_event_read_unlock();
3328 get_total_entries(struct trace_buffer *buf,
3329 unsigned long *total, unsigned long *entries)
3331 unsigned long count;
3337 for_each_tracing_cpu(cpu) {
3338 count = ring_buffer_entries_cpu(buf->buffer, cpu);
3340 * If this buffer has skipped entries, then we hold all
3341 * entries for the trace and we need to ignore the
3342 * ones before the time stamp.
3344 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
3345 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
3346 /* total is the same as the entries */
3350 ring_buffer_overrun_cpu(buf->buffer, cpu);
3355 static void print_lat_help_header(struct seq_file *m)
3357 seq_puts(m, "# _------=> CPU# \n"
3358 "# / _-----=> irqs-off \n"
3359 "# | / _----=> need-resched \n"
3360 "# || / _---=> hardirq/softirq \n"
3361 "# ||| / _--=> preempt-depth \n"
3363 "# cmd pid ||||| time | caller \n"
3364 "# \\ / ||||| \\ | / \n");
3367 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
3369 unsigned long total;
3370 unsigned long entries;
3372 get_total_entries(buf, &total, &entries);
3373 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
3374 entries, total, num_online_cpus());
3378 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m,
3381 bool tgid = flags & TRACE_ITER_RECORD_TGID;
3383 print_event_info(buf, m);
3385 seq_printf(m, "# TASK-PID %s CPU# TIMESTAMP FUNCTION\n", tgid ? "TGID " : "");
3386 seq_printf(m, "# | | %s | | |\n", tgid ? " | " : "");
3389 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m,
3392 bool tgid = flags & TRACE_ITER_RECORD_TGID;
3393 const char tgid_space[] = " ";
3394 const char space[] = " ";
3396 print_event_info(buf, m);
3398 seq_printf(m, "# %s _-----=> irqs-off\n",
3399 tgid ? tgid_space : space);
3400 seq_printf(m, "# %s / _----=> need-resched\n",
3401 tgid ? tgid_space : space);
3402 seq_printf(m, "# %s| / _---=> hardirq/softirq\n",
3403 tgid ? tgid_space : space);
3404 seq_printf(m, "# %s|| / _--=> preempt-depth\n",
3405 tgid ? tgid_space : space);
3406 seq_printf(m, "# %s||| / delay\n",
3407 tgid ? tgid_space : space);
3408 seq_printf(m, "# TASK-PID %sCPU# |||| TIMESTAMP FUNCTION\n",
3409 tgid ? " TGID " : space);
3410 seq_printf(m, "# | | %s | |||| | |\n",
3411 tgid ? " | " : space);
3415 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
3417 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
3418 struct trace_buffer *buf = iter->trace_buffer;
3419 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
3420 struct tracer *type = iter->trace;
3421 unsigned long entries;
3422 unsigned long total;
3423 const char *name = "preemption";
3427 get_total_entries(buf, &total, &entries);
3429 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
3431 seq_puts(m, "# -----------------------------------"
3432 "---------------------------------\n");
3433 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
3434 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
3435 nsecs_to_usecs(data->saved_latency),
3439 #if defined(CONFIG_PREEMPT_NONE)
3441 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
3443 #elif defined(CONFIG_PREEMPT)
3448 /* These are reserved for later use */
3451 seq_printf(m, " #P:%d)\n", num_online_cpus());
3455 seq_puts(m, "# -----------------\n");
3456 seq_printf(m, "# | task: %.16s-%d "
3457 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
3458 data->comm, data->pid,
3459 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
3460 data->policy, data->rt_priority);
3461 seq_puts(m, "# -----------------\n");
3463 if (data->critical_start) {
3464 seq_puts(m, "# => started at: ");
3465 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
3466 trace_print_seq(m, &iter->seq);
3467 seq_puts(m, "\n# => ended at: ");
3468 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
3469 trace_print_seq(m, &iter->seq);
3470 seq_puts(m, "\n#\n");
3476 static void test_cpu_buff_start(struct trace_iterator *iter)
3478 struct trace_seq *s = &iter->seq;
3479 struct trace_array *tr = iter->tr;
3481 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
3484 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
3487 if (cpumask_available(iter->started) &&
3488 cpumask_test_cpu(iter->cpu, iter->started))
3491 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
3494 if (cpumask_available(iter->started))
3495 cpumask_set_cpu(iter->cpu, iter->started);
3497 /* Don't print started cpu buffer for the first entry of the trace */
3499 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
3503 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
3505 struct trace_array *tr = iter->tr;
3506 struct trace_seq *s = &iter->seq;
3507 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
3508 struct trace_entry *entry;
3509 struct trace_event *event;
3513 test_cpu_buff_start(iter);
3515 event = ftrace_find_event(entry->type);
3517 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3518 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3519 trace_print_lat_context(iter);
3521 trace_print_context(iter);
3524 if (trace_seq_has_overflowed(s))
3525 return TRACE_TYPE_PARTIAL_LINE;
3528 return event->funcs->trace(iter, sym_flags, event);
3530 trace_seq_printf(s, "Unknown type %d\n", entry->type);
3532 return trace_handle_return(s);
3535 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
3537 struct trace_array *tr = iter->tr;
3538 struct trace_seq *s = &iter->seq;
3539 struct trace_entry *entry;
3540 struct trace_event *event;
3544 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
3545 trace_seq_printf(s, "%d %d %llu ",
3546 entry->pid, iter->cpu, iter->ts);
3548 if (trace_seq_has_overflowed(s))
3549 return TRACE_TYPE_PARTIAL_LINE;
3551 event = ftrace_find_event(entry->type);
3553 return event->funcs->raw(iter, 0, event);
3555 trace_seq_printf(s, "%d ?\n", entry->type);
3557 return trace_handle_return(s);
3560 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
3562 struct trace_array *tr = iter->tr;
3563 struct trace_seq *s = &iter->seq;
3564 unsigned char newline = '\n';
3565 struct trace_entry *entry;
3566 struct trace_event *event;
3570 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3571 SEQ_PUT_HEX_FIELD(s, entry->pid);
3572 SEQ_PUT_HEX_FIELD(s, iter->cpu);
3573 SEQ_PUT_HEX_FIELD(s, iter->ts);
3574 if (trace_seq_has_overflowed(s))
3575 return TRACE_TYPE_PARTIAL_LINE;
3578 event = ftrace_find_event(entry->type);
3580 enum print_line_t ret = event->funcs->hex(iter, 0, event);
3581 if (ret != TRACE_TYPE_HANDLED)
3585 SEQ_PUT_FIELD(s, newline);
3587 return trace_handle_return(s);
3590 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
3592 struct trace_array *tr = iter->tr;
3593 struct trace_seq *s = &iter->seq;
3594 struct trace_entry *entry;
3595 struct trace_event *event;
3599 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3600 SEQ_PUT_FIELD(s, entry->pid);
3601 SEQ_PUT_FIELD(s, iter->cpu);
3602 SEQ_PUT_FIELD(s, iter->ts);
3603 if (trace_seq_has_overflowed(s))
3604 return TRACE_TYPE_PARTIAL_LINE;
3607 event = ftrace_find_event(entry->type);
3608 return event ? event->funcs->binary(iter, 0, event) :
3612 int trace_empty(struct trace_iterator *iter)
3614 struct ring_buffer_iter *buf_iter;
3617 /* If we are looking at one CPU buffer, only check that one */
3618 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
3619 cpu = iter->cpu_file;
3620 buf_iter = trace_buffer_iter(iter, cpu);
3622 if (!ring_buffer_iter_empty(buf_iter))
3625 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
3631 for_each_tracing_cpu(cpu) {
3632 buf_iter = trace_buffer_iter(iter, cpu);
3634 if (!ring_buffer_iter_empty(buf_iter))
3637 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
3645 /* Called with trace_event_read_lock() held. */
3646 enum print_line_t print_trace_line(struct trace_iterator *iter)
3648 struct trace_array *tr = iter->tr;
3649 unsigned long trace_flags = tr->trace_flags;
3650 enum print_line_t ret;
3652 if (iter->lost_events) {
3653 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
3654 iter->cpu, iter->lost_events);
3655 if (trace_seq_has_overflowed(&iter->seq))
3656 return TRACE_TYPE_PARTIAL_LINE;
3659 if (iter->trace && iter->trace->print_line) {
3660 ret = iter->trace->print_line(iter);
3661 if (ret != TRACE_TYPE_UNHANDLED)
3665 if (iter->ent->type == TRACE_BPUTS &&
3666 trace_flags & TRACE_ITER_PRINTK &&
3667 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3668 return trace_print_bputs_msg_only(iter);
3670 if (iter->ent->type == TRACE_BPRINT &&
3671 trace_flags & TRACE_ITER_PRINTK &&
3672 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3673 return trace_print_bprintk_msg_only(iter);
3675 if (iter->ent->type == TRACE_PRINT &&
3676 trace_flags & TRACE_ITER_PRINTK &&
3677 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3678 return trace_print_printk_msg_only(iter);
3680 if (trace_flags & TRACE_ITER_BIN)
3681 return print_bin_fmt(iter);
3683 if (trace_flags & TRACE_ITER_HEX)
3684 return print_hex_fmt(iter);
3686 if (trace_flags & TRACE_ITER_RAW)
3687 return print_raw_fmt(iter);
3689 return print_trace_fmt(iter);
3692 void trace_latency_header(struct seq_file *m)
3694 struct trace_iterator *iter = m->private;
3695 struct trace_array *tr = iter->tr;
3697 /* print nothing if the buffers are empty */
3698 if (trace_empty(iter))
3701 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3702 print_trace_header(m, iter);
3704 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
3705 print_lat_help_header(m);
3708 void trace_default_header(struct seq_file *m)
3710 struct trace_iterator *iter = m->private;
3711 struct trace_array *tr = iter->tr;
3712 unsigned long trace_flags = tr->trace_flags;
3714 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
3717 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
3718 /* print nothing if the buffers are empty */
3719 if (trace_empty(iter))
3721 print_trace_header(m, iter);
3722 if (!(trace_flags & TRACE_ITER_VERBOSE))
3723 print_lat_help_header(m);
3725 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
3726 if (trace_flags & TRACE_ITER_IRQ_INFO)
3727 print_func_help_header_irq(iter->trace_buffer,
3730 print_func_help_header(iter->trace_buffer, m,
3736 static void test_ftrace_alive(struct seq_file *m)
3738 if (!ftrace_is_dead())
3740 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
3741 "# MAY BE MISSING FUNCTION EVENTS\n");
3744 #ifdef CONFIG_TRACER_MAX_TRACE
3745 static void show_snapshot_main_help(struct seq_file *m)
3747 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
3748 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3749 "# Takes a snapshot of the main buffer.\n"
3750 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
3751 "# (Doesn't have to be '2' works with any number that\n"
3752 "# is not a '0' or '1')\n");
3755 static void show_snapshot_percpu_help(struct seq_file *m)
3757 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
3758 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
3759 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3760 "# Takes a snapshot of the main buffer for this cpu.\n");
3762 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
3763 "# Must use main snapshot file to allocate.\n");
3765 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
3766 "# (Doesn't have to be '2' works with any number that\n"
3767 "# is not a '0' or '1')\n");
3770 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
3772 if (iter->tr->allocated_snapshot)
3773 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
3775 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
3777 seq_puts(m, "# Snapshot commands:\n");
3778 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
3779 show_snapshot_main_help(m);
3781 show_snapshot_percpu_help(m);
3784 /* Should never be called */
3785 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
3788 static int s_show(struct seq_file *m, void *v)
3790 struct trace_iterator *iter = v;
3793 if (iter->ent == NULL) {
3795 seq_printf(m, "# tracer: %s\n", iter->trace->name);
3797 test_ftrace_alive(m);
3799 if (iter->snapshot && trace_empty(iter))
3800 print_snapshot_help(m, iter);
3801 else if (iter->trace && iter->trace->print_header)
3802 iter->trace->print_header(m);
3804 trace_default_header(m);
3806 } else if (iter->leftover) {
3808 * If we filled the seq_file buffer earlier, we
3809 * want to just show it now.
3811 ret = trace_print_seq(m, &iter->seq);
3813 /* ret should this time be zero, but you never know */
3814 iter->leftover = ret;
3817 print_trace_line(iter);
3818 ret = trace_print_seq(m, &iter->seq);
3820 * If we overflow the seq_file buffer, then it will
3821 * ask us for this data again at start up.
3823 * ret is 0 if seq_file write succeeded.
3826 iter->leftover = ret;
3833 * Should be used after trace_array_get(), trace_types_lock
3834 * ensures that i_cdev was already initialized.
3836 static inline int tracing_get_cpu(struct inode *inode)
3838 if (inode->i_cdev) /* See trace_create_cpu_file() */
3839 return (long)inode->i_cdev - 1;
3840 return RING_BUFFER_ALL_CPUS;
3843 static const struct seq_operations tracer_seq_ops = {
3850 static struct trace_iterator *
3851 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
3853 struct trace_array *tr = inode->i_private;
3854 struct trace_iterator *iter;
3857 if (tracing_disabled)
3858 return ERR_PTR(-ENODEV);
3860 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
3862 return ERR_PTR(-ENOMEM);
3864 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
3866 if (!iter->buffer_iter)
3870 * We make a copy of the current tracer to avoid concurrent
3871 * changes on it while we are reading.
3873 mutex_lock(&trace_types_lock);
3874 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
3878 *iter->trace = *tr->current_trace;
3880 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3885 #ifdef CONFIG_TRACER_MAX_TRACE
3886 /* Currently only the top directory has a snapshot */
3887 if (tr->current_trace->print_max || snapshot)
3888 iter->trace_buffer = &tr->max_buffer;
3891 iter->trace_buffer = &tr->trace_buffer;
3892 iter->snapshot = snapshot;
3894 iter->cpu_file = tracing_get_cpu(inode);
3895 mutex_init(&iter->mutex);
3897 /* Notify the tracer early; before we stop tracing. */
3898 if (iter->trace && iter->trace->open)
3899 iter->trace->open(iter);
3901 /* Annotate start of buffers if we had overruns */
3902 if (ring_buffer_overruns(iter->trace_buffer->buffer))
3903 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3905 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3906 if (trace_clocks[tr->clock_id].in_ns)
3907 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3909 /* stop the trace while dumping if we are not opening "snapshot" */
3910 if (!iter->snapshot)
3911 tracing_stop_tr(tr);
3913 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3914 for_each_tracing_cpu(cpu) {
3915 iter->buffer_iter[cpu] =
3916 ring_buffer_read_prepare(iter->trace_buffer->buffer,
3919 ring_buffer_read_prepare_sync();
3920 for_each_tracing_cpu(cpu) {
3921 ring_buffer_read_start(iter->buffer_iter[cpu]);
3922 tracing_iter_reset(iter, cpu);
3925 cpu = iter->cpu_file;
3926 iter->buffer_iter[cpu] =
3927 ring_buffer_read_prepare(iter->trace_buffer->buffer,
3929 ring_buffer_read_prepare_sync();
3930 ring_buffer_read_start(iter->buffer_iter[cpu]);
3931 tracing_iter_reset(iter, cpu);
3934 mutex_unlock(&trace_types_lock);
3939 mutex_unlock(&trace_types_lock);
3941 kfree(iter->buffer_iter);
3943 seq_release_private(inode, file);
3944 return ERR_PTR(-ENOMEM);
3947 int tracing_open_generic(struct inode *inode, struct file *filp)
3949 if (tracing_disabled)
3952 filp->private_data = inode->i_private;
3956 bool tracing_is_disabled(void)
3958 return (tracing_disabled) ? true: false;
3962 * Open and update trace_array ref count.
3963 * Must have the current trace_array passed to it.
3965 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3967 struct trace_array *tr = inode->i_private;
3969 if (tracing_disabled)
3972 if (trace_array_get(tr) < 0)
3975 filp->private_data = inode->i_private;
3980 static int tracing_release(struct inode *inode, struct file *file)
3982 struct trace_array *tr = inode->i_private;
3983 struct seq_file *m = file->private_data;
3984 struct trace_iterator *iter;
3987 if (!(file->f_mode & FMODE_READ)) {
3988 trace_array_put(tr);
3992 /* Writes do not use seq_file */
3994 mutex_lock(&trace_types_lock);
3996 for_each_tracing_cpu(cpu) {
3997 if (iter->buffer_iter[cpu])
3998 ring_buffer_read_finish(iter->buffer_iter[cpu]);
4001 if (iter->trace && iter->trace->close)
4002 iter->trace->close(iter);
4004 if (!iter->snapshot)
4005 /* reenable tracing if it was previously enabled */
4006 tracing_start_tr(tr);
4008 __trace_array_put(tr);
4010 mutex_unlock(&trace_types_lock);
4012 mutex_destroy(&iter->mutex);
4013 free_cpumask_var(iter->started);
4015 kfree(iter->buffer_iter);
4016 seq_release_private(inode, file);
4021 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
4023 struct trace_array *tr = inode->i_private;
4025 trace_array_put(tr);
4029 static int tracing_single_release_tr(struct inode *inode, struct file *file)
4031 struct trace_array *tr = inode->i_private;
4033 trace_array_put(tr);
4035 return single_release(inode, file);
4038 static int tracing_open(struct inode *inode, struct file *file)
4040 struct trace_array *tr = inode->i_private;
4041 struct trace_iterator *iter;
4044 if (trace_array_get(tr) < 0)
4047 /* If this file was open for write, then erase contents */
4048 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
4049 int cpu = tracing_get_cpu(inode);
4050 struct trace_buffer *trace_buf = &tr->trace_buffer;
4052 #ifdef CONFIG_TRACER_MAX_TRACE
4053 if (tr->current_trace->print_max)
4054 trace_buf = &tr->max_buffer;
4057 if (cpu == RING_BUFFER_ALL_CPUS)
4058 tracing_reset_online_cpus(trace_buf);
4060 tracing_reset(trace_buf, cpu);
4063 if (file->f_mode & FMODE_READ) {
4064 iter = __tracing_open(inode, file, false);
4066 ret = PTR_ERR(iter);
4067 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4068 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4072 trace_array_put(tr);
4078 * Some tracers are not suitable for instance buffers.
4079 * A tracer is always available for the global array (toplevel)
4080 * or if it explicitly states that it is.
4083 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
4085 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
4088 /* Find the next tracer that this trace array may use */
4089 static struct tracer *
4090 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
4092 while (t && !trace_ok_for_array(t, tr))
4099 t_next(struct seq_file *m, void *v, loff_t *pos)
4101 struct trace_array *tr = m->private;
4102 struct tracer *t = v;
4107 t = get_tracer_for_array(tr, t->next);
4112 static void *t_start(struct seq_file *m, loff_t *pos)
4114 struct trace_array *tr = m->private;
4118 mutex_lock(&trace_types_lock);
4120 t = get_tracer_for_array(tr, trace_types);
4121 for (; t && l < *pos; t = t_next(m, t, &l))
4127 static void t_stop(struct seq_file *m, void *p)
4129 mutex_unlock(&trace_types_lock);
4132 static int t_show(struct seq_file *m, void *v)
4134 struct tracer *t = v;
4139 seq_puts(m, t->name);
4148 static const struct seq_operations show_traces_seq_ops = {
4155 static int show_traces_open(struct inode *inode, struct file *file)
4157 struct trace_array *tr = inode->i_private;
4161 if (tracing_disabled)
4164 if (trace_array_get(tr) < 0)
4167 ret = seq_open(file, &show_traces_seq_ops);
4169 trace_array_put(tr);
4173 m = file->private_data;
4179 static int show_traces_release(struct inode *inode, struct file *file)
4181 struct trace_array *tr = inode->i_private;
4183 trace_array_put(tr);
4184 return seq_release(inode, file);
4188 tracing_write_stub(struct file *filp, const char __user *ubuf,
4189 size_t count, loff_t *ppos)
4194 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
4198 if (file->f_mode & FMODE_READ)
4199 ret = seq_lseek(file, offset, whence);
4201 file->f_pos = ret = 0;
4206 static const struct file_operations tracing_fops = {
4207 .open = tracing_open,
4209 .write = tracing_write_stub,
4210 .llseek = tracing_lseek,
4211 .release = tracing_release,
4214 static const struct file_operations show_traces_fops = {
4215 .open = show_traces_open,
4217 .llseek = seq_lseek,
4218 .release = show_traces_release,
4222 tracing_cpumask_read(struct file *filp, char __user *ubuf,
4223 size_t count, loff_t *ppos)
4225 struct trace_array *tr = file_inode(filp)->i_private;
4229 len = snprintf(NULL, 0, "%*pb\n",
4230 cpumask_pr_args(tr->tracing_cpumask)) + 1;
4231 mask_str = kmalloc(len, GFP_KERNEL);
4235 len = snprintf(mask_str, len, "%*pb\n",
4236 cpumask_pr_args(tr->tracing_cpumask));
4241 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
4250 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
4251 size_t count, loff_t *ppos)
4253 struct trace_array *tr = file_inode(filp)->i_private;
4254 cpumask_var_t tracing_cpumask_new;
4257 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
4260 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
4264 local_irq_disable();
4265 arch_spin_lock(&tr->max_lock);
4266 for_each_tracing_cpu(cpu) {
4268 * Increase/decrease the disabled counter if we are
4269 * about to flip a bit in the cpumask:
4271 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4272 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4273 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
4274 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
4276 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
4277 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
4278 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
4279 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
4282 arch_spin_unlock(&tr->max_lock);
4285 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
4286 free_cpumask_var(tracing_cpumask_new);
4291 free_cpumask_var(tracing_cpumask_new);
4296 static const struct file_operations tracing_cpumask_fops = {
4297 .open = tracing_open_generic_tr,
4298 .read = tracing_cpumask_read,
4299 .write = tracing_cpumask_write,
4300 .release = tracing_release_generic_tr,
4301 .llseek = generic_file_llseek,
4304 static int tracing_trace_options_show(struct seq_file *m, void *v)
4306 struct tracer_opt *trace_opts;
4307 struct trace_array *tr = m->private;
4311 mutex_lock(&trace_types_lock);
4312 tracer_flags = tr->current_trace->flags->val;
4313 trace_opts = tr->current_trace->flags->opts;
4315 for (i = 0; trace_options[i]; i++) {
4316 if (tr->trace_flags & (1 << i))
4317 seq_printf(m, "%s\n", trace_options[i]);
4319 seq_printf(m, "no%s\n", trace_options[i]);
4322 for (i = 0; trace_opts[i].name; i++) {
4323 if (tracer_flags & trace_opts[i].bit)
4324 seq_printf(m, "%s\n", trace_opts[i].name);
4326 seq_printf(m, "no%s\n", trace_opts[i].name);
4328 mutex_unlock(&trace_types_lock);
4333 static int __set_tracer_option(struct trace_array *tr,
4334 struct tracer_flags *tracer_flags,
4335 struct tracer_opt *opts, int neg)
4337 struct tracer *trace = tracer_flags->trace;
4340 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
4345 tracer_flags->val &= ~opts->bit;
4347 tracer_flags->val |= opts->bit;
4351 /* Try to assign a tracer specific option */
4352 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
4354 struct tracer *trace = tr->current_trace;
4355 struct tracer_flags *tracer_flags = trace->flags;
4356 struct tracer_opt *opts = NULL;
4359 for (i = 0; tracer_flags->opts[i].name; i++) {
4360 opts = &tracer_flags->opts[i];
4362 if (strcmp(cmp, opts->name) == 0)
4363 return __set_tracer_option(tr, trace->flags, opts, neg);
4369 /* Some tracers require overwrite to stay enabled */
4370 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
4372 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
4378 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
4382 if ((mask == TRACE_ITER_RECORD_TGID) ||
4383 (mask == TRACE_ITER_RECORD_CMD))
4384 lockdep_assert_held(&event_mutex);
4386 /* do nothing if flag is already set */
4387 if (!!(tr->trace_flags & mask) == !!enabled)
4390 /* Give the tracer a chance to approve the change */
4391 if (tr->current_trace->flag_changed)
4392 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
4396 tr->trace_flags |= mask;
4398 tr->trace_flags &= ~mask;
4400 if (mask == TRACE_ITER_RECORD_CMD)
4401 trace_event_enable_cmd_record(enabled);
4403 if (mask == TRACE_ITER_RECORD_TGID) {
4405 tgid_map_max = pid_max;
4406 map = kzalloc((tgid_map_max + 1) * sizeof(*tgid_map),
4410 * Pairs with smp_load_acquire() in
4411 * trace_find_tgid_ptr() to ensure that if it observes
4412 * the tgid_map we just allocated then it also observes
4413 * the corresponding tgid_map_max value.
4415 smp_store_release(&tgid_map, map);
4418 tr->trace_flags &= ~TRACE_ITER_RECORD_TGID;
4422 trace_event_enable_tgid_record(enabled);
4425 if (mask == TRACE_ITER_EVENT_FORK)
4426 trace_event_follow_fork(tr, enabled);
4428 if (mask == TRACE_ITER_FUNC_FORK)
4429 ftrace_pid_follow_fork(tr, enabled);
4431 if (mask == TRACE_ITER_OVERWRITE) {
4432 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
4433 #ifdef CONFIG_TRACER_MAX_TRACE
4434 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
4438 if (mask == TRACE_ITER_PRINTK) {
4439 trace_printk_start_stop_comm(enabled);
4440 trace_printk_control(enabled);
4446 static int trace_set_options(struct trace_array *tr, char *option)
4452 size_t orig_len = strlen(option);
4454 cmp = strstrip(option);
4456 if (strncmp(cmp, "no", 2) == 0) {
4461 mutex_lock(&event_mutex);
4462 mutex_lock(&trace_types_lock);
4464 for (i = 0; trace_options[i]; i++) {
4465 if (strcmp(cmp, trace_options[i]) == 0) {
4466 ret = set_tracer_flag(tr, 1 << i, !neg);
4471 /* If no option could be set, test the specific tracer options */
4472 if (!trace_options[i])
4473 ret = set_tracer_option(tr, cmp, neg);
4475 mutex_unlock(&trace_types_lock);
4476 mutex_unlock(&event_mutex);
4479 * If the first trailing whitespace is replaced with '\0' by strstrip,
4480 * turn it back into a space.
4482 if (orig_len > strlen(option))
4483 option[strlen(option)] = ' ';
4488 static void __init apply_trace_boot_options(void)
4490 char *buf = trace_boot_options_buf;
4494 option = strsep(&buf, ",");
4500 trace_set_options(&global_trace, option);
4502 /* Put back the comma to allow this to be called again */
4509 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
4510 size_t cnt, loff_t *ppos)
4512 struct seq_file *m = filp->private_data;
4513 struct trace_array *tr = m->private;
4517 if (cnt >= sizeof(buf))
4520 if (copy_from_user(buf, ubuf, cnt))
4525 ret = trace_set_options(tr, buf);
4534 static int tracing_trace_options_open(struct inode *inode, struct file *file)
4536 struct trace_array *tr = inode->i_private;
4539 if (tracing_disabled)
4542 if (trace_array_get(tr) < 0)
4545 ret = single_open(file, tracing_trace_options_show, inode->i_private);
4547 trace_array_put(tr);
4552 static const struct file_operations tracing_iter_fops = {
4553 .open = tracing_trace_options_open,
4555 .llseek = seq_lseek,
4556 .release = tracing_single_release_tr,
4557 .write = tracing_trace_options_write,
4560 static const char readme_msg[] =
4561 "tracing mini-HOWTO:\n\n"
4562 "# echo 0 > tracing_on : quick way to disable tracing\n"
4563 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
4564 " Important files:\n"
4565 " trace\t\t\t- The static contents of the buffer\n"
4566 "\t\t\t To clear the buffer write into this file: echo > trace\n"
4567 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4568 " current_tracer\t- function and latency tracers\n"
4569 " available_tracers\t- list of configured tracers for current_tracer\n"
4570 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
4571 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
4572 " trace_clock\t\t-change the clock used to order events\n"
4573 " local: Per cpu clock but may not be synced across CPUs\n"
4574 " global: Synced across CPUs but slows tracing down.\n"
4575 " counter: Not a clock, but just an increment\n"
4576 " uptime: Jiffy counter from time of boot\n"
4577 " perf: Same clock that perf events use\n"
4578 #ifdef CONFIG_X86_64
4579 " x86-tsc: TSC cycle counter\n"
4581 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
4582 "\n trace_marker_raw\t\t- Writes into this file writes binary data into the kernel buffer\n"
4583 " tracing_cpumask\t- Limit which CPUs to trace\n"
4584 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4585 "\t\t\t Remove sub-buffer with rmdir\n"
4586 " trace_options\t\t- Set format or modify how tracing happens\n"
4587 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
4588 "\t\t\t option name\n"
4589 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
4590 #ifdef CONFIG_DYNAMIC_FTRACE
4591 "\n available_filter_functions - list of functions that can be filtered on\n"
4592 " set_ftrace_filter\t- echo function name in here to only trace these\n"
4593 "\t\t\t functions\n"
4594 "\t accepts: func_full_name or glob-matching-pattern\n"
4595 "\t modules: Can select a group via module\n"
4596 "\t Format: :mod:<module-name>\n"
4597 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
4598 "\t triggers: a command to perform when function is hit\n"
4599 "\t Format: <function>:<trigger>[:count]\n"
4600 "\t trigger: traceon, traceoff\n"
4601 "\t\t enable_event:<system>:<event>\n"
4602 "\t\t disable_event:<system>:<event>\n"
4603 #ifdef CONFIG_STACKTRACE
4606 #ifdef CONFIG_TRACER_SNAPSHOT
4611 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
4612 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
4613 "\t The first one will disable tracing every time do_fault is hit\n"
4614 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
4615 "\t The first time do trap is hit and it disables tracing, the\n"
4616 "\t counter will decrement to 2. If tracing is already disabled,\n"
4617 "\t the counter will not decrement. It only decrements when the\n"
4618 "\t trigger did work\n"
4619 "\t To remove trigger without count:\n"
4620 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
4621 "\t To remove trigger with a count:\n"
4622 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
4623 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
4624 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4625 "\t modules: Can select a group via module command :mod:\n"
4626 "\t Does not accept triggers\n"
4627 #endif /* CONFIG_DYNAMIC_FTRACE */
4628 #ifdef CONFIG_FUNCTION_TRACER
4629 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
4632 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
4633 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
4634 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
4635 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
4637 #ifdef CONFIG_TRACER_SNAPSHOT
4638 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
4639 "\t\t\t snapshot buffer. Read the contents for more\n"
4640 "\t\t\t information\n"
4642 #ifdef CONFIG_STACK_TRACER
4643 " stack_trace\t\t- Shows the max stack trace when active\n"
4644 " stack_max_size\t- Shows current max stack size that was traced\n"
4645 "\t\t\t Write into this file to reset the max size (trigger a\n"
4646 "\t\t\t new trace)\n"
4647 #ifdef CONFIG_DYNAMIC_FTRACE
4648 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
4651 #endif /* CONFIG_STACK_TRACER */
4652 #ifdef CONFIG_KPROBE_EVENTS
4653 " kprobe_events\t\t- Add/remove/show the kernel dynamic events\n"
4654 "\t\t\t Write into this file to define/undefine new trace events.\n"
4656 #ifdef CONFIG_UPROBE_EVENTS
4657 " uprobe_events\t\t- Add/remove/show the userspace dynamic events\n"
4658 "\t\t\t Write into this file to define/undefine new trace events.\n"
4660 #if defined(CONFIG_KPROBE_EVENTS) || defined(CONFIG_UPROBE_EVENTS)
4661 "\t accepts: event-definitions (one definition per line)\n"
4662 "\t Format: p[:[<group>/]<event>] <place> [<args>]\n"
4663 "\t r[maxactive][:[<group>/]<event>] <place> [<args>]\n"
4664 "\t -:[<group>/]<event>\n"
4665 #ifdef CONFIG_KPROBE_EVENTS
4666 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4667 "place (kretprobe): [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4669 #ifdef CONFIG_UPROBE_EVENTS
4670 "\t place: <path>:<offset>\n"
4672 "\t args: <name>=fetcharg[:type]\n"
4673 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
4674 "\t $stack<index>, $stack, $retval, $comm\n"
4675 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string,\n"
4676 "\t b<bit-width>@<bit-offset>/<container-size>\n"
4678 " events/\t\t- Directory containing all trace event subsystems:\n"
4679 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
4680 " events/<system>/\t- Directory containing all trace events for <system>:\n"
4681 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
4683 " filter\t\t- If set, only events passing filter are traced\n"
4684 " events/<system>/<event>/\t- Directory containing control files for\n"
4686 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
4687 " filter\t\t- If set, only events passing filter are traced\n"
4688 " trigger\t\t- If set, a command to perform when event is hit\n"
4689 "\t Format: <trigger>[:count][if <filter>]\n"
4690 "\t trigger: traceon, traceoff\n"
4691 "\t enable_event:<system>:<event>\n"
4692 "\t disable_event:<system>:<event>\n"
4693 #ifdef CONFIG_HIST_TRIGGERS
4694 "\t enable_hist:<system>:<event>\n"
4695 "\t disable_hist:<system>:<event>\n"
4697 #ifdef CONFIG_STACKTRACE
4700 #ifdef CONFIG_TRACER_SNAPSHOT
4703 #ifdef CONFIG_HIST_TRIGGERS
4704 "\t\t hist (see below)\n"
4706 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
4707 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
4708 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
4709 "\t events/block/block_unplug/trigger\n"
4710 "\t The first disables tracing every time block_unplug is hit.\n"
4711 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
4712 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
4713 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
4714 "\t Like function triggers, the counter is only decremented if it\n"
4715 "\t enabled or disabled tracing.\n"
4716 "\t To remove a trigger without a count:\n"
4717 "\t echo '!<trigger> > <system>/<event>/trigger\n"
4718 "\t To remove a trigger with a count:\n"
4719 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
4720 "\t Filters can be ignored when removing a trigger.\n"
4721 #ifdef CONFIG_HIST_TRIGGERS
4722 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
4723 "\t Format: hist:keys=<field1[,field2,...]>\n"
4724 "\t [:values=<field1[,field2,...]>]\n"
4725 "\t [:sort=<field1[,field2,...]>]\n"
4726 "\t [:size=#entries]\n"
4727 "\t [:pause][:continue][:clear]\n"
4728 "\t [:name=histname1]\n"
4729 "\t [if <filter>]\n\n"
4730 "\t When a matching event is hit, an entry is added to a hash\n"
4731 "\t table using the key(s) and value(s) named, and the value of a\n"
4732 "\t sum called 'hitcount' is incremented. Keys and values\n"
4733 "\t correspond to fields in the event's format description. Keys\n"
4734 "\t can be any field, or the special string 'stacktrace'.\n"
4735 "\t Compound keys consisting of up to two fields can be specified\n"
4736 "\t by the 'keys' keyword. Values must correspond to numeric\n"
4737 "\t fields. Sort keys consisting of up to two fields can be\n"
4738 "\t specified using the 'sort' keyword. The sort direction can\n"
4739 "\t be modified by appending '.descending' or '.ascending' to a\n"
4740 "\t sort field. The 'size' parameter can be used to specify more\n"
4741 "\t or fewer than the default 2048 entries for the hashtable size.\n"
4742 "\t If a hist trigger is given a name using the 'name' parameter,\n"
4743 "\t its histogram data will be shared with other triggers of the\n"
4744 "\t same name, and trigger hits will update this common data.\n\n"
4745 "\t Reading the 'hist' file for the event will dump the hash\n"
4746 "\t table in its entirety to stdout. If there are multiple hist\n"
4747 "\t triggers attached to an event, there will be a table for each\n"
4748 "\t trigger in the output. The table displayed for a named\n"
4749 "\t trigger will be the same as any other instance having the\n"
4750 "\t same name. The default format used to display a given field\n"
4751 "\t can be modified by appending any of the following modifiers\n"
4752 "\t to the field name, as applicable:\n\n"
4753 "\t .hex display a number as a hex value\n"
4754 "\t .sym display an address as a symbol\n"
4755 "\t .sym-offset display an address as a symbol and offset\n"
4756 "\t .execname display a common_pid as a program name\n"
4757 "\t .syscall display a syscall id as a syscall name\n\n"
4758 "\t .log2 display log2 value rather than raw number\n\n"
4759 "\t The 'pause' parameter can be used to pause an existing hist\n"
4760 "\t trigger or to start a hist trigger but not log any events\n"
4761 "\t until told to do so. 'continue' can be used to start or\n"
4762 "\t restart a paused hist trigger.\n\n"
4763 "\t The 'clear' parameter will clear the contents of a running\n"
4764 "\t hist trigger and leave its current paused/active state\n"
4766 "\t The enable_hist and disable_hist triggers can be used to\n"
4767 "\t have one event conditionally start and stop another event's\n"
4768 "\t already-attached hist trigger. The syntax is analagous to\n"
4769 "\t the enable_event and disable_event triggers.\n"
4774 tracing_readme_read(struct file *filp, char __user *ubuf,
4775 size_t cnt, loff_t *ppos)
4777 return simple_read_from_buffer(ubuf, cnt, ppos,
4778 readme_msg, strlen(readme_msg));
4781 static const struct file_operations tracing_readme_fops = {
4782 .open = tracing_open_generic,
4783 .read = tracing_readme_read,
4784 .llseek = generic_file_llseek,
4787 static void *saved_tgids_next(struct seq_file *m, void *v, loff_t *pos)
4791 return trace_find_tgid_ptr(pid);
4794 static void *saved_tgids_start(struct seq_file *m, loff_t *pos)
4798 return trace_find_tgid_ptr(pid);
4801 static void saved_tgids_stop(struct seq_file *m, void *v)
4805 static int saved_tgids_show(struct seq_file *m, void *v)
4807 int *entry = (int *)v;
4808 int pid = entry - tgid_map;
4814 seq_printf(m, "%d %d\n", pid, tgid);
4818 static const struct seq_operations tracing_saved_tgids_seq_ops = {
4819 .start = saved_tgids_start,
4820 .stop = saved_tgids_stop,
4821 .next = saved_tgids_next,
4822 .show = saved_tgids_show,
4825 static int tracing_saved_tgids_open(struct inode *inode, struct file *filp)
4827 if (tracing_disabled)
4830 return seq_open(filp, &tracing_saved_tgids_seq_ops);
4834 static const struct file_operations tracing_saved_tgids_fops = {
4835 .open = tracing_saved_tgids_open,
4837 .llseek = seq_lseek,
4838 .release = seq_release,
4841 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
4843 unsigned int *ptr = v;
4845 if (*pos || m->count)
4850 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
4852 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
4861 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
4867 arch_spin_lock(&trace_cmdline_lock);
4869 v = &savedcmd->map_cmdline_to_pid[0];
4871 v = saved_cmdlines_next(m, v, &l);
4879 static void saved_cmdlines_stop(struct seq_file *m, void *v)
4881 arch_spin_unlock(&trace_cmdline_lock);
4885 static int saved_cmdlines_show(struct seq_file *m, void *v)
4887 char buf[TASK_COMM_LEN];
4888 unsigned int *pid = v;
4890 __trace_find_cmdline(*pid, buf);
4891 seq_printf(m, "%d %s\n", *pid, buf);
4895 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
4896 .start = saved_cmdlines_start,
4897 .next = saved_cmdlines_next,
4898 .stop = saved_cmdlines_stop,
4899 .show = saved_cmdlines_show,
4902 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
4904 if (tracing_disabled)
4907 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
4910 static const struct file_operations tracing_saved_cmdlines_fops = {
4911 .open = tracing_saved_cmdlines_open,
4913 .llseek = seq_lseek,
4914 .release = seq_release,
4918 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
4919 size_t cnt, loff_t *ppos)
4924 arch_spin_lock(&trace_cmdline_lock);
4925 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
4926 arch_spin_unlock(&trace_cmdline_lock);
4928 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4931 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
4933 kfree(s->saved_cmdlines);
4934 kfree(s->map_cmdline_to_pid);
4938 static int tracing_resize_saved_cmdlines(unsigned int val)
4940 struct saved_cmdlines_buffer *s, *savedcmd_temp;
4942 s = kmalloc(sizeof(*s), GFP_KERNEL);
4946 if (allocate_cmdlines_buffer(val, s) < 0) {
4951 arch_spin_lock(&trace_cmdline_lock);
4952 savedcmd_temp = savedcmd;
4954 arch_spin_unlock(&trace_cmdline_lock);
4955 free_saved_cmdlines_buffer(savedcmd_temp);
4961 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
4962 size_t cnt, loff_t *ppos)
4967 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4971 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
4972 if (!val || val > PID_MAX_DEFAULT)
4975 ret = tracing_resize_saved_cmdlines((unsigned int)val);
4984 static const struct file_operations tracing_saved_cmdlines_size_fops = {
4985 .open = tracing_open_generic,
4986 .read = tracing_saved_cmdlines_size_read,
4987 .write = tracing_saved_cmdlines_size_write,
4990 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
4991 static union trace_eval_map_item *
4992 update_eval_map(union trace_eval_map_item *ptr)
4994 if (!ptr->map.eval_string) {
4995 if (ptr->tail.next) {
4996 ptr = ptr->tail.next;
4997 /* Set ptr to the next real item (skip head) */
5005 static void *eval_map_next(struct seq_file *m, void *v, loff_t *pos)
5007 union trace_eval_map_item *ptr = v;
5010 * Paranoid! If ptr points to end, we don't want to increment past it.
5011 * This really should never happen.
5013 ptr = update_eval_map(ptr);
5014 if (WARN_ON_ONCE(!ptr))
5021 ptr = update_eval_map(ptr);
5026 static void *eval_map_start(struct seq_file *m, loff_t *pos)
5028 union trace_eval_map_item *v;
5031 mutex_lock(&trace_eval_mutex);
5033 v = trace_eval_maps;
5037 while (v && l < *pos) {
5038 v = eval_map_next(m, v, &l);
5044 static void eval_map_stop(struct seq_file *m, void *v)
5046 mutex_unlock(&trace_eval_mutex);
5049 static int eval_map_show(struct seq_file *m, void *v)
5051 union trace_eval_map_item *ptr = v;
5053 seq_printf(m, "%s %ld (%s)\n",
5054 ptr->map.eval_string, ptr->map.eval_value,
5060 static const struct seq_operations tracing_eval_map_seq_ops = {
5061 .start = eval_map_start,
5062 .next = eval_map_next,
5063 .stop = eval_map_stop,
5064 .show = eval_map_show,
5067 static int tracing_eval_map_open(struct inode *inode, struct file *filp)
5069 if (tracing_disabled)
5072 return seq_open(filp, &tracing_eval_map_seq_ops);
5075 static const struct file_operations tracing_eval_map_fops = {
5076 .open = tracing_eval_map_open,
5078 .llseek = seq_lseek,
5079 .release = seq_release,
5082 static inline union trace_eval_map_item *
5083 trace_eval_jmp_to_tail(union trace_eval_map_item *ptr)
5085 /* Return tail of array given the head */
5086 return ptr + ptr->head.length + 1;
5090 trace_insert_eval_map_file(struct module *mod, struct trace_eval_map **start,
5093 struct trace_eval_map **stop;
5094 struct trace_eval_map **map;
5095 union trace_eval_map_item *map_array;
5096 union trace_eval_map_item *ptr;
5101 * The trace_eval_maps contains the map plus a head and tail item,
5102 * where the head holds the module and length of array, and the
5103 * tail holds a pointer to the next list.
5105 map_array = kmalloc(sizeof(*map_array) * (len + 2), GFP_KERNEL);
5107 pr_warn("Unable to allocate trace eval mapping\n");
5111 mutex_lock(&trace_eval_mutex);
5113 if (!trace_eval_maps)
5114 trace_eval_maps = map_array;
5116 ptr = trace_eval_maps;
5118 ptr = trace_eval_jmp_to_tail(ptr);
5119 if (!ptr->tail.next)
5121 ptr = ptr->tail.next;
5124 ptr->tail.next = map_array;
5126 map_array->head.mod = mod;
5127 map_array->head.length = len;
5130 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
5131 map_array->map = **map;
5134 memset(map_array, 0, sizeof(*map_array));
5136 mutex_unlock(&trace_eval_mutex);
5139 static void trace_create_eval_file(struct dentry *d_tracer)
5141 trace_create_file("eval_map", 0444, d_tracer,
5142 NULL, &tracing_eval_map_fops);
5145 #else /* CONFIG_TRACE_EVAL_MAP_FILE */
5146 static inline void trace_create_eval_file(struct dentry *d_tracer) { }
5147 static inline void trace_insert_eval_map_file(struct module *mod,
5148 struct trace_eval_map **start, int len) { }
5149 #endif /* !CONFIG_TRACE_EVAL_MAP_FILE */
5151 static void trace_insert_eval_map(struct module *mod,
5152 struct trace_eval_map **start, int len)
5154 struct trace_eval_map **map;
5161 trace_event_eval_update(map, len);
5163 trace_insert_eval_map_file(mod, start, len);
5167 tracing_set_trace_read(struct file *filp, char __user *ubuf,
5168 size_t cnt, loff_t *ppos)
5170 struct trace_array *tr = filp->private_data;
5171 char buf[MAX_TRACER_SIZE+2];
5174 mutex_lock(&trace_types_lock);
5175 r = sprintf(buf, "%s\n", tr->current_trace->name);
5176 mutex_unlock(&trace_types_lock);
5178 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5181 int tracer_init(struct tracer *t, struct trace_array *tr)
5183 tracing_reset_online_cpus(&tr->trace_buffer);
5187 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
5191 for_each_tracing_cpu(cpu)
5192 per_cpu_ptr(buf->data, cpu)->entries = val;
5195 #ifdef CONFIG_TRACER_MAX_TRACE
5196 /* resize @tr's buffer to the size of @size_tr's entries */
5197 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
5198 struct trace_buffer *size_buf, int cpu_id)
5202 if (cpu_id == RING_BUFFER_ALL_CPUS) {
5203 for_each_tracing_cpu(cpu) {
5204 ret = ring_buffer_resize(trace_buf->buffer,
5205 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
5208 per_cpu_ptr(trace_buf->data, cpu)->entries =
5209 per_cpu_ptr(size_buf->data, cpu)->entries;
5212 ret = ring_buffer_resize(trace_buf->buffer,
5213 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
5215 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
5216 per_cpu_ptr(size_buf->data, cpu_id)->entries;
5221 #endif /* CONFIG_TRACER_MAX_TRACE */
5223 static int __tracing_resize_ring_buffer(struct trace_array *tr,
5224 unsigned long size, int cpu)
5229 * If kernel or user changes the size of the ring buffer
5230 * we use the size that was given, and we can forget about
5231 * expanding it later.
5233 ring_buffer_expanded = true;
5235 /* May be called before buffers are initialized */
5236 if (!tr->trace_buffer.buffer)
5239 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
5243 #ifdef CONFIG_TRACER_MAX_TRACE
5244 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
5245 !tr->current_trace->use_max_tr)
5248 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
5250 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
5251 &tr->trace_buffer, cpu);
5254 * AARGH! We are left with different
5255 * size max buffer!!!!
5256 * The max buffer is our "snapshot" buffer.
5257 * When a tracer needs a snapshot (one of the
5258 * latency tracers), it swaps the max buffer
5259 * with the saved snap shot. We succeeded to
5260 * update the size of the main buffer, but failed to
5261 * update the size of the max buffer. But when we tried
5262 * to reset the main buffer to the original size, we
5263 * failed there too. This is very unlikely to
5264 * happen, but if it does, warn and kill all
5268 tracing_disabled = 1;
5273 if (cpu == RING_BUFFER_ALL_CPUS)
5274 set_buffer_entries(&tr->max_buffer, size);
5276 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
5279 #endif /* CONFIG_TRACER_MAX_TRACE */
5281 if (cpu == RING_BUFFER_ALL_CPUS)
5282 set_buffer_entries(&tr->trace_buffer, size);
5284 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
5289 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
5290 unsigned long size, int cpu_id)
5294 mutex_lock(&trace_types_lock);
5296 if (cpu_id != RING_BUFFER_ALL_CPUS) {
5297 /* make sure, this cpu is enabled in the mask */
5298 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
5304 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
5309 mutex_unlock(&trace_types_lock);
5316 * tracing_update_buffers - used by tracing facility to expand ring buffers
5318 * To save on memory when the tracing is never used on a system with it
5319 * configured in. The ring buffers are set to a minimum size. But once
5320 * a user starts to use the tracing facility, then they need to grow
5321 * to their default size.
5323 * This function is to be called when a tracer is about to be used.
5325 int tracing_update_buffers(void)
5329 mutex_lock(&trace_types_lock);
5330 if (!ring_buffer_expanded)
5331 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
5332 RING_BUFFER_ALL_CPUS);
5333 mutex_unlock(&trace_types_lock);
5338 struct trace_option_dentry;
5341 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
5344 * Used to clear out the tracer before deletion of an instance.
5345 * Must have trace_types_lock held.
5347 static void tracing_set_nop(struct trace_array *tr)
5349 if (tr->current_trace == &nop_trace)
5352 tr->current_trace->enabled--;
5354 if (tr->current_trace->reset)
5355 tr->current_trace->reset(tr);
5357 tr->current_trace = &nop_trace;
5360 static void add_tracer_options(struct trace_array *tr, struct tracer *t)
5362 /* Only enable if the directory has been created already. */
5366 create_trace_option_files(tr, t);
5369 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
5372 #ifdef CONFIG_TRACER_MAX_TRACE
5377 mutex_lock(&trace_types_lock);
5379 if (!ring_buffer_expanded) {
5380 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
5381 RING_BUFFER_ALL_CPUS);
5387 for (t = trace_types; t; t = t->next) {
5388 if (strcmp(t->name, buf) == 0)
5395 if (t == tr->current_trace)
5398 /* Some tracers won't work on kernel command line */
5399 if (system_state < SYSTEM_RUNNING && t->noboot) {
5400 pr_warn("Tracer '%s' is not allowed on command line, ignored\n",
5405 /* Some tracers are only allowed for the top level buffer */
5406 if (!trace_ok_for_array(t, tr)) {
5411 /* If trace pipe files are being read, we can't change the tracer */
5412 if (tr->current_trace->ref) {
5417 trace_branch_disable();
5419 tr->current_trace->enabled--;
5421 if (tr->current_trace->reset)
5422 tr->current_trace->reset(tr);
5424 /* Current trace needs to be nop_trace before synchronize_sched */
5425 tr->current_trace = &nop_trace;
5427 #ifdef CONFIG_TRACER_MAX_TRACE
5428 had_max_tr = tr->allocated_snapshot;
5430 if (had_max_tr && !t->use_max_tr) {
5432 * We need to make sure that the update_max_tr sees that
5433 * current_trace changed to nop_trace to keep it from
5434 * swapping the buffers after we resize it.
5435 * The update_max_tr is called from interrupts disabled
5436 * so a synchronized_sched() is sufficient.
5438 synchronize_sched();
5443 #ifdef CONFIG_TRACER_MAX_TRACE
5444 if (t->use_max_tr && !had_max_tr) {
5445 ret = tracing_alloc_snapshot_instance(tr);
5452 ret = tracer_init(t, tr);
5457 tr->current_trace = t;
5458 tr->current_trace->enabled++;
5459 trace_branch_enable(tr);
5461 mutex_unlock(&trace_types_lock);
5467 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
5468 size_t cnt, loff_t *ppos)
5470 struct trace_array *tr = filp->private_data;
5471 char buf[MAX_TRACER_SIZE+1];
5478 if (cnt > MAX_TRACER_SIZE)
5479 cnt = MAX_TRACER_SIZE;
5481 if (copy_from_user(buf, ubuf, cnt))
5486 /* strip ending whitespace. */
5487 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
5490 err = tracing_set_tracer(tr, buf);
5500 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
5501 size_t cnt, loff_t *ppos)
5506 r = snprintf(buf, sizeof(buf), "%ld\n",
5507 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
5508 if (r > sizeof(buf))
5510 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5514 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
5515 size_t cnt, loff_t *ppos)
5520 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5530 tracing_thresh_read(struct file *filp, char __user *ubuf,
5531 size_t cnt, loff_t *ppos)
5533 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
5537 tracing_thresh_write(struct file *filp, const char __user *ubuf,
5538 size_t cnt, loff_t *ppos)
5540 struct trace_array *tr = filp->private_data;
5543 mutex_lock(&trace_types_lock);
5544 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
5548 if (tr->current_trace->update_thresh) {
5549 ret = tr->current_trace->update_thresh(tr);
5556 mutex_unlock(&trace_types_lock);
5561 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
5564 tracing_max_lat_read(struct file *filp, char __user *ubuf,
5565 size_t cnt, loff_t *ppos)
5567 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
5571 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
5572 size_t cnt, loff_t *ppos)
5574 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
5579 static int tracing_open_pipe(struct inode *inode, struct file *filp)
5581 struct trace_array *tr = inode->i_private;
5582 struct trace_iterator *iter;
5585 if (tracing_disabled)
5588 if (trace_array_get(tr) < 0)
5591 mutex_lock(&trace_types_lock);
5593 /* create a buffer to store the information to pass to userspace */
5594 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5597 __trace_array_put(tr);
5601 trace_seq_init(&iter->seq);
5602 iter->trace = tr->current_trace;
5604 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
5609 /* trace pipe does not show start of buffer */
5610 cpumask_setall(iter->started);
5612 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
5613 iter->iter_flags |= TRACE_FILE_LAT_FMT;
5615 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
5616 if (trace_clocks[tr->clock_id].in_ns)
5617 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
5620 iter->trace_buffer = &tr->trace_buffer;
5621 iter->cpu_file = tracing_get_cpu(inode);
5622 mutex_init(&iter->mutex);
5623 filp->private_data = iter;
5625 if (iter->trace->pipe_open)
5626 iter->trace->pipe_open(iter);
5628 nonseekable_open(inode, filp);
5630 tr->current_trace->ref++;
5632 mutex_unlock(&trace_types_lock);
5637 __trace_array_put(tr);
5638 mutex_unlock(&trace_types_lock);
5642 static int tracing_release_pipe(struct inode *inode, struct file *file)
5644 struct trace_iterator *iter = file->private_data;
5645 struct trace_array *tr = inode->i_private;
5647 mutex_lock(&trace_types_lock);
5649 tr->current_trace->ref--;
5651 if (iter->trace->pipe_close)
5652 iter->trace->pipe_close(iter);
5654 mutex_unlock(&trace_types_lock);
5656 free_cpumask_var(iter->started);
5657 mutex_destroy(&iter->mutex);
5660 trace_array_put(tr);
5666 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
5668 struct trace_array *tr = iter->tr;
5670 /* Iterators are static, they should be filled or empty */
5671 if (trace_buffer_iter(iter, iter->cpu_file))
5672 return POLLIN | POLLRDNORM;
5674 if (tr->trace_flags & TRACE_ITER_BLOCK)
5676 * Always select as readable when in blocking mode
5678 return POLLIN | POLLRDNORM;
5680 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
5685 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
5687 struct trace_iterator *iter = filp->private_data;
5689 return trace_poll(iter, filp, poll_table);
5692 /* Must be called with iter->mutex held. */
5693 static int tracing_wait_pipe(struct file *filp)
5695 struct trace_iterator *iter = filp->private_data;
5698 while (trace_empty(iter)) {
5700 if ((filp->f_flags & O_NONBLOCK)) {
5705 * We block until we read something and tracing is disabled.
5706 * We still block if tracing is disabled, but we have never
5707 * read anything. This allows a user to cat this file, and
5708 * then enable tracing. But after we have read something,
5709 * we give an EOF when tracing is again disabled.
5711 * iter->pos will be 0 if we haven't read anything.
5713 if (!tracer_tracing_is_on(iter->tr) && iter->pos)
5716 mutex_unlock(&iter->mutex);
5718 ret = wait_on_pipe(iter, false);
5720 mutex_lock(&iter->mutex);
5733 tracing_read_pipe(struct file *filp, char __user *ubuf,
5734 size_t cnt, loff_t *ppos)
5736 struct trace_iterator *iter = filp->private_data;
5740 * Avoid more than one consumer on a single file descriptor
5741 * This is just a matter of traces coherency, the ring buffer itself
5744 mutex_lock(&iter->mutex);
5746 /* return any leftover data */
5747 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5751 trace_seq_init(&iter->seq);
5753 if (iter->trace->read) {
5754 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
5760 sret = tracing_wait_pipe(filp);
5764 /* stop when tracing is finished */
5765 if (trace_empty(iter)) {
5770 if (cnt >= PAGE_SIZE)
5771 cnt = PAGE_SIZE - 1;
5773 /* reset all but tr, trace, and overruns */
5774 memset(&iter->seq, 0,
5775 sizeof(struct trace_iterator) -
5776 offsetof(struct trace_iterator, seq));
5777 cpumask_clear(iter->started);
5778 trace_seq_init(&iter->seq);
5781 trace_event_read_lock();
5782 trace_access_lock(iter->cpu_file);
5783 while (trace_find_next_entry_inc(iter) != NULL) {
5784 enum print_line_t ret;
5785 int save_len = iter->seq.seq.len;
5787 ret = print_trace_line(iter);
5788 if (ret == TRACE_TYPE_PARTIAL_LINE) {
5789 /* don't print partial lines */
5790 iter->seq.seq.len = save_len;
5793 if (ret != TRACE_TYPE_NO_CONSUME)
5794 trace_consume(iter);
5796 if (trace_seq_used(&iter->seq) >= cnt)
5800 * Setting the full flag means we reached the trace_seq buffer
5801 * size and we should leave by partial output condition above.
5802 * One of the trace_seq_* functions is not used properly.
5804 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
5807 trace_access_unlock(iter->cpu_file);
5808 trace_event_read_unlock();
5810 /* Now copy what we have to the user */
5811 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5812 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
5813 trace_seq_init(&iter->seq);
5816 * If there was nothing to send to user, in spite of consuming trace
5817 * entries, go back to wait for more entries.
5823 mutex_unlock(&iter->mutex);
5828 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
5831 __free_page(spd->pages[idx]);
5834 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
5836 .confirm = generic_pipe_buf_confirm,
5837 .release = generic_pipe_buf_release,
5838 .steal = generic_pipe_buf_steal,
5839 .get = generic_pipe_buf_get,
5843 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
5849 /* Seq buffer is page-sized, exactly what we need. */
5851 save_len = iter->seq.seq.len;
5852 ret = print_trace_line(iter);
5854 if (trace_seq_has_overflowed(&iter->seq)) {
5855 iter->seq.seq.len = save_len;
5860 * This should not be hit, because it should only
5861 * be set if the iter->seq overflowed. But check it
5862 * anyway to be safe.
5864 if (ret == TRACE_TYPE_PARTIAL_LINE) {
5865 iter->seq.seq.len = save_len;
5869 count = trace_seq_used(&iter->seq) - save_len;
5872 iter->seq.seq.len = save_len;
5876 if (ret != TRACE_TYPE_NO_CONSUME)
5877 trace_consume(iter);
5879 if (!trace_find_next_entry_inc(iter)) {
5889 static ssize_t tracing_splice_read_pipe(struct file *filp,
5891 struct pipe_inode_info *pipe,
5895 struct page *pages_def[PIPE_DEF_BUFFERS];
5896 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5897 struct trace_iterator *iter = filp->private_data;
5898 struct splice_pipe_desc spd = {
5900 .partial = partial_def,
5901 .nr_pages = 0, /* This gets updated below. */
5902 .nr_pages_max = PIPE_DEF_BUFFERS,
5903 .ops = &tracing_pipe_buf_ops,
5904 .spd_release = tracing_spd_release_pipe,
5910 if (splice_grow_spd(pipe, &spd))
5913 mutex_lock(&iter->mutex);
5915 if (iter->trace->splice_read) {
5916 ret = iter->trace->splice_read(iter, filp,
5917 ppos, pipe, len, flags);
5922 ret = tracing_wait_pipe(filp);
5926 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
5931 trace_event_read_lock();
5932 trace_access_lock(iter->cpu_file);
5934 /* Fill as many pages as possible. */
5935 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
5936 spd.pages[i] = alloc_page(GFP_KERNEL);
5940 rem = tracing_fill_pipe_page(rem, iter);
5942 /* Copy the data into the page, so we can start over. */
5943 ret = trace_seq_to_buffer(&iter->seq,
5944 page_address(spd.pages[i]),
5945 trace_seq_used(&iter->seq));
5947 __free_page(spd.pages[i]);
5950 spd.partial[i].offset = 0;
5951 spd.partial[i].len = trace_seq_used(&iter->seq);
5953 trace_seq_init(&iter->seq);
5956 trace_access_unlock(iter->cpu_file);
5957 trace_event_read_unlock();
5958 mutex_unlock(&iter->mutex);
5963 ret = splice_to_pipe(pipe, &spd);
5967 splice_shrink_spd(&spd);
5971 mutex_unlock(&iter->mutex);
5976 tracing_entries_read(struct file *filp, char __user *ubuf,
5977 size_t cnt, loff_t *ppos)
5979 struct inode *inode = file_inode(filp);
5980 struct trace_array *tr = inode->i_private;
5981 int cpu = tracing_get_cpu(inode);
5986 mutex_lock(&trace_types_lock);
5988 if (cpu == RING_BUFFER_ALL_CPUS) {
5989 int cpu, buf_size_same;
5994 /* check if all cpu sizes are same */
5995 for_each_tracing_cpu(cpu) {
5996 /* fill in the size from first enabled cpu */
5998 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
5999 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
6005 if (buf_size_same) {
6006 if (!ring_buffer_expanded)
6007 r = sprintf(buf, "%lu (expanded: %lu)\n",
6009 trace_buf_size >> 10);
6011 r = sprintf(buf, "%lu\n", size >> 10);
6013 r = sprintf(buf, "X\n");
6015 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
6017 mutex_unlock(&trace_types_lock);
6019 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6024 tracing_entries_write(struct file *filp, const char __user *ubuf,
6025 size_t cnt, loff_t *ppos)
6027 struct inode *inode = file_inode(filp);
6028 struct trace_array *tr = inode->i_private;
6032 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6036 /* must have at least 1 entry */
6040 /* value is in KB */
6042 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
6052 tracing_total_entries_read(struct file *filp, char __user *ubuf,
6053 size_t cnt, loff_t *ppos)
6055 struct trace_array *tr = filp->private_data;
6058 unsigned long size = 0, expanded_size = 0;
6060 mutex_lock(&trace_types_lock);
6061 for_each_tracing_cpu(cpu) {
6062 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
6063 if (!ring_buffer_expanded)
6064 expanded_size += trace_buf_size >> 10;
6066 if (ring_buffer_expanded)
6067 r = sprintf(buf, "%lu\n", size);
6069 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
6070 mutex_unlock(&trace_types_lock);
6072 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6076 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
6077 size_t cnt, loff_t *ppos)
6080 * There is no need to read what the user has written, this function
6081 * is just to make sure that there is no error when "echo" is used
6090 tracing_free_buffer_release(struct inode *inode, struct file *filp)
6092 struct trace_array *tr = inode->i_private;
6094 /* disable tracing ? */
6095 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
6096 tracer_tracing_off(tr);
6097 /* resize the ring buffer to 0 */
6098 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
6100 trace_array_put(tr);
6106 tracing_mark_write(struct file *filp, const char __user *ubuf,
6107 size_t cnt, loff_t *fpos)
6109 struct trace_array *tr = filp->private_data;
6110 struct ring_buffer_event *event;
6111 struct ring_buffer *buffer;
6112 struct print_entry *entry;
6113 unsigned long irq_flags;
6114 const char faulted[] = "<faulted>";
6119 /* Used in tracing_mark_raw_write() as well */
6120 #define FAULTED_SIZE (sizeof(faulted) - 1) /* '\0' is already accounted for */
6122 if (tracing_disabled)
6125 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6128 if (cnt > TRACE_BUF_SIZE)
6129 cnt = TRACE_BUF_SIZE;
6131 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6133 local_save_flags(irq_flags);
6134 size = sizeof(*entry) + cnt + 2; /* add '\0' and possible '\n' */
6136 /* If less than "<faulted>", then make sure we can still add that */
6137 if (cnt < FAULTED_SIZE)
6138 size += FAULTED_SIZE - cnt;
6140 buffer = tr->trace_buffer.buffer;
6141 event = __trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
6142 irq_flags, preempt_count());
6143 if (unlikely(!event))
6144 /* Ring buffer disabled, return as if not open for write */
6147 entry = ring_buffer_event_data(event);
6148 entry->ip = _THIS_IP_;
6150 len = __copy_from_user_inatomic(&entry->buf, ubuf, cnt);
6152 memcpy(&entry->buf, faulted, FAULTED_SIZE);
6159 if (entry->buf[cnt - 1] != '\n') {
6160 entry->buf[cnt] = '\n';
6161 entry->buf[cnt + 1] = '\0';
6163 entry->buf[cnt] = '\0';
6165 __buffer_unlock_commit(buffer, event);
6173 /* Limit it for now to 3K (including tag) */
6174 #define RAW_DATA_MAX_SIZE (1024*3)
6177 tracing_mark_raw_write(struct file *filp, const char __user *ubuf,
6178 size_t cnt, loff_t *fpos)
6180 struct trace_array *tr = filp->private_data;
6181 struct ring_buffer_event *event;
6182 struct ring_buffer *buffer;
6183 struct raw_data_entry *entry;
6184 const char faulted[] = "<faulted>";
6185 unsigned long irq_flags;
6190 #define FAULT_SIZE_ID (FAULTED_SIZE + sizeof(int))
6192 if (tracing_disabled)
6195 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
6198 /* The marker must at least have a tag id */
6199 if (cnt < sizeof(unsigned int) || cnt > RAW_DATA_MAX_SIZE)
6202 if (cnt > TRACE_BUF_SIZE)
6203 cnt = TRACE_BUF_SIZE;
6205 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
6207 local_save_flags(irq_flags);
6208 size = sizeof(*entry) + cnt;
6209 if (cnt < FAULT_SIZE_ID)
6210 size += FAULT_SIZE_ID - cnt;
6212 buffer = tr->trace_buffer.buffer;
6213 event = __trace_buffer_lock_reserve(buffer, TRACE_RAW_DATA, size,
6214 irq_flags, preempt_count());
6216 /* Ring buffer disabled, return as if not open for write */
6219 entry = ring_buffer_event_data(event);
6221 len = __copy_from_user_inatomic(&entry->id, ubuf, cnt);
6224 memcpy(&entry->buf, faulted, FAULTED_SIZE);
6229 __buffer_unlock_commit(buffer, event);
6237 static int tracing_clock_show(struct seq_file *m, void *v)
6239 struct trace_array *tr = m->private;
6242 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
6244 "%s%s%s%s", i ? " " : "",
6245 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
6246 i == tr->clock_id ? "]" : "");
6252 static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
6256 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
6257 if (strcmp(trace_clocks[i].name, clockstr) == 0)
6260 if (i == ARRAY_SIZE(trace_clocks))
6263 mutex_lock(&trace_types_lock);
6267 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
6270 * New clock may not be consistent with the previous clock.
6271 * Reset the buffer so that it doesn't have incomparable timestamps.
6273 tracing_reset_online_cpus(&tr->trace_buffer);
6275 #ifdef CONFIG_TRACER_MAX_TRACE
6276 if (tr->max_buffer.buffer)
6277 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
6278 tracing_reset_online_cpus(&tr->max_buffer);
6281 mutex_unlock(&trace_types_lock);
6286 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
6287 size_t cnt, loff_t *fpos)
6289 struct seq_file *m = filp->private_data;
6290 struct trace_array *tr = m->private;
6292 const char *clockstr;
6295 if (cnt >= sizeof(buf))
6298 if (copy_from_user(buf, ubuf, cnt))
6303 clockstr = strstrip(buf);
6305 ret = tracing_set_clock(tr, clockstr);
6314 static int tracing_clock_open(struct inode *inode, struct file *file)
6316 struct trace_array *tr = inode->i_private;
6319 if (tracing_disabled)
6322 if (trace_array_get(tr))
6325 ret = single_open(file, tracing_clock_show, inode->i_private);
6327 trace_array_put(tr);
6332 struct ftrace_buffer_info {
6333 struct trace_iterator iter;
6335 unsigned int spare_cpu;
6339 #ifdef CONFIG_TRACER_SNAPSHOT
6340 static int tracing_snapshot_open(struct inode *inode, struct file *file)
6342 struct trace_array *tr = inode->i_private;
6343 struct trace_iterator *iter;
6347 if (trace_array_get(tr) < 0)
6350 if (file->f_mode & FMODE_READ) {
6351 iter = __tracing_open(inode, file, true);
6353 ret = PTR_ERR(iter);
6355 /* Writes still need the seq_file to hold the private data */
6357 m = kzalloc(sizeof(*m), GFP_KERNEL);
6360 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
6368 iter->trace_buffer = &tr->max_buffer;
6369 iter->cpu_file = tracing_get_cpu(inode);
6371 file->private_data = m;
6375 trace_array_put(tr);
6381 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
6384 struct seq_file *m = filp->private_data;
6385 struct trace_iterator *iter = m->private;
6386 struct trace_array *tr = iter->tr;
6390 ret = tracing_update_buffers();
6394 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6398 mutex_lock(&trace_types_lock);
6400 if (tr->current_trace->use_max_tr) {
6407 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6411 if (tr->allocated_snapshot)
6415 /* Only allow per-cpu swap if the ring buffer supports it */
6416 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
6417 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
6422 if (tr->allocated_snapshot)
6423 ret = resize_buffer_duplicate_size(&tr->max_buffer,
6424 &tr->trace_buffer, iter->cpu_file);
6426 ret = tracing_alloc_snapshot_instance(tr);
6429 local_irq_disable();
6430 /* Now, we're going to swap */
6431 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6432 update_max_tr(tr, current, smp_processor_id());
6434 update_max_tr_single(tr, current, iter->cpu_file);
6438 if (tr->allocated_snapshot) {
6439 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
6440 tracing_reset_online_cpus(&tr->max_buffer);
6442 tracing_reset(&tr->max_buffer, iter->cpu_file);
6452 mutex_unlock(&trace_types_lock);
6456 static int tracing_snapshot_release(struct inode *inode, struct file *file)
6458 struct seq_file *m = file->private_data;
6461 ret = tracing_release(inode, file);
6463 if (file->f_mode & FMODE_READ)
6466 /* If write only, the seq_file is just a stub */
6474 static int tracing_buffers_open(struct inode *inode, struct file *filp);
6475 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
6476 size_t count, loff_t *ppos);
6477 static int tracing_buffers_release(struct inode *inode, struct file *file);
6478 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
6479 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
6481 static int snapshot_raw_open(struct inode *inode, struct file *filp)
6483 struct ftrace_buffer_info *info;
6486 ret = tracing_buffers_open(inode, filp);
6490 info = filp->private_data;
6492 if (info->iter.trace->use_max_tr) {
6493 tracing_buffers_release(inode, filp);
6497 info->iter.snapshot = true;
6498 info->iter.trace_buffer = &info->iter.tr->max_buffer;
6503 #endif /* CONFIG_TRACER_SNAPSHOT */
6506 static const struct file_operations tracing_thresh_fops = {
6507 .open = tracing_open_generic,
6508 .read = tracing_thresh_read,
6509 .write = tracing_thresh_write,
6510 .llseek = generic_file_llseek,
6513 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
6514 static const struct file_operations tracing_max_lat_fops = {
6515 .open = tracing_open_generic,
6516 .read = tracing_max_lat_read,
6517 .write = tracing_max_lat_write,
6518 .llseek = generic_file_llseek,
6522 static const struct file_operations set_tracer_fops = {
6523 .open = tracing_open_generic,
6524 .read = tracing_set_trace_read,
6525 .write = tracing_set_trace_write,
6526 .llseek = generic_file_llseek,
6529 static const struct file_operations tracing_pipe_fops = {
6530 .open = tracing_open_pipe,
6531 .poll = tracing_poll_pipe,
6532 .read = tracing_read_pipe,
6533 .splice_read = tracing_splice_read_pipe,
6534 .release = tracing_release_pipe,
6535 .llseek = no_llseek,
6538 static const struct file_operations tracing_entries_fops = {
6539 .open = tracing_open_generic_tr,
6540 .read = tracing_entries_read,
6541 .write = tracing_entries_write,
6542 .llseek = generic_file_llseek,
6543 .release = tracing_release_generic_tr,
6546 static const struct file_operations tracing_total_entries_fops = {
6547 .open = tracing_open_generic_tr,
6548 .read = tracing_total_entries_read,
6549 .llseek = generic_file_llseek,
6550 .release = tracing_release_generic_tr,
6553 static const struct file_operations tracing_free_buffer_fops = {
6554 .open = tracing_open_generic_tr,
6555 .write = tracing_free_buffer_write,
6556 .release = tracing_free_buffer_release,
6559 static const struct file_operations tracing_mark_fops = {
6560 .open = tracing_open_generic_tr,
6561 .write = tracing_mark_write,
6562 .llseek = generic_file_llseek,
6563 .release = tracing_release_generic_tr,
6566 static const struct file_operations tracing_mark_raw_fops = {
6567 .open = tracing_open_generic_tr,
6568 .write = tracing_mark_raw_write,
6569 .llseek = generic_file_llseek,
6570 .release = tracing_release_generic_tr,
6573 static const struct file_operations trace_clock_fops = {
6574 .open = tracing_clock_open,
6576 .llseek = seq_lseek,
6577 .release = tracing_single_release_tr,
6578 .write = tracing_clock_write,
6581 #ifdef CONFIG_TRACER_SNAPSHOT
6582 static const struct file_operations snapshot_fops = {
6583 .open = tracing_snapshot_open,
6585 .write = tracing_snapshot_write,
6586 .llseek = tracing_lseek,
6587 .release = tracing_snapshot_release,
6590 static const struct file_operations snapshot_raw_fops = {
6591 .open = snapshot_raw_open,
6592 .read = tracing_buffers_read,
6593 .release = tracing_buffers_release,
6594 .splice_read = tracing_buffers_splice_read,
6595 .llseek = no_llseek,
6598 #endif /* CONFIG_TRACER_SNAPSHOT */
6600 static int tracing_buffers_open(struct inode *inode, struct file *filp)
6602 struct trace_array *tr = inode->i_private;
6603 struct ftrace_buffer_info *info;
6606 if (tracing_disabled)
6609 if (trace_array_get(tr) < 0)
6612 info = kzalloc(sizeof(*info), GFP_KERNEL);
6614 trace_array_put(tr);
6618 mutex_lock(&trace_types_lock);
6621 info->iter.cpu_file = tracing_get_cpu(inode);
6622 info->iter.trace = tr->current_trace;
6623 info->iter.trace_buffer = &tr->trace_buffer;
6625 /* Force reading ring buffer for first read */
6626 info->read = (unsigned int)-1;
6628 filp->private_data = info;
6630 tr->current_trace->ref++;
6632 mutex_unlock(&trace_types_lock);
6634 ret = nonseekable_open(inode, filp);
6636 trace_array_put(tr);
6642 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
6644 struct ftrace_buffer_info *info = filp->private_data;
6645 struct trace_iterator *iter = &info->iter;
6647 return trace_poll(iter, filp, poll_table);
6651 tracing_buffers_read(struct file *filp, char __user *ubuf,
6652 size_t count, loff_t *ppos)
6654 struct ftrace_buffer_info *info = filp->private_data;
6655 struct trace_iterator *iter = &info->iter;
6662 #ifdef CONFIG_TRACER_MAX_TRACE
6663 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
6668 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
6670 if (IS_ERR(info->spare)) {
6671 ret = PTR_ERR(info->spare);
6674 info->spare_cpu = iter->cpu_file;
6680 /* Do we have previous read data to read? */
6681 if (info->read < PAGE_SIZE)
6685 trace_access_lock(iter->cpu_file);
6686 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
6690 trace_access_unlock(iter->cpu_file);
6693 if (trace_empty(iter)) {
6694 if ((filp->f_flags & O_NONBLOCK))
6697 ret = wait_on_pipe(iter, false);
6708 size = PAGE_SIZE - info->read;
6712 ret = copy_to_user(ubuf, info->spare + info->read, size);
6724 static int tracing_buffers_release(struct inode *inode, struct file *file)
6726 struct ftrace_buffer_info *info = file->private_data;
6727 struct trace_iterator *iter = &info->iter;
6729 mutex_lock(&trace_types_lock);
6731 iter->tr->current_trace->ref--;
6733 __trace_array_put(iter->tr);
6736 ring_buffer_free_read_page(iter->trace_buffer->buffer,
6737 info->spare_cpu, info->spare);
6740 mutex_unlock(&trace_types_lock);
6746 struct ring_buffer *buffer;
6749 refcount_t refcount;
6752 static void buffer_ref_release(struct buffer_ref *ref)
6754 if (!refcount_dec_and_test(&ref->refcount))
6756 ring_buffer_free_read_page(ref->buffer, ref->cpu, ref->page);
6760 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
6761 struct pipe_buffer *buf)
6763 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
6765 buffer_ref_release(ref);
6769 static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
6770 struct pipe_buffer *buf)
6772 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
6774 if (refcount_read(&ref->refcount) > INT_MAX/2)
6777 refcount_inc(&ref->refcount);
6781 /* Pipe buffer operations for a buffer. */
6782 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
6784 .confirm = generic_pipe_buf_confirm,
6785 .release = buffer_pipe_buf_release,
6786 .steal = generic_pipe_buf_nosteal,
6787 .get = buffer_pipe_buf_get,
6791 * Callback from splice_to_pipe(), if we need to release some pages
6792 * at the end of the spd in case we error'ed out in filling the pipe.
6794 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
6796 struct buffer_ref *ref =
6797 (struct buffer_ref *)spd->partial[i].private;
6799 buffer_ref_release(ref);
6800 spd->partial[i].private = 0;
6804 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
6805 struct pipe_inode_info *pipe, size_t len,
6808 struct ftrace_buffer_info *info = file->private_data;
6809 struct trace_iterator *iter = &info->iter;
6810 struct partial_page partial_def[PIPE_DEF_BUFFERS];
6811 struct page *pages_def[PIPE_DEF_BUFFERS];
6812 struct splice_pipe_desc spd = {
6814 .partial = partial_def,
6815 .nr_pages_max = PIPE_DEF_BUFFERS,
6816 .ops = &buffer_pipe_buf_ops,
6817 .spd_release = buffer_spd_release,
6819 struct buffer_ref *ref;
6823 #ifdef CONFIG_TRACER_MAX_TRACE
6824 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
6828 if (*ppos & (PAGE_SIZE - 1))
6831 if (len & (PAGE_SIZE - 1)) {
6832 if (len < PAGE_SIZE)
6837 if (splice_grow_spd(pipe, &spd))
6841 trace_access_lock(iter->cpu_file);
6842 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
6844 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
6848 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
6854 refcount_set(&ref->refcount, 1);
6855 ref->buffer = iter->trace_buffer->buffer;
6856 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
6857 if (IS_ERR(ref->page)) {
6858 ret = PTR_ERR(ref->page);
6863 ref->cpu = iter->cpu_file;
6865 r = ring_buffer_read_page(ref->buffer, &ref->page,
6866 len, iter->cpu_file, 1);
6868 ring_buffer_free_read_page(ref->buffer, ref->cpu,
6874 page = virt_to_page(ref->page);
6876 spd.pages[i] = page;
6877 spd.partial[i].len = PAGE_SIZE;
6878 spd.partial[i].offset = 0;
6879 spd.partial[i].private = (unsigned long)ref;
6883 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
6886 trace_access_unlock(iter->cpu_file);
6889 /* did we read anything? */
6890 if (!spd.nr_pages) {
6895 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
6898 ret = wait_on_pipe(iter, true);
6905 ret = splice_to_pipe(pipe, &spd);
6907 splice_shrink_spd(&spd);
6912 static const struct file_operations tracing_buffers_fops = {
6913 .open = tracing_buffers_open,
6914 .read = tracing_buffers_read,
6915 .poll = tracing_buffers_poll,
6916 .release = tracing_buffers_release,
6917 .splice_read = tracing_buffers_splice_read,
6918 .llseek = no_llseek,
6922 tracing_stats_read(struct file *filp, char __user *ubuf,
6923 size_t count, loff_t *ppos)
6925 struct inode *inode = file_inode(filp);
6926 struct trace_array *tr = inode->i_private;
6927 struct trace_buffer *trace_buf = &tr->trace_buffer;
6928 int cpu = tracing_get_cpu(inode);
6929 struct trace_seq *s;
6931 unsigned long long t;
6932 unsigned long usec_rem;
6934 s = kmalloc(sizeof(*s), GFP_KERNEL);
6940 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
6941 trace_seq_printf(s, "entries: %ld\n", cnt);
6943 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
6944 trace_seq_printf(s, "overrun: %ld\n", cnt);
6946 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
6947 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
6949 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
6950 trace_seq_printf(s, "bytes: %ld\n", cnt);
6952 if (trace_clocks[tr->clock_id].in_ns) {
6953 /* local or global for trace_clock */
6954 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
6955 usec_rem = do_div(t, USEC_PER_SEC);
6956 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
6959 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
6960 usec_rem = do_div(t, USEC_PER_SEC);
6961 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
6963 /* counter or tsc mode for trace_clock */
6964 trace_seq_printf(s, "oldest event ts: %llu\n",
6965 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
6967 trace_seq_printf(s, "now ts: %llu\n",
6968 ring_buffer_time_stamp(trace_buf->buffer, cpu));
6971 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
6972 trace_seq_printf(s, "dropped events: %ld\n", cnt);
6974 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
6975 trace_seq_printf(s, "read events: %ld\n", cnt);
6977 count = simple_read_from_buffer(ubuf, count, ppos,
6978 s->buffer, trace_seq_used(s));
6985 static const struct file_operations tracing_stats_fops = {
6986 .open = tracing_open_generic_tr,
6987 .read = tracing_stats_read,
6988 .llseek = generic_file_llseek,
6989 .release = tracing_release_generic_tr,
6992 #ifdef CONFIG_DYNAMIC_FTRACE
6995 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
6996 size_t cnt, loff_t *ppos)
6998 unsigned long *p = filp->private_data;
6999 char buf[64]; /* Not too big for a shallow stack */
7002 r = scnprintf(buf, 63, "%ld", *p);
7005 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7008 static const struct file_operations tracing_dyn_info_fops = {
7009 .open = tracing_open_generic,
7010 .read = tracing_read_dyn_info,
7011 .llseek = generic_file_llseek,
7013 #endif /* CONFIG_DYNAMIC_FTRACE */
7015 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
7017 ftrace_snapshot(unsigned long ip, unsigned long parent_ip,
7018 struct trace_array *tr, struct ftrace_probe_ops *ops,
7021 tracing_snapshot_instance(tr);
7025 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip,
7026 struct trace_array *tr, struct ftrace_probe_ops *ops,
7029 struct ftrace_func_mapper *mapper = data;
7033 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7043 tracing_snapshot_instance(tr);
7047 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
7048 struct ftrace_probe_ops *ops, void *data)
7050 struct ftrace_func_mapper *mapper = data;
7053 seq_printf(m, "%ps:", (void *)ip);
7055 seq_puts(m, "snapshot");
7058 count = (long *)ftrace_func_mapper_find_ip(mapper, ip);
7061 seq_printf(m, ":count=%ld\n", *count);
7063 seq_puts(m, ":unlimited\n");
7069 ftrace_snapshot_init(struct ftrace_probe_ops *ops, struct trace_array *tr,
7070 unsigned long ip, void *init_data, void **data)
7072 struct ftrace_func_mapper *mapper = *data;
7075 mapper = allocate_ftrace_func_mapper();
7081 return ftrace_func_mapper_add_ip(mapper, ip, init_data);
7085 ftrace_snapshot_free(struct ftrace_probe_ops *ops, struct trace_array *tr,
7086 unsigned long ip, void *data)
7088 struct ftrace_func_mapper *mapper = data;
7093 free_ftrace_func_mapper(mapper, NULL);
7097 ftrace_func_mapper_remove_ip(mapper, ip);
7100 static struct ftrace_probe_ops snapshot_probe_ops = {
7101 .func = ftrace_snapshot,
7102 .print = ftrace_snapshot_print,
7105 static struct ftrace_probe_ops snapshot_count_probe_ops = {
7106 .func = ftrace_count_snapshot,
7107 .print = ftrace_snapshot_print,
7108 .init = ftrace_snapshot_init,
7109 .free = ftrace_snapshot_free,
7113 ftrace_trace_snapshot_callback(struct trace_array *tr, struct ftrace_hash *hash,
7114 char *glob, char *cmd, char *param, int enable)
7116 struct ftrace_probe_ops *ops;
7117 void *count = (void *)-1;
7124 /* hash funcs only work with set_ftrace_filter */
7128 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
7131 return unregister_ftrace_function_probe_func(glob+1, tr, ops);
7136 number = strsep(¶m, ":");
7138 if (!strlen(number))
7142 * We use the callback data field (which is a pointer)
7145 ret = kstrtoul(number, 0, (unsigned long *)&count);
7150 ret = tracing_alloc_snapshot_instance(tr);
7154 ret = register_ftrace_function_probe(glob, tr, ops, count);
7157 return ret < 0 ? ret : 0;
7160 static struct ftrace_func_command ftrace_snapshot_cmd = {
7162 .func = ftrace_trace_snapshot_callback,
7165 static __init int register_snapshot_cmd(void)
7167 return register_ftrace_command(&ftrace_snapshot_cmd);
7170 static inline __init int register_snapshot_cmd(void) { return 0; }
7171 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
7173 static struct dentry *tracing_get_dentry(struct trace_array *tr)
7175 if (WARN_ON(!tr->dir))
7176 return ERR_PTR(-ENODEV);
7178 /* Top directory uses NULL as the parent */
7179 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
7182 /* All sub buffers have a descriptor */
7186 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
7188 struct dentry *d_tracer;
7191 return tr->percpu_dir;
7193 d_tracer = tracing_get_dentry(tr);
7194 if (IS_ERR(d_tracer))
7197 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
7199 WARN_ONCE(!tr->percpu_dir,
7200 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
7202 return tr->percpu_dir;
7205 static struct dentry *
7206 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
7207 void *data, long cpu, const struct file_operations *fops)
7209 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
7211 if (ret) /* See tracing_get_cpu() */
7212 d_inode(ret)->i_cdev = (void *)(cpu + 1);
7217 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
7219 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
7220 struct dentry *d_cpu;
7221 char cpu_dir[30]; /* 30 characters should be more than enough */
7226 snprintf(cpu_dir, 30, "cpu%ld", cpu);
7227 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
7229 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
7233 /* per cpu trace_pipe */
7234 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
7235 tr, cpu, &tracing_pipe_fops);
7238 trace_create_cpu_file("trace", 0644, d_cpu,
7239 tr, cpu, &tracing_fops);
7241 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
7242 tr, cpu, &tracing_buffers_fops);
7244 trace_create_cpu_file("stats", 0444, d_cpu,
7245 tr, cpu, &tracing_stats_fops);
7247 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
7248 tr, cpu, &tracing_entries_fops);
7250 #ifdef CONFIG_TRACER_SNAPSHOT
7251 trace_create_cpu_file("snapshot", 0644, d_cpu,
7252 tr, cpu, &snapshot_fops);
7254 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
7255 tr, cpu, &snapshot_raw_fops);
7259 #ifdef CONFIG_FTRACE_SELFTEST
7260 /* Let selftest have access to static functions in this file */
7261 #include "trace_selftest.c"
7265 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
7268 struct trace_option_dentry *topt = filp->private_data;
7271 if (topt->flags->val & topt->opt->bit)
7276 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
7280 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
7283 struct trace_option_dentry *topt = filp->private_data;
7287 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7291 if (val != 0 && val != 1)
7294 if (!!(topt->flags->val & topt->opt->bit) != val) {
7295 mutex_lock(&trace_types_lock);
7296 ret = __set_tracer_option(topt->tr, topt->flags,
7298 mutex_unlock(&trace_types_lock);
7309 static const struct file_operations trace_options_fops = {
7310 .open = tracing_open_generic,
7311 .read = trace_options_read,
7312 .write = trace_options_write,
7313 .llseek = generic_file_llseek,
7317 * In order to pass in both the trace_array descriptor as well as the index
7318 * to the flag that the trace option file represents, the trace_array
7319 * has a character array of trace_flags_index[], which holds the index
7320 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
7321 * The address of this character array is passed to the flag option file
7322 * read/write callbacks.
7324 * In order to extract both the index and the trace_array descriptor,
7325 * get_tr_index() uses the following algorithm.
7329 * As the pointer itself contains the address of the index (remember
7332 * Then to get the trace_array descriptor, by subtracting that index
7333 * from the ptr, we get to the start of the index itself.
7335 * ptr - idx == &index[0]
7337 * Then a simple container_of() from that pointer gets us to the
7338 * trace_array descriptor.
7340 static void get_tr_index(void *data, struct trace_array **ptr,
7341 unsigned int *pindex)
7343 *pindex = *(unsigned char *)data;
7345 *ptr = container_of(data - *pindex, struct trace_array,
7350 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
7353 void *tr_index = filp->private_data;
7354 struct trace_array *tr;
7358 get_tr_index(tr_index, &tr, &index);
7360 if (tr->trace_flags & (1 << index))
7365 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
7369 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
7372 void *tr_index = filp->private_data;
7373 struct trace_array *tr;
7378 get_tr_index(tr_index, &tr, &index);
7380 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7384 if (val != 0 && val != 1)
7387 mutex_lock(&event_mutex);
7388 mutex_lock(&trace_types_lock);
7389 ret = set_tracer_flag(tr, 1 << index, val);
7390 mutex_unlock(&trace_types_lock);
7391 mutex_unlock(&event_mutex);
7401 static const struct file_operations trace_options_core_fops = {
7402 .open = tracing_open_generic,
7403 .read = trace_options_core_read,
7404 .write = trace_options_core_write,
7405 .llseek = generic_file_llseek,
7408 struct dentry *trace_create_file(const char *name,
7410 struct dentry *parent,
7412 const struct file_operations *fops)
7416 ret = tracefs_create_file(name, mode, parent, data, fops);
7418 pr_warn("Could not create tracefs '%s' entry\n", name);
7424 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
7426 struct dentry *d_tracer;
7431 d_tracer = tracing_get_dentry(tr);
7432 if (IS_ERR(d_tracer))
7435 tr->options = tracefs_create_dir("options", d_tracer);
7437 pr_warn("Could not create tracefs directory 'options'\n");
7445 create_trace_option_file(struct trace_array *tr,
7446 struct trace_option_dentry *topt,
7447 struct tracer_flags *flags,
7448 struct tracer_opt *opt)
7450 struct dentry *t_options;
7452 t_options = trace_options_init_dentry(tr);
7456 topt->flags = flags;
7460 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
7461 &trace_options_fops);
7466 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
7468 struct trace_option_dentry *topts;
7469 struct trace_options *tr_topts;
7470 struct tracer_flags *flags;
7471 struct tracer_opt *opts;
7478 flags = tracer->flags;
7480 if (!flags || !flags->opts)
7484 * If this is an instance, only create flags for tracers
7485 * the instance may have.
7487 if (!trace_ok_for_array(tracer, tr))
7490 for (i = 0; i < tr->nr_topts; i++) {
7491 /* Make sure there's no duplicate flags. */
7492 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
7498 for (cnt = 0; opts[cnt].name; cnt++)
7501 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
7505 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
7512 tr->topts = tr_topts;
7513 tr->topts[tr->nr_topts].tracer = tracer;
7514 tr->topts[tr->nr_topts].topts = topts;
7517 for (cnt = 0; opts[cnt].name; cnt++) {
7518 create_trace_option_file(tr, &topts[cnt], flags,
7520 WARN_ONCE(topts[cnt].entry == NULL,
7521 "Failed to create trace option: %s",
7526 static struct dentry *
7527 create_trace_option_core_file(struct trace_array *tr,
7528 const char *option, long index)
7530 struct dentry *t_options;
7532 t_options = trace_options_init_dentry(tr);
7536 return trace_create_file(option, 0644, t_options,
7537 (void *)&tr->trace_flags_index[index],
7538 &trace_options_core_fops);
7541 static void create_trace_options_dir(struct trace_array *tr)
7543 struct dentry *t_options;
7544 bool top_level = tr == &global_trace;
7547 t_options = trace_options_init_dentry(tr);
7551 for (i = 0; trace_options[i]; i++) {
7553 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
7554 create_trace_option_core_file(tr, trace_options[i], i);
7559 rb_simple_read(struct file *filp, char __user *ubuf,
7560 size_t cnt, loff_t *ppos)
7562 struct trace_array *tr = filp->private_data;
7566 r = tracer_tracing_is_on(tr);
7567 r = sprintf(buf, "%d\n", r);
7569 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
7573 rb_simple_write(struct file *filp, const char __user *ubuf,
7574 size_t cnt, loff_t *ppos)
7576 struct trace_array *tr = filp->private_data;
7577 struct ring_buffer *buffer = tr->trace_buffer.buffer;
7581 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
7586 mutex_lock(&trace_types_lock);
7587 if (!!val == tracer_tracing_is_on(tr)) {
7588 val = 0; /* do nothing */
7590 tracer_tracing_on(tr);
7591 if (tr->current_trace->start)
7592 tr->current_trace->start(tr);
7594 tracer_tracing_off(tr);
7595 if (tr->current_trace->stop)
7596 tr->current_trace->stop(tr);
7598 mutex_unlock(&trace_types_lock);
7606 static const struct file_operations rb_simple_fops = {
7607 .open = tracing_open_generic_tr,
7608 .read = rb_simple_read,
7609 .write = rb_simple_write,
7610 .release = tracing_release_generic_tr,
7611 .llseek = default_llseek,
7614 struct dentry *trace_instance_dir;
7617 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
7620 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
7622 enum ring_buffer_flags rb_flags;
7624 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
7628 buf->buffer = ring_buffer_alloc(size, rb_flags);
7632 buf->data = alloc_percpu(struct trace_array_cpu);
7634 ring_buffer_free(buf->buffer);
7639 /* Allocate the first page for all buffers */
7640 set_buffer_entries(&tr->trace_buffer,
7641 ring_buffer_size(tr->trace_buffer.buffer, 0));
7646 static int allocate_trace_buffers(struct trace_array *tr, int size)
7650 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
7654 #ifdef CONFIG_TRACER_MAX_TRACE
7655 ret = allocate_trace_buffer(tr, &tr->max_buffer,
7656 allocate_snapshot ? size : 1);
7658 ring_buffer_free(tr->trace_buffer.buffer);
7659 tr->trace_buffer.buffer = NULL;
7660 free_percpu(tr->trace_buffer.data);
7661 tr->trace_buffer.data = NULL;
7664 tr->allocated_snapshot = allocate_snapshot;
7667 * Only the top level trace array gets its snapshot allocated
7668 * from the kernel command line.
7670 allocate_snapshot = false;
7674 * Because of some magic with the way alloc_percpu() works on
7675 * x86_64, we need to synchronize the pgd of all the tables,
7676 * otherwise the trace events that happen in x86_64 page fault
7677 * handlers can't cope with accessing the chance that a
7678 * alloc_percpu()'d memory might be touched in the page fault trace
7679 * event. Oh, and we need to audit all other alloc_percpu() and vmalloc()
7680 * calls in tracing, because something might get triggered within a
7681 * page fault trace event!
7683 vmalloc_sync_mappings();
7688 static void free_trace_buffer(struct trace_buffer *buf)
7691 ring_buffer_free(buf->buffer);
7693 free_percpu(buf->data);
7698 static void free_trace_buffers(struct trace_array *tr)
7703 free_trace_buffer(&tr->trace_buffer);
7705 #ifdef CONFIG_TRACER_MAX_TRACE
7706 free_trace_buffer(&tr->max_buffer);
7710 static void init_trace_flags_index(struct trace_array *tr)
7714 /* Used by the trace options files */
7715 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
7716 tr->trace_flags_index[i] = i;
7719 static void __update_tracer_options(struct trace_array *tr)
7723 for (t = trace_types; t; t = t->next)
7724 add_tracer_options(tr, t);
7727 static void update_tracer_options(struct trace_array *tr)
7729 mutex_lock(&trace_types_lock);
7730 __update_tracer_options(tr);
7731 mutex_unlock(&trace_types_lock);
7734 static int instance_mkdir(const char *name)
7736 struct trace_array *tr;
7739 mutex_lock(&event_mutex);
7740 mutex_lock(&trace_types_lock);
7743 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7744 if (tr->name && strcmp(tr->name, name) == 0)
7749 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
7753 tr->name = kstrdup(name, GFP_KERNEL);
7757 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
7760 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
7762 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
7764 raw_spin_lock_init(&tr->start_lock);
7766 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7768 tr->current_trace = &nop_trace;
7770 INIT_LIST_HEAD(&tr->systems);
7771 INIT_LIST_HEAD(&tr->events);
7773 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
7776 tr->dir = tracefs_create_dir(name, trace_instance_dir);
7780 ret = event_trace_add_tracer(tr->dir, tr);
7782 tracefs_remove_recursive(tr->dir);
7786 ftrace_init_trace_array(tr);
7788 init_tracer_tracefs(tr, tr->dir);
7789 init_trace_flags_index(tr);
7790 __update_tracer_options(tr);
7792 list_add(&tr->list, &ftrace_trace_arrays);
7794 mutex_unlock(&trace_types_lock);
7795 mutex_unlock(&event_mutex);
7800 free_trace_buffers(tr);
7801 free_cpumask_var(tr->tracing_cpumask);
7806 mutex_unlock(&trace_types_lock);
7807 mutex_unlock(&event_mutex);
7813 static int instance_rmdir(const char *name)
7815 struct trace_array *tr;
7820 mutex_lock(&event_mutex);
7821 mutex_lock(&trace_types_lock);
7824 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7825 if (tr->name && strcmp(tr->name, name) == 0) {
7834 if (tr->ref || (tr->current_trace && tr->current_trace->ref))
7837 list_del(&tr->list);
7839 /* Disable all the flags that were enabled coming in */
7840 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
7841 if ((1 << i) & ZEROED_TRACE_FLAGS)
7842 set_tracer_flag(tr, 1 << i, 0);
7845 tracing_set_nop(tr);
7846 clear_ftrace_function_probes(tr);
7847 event_trace_del_tracer(tr);
7848 ftrace_clear_pids(tr);
7849 ftrace_destroy_function_files(tr);
7850 tracefs_remove_recursive(tr->dir);
7851 free_trace_buffers(tr);
7853 for (i = 0; i < tr->nr_topts; i++) {
7854 kfree(tr->topts[i].topts);
7858 free_cpumask_var(tr->tracing_cpumask);
7865 mutex_unlock(&trace_types_lock);
7866 mutex_unlock(&event_mutex);
7871 static __init void create_trace_instances(struct dentry *d_tracer)
7873 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
7876 if (WARN_ON(!trace_instance_dir))
7881 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
7885 trace_create_file("available_tracers", 0444, d_tracer,
7886 tr, &show_traces_fops);
7888 trace_create_file("current_tracer", 0644, d_tracer,
7889 tr, &set_tracer_fops);
7891 trace_create_file("tracing_cpumask", 0644, d_tracer,
7892 tr, &tracing_cpumask_fops);
7894 trace_create_file("trace_options", 0644, d_tracer,
7895 tr, &tracing_iter_fops);
7897 trace_create_file("trace", 0644, d_tracer,
7900 trace_create_file("trace_pipe", 0444, d_tracer,
7901 tr, &tracing_pipe_fops);
7903 trace_create_file("buffer_size_kb", 0644, d_tracer,
7904 tr, &tracing_entries_fops);
7906 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
7907 tr, &tracing_total_entries_fops);
7909 trace_create_file("free_buffer", 0200, d_tracer,
7910 tr, &tracing_free_buffer_fops);
7912 trace_create_file("trace_marker", 0220, d_tracer,
7913 tr, &tracing_mark_fops);
7915 trace_create_file("trace_marker_raw", 0220, d_tracer,
7916 tr, &tracing_mark_raw_fops);
7918 trace_create_file("trace_clock", 0644, d_tracer, tr,
7921 trace_create_file("tracing_on", 0644, d_tracer,
7922 tr, &rb_simple_fops);
7924 create_trace_options_dir(tr);
7926 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
7927 trace_create_file("tracing_max_latency", 0644, d_tracer,
7928 &tr->max_latency, &tracing_max_lat_fops);
7931 if (ftrace_create_function_files(tr, d_tracer))
7932 WARN(1, "Could not allocate function filter files");
7934 #ifdef CONFIG_TRACER_SNAPSHOT
7935 trace_create_file("snapshot", 0644, d_tracer,
7936 tr, &snapshot_fops);
7939 for_each_tracing_cpu(cpu)
7940 tracing_init_tracefs_percpu(tr, cpu);
7942 ftrace_init_tracefs(tr, d_tracer);
7945 static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
7947 struct vfsmount *mnt;
7948 struct file_system_type *type;
7951 * To maintain backward compatibility for tools that mount
7952 * debugfs to get to the tracing facility, tracefs is automatically
7953 * mounted to the debugfs/tracing directory.
7955 type = get_fs_type("tracefs");
7958 mnt = vfs_submount(mntpt, type, "tracefs", NULL);
7959 put_filesystem(type);
7968 * tracing_init_dentry - initialize top level trace array
7970 * This is called when creating files or directories in the tracing
7971 * directory. It is called via fs_initcall() by any of the boot up code
7972 * and expects to return the dentry of the top level tracing directory.
7974 struct dentry *tracing_init_dentry(void)
7976 struct trace_array *tr = &global_trace;
7978 /* The top level trace array uses NULL as parent */
7982 if (WARN_ON(!tracefs_initialized()) ||
7983 (IS_ENABLED(CONFIG_DEBUG_FS) &&
7984 WARN_ON(!debugfs_initialized())))
7985 return ERR_PTR(-ENODEV);
7988 * As there may still be users that expect the tracing
7989 * files to exist in debugfs/tracing, we must automount
7990 * the tracefs file system there, so older tools still
7991 * work with the newer kerenl.
7993 tr->dir = debugfs_create_automount("tracing", NULL,
7994 trace_automount, NULL);
7996 pr_warn_once("Could not create debugfs directory 'tracing'\n");
7997 return ERR_PTR(-ENOMEM);
8003 extern struct trace_eval_map *__start_ftrace_eval_maps[];
8004 extern struct trace_eval_map *__stop_ftrace_eval_maps[];
8006 static void __init trace_eval_init(void)
8010 len = __stop_ftrace_eval_maps - __start_ftrace_eval_maps;
8011 trace_insert_eval_map(NULL, __start_ftrace_eval_maps, len);
8014 #ifdef CONFIG_MODULES
8015 static void trace_module_add_evals(struct module *mod)
8017 if (!mod->num_trace_evals)
8021 * Modules with bad taint do not have events created, do
8022 * not bother with enums either.
8024 if (trace_module_has_bad_taint(mod))
8027 trace_insert_eval_map(mod, mod->trace_evals, mod->num_trace_evals);
8030 #ifdef CONFIG_TRACE_EVAL_MAP_FILE
8031 static void trace_module_remove_evals(struct module *mod)
8033 union trace_eval_map_item *map;
8034 union trace_eval_map_item **last = &trace_eval_maps;
8036 if (!mod->num_trace_evals)
8039 mutex_lock(&trace_eval_mutex);
8041 map = trace_eval_maps;
8044 if (map->head.mod == mod)
8046 map = trace_eval_jmp_to_tail(map);
8047 last = &map->tail.next;
8048 map = map->tail.next;
8053 *last = trace_eval_jmp_to_tail(map)->tail.next;
8056 mutex_unlock(&trace_eval_mutex);
8059 static inline void trace_module_remove_evals(struct module *mod) { }
8060 #endif /* CONFIG_TRACE_EVAL_MAP_FILE */
8062 static int trace_module_notify(struct notifier_block *self,
8063 unsigned long val, void *data)
8065 struct module *mod = data;
8068 case MODULE_STATE_COMING:
8069 trace_module_add_evals(mod);
8071 case MODULE_STATE_GOING:
8072 trace_module_remove_evals(mod);
8079 static struct notifier_block trace_module_nb = {
8080 .notifier_call = trace_module_notify,
8083 #endif /* CONFIG_MODULES */
8085 static __init int tracer_init_tracefs(void)
8087 struct dentry *d_tracer;
8089 trace_access_lock_init();
8091 d_tracer = tracing_init_dentry();
8092 if (IS_ERR(d_tracer))
8095 init_tracer_tracefs(&global_trace, d_tracer);
8096 ftrace_init_tracefs_toplevel(&global_trace, d_tracer);
8098 trace_create_file("tracing_thresh", 0644, d_tracer,
8099 &global_trace, &tracing_thresh_fops);
8101 trace_create_file("README", 0444, d_tracer,
8102 NULL, &tracing_readme_fops);
8104 trace_create_file("saved_cmdlines", 0444, d_tracer,
8105 NULL, &tracing_saved_cmdlines_fops);
8107 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
8108 NULL, &tracing_saved_cmdlines_size_fops);
8110 trace_create_file("saved_tgids", 0444, d_tracer,
8111 NULL, &tracing_saved_tgids_fops);
8115 trace_create_eval_file(d_tracer);
8117 #ifdef CONFIG_MODULES
8118 register_module_notifier(&trace_module_nb);
8121 #ifdef CONFIG_DYNAMIC_FTRACE
8122 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
8123 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
8126 create_trace_instances(d_tracer);
8128 update_tracer_options(&global_trace);
8133 static int trace_panic_handler(struct notifier_block *this,
8134 unsigned long event, void *unused)
8136 if (ftrace_dump_on_oops)
8137 ftrace_dump(ftrace_dump_on_oops);
8141 static struct notifier_block trace_panic_notifier = {
8142 .notifier_call = trace_panic_handler,
8144 .priority = 150 /* priority: INT_MAX >= x >= 0 */
8147 static int trace_die_handler(struct notifier_block *self,
8153 if (ftrace_dump_on_oops)
8154 ftrace_dump(ftrace_dump_on_oops);
8162 static struct notifier_block trace_die_notifier = {
8163 .notifier_call = trace_die_handler,
8168 * printk is set to max of 1024, we really don't need it that big.
8169 * Nothing should be printing 1000 characters anyway.
8171 #define TRACE_MAX_PRINT 1000
8174 * Define here KERN_TRACE so that we have one place to modify
8175 * it if we decide to change what log level the ftrace dump
8178 #define KERN_TRACE KERN_EMERG
8181 trace_printk_seq(struct trace_seq *s)
8183 /* Probably should print a warning here. */
8184 if (s->seq.len >= TRACE_MAX_PRINT)
8185 s->seq.len = TRACE_MAX_PRINT;
8188 * More paranoid code. Although the buffer size is set to
8189 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
8190 * an extra layer of protection.
8192 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
8193 s->seq.len = s->seq.size - 1;
8195 /* should be zero ended, but we are paranoid. */
8196 s->buffer[s->seq.len] = 0;
8198 printk(KERN_TRACE "%s", s->buffer);
8203 void trace_init_global_iter(struct trace_iterator *iter)
8205 iter->tr = &global_trace;
8206 iter->trace = iter->tr->current_trace;
8207 iter->cpu_file = RING_BUFFER_ALL_CPUS;
8208 iter->trace_buffer = &global_trace.trace_buffer;
8210 if (iter->trace && iter->trace->open)
8211 iter->trace->open(iter);
8213 /* Annotate start of buffers if we had overruns */
8214 if (ring_buffer_overruns(iter->trace_buffer->buffer))
8215 iter->iter_flags |= TRACE_FILE_ANNOTATE;
8217 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
8218 if (trace_clocks[iter->tr->clock_id].in_ns)
8219 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
8222 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
8224 /* use static because iter can be a bit big for the stack */
8225 static struct trace_iterator iter;
8226 static atomic_t dump_running;
8227 struct trace_array *tr = &global_trace;
8228 unsigned int old_userobj;
8229 unsigned long flags;
8232 /* Only allow one dump user at a time. */
8233 if (atomic_inc_return(&dump_running) != 1) {
8234 atomic_dec(&dump_running);
8239 * Always turn off tracing when we dump.
8240 * We don't need to show trace output of what happens
8241 * between multiple crashes.
8243 * If the user does a sysrq-z, then they can re-enable
8244 * tracing with echo 1 > tracing_on.
8248 local_irq_save(flags);
8249 printk_nmi_direct_enter();
8251 /* Simulate the iterator */
8252 trace_init_global_iter(&iter);
8254 for_each_tracing_cpu(cpu) {
8255 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
8258 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
8260 /* don't look at user memory in panic mode */
8261 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
8263 switch (oops_dump_mode) {
8265 iter.cpu_file = RING_BUFFER_ALL_CPUS;
8268 iter.cpu_file = raw_smp_processor_id();
8273 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
8274 iter.cpu_file = RING_BUFFER_ALL_CPUS;
8277 printk(KERN_TRACE "Dumping ftrace buffer:\n");
8279 /* Did function tracer already get disabled? */
8280 if (ftrace_is_dead()) {
8281 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
8282 printk("# MAY BE MISSING FUNCTION EVENTS\n");
8286 * We need to stop all tracing on all CPUS to read the
8287 * the next buffer. This is a bit expensive, but is
8288 * not done often. We fill all what we can read,
8289 * and then release the locks again.
8292 while (!trace_empty(&iter)) {
8295 printk(KERN_TRACE "---------------------------------\n");
8299 trace_iterator_reset(&iter);
8300 iter.iter_flags |= TRACE_FILE_LAT_FMT;
8302 if (trace_find_next_entry_inc(&iter) != NULL) {
8305 ret = print_trace_line(&iter);
8306 if (ret != TRACE_TYPE_NO_CONSUME)
8307 trace_consume(&iter);
8309 touch_nmi_watchdog();
8311 trace_printk_seq(&iter.seq);
8315 printk(KERN_TRACE " (ftrace buffer empty)\n");
8317 printk(KERN_TRACE "---------------------------------\n");
8320 tr->trace_flags |= old_userobj;
8322 for_each_tracing_cpu(cpu) {
8323 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
8325 atomic_dec(&dump_running);
8326 printk_nmi_direct_exit();
8327 local_irq_restore(flags);
8329 EXPORT_SYMBOL_GPL(ftrace_dump);
8331 __init static int tracer_alloc_buffers(void)
8337 * Make sure we don't accidently add more trace options
8338 * than we have bits for.
8340 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
8342 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
8345 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
8346 goto out_free_buffer_mask;
8348 /* Only allocate trace_printk buffers if a trace_printk exists */
8349 if (&__stop___trace_bprintk_fmt != &__start___trace_bprintk_fmt)
8350 /* Must be called before global_trace.buffer is allocated */
8351 trace_printk_init_buffers();
8353 /* To save memory, keep the ring buffer size to its minimum */
8354 if (ring_buffer_expanded)
8355 ring_buf_size = trace_buf_size;
8359 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
8360 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
8362 raw_spin_lock_init(&global_trace.start_lock);
8365 * The prepare callbacks allocates some memory for the ring buffer. We
8366 * don't free the buffer if the if the CPU goes down. If we were to free
8367 * the buffer, then the user would lose any trace that was in the
8368 * buffer. The memory will be removed once the "instance" is removed.
8370 ret = cpuhp_setup_state_multi(CPUHP_TRACE_RB_PREPARE,
8371 "trace/RB:preapre", trace_rb_cpu_prepare,
8374 goto out_free_cpumask;
8375 /* Used for event triggers */
8377 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
8379 goto out_rm_hp_state;
8381 if (trace_create_savedcmd() < 0)
8382 goto out_free_temp_buffer;
8384 /* TODO: make the number of buffers hot pluggable with CPUS */
8385 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
8386 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
8388 goto out_free_savedcmd;
8391 if (global_trace.buffer_disabled)
8394 if (trace_boot_clock) {
8395 ret = tracing_set_clock(&global_trace, trace_boot_clock);
8397 pr_warn("Trace clock %s not defined, going back to default\n",
8402 * register_tracer() might reference current_trace, so it
8403 * needs to be set before we register anything. This is
8404 * just a bootstrap of current_trace anyway.
8406 global_trace.current_trace = &nop_trace;
8408 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
8410 ftrace_init_global_array_ops(&global_trace);
8412 init_trace_flags_index(&global_trace);
8414 register_tracer(&nop_trace);
8416 /* Function tracing may start here (via kernel command line) */
8417 init_function_trace();
8419 /* All seems OK, enable tracing */
8420 tracing_disabled = 0;
8422 atomic_notifier_chain_register(&panic_notifier_list,
8423 &trace_panic_notifier);
8425 register_die_notifier(&trace_die_notifier);
8427 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
8429 INIT_LIST_HEAD(&global_trace.systems);
8430 INIT_LIST_HEAD(&global_trace.events);
8431 list_add(&global_trace.list, &ftrace_trace_arrays);
8433 apply_trace_boot_options();
8435 register_snapshot_cmd();
8440 free_saved_cmdlines_buffer(savedcmd);
8441 out_free_temp_buffer:
8442 ring_buffer_free(temp_buffer);
8444 cpuhp_remove_multi_state(CPUHP_TRACE_RB_PREPARE);
8446 free_cpumask_var(global_trace.tracing_cpumask);
8447 out_free_buffer_mask:
8448 free_cpumask_var(tracing_buffer_mask);
8453 void __init early_trace_init(void)
8455 if (tracepoint_printk) {
8456 tracepoint_print_iter =
8457 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
8458 if (WARN_ON(!tracepoint_print_iter))
8459 tracepoint_printk = 0;
8461 static_key_enable(&tracepoint_printk_key.key);
8463 tracer_alloc_buffers();
8466 void __init trace_init(void)
8471 __init static int clear_boot_tracer(void)
8474 * The default tracer at boot buffer is an init section.
8475 * This function is called in lateinit. If we did not
8476 * find the boot tracer, then clear it out, to prevent
8477 * later registration from accessing the buffer that is
8478 * about to be freed.
8480 if (!default_bootup_tracer)
8483 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
8484 default_bootup_tracer);
8485 default_bootup_tracer = NULL;
8490 fs_initcall(tracer_init_tracefs);
8491 late_initcall_sync(clear_boot_tracer);