2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/tracefs.h>
24 #include <linux/pagemap.h>
25 #include <linux/hardirq.h>
26 #include <linux/linkage.h>
27 #include <linux/uaccess.h>
28 #include <linux/kprobes.h>
29 #include <linux/vmalloc.h>
30 #include <linux/ftrace.h>
31 #include <linux/module.h>
32 #include <linux/percpu.h>
33 #include <linux/splice.h>
34 #include <linux/kdebug.h>
35 #include <linux/string.h>
36 #include <linux/mount.h>
37 #include <linux/rwsem.h>
38 #include <linux/slab.h>
39 #include <linux/ctype.h>
40 #include <linux/init.h>
41 #include <linux/poll.h>
42 #include <linux/nmi.h>
44 #include <linux/sched/rt.h>
47 #include "trace_output.h"
50 * On boot up, the ring buffer is set to the minimum size, so that
51 * we do not waste memory on systems that are not using tracing.
53 bool ring_buffer_expanded;
56 * We need to change this state when a selftest is running.
57 * A selftest will lurk into the ring-buffer to count the
58 * entries inserted during the selftest although some concurrent
59 * insertions into the ring-buffer such as trace_printk could occurred
60 * at the same time, giving false positive or negative results.
62 static bool __read_mostly tracing_selftest_running;
65 * If a tracer is running, we do not want to run SELFTEST.
67 bool __read_mostly tracing_selftest_disabled;
69 /* Pipe tracepoints to printk */
70 struct trace_iterator *tracepoint_print_iter;
71 int tracepoint_printk;
73 /* For tracers that don't implement custom flags */
74 static struct tracer_opt dummy_tracer_opt[] = {
78 static struct tracer_flags dummy_tracer_flags = {
80 .opts = dummy_tracer_opt
84 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
90 * To prevent the comm cache from being overwritten when no
91 * tracing is active, only save the comm when a trace event
94 static DEFINE_PER_CPU(bool, trace_cmdline_save);
97 * Kill all tracing for good (never come back).
98 * It is initialized to 1 but will turn to zero if the initialization
99 * of the tracer is successful. But that is the only place that sets
102 static int tracing_disabled = 1;
104 cpumask_var_t __read_mostly tracing_buffer_mask;
107 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
109 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
110 * is set, then ftrace_dump is called. This will output the contents
111 * of the ftrace buffers to the console. This is very useful for
112 * capturing traces that lead to crashes and outputing it to a
115 * It is default off, but you can enable it with either specifying
116 * "ftrace_dump_on_oops" in the kernel command line, or setting
117 * /proc/sys/kernel/ftrace_dump_on_oops
118 * Set 1 if you want to dump buffers of all CPUs
119 * Set 2 if you want to dump the buffer of the CPU that triggered oops
122 enum ftrace_dump_mode ftrace_dump_on_oops;
124 /* When set, tracing will stop when a WARN*() is hit */
125 int __disable_trace_on_warning;
127 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
128 /* Map of enums to their values, for "enum_map" file */
129 struct trace_enum_map_head {
131 unsigned long length;
134 union trace_enum_map_item;
136 struct trace_enum_map_tail {
138 * "end" is first and points to NULL as it must be different
139 * than "mod" or "enum_string"
141 union trace_enum_map_item *next;
142 const char *end; /* points to NULL */
145 static DEFINE_MUTEX(trace_enum_mutex);
148 * The trace_enum_maps are saved in an array with two extra elements,
149 * one at the beginning, and one at the end. The beginning item contains
150 * the count of the saved maps (head.length), and the module they
151 * belong to if not built in (head.mod). The ending item contains a
152 * pointer to the next array of saved enum_map items.
154 union trace_enum_map_item {
155 struct trace_enum_map map;
156 struct trace_enum_map_head head;
157 struct trace_enum_map_tail tail;
160 static union trace_enum_map_item *trace_enum_maps;
161 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
163 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
165 #define MAX_TRACER_SIZE 100
166 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
167 static char *default_bootup_tracer;
169 static bool allocate_snapshot;
171 static int __init set_cmdline_ftrace(char *str)
173 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
174 default_bootup_tracer = bootup_tracer_buf;
175 /* We are using ftrace early, expand it */
176 ring_buffer_expanded = true;
179 __setup("ftrace=", set_cmdline_ftrace);
181 static int __init set_ftrace_dump_on_oops(char *str)
183 if (*str++ != '=' || !*str) {
184 ftrace_dump_on_oops = DUMP_ALL;
188 if (!strcmp("orig_cpu", str)) {
189 ftrace_dump_on_oops = DUMP_ORIG;
195 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
197 static int __init stop_trace_on_warning(char *str)
199 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
200 __disable_trace_on_warning = 1;
203 __setup("traceoff_on_warning", stop_trace_on_warning);
205 static int __init boot_alloc_snapshot(char *str)
207 allocate_snapshot = true;
208 /* We also need the main ring buffer expanded */
209 ring_buffer_expanded = true;
212 __setup("alloc_snapshot", boot_alloc_snapshot);
215 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
217 static int __init set_trace_boot_options(char *str)
219 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
222 __setup("trace_options=", set_trace_boot_options);
224 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
225 static char *trace_boot_clock __initdata;
227 static int __init set_trace_boot_clock(char *str)
229 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
230 trace_boot_clock = trace_boot_clock_buf;
233 __setup("trace_clock=", set_trace_boot_clock);
235 static int __init set_tracepoint_printk(char *str)
237 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
238 tracepoint_printk = 1;
241 __setup("tp_printk", set_tracepoint_printk);
243 unsigned long long ns2usecs(cycle_t nsec)
250 /* trace_flags holds trace_options default values */
251 #define TRACE_DEFAULT_FLAGS \
252 (FUNCTION_DEFAULT_FLAGS | \
253 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
254 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
255 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
256 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
258 /* trace_options that are only supported by global_trace */
259 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
260 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
264 * The global_trace is the descriptor that holds the tracing
265 * buffers for the live tracing. For each CPU, it contains
266 * a link list of pages that will store trace entries. The
267 * page descriptor of the pages in the memory is used to hold
268 * the link list by linking the lru item in the page descriptor
269 * to each of the pages in the buffer per CPU.
271 * For each active CPU there is a data field that holds the
272 * pages for the buffer for that CPU. Each CPU has the same number
273 * of pages allocated for its buffer.
275 static struct trace_array global_trace = {
276 .trace_flags = TRACE_DEFAULT_FLAGS,
279 LIST_HEAD(ftrace_trace_arrays);
281 int trace_array_get(struct trace_array *this_tr)
283 struct trace_array *tr;
286 mutex_lock(&trace_types_lock);
287 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
294 mutex_unlock(&trace_types_lock);
299 static void __trace_array_put(struct trace_array *this_tr)
301 WARN_ON(!this_tr->ref);
305 void trace_array_put(struct trace_array *this_tr)
307 mutex_lock(&trace_types_lock);
308 __trace_array_put(this_tr);
309 mutex_unlock(&trace_types_lock);
312 int filter_check_discard(struct trace_event_file *file, void *rec,
313 struct ring_buffer *buffer,
314 struct ring_buffer_event *event)
316 if (unlikely(file->flags & EVENT_FILE_FL_FILTERED) &&
317 !filter_match_preds(file->filter, rec)) {
318 ring_buffer_discard_commit(buffer, event);
324 EXPORT_SYMBOL_GPL(filter_check_discard);
326 int call_filter_check_discard(struct trace_event_call *call, void *rec,
327 struct ring_buffer *buffer,
328 struct ring_buffer_event *event)
330 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
331 !filter_match_preds(call->filter, rec)) {
332 ring_buffer_discard_commit(buffer, event);
338 EXPORT_SYMBOL_GPL(call_filter_check_discard);
340 static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
344 /* Early boot up does not have a buffer yet */
346 return trace_clock_local();
348 ts = ring_buffer_time_stamp(buf->buffer, cpu);
349 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
354 cycle_t ftrace_now(int cpu)
356 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
360 * tracing_is_enabled - Show if global_trace has been disabled
362 * Shows if the global trace has been enabled or not. It uses the
363 * mirror flag "buffer_disabled" to be used in fast paths such as for
364 * the irqsoff tracer. But it may be inaccurate due to races. If you
365 * need to know the accurate state, use tracing_is_on() which is a little
366 * slower, but accurate.
368 int tracing_is_enabled(void)
371 * For quick access (irqsoff uses this in fast path), just
372 * return the mirror variable of the state of the ring buffer.
373 * It's a little racy, but we don't really care.
376 return !global_trace.buffer_disabled;
380 * trace_buf_size is the size in bytes that is allocated
381 * for a buffer. Note, the number of bytes is always rounded
384 * This number is purposely set to a low number of 16384.
385 * If the dump on oops happens, it will be much appreciated
386 * to not have to wait for all that output. Anyway this can be
387 * boot time and run time configurable.
389 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
391 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
393 /* trace_types holds a link list of available tracers. */
394 static struct tracer *trace_types __read_mostly;
397 * trace_types_lock is used to protect the trace_types list.
399 DEFINE_MUTEX(trace_types_lock);
402 * serialize the access of the ring buffer
404 * ring buffer serializes readers, but it is low level protection.
405 * The validity of the events (which returns by ring_buffer_peek() ..etc)
406 * are not protected by ring buffer.
408 * The content of events may become garbage if we allow other process consumes
409 * these events concurrently:
410 * A) the page of the consumed events may become a normal page
411 * (not reader page) in ring buffer, and this page will be rewrited
412 * by events producer.
413 * B) The page of the consumed events may become a page for splice_read,
414 * and this page will be returned to system.
416 * These primitives allow multi process access to different cpu ring buffer
419 * These primitives don't distinguish read-only and read-consume access.
420 * Multi read-only access are also serialized.
424 static DECLARE_RWSEM(all_cpu_access_lock);
425 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
427 static inline void trace_access_lock(int cpu)
429 if (cpu == RING_BUFFER_ALL_CPUS) {
430 /* gain it for accessing the whole ring buffer. */
431 down_write(&all_cpu_access_lock);
433 /* gain it for accessing a cpu ring buffer. */
435 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
436 down_read(&all_cpu_access_lock);
438 /* Secondly block other access to this @cpu ring buffer. */
439 mutex_lock(&per_cpu(cpu_access_lock, cpu));
443 static inline void trace_access_unlock(int cpu)
445 if (cpu == RING_BUFFER_ALL_CPUS) {
446 up_write(&all_cpu_access_lock);
448 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
449 up_read(&all_cpu_access_lock);
453 static inline void trace_access_lock_init(void)
457 for_each_possible_cpu(cpu)
458 mutex_init(&per_cpu(cpu_access_lock, cpu));
463 static DEFINE_MUTEX(access_lock);
465 static inline void trace_access_lock(int cpu)
468 mutex_lock(&access_lock);
471 static inline void trace_access_unlock(int cpu)
474 mutex_unlock(&access_lock);
477 static inline void trace_access_lock_init(void)
483 #ifdef CONFIG_STACKTRACE
484 static void __ftrace_trace_stack(struct ring_buffer *buffer,
486 int skip, int pc, struct pt_regs *regs);
487 static inline void ftrace_trace_stack(struct trace_array *tr,
488 struct ring_buffer *buffer,
490 int skip, int pc, struct pt_regs *regs);
493 static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
495 int skip, int pc, struct pt_regs *regs)
498 static inline void ftrace_trace_stack(struct trace_array *tr,
499 struct ring_buffer *buffer,
501 int skip, int pc, struct pt_regs *regs)
507 static void tracer_tracing_on(struct trace_array *tr)
509 if (tr->trace_buffer.buffer)
510 ring_buffer_record_on(tr->trace_buffer.buffer);
512 * This flag is looked at when buffers haven't been allocated
513 * yet, or by some tracers (like irqsoff), that just want to
514 * know if the ring buffer has been disabled, but it can handle
515 * races of where it gets disabled but we still do a record.
516 * As the check is in the fast path of the tracers, it is more
517 * important to be fast than accurate.
519 tr->buffer_disabled = 0;
520 /* Make the flag seen by readers */
525 * tracing_on - enable tracing buffers
527 * This function enables tracing buffers that may have been
528 * disabled with tracing_off.
530 void tracing_on(void)
532 tracer_tracing_on(&global_trace);
534 EXPORT_SYMBOL_GPL(tracing_on);
537 * __trace_puts - write a constant string into the trace buffer.
538 * @ip: The address of the caller
539 * @str: The constant string to write
540 * @size: The size of the string.
542 int __trace_puts(unsigned long ip, const char *str, int size)
544 struct ring_buffer_event *event;
545 struct ring_buffer *buffer;
546 struct print_entry *entry;
547 unsigned long irq_flags;
551 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
554 pc = preempt_count();
556 if (unlikely(tracing_selftest_running || tracing_disabled))
559 alloc = sizeof(*entry) + size + 2; /* possible \n added */
561 local_save_flags(irq_flags);
562 buffer = global_trace.trace_buffer.buffer;
563 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
568 entry = ring_buffer_event_data(event);
571 memcpy(&entry->buf, str, size);
573 /* Add a newline if necessary */
574 if (entry->buf[size - 1] != '\n') {
575 entry->buf[size] = '\n';
576 entry->buf[size + 1] = '\0';
578 entry->buf[size] = '\0';
580 __buffer_unlock_commit(buffer, event);
581 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
585 EXPORT_SYMBOL_GPL(__trace_puts);
588 * __trace_bputs - write the pointer to a constant string into trace buffer
589 * @ip: The address of the caller
590 * @str: The constant string to write to the buffer to
592 int __trace_bputs(unsigned long ip, const char *str)
594 struct ring_buffer_event *event;
595 struct ring_buffer *buffer;
596 struct bputs_entry *entry;
597 unsigned long irq_flags;
598 int size = sizeof(struct bputs_entry);
601 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
604 pc = preempt_count();
606 if (unlikely(tracing_selftest_running || tracing_disabled))
609 local_save_flags(irq_flags);
610 buffer = global_trace.trace_buffer.buffer;
611 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
616 entry = ring_buffer_event_data(event);
620 __buffer_unlock_commit(buffer, event);
621 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
625 EXPORT_SYMBOL_GPL(__trace_bputs);
627 #ifdef CONFIG_TRACER_SNAPSHOT
629 * trace_snapshot - take a snapshot of the current buffer.
631 * This causes a swap between the snapshot buffer and the current live
632 * tracing buffer. You can use this to take snapshots of the live
633 * trace when some condition is triggered, but continue to trace.
635 * Note, make sure to allocate the snapshot with either
636 * a tracing_snapshot_alloc(), or by doing it manually
637 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
639 * If the snapshot buffer is not allocated, it will stop tracing.
640 * Basically making a permanent snapshot.
642 void tracing_snapshot(void)
644 struct trace_array *tr = &global_trace;
645 struct tracer *tracer = tr->current_trace;
649 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
650 internal_trace_puts("*** snapshot is being ignored ***\n");
654 if (!tr->allocated_snapshot) {
655 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
656 internal_trace_puts("*** stopping trace here! ***\n");
661 /* Note, snapshot can not be used when the tracer uses it */
662 if (tracer->use_max_tr) {
663 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
664 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
668 local_irq_save(flags);
669 update_max_tr(tr, current, smp_processor_id());
670 local_irq_restore(flags);
672 EXPORT_SYMBOL_GPL(tracing_snapshot);
674 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
675 struct trace_buffer *size_buf, int cpu_id);
676 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
678 static int alloc_snapshot(struct trace_array *tr)
682 if (!tr->allocated_snapshot) {
684 /* allocate spare buffer */
685 ret = resize_buffer_duplicate_size(&tr->max_buffer,
686 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
690 tr->allocated_snapshot = true;
696 static void free_snapshot(struct trace_array *tr)
699 * We don't free the ring buffer. instead, resize it because
700 * The max_tr ring buffer has some state (e.g. ring->clock) and
701 * we want preserve it.
703 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
704 set_buffer_entries(&tr->max_buffer, 1);
705 tracing_reset_online_cpus(&tr->max_buffer);
706 tr->allocated_snapshot = false;
710 * tracing_alloc_snapshot - allocate snapshot buffer.
712 * This only allocates the snapshot buffer if it isn't already
713 * allocated - it doesn't also take a snapshot.
715 * This is meant to be used in cases where the snapshot buffer needs
716 * to be set up for events that can't sleep but need to be able to
717 * trigger a snapshot.
719 int tracing_alloc_snapshot(void)
721 struct trace_array *tr = &global_trace;
724 ret = alloc_snapshot(tr);
729 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
732 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
734 * This is similar to trace_snapshot(), but it will allocate the
735 * snapshot buffer if it isn't already allocated. Use this only
736 * where it is safe to sleep, as the allocation may sleep.
738 * This causes a swap between the snapshot buffer and the current live
739 * tracing buffer. You can use this to take snapshots of the live
740 * trace when some condition is triggered, but continue to trace.
742 void tracing_snapshot_alloc(void)
746 ret = tracing_alloc_snapshot();
752 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
754 void tracing_snapshot(void)
756 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
758 EXPORT_SYMBOL_GPL(tracing_snapshot);
759 int tracing_alloc_snapshot(void)
761 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
764 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
765 void tracing_snapshot_alloc(void)
770 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
771 #endif /* CONFIG_TRACER_SNAPSHOT */
773 static void tracer_tracing_off(struct trace_array *tr)
775 if (tr->trace_buffer.buffer)
776 ring_buffer_record_off(tr->trace_buffer.buffer);
778 * This flag is looked at when buffers haven't been allocated
779 * yet, or by some tracers (like irqsoff), that just want to
780 * know if the ring buffer has been disabled, but it can handle
781 * races of where it gets disabled but we still do a record.
782 * As the check is in the fast path of the tracers, it is more
783 * important to be fast than accurate.
785 tr->buffer_disabled = 1;
786 /* Make the flag seen by readers */
791 * tracing_off - turn off tracing buffers
793 * This function stops the tracing buffers from recording data.
794 * It does not disable any overhead the tracers themselves may
795 * be causing. This function simply causes all recording to
796 * the ring buffers to fail.
798 void tracing_off(void)
800 tracer_tracing_off(&global_trace);
802 EXPORT_SYMBOL_GPL(tracing_off);
804 void disable_trace_on_warning(void)
806 if (__disable_trace_on_warning)
811 * tracer_tracing_is_on - show real state of ring buffer enabled
812 * @tr : the trace array to know if ring buffer is enabled
814 * Shows real state of the ring buffer if it is enabled or not.
816 static int tracer_tracing_is_on(struct trace_array *tr)
818 if (tr->trace_buffer.buffer)
819 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
820 return !tr->buffer_disabled;
824 * tracing_is_on - show state of ring buffers enabled
826 int tracing_is_on(void)
828 return tracer_tracing_is_on(&global_trace);
830 EXPORT_SYMBOL_GPL(tracing_is_on);
832 static int __init set_buf_size(char *str)
834 unsigned long buf_size;
838 buf_size = memparse(str, &str);
839 /* nr_entries can not be zero */
842 trace_buf_size = buf_size;
845 __setup("trace_buf_size=", set_buf_size);
847 static int __init set_tracing_thresh(char *str)
849 unsigned long threshold;
854 ret = kstrtoul(str, 0, &threshold);
857 tracing_thresh = threshold * 1000;
860 __setup("tracing_thresh=", set_tracing_thresh);
862 unsigned long nsecs_to_usecs(unsigned long nsecs)
868 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
869 * It uses C(a, b) where 'a' is the enum name and 'b' is the string that
870 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
871 * of strings in the order that the enums were defined.
876 /* These must match the bit postions in trace_iterator_flags */
877 static const char *trace_options[] = {
885 int in_ns; /* is this clock in nanoseconds? */
887 { trace_clock_local, "local", 1 },
888 { trace_clock_global, "global", 1 },
889 { trace_clock_counter, "counter", 0 },
890 { trace_clock_jiffies, "uptime", 0 },
891 { trace_clock, "perf", 1 },
892 { ktime_get_mono_fast_ns, "mono", 1 },
893 { ktime_get_raw_fast_ns, "mono_raw", 1 },
898 * trace_parser_get_init - gets the buffer for trace parser
900 int trace_parser_get_init(struct trace_parser *parser, int size)
902 memset(parser, 0, sizeof(*parser));
904 parser->buffer = kmalloc(size, GFP_KERNEL);
913 * trace_parser_put - frees the buffer for trace parser
915 void trace_parser_put(struct trace_parser *parser)
917 kfree(parser->buffer);
921 * trace_get_user - reads the user input string separated by space
922 * (matched by isspace(ch))
924 * For each string found the 'struct trace_parser' is updated,
925 * and the function returns.
927 * Returns number of bytes read.
929 * See kernel/trace/trace.h for 'struct trace_parser' details.
931 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
932 size_t cnt, loff_t *ppos)
939 trace_parser_clear(parser);
941 ret = get_user(ch, ubuf++);
949 * The parser is not finished with the last write,
950 * continue reading the user input without skipping spaces.
953 /* skip white space */
954 while (cnt && isspace(ch)) {
955 ret = get_user(ch, ubuf++);
962 /* only spaces were written */
972 /* read the non-space input */
973 while (cnt && !isspace(ch)) {
974 if (parser->idx < parser->size - 1)
975 parser->buffer[parser->idx++] = ch;
980 ret = get_user(ch, ubuf++);
987 /* We either got finished input or we have to wait for another call. */
989 parser->buffer[parser->idx] = 0;
990 parser->cont = false;
991 } else if (parser->idx < parser->size - 1) {
993 parser->buffer[parser->idx++] = ch;
1006 /* TODO add a seq_buf_to_buffer() */
1007 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1011 if (trace_seq_used(s) <= s->seq.readpos)
1014 len = trace_seq_used(s) - s->seq.readpos;
1017 memcpy(buf, s->buffer + s->seq.readpos, cnt);
1019 s->seq.readpos += cnt;
1023 unsigned long __read_mostly tracing_thresh;
1025 #ifdef CONFIG_TRACER_MAX_TRACE
1027 * Copy the new maximum trace into the separate maximum-trace
1028 * structure. (this way the maximum trace is permanently saved,
1029 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
1032 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1034 struct trace_buffer *trace_buf = &tr->trace_buffer;
1035 struct trace_buffer *max_buf = &tr->max_buffer;
1036 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1037 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1040 max_buf->time_start = data->preempt_timestamp;
1042 max_data->saved_latency = tr->max_latency;
1043 max_data->critical_start = data->critical_start;
1044 max_data->critical_end = data->critical_end;
1046 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1047 max_data->pid = tsk->pid;
1049 * If tsk == current, then use current_uid(), as that does not use
1050 * RCU. The irq tracer can be called out of RCU scope.
1053 max_data->uid = current_uid();
1055 max_data->uid = task_uid(tsk);
1057 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1058 max_data->policy = tsk->policy;
1059 max_data->rt_priority = tsk->rt_priority;
1061 /* record this tasks comm */
1062 tracing_record_cmdline(tsk);
1066 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1068 * @tsk: the task with the latency
1069 * @cpu: The cpu that initiated the trace.
1071 * Flip the buffers between the @tr and the max_tr and record information
1072 * about which task was the cause of this latency.
1075 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1077 struct ring_buffer *buf;
1082 WARN_ON_ONCE(!irqs_disabled());
1084 if (!tr->allocated_snapshot) {
1085 /* Only the nop tracer should hit this when disabling */
1086 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1090 arch_spin_lock(&tr->max_lock);
1092 /* Inherit the recordable setting from trace_buffer */
1093 if (ring_buffer_record_is_set_on(tr->trace_buffer.buffer))
1094 ring_buffer_record_on(tr->max_buffer.buffer);
1096 ring_buffer_record_off(tr->max_buffer.buffer);
1098 buf = tr->trace_buffer.buffer;
1099 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1100 tr->max_buffer.buffer = buf;
1102 __update_max_tr(tr, tsk, cpu);
1103 arch_spin_unlock(&tr->max_lock);
1107 * update_max_tr_single - only copy one trace over, and reset the rest
1109 * @tsk - task with the latency
1110 * @cpu - the cpu of the buffer to copy.
1112 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1115 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1122 WARN_ON_ONCE(!irqs_disabled());
1123 if (!tr->allocated_snapshot) {
1124 /* Only the nop tracer should hit this when disabling */
1125 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1129 arch_spin_lock(&tr->max_lock);
1131 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1133 if (ret == -EBUSY) {
1135 * We failed to swap the buffer due to a commit taking
1136 * place on this CPU. We fail to record, but we reset
1137 * the max trace buffer (no one writes directly to it)
1138 * and flag that it failed.
1140 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1141 "Failed to swap buffers due to commit in progress\n");
1144 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1146 __update_max_tr(tr, tsk, cpu);
1147 arch_spin_unlock(&tr->max_lock);
1149 #endif /* CONFIG_TRACER_MAX_TRACE */
1151 static int wait_on_pipe(struct trace_iterator *iter, bool full)
1153 /* Iterators are static, they should be filled or empty */
1154 if (trace_buffer_iter(iter, iter->cpu_file))
1157 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1161 #ifdef CONFIG_FTRACE_STARTUP_TEST
1162 static int run_tracer_selftest(struct tracer *type)
1164 struct trace_array *tr = &global_trace;
1165 struct tracer *saved_tracer = tr->current_trace;
1168 if (!type->selftest || tracing_selftest_disabled)
1172 * Run a selftest on this tracer.
1173 * Here we reset the trace buffer, and set the current
1174 * tracer to be this tracer. The tracer can then run some
1175 * internal tracing to verify that everything is in order.
1176 * If we fail, we do not register this tracer.
1178 tracing_reset_online_cpus(&tr->trace_buffer);
1180 tr->current_trace = type;
1182 #ifdef CONFIG_TRACER_MAX_TRACE
1183 if (type->use_max_tr) {
1184 /* If we expanded the buffers, make sure the max is expanded too */
1185 if (ring_buffer_expanded)
1186 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1187 RING_BUFFER_ALL_CPUS);
1188 tr->allocated_snapshot = true;
1192 /* the test is responsible for initializing and enabling */
1193 pr_info("Testing tracer %s: ", type->name);
1194 ret = type->selftest(type, tr);
1195 /* the test is responsible for resetting too */
1196 tr->current_trace = saved_tracer;
1198 printk(KERN_CONT "FAILED!\n");
1199 /* Add the warning after printing 'FAILED' */
1203 /* Only reset on passing, to avoid touching corrupted buffers */
1204 tracing_reset_online_cpus(&tr->trace_buffer);
1206 #ifdef CONFIG_TRACER_MAX_TRACE
1207 if (type->use_max_tr) {
1208 tr->allocated_snapshot = false;
1210 /* Shrink the max buffer again */
1211 if (ring_buffer_expanded)
1212 ring_buffer_resize(tr->max_buffer.buffer, 1,
1213 RING_BUFFER_ALL_CPUS);
1217 printk(KERN_CONT "PASSED\n");
1221 static inline int run_tracer_selftest(struct tracer *type)
1225 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1227 static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1229 static void __init apply_trace_boot_options(void);
1232 * register_tracer - register a tracer with the ftrace system.
1233 * @type - the plugin for the tracer
1235 * Register a new plugin tracer.
1237 int __init register_tracer(struct tracer *type)
1243 pr_info("Tracer must have a name\n");
1247 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1248 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1252 mutex_lock(&trace_types_lock);
1254 tracing_selftest_running = true;
1256 for (t = trace_types; t; t = t->next) {
1257 if (strcmp(type->name, t->name) == 0) {
1259 pr_info("Tracer %s already registered\n",
1266 if (!type->set_flag)
1267 type->set_flag = &dummy_set_flag;
1269 type->flags = &dummy_tracer_flags;
1271 if (!type->flags->opts)
1272 type->flags->opts = dummy_tracer_opt;
1274 ret = run_tracer_selftest(type);
1278 type->next = trace_types;
1280 add_tracer_options(&global_trace, type);
1283 tracing_selftest_running = false;
1284 mutex_unlock(&trace_types_lock);
1286 if (ret || !default_bootup_tracer)
1289 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1292 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1293 /* Do we want this tracer to start on bootup? */
1294 tracing_set_tracer(&global_trace, type->name);
1295 default_bootup_tracer = NULL;
1297 apply_trace_boot_options();
1299 /* disable other selftests, since this will break it. */
1300 tracing_selftest_disabled = true;
1301 #ifdef CONFIG_FTRACE_STARTUP_TEST
1302 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1310 void tracing_reset(struct trace_buffer *buf, int cpu)
1312 struct ring_buffer *buffer = buf->buffer;
1317 ring_buffer_record_disable(buffer);
1319 /* Make sure all commits have finished */
1320 synchronize_sched();
1321 ring_buffer_reset_cpu(buffer, cpu);
1323 ring_buffer_record_enable(buffer);
1326 void tracing_reset_online_cpus(struct trace_buffer *buf)
1328 struct ring_buffer *buffer = buf->buffer;
1334 ring_buffer_record_disable(buffer);
1336 /* Make sure all commits have finished */
1337 synchronize_sched();
1339 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1341 for_each_online_cpu(cpu)
1342 ring_buffer_reset_cpu(buffer, cpu);
1344 ring_buffer_record_enable(buffer);
1347 /* Must have trace_types_lock held */
1348 void tracing_reset_all_online_cpus(void)
1350 struct trace_array *tr;
1352 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1353 tracing_reset_online_cpus(&tr->trace_buffer);
1354 #ifdef CONFIG_TRACER_MAX_TRACE
1355 tracing_reset_online_cpus(&tr->max_buffer);
1360 #define SAVED_CMDLINES_DEFAULT 128
1361 #define NO_CMDLINE_MAP UINT_MAX
1362 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1363 struct saved_cmdlines_buffer {
1364 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1365 unsigned *map_cmdline_to_pid;
1366 unsigned cmdline_num;
1368 char *saved_cmdlines;
1370 static struct saved_cmdlines_buffer *savedcmd;
1372 static inline char *get_saved_cmdlines(int idx)
1374 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1377 static inline void set_cmdline(int idx, const char *cmdline)
1379 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1382 static int allocate_cmdlines_buffer(unsigned int val,
1383 struct saved_cmdlines_buffer *s)
1385 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1387 if (!s->map_cmdline_to_pid)
1390 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1391 if (!s->saved_cmdlines) {
1392 kfree(s->map_cmdline_to_pid);
1397 s->cmdline_num = val;
1398 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1399 sizeof(s->map_pid_to_cmdline));
1400 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1401 val * sizeof(*s->map_cmdline_to_pid));
1406 static int trace_create_savedcmd(void)
1410 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1414 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1424 int is_tracing_stopped(void)
1426 return global_trace.stop_count;
1430 * tracing_start - quick start of the tracer
1432 * If tracing is enabled but was stopped by tracing_stop,
1433 * this will start the tracer back up.
1435 void tracing_start(void)
1437 struct ring_buffer *buffer;
1438 unsigned long flags;
1440 if (tracing_disabled)
1443 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1444 if (--global_trace.stop_count) {
1445 if (global_trace.stop_count < 0) {
1446 /* Someone screwed up their debugging */
1448 global_trace.stop_count = 0;
1453 /* Prevent the buffers from switching */
1454 arch_spin_lock(&global_trace.max_lock);
1456 buffer = global_trace.trace_buffer.buffer;
1458 ring_buffer_record_enable(buffer);
1460 #ifdef CONFIG_TRACER_MAX_TRACE
1461 buffer = global_trace.max_buffer.buffer;
1463 ring_buffer_record_enable(buffer);
1466 arch_spin_unlock(&global_trace.max_lock);
1469 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1472 static void tracing_start_tr(struct trace_array *tr)
1474 struct ring_buffer *buffer;
1475 unsigned long flags;
1477 if (tracing_disabled)
1480 /* If global, we need to also start the max tracer */
1481 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1482 return tracing_start();
1484 raw_spin_lock_irqsave(&tr->start_lock, flags);
1486 if (--tr->stop_count) {
1487 if (tr->stop_count < 0) {
1488 /* Someone screwed up their debugging */
1495 buffer = tr->trace_buffer.buffer;
1497 ring_buffer_record_enable(buffer);
1500 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1504 * tracing_stop - quick stop of the tracer
1506 * Light weight way to stop tracing. Use in conjunction with
1509 void tracing_stop(void)
1511 struct ring_buffer *buffer;
1512 unsigned long flags;
1514 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1515 if (global_trace.stop_count++)
1518 /* Prevent the buffers from switching */
1519 arch_spin_lock(&global_trace.max_lock);
1521 buffer = global_trace.trace_buffer.buffer;
1523 ring_buffer_record_disable(buffer);
1525 #ifdef CONFIG_TRACER_MAX_TRACE
1526 buffer = global_trace.max_buffer.buffer;
1528 ring_buffer_record_disable(buffer);
1531 arch_spin_unlock(&global_trace.max_lock);
1534 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1537 static void tracing_stop_tr(struct trace_array *tr)
1539 struct ring_buffer *buffer;
1540 unsigned long flags;
1542 /* If global, we need to also stop the max tracer */
1543 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1544 return tracing_stop();
1546 raw_spin_lock_irqsave(&tr->start_lock, flags);
1547 if (tr->stop_count++)
1550 buffer = tr->trace_buffer.buffer;
1552 ring_buffer_record_disable(buffer);
1555 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1558 void trace_stop_cmdline_recording(void);
1560 static int trace_save_cmdline(struct task_struct *tsk)
1564 /* treat recording of idle task as a success */
1568 tpid = tsk->pid & (PID_MAX_DEFAULT - 1);
1571 * It's not the end of the world if we don't get
1572 * the lock, but we also don't want to spin
1573 * nor do we want to disable interrupts,
1574 * so if we miss here, then better luck next time.
1576 if (!arch_spin_trylock(&trace_cmdline_lock))
1579 idx = savedcmd->map_pid_to_cmdline[tpid];
1580 if (idx == NO_CMDLINE_MAP) {
1581 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1583 savedcmd->map_pid_to_cmdline[tpid] = idx;
1584 savedcmd->cmdline_idx = idx;
1587 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1588 set_cmdline(idx, tsk->comm);
1590 arch_spin_unlock(&trace_cmdline_lock);
1595 static void __trace_find_cmdline(int pid, char comm[])
1601 strcpy(comm, "<idle>");
1605 if (WARN_ON_ONCE(pid < 0)) {
1606 strcpy(comm, "<XXX>");
1610 tpid = pid & (PID_MAX_DEFAULT - 1);
1611 map = savedcmd->map_pid_to_cmdline[tpid];
1612 if (map != NO_CMDLINE_MAP) {
1613 tpid = savedcmd->map_cmdline_to_pid[map];
1615 strlcpy(comm, get_saved_cmdlines(map), TASK_COMM_LEN);
1619 strcpy(comm, "<...>");
1622 void trace_find_cmdline(int pid, char comm[])
1625 arch_spin_lock(&trace_cmdline_lock);
1627 __trace_find_cmdline(pid, comm);
1629 arch_spin_unlock(&trace_cmdline_lock);
1633 void tracing_record_cmdline(struct task_struct *tsk)
1635 if (!__this_cpu_read(trace_cmdline_save))
1638 if (trace_save_cmdline(tsk))
1639 __this_cpu_write(trace_cmdline_save, false);
1643 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1646 struct task_struct *tsk = current;
1648 entry->preempt_count = pc & 0xff;
1649 entry->pid = (tsk) ? tsk->pid : 0;
1651 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1652 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1654 TRACE_FLAG_IRQS_NOSUPPORT |
1656 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1657 ((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) |
1658 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1659 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1661 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1663 struct ring_buffer_event *
1664 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1667 unsigned long flags, int pc)
1669 struct ring_buffer_event *event;
1671 event = ring_buffer_lock_reserve(buffer, len);
1672 if (event != NULL) {
1673 struct trace_entry *ent = ring_buffer_event_data(event);
1675 tracing_generic_entry_update(ent, flags, pc);
1683 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1685 __this_cpu_write(trace_cmdline_save, true);
1686 ring_buffer_unlock_commit(buffer, event);
1689 void trace_buffer_unlock_commit(struct trace_array *tr,
1690 struct ring_buffer *buffer,
1691 struct ring_buffer_event *event,
1692 unsigned long flags, int pc)
1694 __buffer_unlock_commit(buffer, event);
1696 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
1697 ftrace_trace_userstack(tr, buffer, flags, pc);
1699 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1701 static struct ring_buffer *temp_buffer;
1703 struct ring_buffer_event *
1704 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1705 struct trace_event_file *trace_file,
1706 int type, unsigned long len,
1707 unsigned long flags, int pc)
1709 struct ring_buffer_event *entry;
1711 *current_rb = trace_file->tr->trace_buffer.buffer;
1712 entry = trace_buffer_lock_reserve(*current_rb,
1713 type, len, flags, pc);
1715 * If tracing is off, but we have triggers enabled
1716 * we still need to look at the event data. Use the temp_buffer
1717 * to store the trace event for the tigger to use. It's recusive
1718 * safe and will not be recorded anywhere.
1720 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
1721 *current_rb = temp_buffer;
1722 entry = trace_buffer_lock_reserve(*current_rb,
1723 type, len, flags, pc);
1727 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1729 struct ring_buffer_event *
1730 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1731 int type, unsigned long len,
1732 unsigned long flags, int pc)
1734 *current_rb = global_trace.trace_buffer.buffer;
1735 return trace_buffer_lock_reserve(*current_rb,
1736 type, len, flags, pc);
1738 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1740 void trace_buffer_unlock_commit_regs(struct trace_array *tr,
1741 struct ring_buffer *buffer,
1742 struct ring_buffer_event *event,
1743 unsigned long flags, int pc,
1744 struct pt_regs *regs)
1746 __buffer_unlock_commit(buffer, event);
1749 * If regs is not set, then skip the following callers:
1750 * trace_buffer_unlock_commit_regs
1751 * event_trigger_unlock_commit
1752 * trace_event_buffer_commit
1753 * trace_event_raw_event_sched_switch
1754 * Note, we can still get here via blktrace, wakeup tracer
1755 * and mmiotrace, but that's ok if they lose a function or
1756 * two. They are that meaningful.
1758 ftrace_trace_stack(tr, buffer, flags, regs ? 0 : 4, pc, regs);
1759 ftrace_trace_userstack(tr, buffer, flags, pc);
1761 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1763 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1764 struct ring_buffer_event *event)
1766 ring_buffer_discard_commit(buffer, event);
1768 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1771 trace_function(struct trace_array *tr,
1772 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1775 struct trace_event_call *call = &event_function;
1776 struct ring_buffer *buffer = tr->trace_buffer.buffer;
1777 struct ring_buffer_event *event;
1778 struct ftrace_entry *entry;
1780 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1784 entry = ring_buffer_event_data(event);
1786 entry->parent_ip = parent_ip;
1788 if (!call_filter_check_discard(call, entry, buffer, event))
1789 __buffer_unlock_commit(buffer, event);
1792 #ifdef CONFIG_STACKTRACE
1794 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1795 struct ftrace_stack {
1796 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1799 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1800 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1802 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1803 unsigned long flags,
1804 int skip, int pc, struct pt_regs *regs)
1806 struct trace_event_call *call = &event_kernel_stack;
1807 struct ring_buffer_event *event;
1808 struct stack_entry *entry;
1809 struct stack_trace trace;
1811 int size = FTRACE_STACK_ENTRIES;
1813 trace.nr_entries = 0;
1817 * Add two, for this function and the call to save_stack_trace()
1818 * If regs is set, then these functions will not be in the way.
1824 * Since events can happen in NMIs there's no safe way to
1825 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1826 * or NMI comes in, it will just have to use the default
1827 * FTRACE_STACK_SIZE.
1829 preempt_disable_notrace();
1831 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1833 * We don't need any atomic variables, just a barrier.
1834 * If an interrupt comes in, we don't care, because it would
1835 * have exited and put the counter back to what we want.
1836 * We just need a barrier to keep gcc from moving things
1840 if (use_stack == 1) {
1841 trace.entries = this_cpu_ptr(ftrace_stack.calls);
1842 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1845 save_stack_trace_regs(regs, &trace);
1847 save_stack_trace(&trace);
1849 if (trace.nr_entries > size)
1850 size = trace.nr_entries;
1852 /* From now on, use_stack is a boolean */
1855 size *= sizeof(unsigned long);
1857 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1858 (sizeof(*entry) - sizeof(entry->caller)) + size,
1862 entry = ring_buffer_event_data(event);
1864 memset(&entry->caller, 0, size);
1867 memcpy(&entry->caller, trace.entries,
1868 trace.nr_entries * sizeof(unsigned long));
1870 trace.max_entries = FTRACE_STACK_ENTRIES;
1871 trace.entries = entry->caller;
1873 save_stack_trace_regs(regs, &trace);
1875 save_stack_trace(&trace);
1878 entry->size = trace.nr_entries;
1880 if (!call_filter_check_discard(call, entry, buffer, event))
1881 __buffer_unlock_commit(buffer, event);
1884 /* Again, don't let gcc optimize things here */
1886 __this_cpu_dec(ftrace_stack_reserve);
1887 preempt_enable_notrace();
1891 static inline void ftrace_trace_stack(struct trace_array *tr,
1892 struct ring_buffer *buffer,
1893 unsigned long flags,
1894 int skip, int pc, struct pt_regs *regs)
1896 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
1899 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1902 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1905 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1909 * trace_dump_stack - record a stack back trace in the trace buffer
1910 * @skip: Number of functions to skip (helper handlers)
1912 void trace_dump_stack(int skip)
1914 unsigned long flags;
1916 if (tracing_disabled || tracing_selftest_running)
1919 local_save_flags(flags);
1922 * Skip 3 more, seems to get us at the caller of
1926 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1927 flags, skip, preempt_count(), NULL);
1930 static DEFINE_PER_CPU(int, user_stack_count);
1933 ftrace_trace_userstack(struct trace_array *tr,
1934 struct ring_buffer *buffer, unsigned long flags, int pc)
1936 struct trace_event_call *call = &event_user_stack;
1937 struct ring_buffer_event *event;
1938 struct userstack_entry *entry;
1939 struct stack_trace trace;
1941 if (!(tr->trace_flags & TRACE_ITER_USERSTACKTRACE))
1945 * NMIs can not handle page faults, even with fix ups.
1946 * The save user stack can (and often does) fault.
1948 if (unlikely(in_nmi()))
1952 * prevent recursion, since the user stack tracing may
1953 * trigger other kernel events.
1956 if (__this_cpu_read(user_stack_count))
1959 __this_cpu_inc(user_stack_count);
1961 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1962 sizeof(*entry), flags, pc);
1964 goto out_drop_count;
1965 entry = ring_buffer_event_data(event);
1967 entry->tgid = current->tgid;
1968 memset(&entry->caller, 0, sizeof(entry->caller));
1970 trace.nr_entries = 0;
1971 trace.max_entries = FTRACE_STACK_ENTRIES;
1973 trace.entries = entry->caller;
1975 save_stack_trace_user(&trace);
1976 if (!call_filter_check_discard(call, entry, buffer, event))
1977 __buffer_unlock_commit(buffer, event);
1980 __this_cpu_dec(user_stack_count);
1986 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1988 ftrace_trace_userstack(tr, flags, preempt_count());
1992 #endif /* CONFIG_STACKTRACE */
1994 /* created for use with alloc_percpu */
1995 struct trace_buffer_struct {
1996 char buffer[TRACE_BUF_SIZE];
1999 static struct trace_buffer_struct *trace_percpu_buffer;
2000 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
2001 static struct trace_buffer_struct *trace_percpu_irq_buffer;
2002 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
2005 * The buffer used is dependent on the context. There is a per cpu
2006 * buffer for normal context, softirq contex, hard irq context and
2007 * for NMI context. Thise allows for lockless recording.
2009 * Note, if the buffers failed to be allocated, then this returns NULL
2011 static char *get_trace_buf(void)
2013 struct trace_buffer_struct *percpu_buffer;
2016 * If we have allocated per cpu buffers, then we do not
2017 * need to do any locking.
2020 percpu_buffer = trace_percpu_nmi_buffer;
2022 percpu_buffer = trace_percpu_irq_buffer;
2023 else if (in_softirq())
2024 percpu_buffer = trace_percpu_sirq_buffer;
2026 percpu_buffer = trace_percpu_buffer;
2031 return this_cpu_ptr(&percpu_buffer->buffer[0]);
2034 static int alloc_percpu_trace_buffer(void)
2036 struct trace_buffer_struct *buffers;
2037 struct trace_buffer_struct *sirq_buffers;
2038 struct trace_buffer_struct *irq_buffers;
2039 struct trace_buffer_struct *nmi_buffers;
2041 buffers = alloc_percpu(struct trace_buffer_struct);
2045 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
2049 irq_buffers = alloc_percpu(struct trace_buffer_struct);
2053 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
2057 trace_percpu_buffer = buffers;
2058 trace_percpu_sirq_buffer = sirq_buffers;
2059 trace_percpu_irq_buffer = irq_buffers;
2060 trace_percpu_nmi_buffer = nmi_buffers;
2065 free_percpu(irq_buffers);
2067 free_percpu(sirq_buffers);
2069 free_percpu(buffers);
2071 WARN(1, "Could not allocate percpu trace_printk buffer");
2075 static int buffers_allocated;
2077 void trace_printk_init_buffers(void)
2079 if (buffers_allocated)
2082 if (alloc_percpu_trace_buffer())
2085 /* trace_printk() is for debug use only. Don't use it in production. */
2088 pr_warning("**********************************************************\n");
2089 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2090 pr_warning("** **\n");
2091 pr_warning("** trace_printk() being used. Allocating extra memory. **\n");
2092 pr_warning("** **\n");
2093 pr_warning("** This means that this is a DEBUG kernel and it is **\n");
2094 pr_warning("** unsafe for production use. **\n");
2095 pr_warning("** **\n");
2096 pr_warning("** If you see this message and you are not debugging **\n");
2097 pr_warning("** the kernel, report this immediately to your vendor! **\n");
2098 pr_warning("** **\n");
2099 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2100 pr_warning("**********************************************************\n");
2102 /* Expand the buffers to set size */
2103 tracing_update_buffers();
2105 buffers_allocated = 1;
2108 * trace_printk_init_buffers() can be called by modules.
2109 * If that happens, then we need to start cmdline recording
2110 * directly here. If the global_trace.buffer is already
2111 * allocated here, then this was called by module code.
2113 if (global_trace.trace_buffer.buffer)
2114 tracing_start_cmdline_record();
2117 void trace_printk_start_comm(void)
2119 /* Start tracing comms if trace printk is set */
2120 if (!buffers_allocated)
2122 tracing_start_cmdline_record();
2125 static void trace_printk_start_stop_comm(int enabled)
2127 if (!buffers_allocated)
2131 tracing_start_cmdline_record();
2133 tracing_stop_cmdline_record();
2137 * trace_vbprintk - write binary msg to tracing buffer
2140 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2142 struct trace_event_call *call = &event_bprint;
2143 struct ring_buffer_event *event;
2144 struct ring_buffer *buffer;
2145 struct trace_array *tr = &global_trace;
2146 struct bprint_entry *entry;
2147 unsigned long flags;
2149 int len = 0, size, pc;
2151 if (unlikely(tracing_selftest_running || tracing_disabled))
2154 /* Don't pollute graph traces with trace_vprintk internals */
2155 pause_graph_tracing();
2157 pc = preempt_count();
2158 preempt_disable_notrace();
2160 tbuffer = get_trace_buf();
2166 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2168 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2171 local_save_flags(flags);
2172 size = sizeof(*entry) + sizeof(u32) * len;
2173 buffer = tr->trace_buffer.buffer;
2174 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2178 entry = ring_buffer_event_data(event);
2182 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2183 if (!call_filter_check_discard(call, entry, buffer, event)) {
2184 __buffer_unlock_commit(buffer, event);
2185 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
2189 preempt_enable_notrace();
2190 unpause_graph_tracing();
2194 EXPORT_SYMBOL_GPL(trace_vbprintk);
2198 __trace_array_vprintk(struct ring_buffer *buffer,
2199 unsigned long ip, const char *fmt, va_list args)
2201 struct trace_event_call *call = &event_print;
2202 struct ring_buffer_event *event;
2203 int len = 0, size, pc;
2204 struct print_entry *entry;
2205 unsigned long flags;
2208 if (tracing_disabled || tracing_selftest_running)
2211 /* Don't pollute graph traces with trace_vprintk internals */
2212 pause_graph_tracing();
2214 pc = preempt_count();
2215 preempt_disable_notrace();
2218 tbuffer = get_trace_buf();
2224 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2226 local_save_flags(flags);
2227 size = sizeof(*entry) + len + 1;
2228 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2232 entry = ring_buffer_event_data(event);
2235 memcpy(&entry->buf, tbuffer, len + 1);
2236 if (!call_filter_check_discard(call, entry, buffer, event)) {
2237 __buffer_unlock_commit(buffer, event);
2238 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
2241 preempt_enable_notrace();
2242 unpause_graph_tracing();
2248 int trace_array_vprintk(struct trace_array *tr,
2249 unsigned long ip, const char *fmt, va_list args)
2251 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2255 int trace_array_printk(struct trace_array *tr,
2256 unsigned long ip, const char *fmt, ...)
2261 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
2268 ret = trace_array_vprintk(tr, ip, fmt, ap);
2274 int trace_array_printk_buf(struct ring_buffer *buffer,
2275 unsigned long ip, const char *fmt, ...)
2280 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
2284 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2290 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2292 return trace_array_vprintk(&global_trace, ip, fmt, args);
2294 EXPORT_SYMBOL_GPL(trace_vprintk);
2296 static void trace_iterator_increment(struct trace_iterator *iter)
2298 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2302 ring_buffer_read(buf_iter, NULL);
2305 static struct trace_entry *
2306 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2307 unsigned long *lost_events)
2309 struct ring_buffer_event *event;
2310 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2313 event = ring_buffer_iter_peek(buf_iter, ts);
2315 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2319 iter->ent_size = ring_buffer_event_length(event);
2320 return ring_buffer_event_data(event);
2326 static struct trace_entry *
2327 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2328 unsigned long *missing_events, u64 *ent_ts)
2330 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2331 struct trace_entry *ent, *next = NULL;
2332 unsigned long lost_events = 0, next_lost = 0;
2333 int cpu_file = iter->cpu_file;
2334 u64 next_ts = 0, ts;
2340 * If we are in a per_cpu trace file, don't bother by iterating over
2341 * all cpu and peek directly.
2343 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2344 if (ring_buffer_empty_cpu(buffer, cpu_file))
2346 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2348 *ent_cpu = cpu_file;
2353 for_each_tracing_cpu(cpu) {
2355 if (ring_buffer_empty_cpu(buffer, cpu))
2358 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2361 * Pick the entry with the smallest timestamp:
2363 if (ent && (!next || ts < next_ts)) {
2367 next_lost = lost_events;
2368 next_size = iter->ent_size;
2372 iter->ent_size = next_size;
2375 *ent_cpu = next_cpu;
2381 *missing_events = next_lost;
2386 /* Find the next real entry, without updating the iterator itself */
2387 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2388 int *ent_cpu, u64 *ent_ts)
2390 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2393 /* Find the next real entry, and increment the iterator to the next entry */
2394 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2396 iter->ent = __find_next_entry(iter, &iter->cpu,
2397 &iter->lost_events, &iter->ts);
2400 trace_iterator_increment(iter);
2402 return iter->ent ? iter : NULL;
2405 static void trace_consume(struct trace_iterator *iter)
2407 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2408 &iter->lost_events);
2411 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2413 struct trace_iterator *iter = m->private;
2417 WARN_ON_ONCE(iter->leftover);
2421 /* can't go backwards */
2426 ent = trace_find_next_entry_inc(iter);
2430 while (ent && iter->idx < i)
2431 ent = trace_find_next_entry_inc(iter);
2438 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2440 struct ring_buffer_event *event;
2441 struct ring_buffer_iter *buf_iter;
2442 unsigned long entries = 0;
2445 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2447 buf_iter = trace_buffer_iter(iter, cpu);
2451 ring_buffer_iter_reset(buf_iter);
2454 * We could have the case with the max latency tracers
2455 * that a reset never took place on a cpu. This is evident
2456 * by the timestamp being before the start of the buffer.
2458 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2459 if (ts >= iter->trace_buffer->time_start)
2462 ring_buffer_read(buf_iter, NULL);
2465 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2469 * The current tracer is copied to avoid a global locking
2472 static void *s_start(struct seq_file *m, loff_t *pos)
2474 struct trace_iterator *iter = m->private;
2475 struct trace_array *tr = iter->tr;
2476 int cpu_file = iter->cpu_file;
2482 * copy the tracer to avoid using a global lock all around.
2483 * iter->trace is a copy of current_trace, the pointer to the
2484 * name may be used instead of a strcmp(), as iter->trace->name
2485 * will point to the same string as current_trace->name.
2487 mutex_lock(&trace_types_lock);
2488 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2489 *iter->trace = *tr->current_trace;
2490 mutex_unlock(&trace_types_lock);
2492 #ifdef CONFIG_TRACER_MAX_TRACE
2493 if (iter->snapshot && iter->trace->use_max_tr)
2494 return ERR_PTR(-EBUSY);
2497 if (*pos != iter->pos) {
2502 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2503 for_each_tracing_cpu(cpu)
2504 tracing_iter_reset(iter, cpu);
2506 tracing_iter_reset(iter, cpu_file);
2509 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2514 * If we overflowed the seq_file before, then we want
2515 * to just reuse the trace_seq buffer again.
2521 p = s_next(m, p, &l);
2525 trace_event_read_lock();
2526 trace_access_lock(cpu_file);
2530 static void s_stop(struct seq_file *m, void *p)
2532 struct trace_iterator *iter = m->private;
2534 #ifdef CONFIG_TRACER_MAX_TRACE
2535 if (iter->snapshot && iter->trace->use_max_tr)
2539 trace_access_unlock(iter->cpu_file);
2540 trace_event_read_unlock();
2544 get_total_entries(struct trace_buffer *buf,
2545 unsigned long *total, unsigned long *entries)
2547 unsigned long count;
2553 for_each_tracing_cpu(cpu) {
2554 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2556 * If this buffer has skipped entries, then we hold all
2557 * entries for the trace and we need to ignore the
2558 * ones before the time stamp.
2560 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2561 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2562 /* total is the same as the entries */
2566 ring_buffer_overrun_cpu(buf->buffer, cpu);
2571 static void print_lat_help_header(struct seq_file *m)
2573 seq_puts(m, "# _------=> CPU# \n"
2574 "# / _-----=> irqs-off \n"
2575 "# | / _----=> need-resched \n"
2576 "# || / _---=> hardirq/softirq \n"
2577 "# ||| / _--=> preempt-depth \n"
2579 "# cmd pid ||||| time | caller \n"
2580 "# \\ / ||||| \\ | / \n");
2583 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2585 unsigned long total;
2586 unsigned long entries;
2588 get_total_entries(buf, &total, &entries);
2589 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2590 entries, total, num_online_cpus());
2594 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2596 print_event_info(buf, m);
2597 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2601 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2603 print_event_info(buf, m);
2604 seq_puts(m, "# _-----=> irqs-off\n"
2605 "# / _----=> need-resched\n"
2606 "# | / _---=> hardirq/softirq\n"
2607 "# || / _--=> preempt-depth\n"
2609 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
2610 "# | | | |||| | |\n");
2614 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2616 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
2617 struct trace_buffer *buf = iter->trace_buffer;
2618 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2619 struct tracer *type = iter->trace;
2620 unsigned long entries;
2621 unsigned long total;
2622 const char *name = "preemption";
2626 get_total_entries(buf, &total, &entries);
2628 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2630 seq_puts(m, "# -----------------------------------"
2631 "---------------------------------\n");
2632 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2633 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2634 nsecs_to_usecs(data->saved_latency),
2638 #if defined(CONFIG_PREEMPT_NONE)
2640 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2642 #elif defined(CONFIG_PREEMPT)
2647 /* These are reserved for later use */
2650 seq_printf(m, " #P:%d)\n", num_online_cpus());
2654 seq_puts(m, "# -----------------\n");
2655 seq_printf(m, "# | task: %.16s-%d "
2656 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2657 data->comm, data->pid,
2658 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2659 data->policy, data->rt_priority);
2660 seq_puts(m, "# -----------------\n");
2662 if (data->critical_start) {
2663 seq_puts(m, "# => started at: ");
2664 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2665 trace_print_seq(m, &iter->seq);
2666 seq_puts(m, "\n# => ended at: ");
2667 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2668 trace_print_seq(m, &iter->seq);
2669 seq_puts(m, "\n#\n");
2675 static void test_cpu_buff_start(struct trace_iterator *iter)
2677 struct trace_seq *s = &iter->seq;
2678 struct trace_array *tr = iter->tr;
2680 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
2683 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2686 if (iter->started && cpumask_test_cpu(iter->cpu, iter->started))
2689 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2693 cpumask_set_cpu(iter->cpu, iter->started);
2695 /* Don't print started cpu buffer for the first entry of the trace */
2697 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2701 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2703 struct trace_array *tr = iter->tr;
2704 struct trace_seq *s = &iter->seq;
2705 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
2706 struct trace_entry *entry;
2707 struct trace_event *event;
2711 test_cpu_buff_start(iter);
2713 event = ftrace_find_event(entry->type);
2715 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
2716 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2717 trace_print_lat_context(iter);
2719 trace_print_context(iter);
2722 if (trace_seq_has_overflowed(s))
2723 return TRACE_TYPE_PARTIAL_LINE;
2726 return event->funcs->trace(iter, sym_flags, event);
2728 trace_seq_printf(s, "Unknown type %d\n", entry->type);
2730 return trace_handle_return(s);
2733 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2735 struct trace_array *tr = iter->tr;
2736 struct trace_seq *s = &iter->seq;
2737 struct trace_entry *entry;
2738 struct trace_event *event;
2742 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
2743 trace_seq_printf(s, "%d %d %llu ",
2744 entry->pid, iter->cpu, iter->ts);
2746 if (trace_seq_has_overflowed(s))
2747 return TRACE_TYPE_PARTIAL_LINE;
2749 event = ftrace_find_event(entry->type);
2751 return event->funcs->raw(iter, 0, event);
2753 trace_seq_printf(s, "%d ?\n", entry->type);
2755 return trace_handle_return(s);
2758 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2760 struct trace_array *tr = iter->tr;
2761 struct trace_seq *s = &iter->seq;
2762 unsigned char newline = '\n';
2763 struct trace_entry *entry;
2764 struct trace_event *event;
2768 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
2769 SEQ_PUT_HEX_FIELD(s, entry->pid);
2770 SEQ_PUT_HEX_FIELD(s, iter->cpu);
2771 SEQ_PUT_HEX_FIELD(s, iter->ts);
2772 if (trace_seq_has_overflowed(s))
2773 return TRACE_TYPE_PARTIAL_LINE;
2776 event = ftrace_find_event(entry->type);
2778 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2779 if (ret != TRACE_TYPE_HANDLED)
2783 SEQ_PUT_FIELD(s, newline);
2785 return trace_handle_return(s);
2788 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2790 struct trace_array *tr = iter->tr;
2791 struct trace_seq *s = &iter->seq;
2792 struct trace_entry *entry;
2793 struct trace_event *event;
2797 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
2798 SEQ_PUT_FIELD(s, entry->pid);
2799 SEQ_PUT_FIELD(s, iter->cpu);
2800 SEQ_PUT_FIELD(s, iter->ts);
2801 if (trace_seq_has_overflowed(s))
2802 return TRACE_TYPE_PARTIAL_LINE;
2805 event = ftrace_find_event(entry->type);
2806 return event ? event->funcs->binary(iter, 0, event) :
2810 int trace_empty(struct trace_iterator *iter)
2812 struct ring_buffer_iter *buf_iter;
2815 /* If we are looking at one CPU buffer, only check that one */
2816 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2817 cpu = iter->cpu_file;
2818 buf_iter = trace_buffer_iter(iter, cpu);
2820 if (!ring_buffer_iter_empty(buf_iter))
2823 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2829 for_each_tracing_cpu(cpu) {
2830 buf_iter = trace_buffer_iter(iter, cpu);
2832 if (!ring_buffer_iter_empty(buf_iter))
2835 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2843 /* Called with trace_event_read_lock() held. */
2844 enum print_line_t print_trace_line(struct trace_iterator *iter)
2846 struct trace_array *tr = iter->tr;
2847 unsigned long trace_flags = tr->trace_flags;
2848 enum print_line_t ret;
2850 if (iter->lost_events) {
2851 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2852 iter->cpu, iter->lost_events);
2853 if (trace_seq_has_overflowed(&iter->seq))
2854 return TRACE_TYPE_PARTIAL_LINE;
2857 if (iter->trace && iter->trace->print_line) {
2858 ret = iter->trace->print_line(iter);
2859 if (ret != TRACE_TYPE_UNHANDLED)
2863 if (iter->ent->type == TRACE_BPUTS &&
2864 trace_flags & TRACE_ITER_PRINTK &&
2865 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2866 return trace_print_bputs_msg_only(iter);
2868 if (iter->ent->type == TRACE_BPRINT &&
2869 trace_flags & TRACE_ITER_PRINTK &&
2870 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2871 return trace_print_bprintk_msg_only(iter);
2873 if (iter->ent->type == TRACE_PRINT &&
2874 trace_flags & TRACE_ITER_PRINTK &&
2875 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2876 return trace_print_printk_msg_only(iter);
2878 if (trace_flags & TRACE_ITER_BIN)
2879 return print_bin_fmt(iter);
2881 if (trace_flags & TRACE_ITER_HEX)
2882 return print_hex_fmt(iter);
2884 if (trace_flags & TRACE_ITER_RAW)
2885 return print_raw_fmt(iter);
2887 return print_trace_fmt(iter);
2890 void trace_latency_header(struct seq_file *m)
2892 struct trace_iterator *iter = m->private;
2893 struct trace_array *tr = iter->tr;
2895 /* print nothing if the buffers are empty */
2896 if (trace_empty(iter))
2899 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2900 print_trace_header(m, iter);
2902 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
2903 print_lat_help_header(m);
2906 void trace_default_header(struct seq_file *m)
2908 struct trace_iterator *iter = m->private;
2909 struct trace_array *tr = iter->tr;
2910 unsigned long trace_flags = tr->trace_flags;
2912 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2915 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2916 /* print nothing if the buffers are empty */
2917 if (trace_empty(iter))
2919 print_trace_header(m, iter);
2920 if (!(trace_flags & TRACE_ITER_VERBOSE))
2921 print_lat_help_header(m);
2923 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2924 if (trace_flags & TRACE_ITER_IRQ_INFO)
2925 print_func_help_header_irq(iter->trace_buffer, m);
2927 print_func_help_header(iter->trace_buffer, m);
2932 static void test_ftrace_alive(struct seq_file *m)
2934 if (!ftrace_is_dead())
2936 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
2937 "# MAY BE MISSING FUNCTION EVENTS\n");
2940 #ifdef CONFIG_TRACER_MAX_TRACE
2941 static void show_snapshot_main_help(struct seq_file *m)
2943 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
2944 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2945 "# Takes a snapshot of the main buffer.\n"
2946 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
2947 "# (Doesn't have to be '2' works with any number that\n"
2948 "# is not a '0' or '1')\n");
2951 static void show_snapshot_percpu_help(struct seq_file *m)
2953 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2954 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2955 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2956 "# Takes a snapshot of the main buffer for this cpu.\n");
2958 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
2959 "# Must use main snapshot file to allocate.\n");
2961 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
2962 "# (Doesn't have to be '2' works with any number that\n"
2963 "# is not a '0' or '1')\n");
2966 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2968 if (iter->tr->allocated_snapshot)
2969 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
2971 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
2973 seq_puts(m, "# Snapshot commands:\n");
2974 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2975 show_snapshot_main_help(m);
2977 show_snapshot_percpu_help(m);
2980 /* Should never be called */
2981 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2984 static int s_show(struct seq_file *m, void *v)
2986 struct trace_iterator *iter = v;
2989 if (iter->ent == NULL) {
2991 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2993 test_ftrace_alive(m);
2995 if (iter->snapshot && trace_empty(iter))
2996 print_snapshot_help(m, iter);
2997 else if (iter->trace && iter->trace->print_header)
2998 iter->trace->print_header(m);
3000 trace_default_header(m);
3002 } else if (iter->leftover) {
3004 * If we filled the seq_file buffer earlier, we
3005 * want to just show it now.
3007 ret = trace_print_seq(m, &iter->seq);
3009 /* ret should this time be zero, but you never know */
3010 iter->leftover = ret;
3013 print_trace_line(iter);
3014 ret = trace_print_seq(m, &iter->seq);
3016 * If we overflow the seq_file buffer, then it will
3017 * ask us for this data again at start up.
3019 * ret is 0 if seq_file write succeeded.
3022 iter->leftover = ret;
3029 * Should be used after trace_array_get(), trace_types_lock
3030 * ensures that i_cdev was already initialized.
3032 static inline int tracing_get_cpu(struct inode *inode)
3034 if (inode->i_cdev) /* See trace_create_cpu_file() */
3035 return (long)inode->i_cdev - 1;
3036 return RING_BUFFER_ALL_CPUS;
3039 static const struct seq_operations tracer_seq_ops = {
3046 static struct trace_iterator *
3047 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
3049 struct trace_array *tr = inode->i_private;
3050 struct trace_iterator *iter;
3053 if (tracing_disabled)
3054 return ERR_PTR(-ENODEV);
3056 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
3058 return ERR_PTR(-ENOMEM);
3060 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
3062 if (!iter->buffer_iter)
3066 * We make a copy of the current tracer to avoid concurrent
3067 * changes on it while we are reading.
3069 mutex_lock(&trace_types_lock);
3070 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
3074 *iter->trace = *tr->current_trace;
3076 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3081 #ifdef CONFIG_TRACER_MAX_TRACE
3082 /* Currently only the top directory has a snapshot */
3083 if (tr->current_trace->print_max || snapshot)
3084 iter->trace_buffer = &tr->max_buffer;
3087 iter->trace_buffer = &tr->trace_buffer;
3088 iter->snapshot = snapshot;
3090 iter->cpu_file = tracing_get_cpu(inode);
3091 mutex_init(&iter->mutex);
3093 /* Notify the tracer early; before we stop tracing. */
3094 if (iter->trace && iter->trace->open)
3095 iter->trace->open(iter);
3097 /* Annotate start of buffers if we had overruns */
3098 if (ring_buffer_overruns(iter->trace_buffer->buffer))
3099 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3101 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3102 if (trace_clocks[tr->clock_id].in_ns)
3103 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3105 /* stop the trace while dumping if we are not opening "snapshot" */
3106 if (!iter->snapshot)
3107 tracing_stop_tr(tr);
3109 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3110 for_each_tracing_cpu(cpu) {
3111 iter->buffer_iter[cpu] =
3112 ring_buffer_read_prepare(iter->trace_buffer->buffer,
3115 ring_buffer_read_prepare_sync();
3116 for_each_tracing_cpu(cpu) {
3117 ring_buffer_read_start(iter->buffer_iter[cpu]);
3118 tracing_iter_reset(iter, cpu);
3121 cpu = iter->cpu_file;
3122 iter->buffer_iter[cpu] =
3123 ring_buffer_read_prepare(iter->trace_buffer->buffer,
3125 ring_buffer_read_prepare_sync();
3126 ring_buffer_read_start(iter->buffer_iter[cpu]);
3127 tracing_iter_reset(iter, cpu);
3130 mutex_unlock(&trace_types_lock);
3135 mutex_unlock(&trace_types_lock);
3137 kfree(iter->buffer_iter);
3139 seq_release_private(inode, file);
3140 return ERR_PTR(-ENOMEM);
3143 int tracing_open_generic(struct inode *inode, struct file *filp)
3145 if (tracing_disabled)
3148 filp->private_data = inode->i_private;
3152 bool tracing_is_disabled(void)
3154 return (tracing_disabled) ? true: false;
3158 * Open and update trace_array ref count.
3159 * Must have the current trace_array passed to it.
3161 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3163 struct trace_array *tr = inode->i_private;
3165 if (tracing_disabled)
3168 if (trace_array_get(tr) < 0)
3171 filp->private_data = inode->i_private;
3176 static int tracing_release(struct inode *inode, struct file *file)
3178 struct trace_array *tr = inode->i_private;
3179 struct seq_file *m = file->private_data;
3180 struct trace_iterator *iter;
3183 if (!(file->f_mode & FMODE_READ)) {
3184 trace_array_put(tr);
3188 /* Writes do not use seq_file */
3190 mutex_lock(&trace_types_lock);
3192 for_each_tracing_cpu(cpu) {
3193 if (iter->buffer_iter[cpu])
3194 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3197 if (iter->trace && iter->trace->close)
3198 iter->trace->close(iter);
3200 if (!iter->snapshot)
3201 /* reenable tracing if it was previously enabled */
3202 tracing_start_tr(tr);
3204 __trace_array_put(tr);
3206 mutex_unlock(&trace_types_lock);
3208 mutex_destroy(&iter->mutex);
3209 free_cpumask_var(iter->started);
3211 kfree(iter->buffer_iter);
3212 seq_release_private(inode, file);
3217 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3219 struct trace_array *tr = inode->i_private;
3221 trace_array_put(tr);
3225 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3227 struct trace_array *tr = inode->i_private;
3229 trace_array_put(tr);
3231 return single_release(inode, file);
3234 static int tracing_open(struct inode *inode, struct file *file)
3236 struct trace_array *tr = inode->i_private;
3237 struct trace_iterator *iter;
3240 if (trace_array_get(tr) < 0)
3243 /* If this file was open for write, then erase contents */
3244 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3245 int cpu = tracing_get_cpu(inode);
3246 struct trace_buffer *trace_buf = &tr->trace_buffer;
3248 #ifdef CONFIG_TRACER_MAX_TRACE
3249 if (tr->current_trace->print_max)
3250 trace_buf = &tr->max_buffer;
3253 if (cpu == RING_BUFFER_ALL_CPUS)
3254 tracing_reset_online_cpus(trace_buf);
3256 tracing_reset(trace_buf, cpu);
3259 if (file->f_mode & FMODE_READ) {
3260 iter = __tracing_open(inode, file, false);
3262 ret = PTR_ERR(iter);
3263 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
3264 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3268 trace_array_put(tr);
3274 * Some tracers are not suitable for instance buffers.
3275 * A tracer is always available for the global array (toplevel)
3276 * or if it explicitly states that it is.
3279 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3281 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3284 /* Find the next tracer that this trace array may use */
3285 static struct tracer *
3286 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3288 while (t && !trace_ok_for_array(t, tr))
3295 t_next(struct seq_file *m, void *v, loff_t *pos)
3297 struct trace_array *tr = m->private;
3298 struct tracer *t = v;
3303 t = get_tracer_for_array(tr, t->next);
3308 static void *t_start(struct seq_file *m, loff_t *pos)
3310 struct trace_array *tr = m->private;
3314 mutex_lock(&trace_types_lock);
3316 t = get_tracer_for_array(tr, trace_types);
3317 for (; t && l < *pos; t = t_next(m, t, &l))
3323 static void t_stop(struct seq_file *m, void *p)
3325 mutex_unlock(&trace_types_lock);
3328 static int t_show(struct seq_file *m, void *v)
3330 struct tracer *t = v;
3335 seq_puts(m, t->name);
3344 static const struct seq_operations show_traces_seq_ops = {
3351 static int show_traces_open(struct inode *inode, struct file *file)
3353 struct trace_array *tr = inode->i_private;
3357 if (tracing_disabled)
3360 if (trace_array_get(tr) < 0)
3363 ret = seq_open(file, &show_traces_seq_ops);
3365 trace_array_put(tr);
3369 m = file->private_data;
3375 static int show_traces_release(struct inode *inode, struct file *file)
3377 struct trace_array *tr = inode->i_private;
3379 trace_array_put(tr);
3380 return seq_release(inode, file);
3384 tracing_write_stub(struct file *filp, const char __user *ubuf,
3385 size_t count, loff_t *ppos)
3390 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3394 if (file->f_mode & FMODE_READ)
3395 ret = seq_lseek(file, offset, whence);
3397 file->f_pos = ret = 0;
3402 static const struct file_operations tracing_fops = {
3403 .open = tracing_open,
3405 .write = tracing_write_stub,
3406 .llseek = tracing_lseek,
3407 .release = tracing_release,
3410 static const struct file_operations show_traces_fops = {
3411 .open = show_traces_open,
3413 .llseek = seq_lseek,
3414 .release = show_traces_release,
3418 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3419 size_t count, loff_t *ppos)
3421 struct trace_array *tr = file_inode(filp)->i_private;
3425 len = snprintf(NULL, 0, "%*pb\n",
3426 cpumask_pr_args(tr->tracing_cpumask)) + 1;
3427 mask_str = kmalloc(len, GFP_KERNEL);
3431 len = snprintf(mask_str, len, "%*pb\n",
3432 cpumask_pr_args(tr->tracing_cpumask));
3437 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
3446 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3447 size_t count, loff_t *ppos)
3449 struct trace_array *tr = file_inode(filp)->i_private;
3450 cpumask_var_t tracing_cpumask_new;
3453 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3456 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3460 local_irq_disable();
3461 arch_spin_lock(&tr->max_lock);
3462 for_each_tracing_cpu(cpu) {
3464 * Increase/decrease the disabled counter if we are
3465 * about to flip a bit in the cpumask:
3467 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3468 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3469 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3470 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3472 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3473 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3474 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3475 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3478 arch_spin_unlock(&tr->max_lock);
3481 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3482 free_cpumask_var(tracing_cpumask_new);
3487 free_cpumask_var(tracing_cpumask_new);
3492 static const struct file_operations tracing_cpumask_fops = {
3493 .open = tracing_open_generic_tr,
3494 .read = tracing_cpumask_read,
3495 .write = tracing_cpumask_write,
3496 .release = tracing_release_generic_tr,
3497 .llseek = generic_file_llseek,
3500 static int tracing_trace_options_show(struct seq_file *m, void *v)
3502 struct tracer_opt *trace_opts;
3503 struct trace_array *tr = m->private;
3507 mutex_lock(&trace_types_lock);
3508 tracer_flags = tr->current_trace->flags->val;
3509 trace_opts = tr->current_trace->flags->opts;
3511 for (i = 0; trace_options[i]; i++) {
3512 if (tr->trace_flags & (1 << i))
3513 seq_printf(m, "%s\n", trace_options[i]);
3515 seq_printf(m, "no%s\n", trace_options[i]);
3518 for (i = 0; trace_opts[i].name; i++) {
3519 if (tracer_flags & trace_opts[i].bit)
3520 seq_printf(m, "%s\n", trace_opts[i].name);
3522 seq_printf(m, "no%s\n", trace_opts[i].name);
3524 mutex_unlock(&trace_types_lock);
3529 static int __set_tracer_option(struct trace_array *tr,
3530 struct tracer_flags *tracer_flags,
3531 struct tracer_opt *opts, int neg)
3533 struct tracer *trace = tr->current_trace;
3536 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3541 tracer_flags->val &= ~opts->bit;
3543 tracer_flags->val |= opts->bit;
3547 /* Try to assign a tracer specific option */
3548 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3550 struct tracer *trace = tr->current_trace;
3551 struct tracer_flags *tracer_flags = trace->flags;
3552 struct tracer_opt *opts = NULL;
3555 for (i = 0; tracer_flags->opts[i].name; i++) {
3556 opts = &tracer_flags->opts[i];
3558 if (strcmp(cmp, opts->name) == 0)
3559 return __set_tracer_option(tr, trace->flags, opts, neg);
3565 /* Some tracers require overwrite to stay enabled */
3566 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3568 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3574 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3576 /* do nothing if flag is already set */
3577 if (!!(tr->trace_flags & mask) == !!enabled)
3580 /* Give the tracer a chance to approve the change */
3581 if (tr->current_trace->flag_changed)
3582 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3586 tr->trace_flags |= mask;
3588 tr->trace_flags &= ~mask;
3590 if (mask == TRACE_ITER_RECORD_CMD)
3591 trace_event_enable_cmd_record(enabled);
3593 if (mask == TRACE_ITER_OVERWRITE) {
3594 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3595 #ifdef CONFIG_TRACER_MAX_TRACE
3596 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3600 if (mask == TRACE_ITER_PRINTK) {
3601 trace_printk_start_stop_comm(enabled);
3602 trace_printk_control(enabled);
3608 static int trace_set_options(struct trace_array *tr, char *option)
3614 size_t orig_len = strlen(option);
3616 cmp = strstrip(option);
3618 if (strncmp(cmp, "no", 2) == 0) {
3623 mutex_lock(&trace_types_lock);
3625 for (i = 0; trace_options[i]; i++) {
3626 if (strcmp(cmp, trace_options[i]) == 0) {
3627 ret = set_tracer_flag(tr, 1 << i, !neg);
3632 /* If no option could be set, test the specific tracer options */
3633 if (!trace_options[i])
3634 ret = set_tracer_option(tr, cmp, neg);
3636 mutex_unlock(&trace_types_lock);
3639 * If the first trailing whitespace is replaced with '\0' by strstrip,
3640 * turn it back into a space.
3642 if (orig_len > strlen(option))
3643 option[strlen(option)] = ' ';
3648 static void __init apply_trace_boot_options(void)
3650 char *buf = trace_boot_options_buf;
3654 option = strsep(&buf, ",");
3660 trace_set_options(&global_trace, option);
3662 /* Put back the comma to allow this to be called again */
3669 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3670 size_t cnt, loff_t *ppos)
3672 struct seq_file *m = filp->private_data;
3673 struct trace_array *tr = m->private;
3677 if (cnt >= sizeof(buf))
3680 if (copy_from_user(&buf, ubuf, cnt))
3685 ret = trace_set_options(tr, buf);
3694 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3696 struct trace_array *tr = inode->i_private;
3699 if (tracing_disabled)
3702 if (trace_array_get(tr) < 0)
3705 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3707 trace_array_put(tr);
3712 static const struct file_operations tracing_iter_fops = {
3713 .open = tracing_trace_options_open,
3715 .llseek = seq_lseek,
3716 .release = tracing_single_release_tr,
3717 .write = tracing_trace_options_write,
3720 static const char readme_msg[] =
3721 "tracing mini-HOWTO:\n\n"
3722 "# echo 0 > tracing_on : quick way to disable tracing\n"
3723 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3724 " Important files:\n"
3725 " trace\t\t\t- The static contents of the buffer\n"
3726 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3727 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3728 " current_tracer\t- function and latency tracers\n"
3729 " available_tracers\t- list of configured tracers for current_tracer\n"
3730 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3731 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3732 " trace_clock\t\t-change the clock used to order events\n"
3733 " local: Per cpu clock but may not be synced across CPUs\n"
3734 " global: Synced across CPUs but slows tracing down.\n"
3735 " counter: Not a clock, but just an increment\n"
3736 " uptime: Jiffy counter from time of boot\n"
3737 " perf: Same clock that perf events use\n"
3738 #ifdef CONFIG_X86_64
3739 " x86-tsc: TSC cycle counter\n"
3741 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3742 " tracing_cpumask\t- Limit which CPUs to trace\n"
3743 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3744 "\t\t\t Remove sub-buffer with rmdir\n"
3745 " trace_options\t\t- Set format or modify how tracing happens\n"
3746 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3747 "\t\t\t option name\n"
3748 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
3749 #ifdef CONFIG_DYNAMIC_FTRACE
3750 "\n available_filter_functions - list of functions that can be filtered on\n"
3751 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3752 "\t\t\t functions\n"
3753 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3754 "\t modules: Can select a group via module\n"
3755 "\t Format: :mod:<module-name>\n"
3756 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3757 "\t triggers: a command to perform when function is hit\n"
3758 "\t Format: <function>:<trigger>[:count]\n"
3759 "\t trigger: traceon, traceoff\n"
3760 "\t\t enable_event:<system>:<event>\n"
3761 "\t\t disable_event:<system>:<event>\n"
3762 #ifdef CONFIG_STACKTRACE
3765 #ifdef CONFIG_TRACER_SNAPSHOT
3770 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3771 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3772 "\t The first one will disable tracing every time do_fault is hit\n"
3773 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3774 "\t The first time do trap is hit and it disables tracing, the\n"
3775 "\t counter will decrement to 2. If tracing is already disabled,\n"
3776 "\t the counter will not decrement. It only decrements when the\n"
3777 "\t trigger did work\n"
3778 "\t To remove trigger without count:\n"
3779 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3780 "\t To remove trigger with a count:\n"
3781 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3782 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3783 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3784 "\t modules: Can select a group via module command :mod:\n"
3785 "\t Does not accept triggers\n"
3786 #endif /* CONFIG_DYNAMIC_FTRACE */
3787 #ifdef CONFIG_FUNCTION_TRACER
3788 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3791 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3792 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3793 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
3794 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3796 #ifdef CONFIG_TRACER_SNAPSHOT
3797 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3798 "\t\t\t snapshot buffer. Read the contents for more\n"
3799 "\t\t\t information\n"
3801 #ifdef CONFIG_STACK_TRACER
3802 " stack_trace\t\t- Shows the max stack trace when active\n"
3803 " stack_max_size\t- Shows current max stack size that was traced\n"
3804 "\t\t\t Write into this file to reset the max size (trigger a\n"
3805 "\t\t\t new trace)\n"
3806 #ifdef CONFIG_DYNAMIC_FTRACE
3807 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3810 #endif /* CONFIG_STACK_TRACER */
3811 " events/\t\t- Directory containing all trace event subsystems:\n"
3812 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3813 " events/<system>/\t- Directory containing all trace events for <system>:\n"
3814 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3816 " filter\t\t- If set, only events passing filter are traced\n"
3817 " events/<system>/<event>/\t- Directory containing control files for\n"
3819 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3820 " filter\t\t- If set, only events passing filter are traced\n"
3821 " trigger\t\t- If set, a command to perform when event is hit\n"
3822 "\t Format: <trigger>[:count][if <filter>]\n"
3823 "\t trigger: traceon, traceoff\n"
3824 "\t enable_event:<system>:<event>\n"
3825 "\t disable_event:<system>:<event>\n"
3826 #ifdef CONFIG_STACKTRACE
3829 #ifdef CONFIG_TRACER_SNAPSHOT
3832 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3833 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3834 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3835 "\t events/block/block_unplug/trigger\n"
3836 "\t The first disables tracing every time block_unplug is hit.\n"
3837 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3838 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3839 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3840 "\t Like function triggers, the counter is only decremented if it\n"
3841 "\t enabled or disabled tracing.\n"
3842 "\t To remove a trigger without a count:\n"
3843 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3844 "\t To remove a trigger with a count:\n"
3845 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3846 "\t Filters can be ignored when removing a trigger.\n"
3850 tracing_readme_read(struct file *filp, char __user *ubuf,
3851 size_t cnt, loff_t *ppos)
3853 return simple_read_from_buffer(ubuf, cnt, ppos,
3854 readme_msg, strlen(readme_msg));
3857 static const struct file_operations tracing_readme_fops = {
3858 .open = tracing_open_generic,
3859 .read = tracing_readme_read,
3860 .llseek = generic_file_llseek,
3863 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
3865 unsigned int *ptr = v;
3867 if (*pos || m->count)
3872 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
3874 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
3883 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
3889 arch_spin_lock(&trace_cmdline_lock);
3891 v = &savedcmd->map_cmdline_to_pid[0];
3893 v = saved_cmdlines_next(m, v, &l);
3901 static void saved_cmdlines_stop(struct seq_file *m, void *v)
3903 arch_spin_unlock(&trace_cmdline_lock);
3907 static int saved_cmdlines_show(struct seq_file *m, void *v)
3909 char buf[TASK_COMM_LEN];
3910 unsigned int *pid = v;
3912 __trace_find_cmdline(*pid, buf);
3913 seq_printf(m, "%d %s\n", *pid, buf);
3917 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
3918 .start = saved_cmdlines_start,
3919 .next = saved_cmdlines_next,
3920 .stop = saved_cmdlines_stop,
3921 .show = saved_cmdlines_show,
3924 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
3926 if (tracing_disabled)
3929 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
3932 static const struct file_operations tracing_saved_cmdlines_fops = {
3933 .open = tracing_saved_cmdlines_open,
3935 .llseek = seq_lseek,
3936 .release = seq_release,
3940 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
3941 size_t cnt, loff_t *ppos)
3946 arch_spin_lock(&trace_cmdline_lock);
3947 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
3948 arch_spin_unlock(&trace_cmdline_lock);
3950 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3953 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
3955 kfree(s->saved_cmdlines);
3956 kfree(s->map_cmdline_to_pid);
3960 static int tracing_resize_saved_cmdlines(unsigned int val)
3962 struct saved_cmdlines_buffer *s, *savedcmd_temp;
3964 s = kmalloc(sizeof(*s), GFP_KERNEL);
3968 if (allocate_cmdlines_buffer(val, s) < 0) {
3973 arch_spin_lock(&trace_cmdline_lock);
3974 savedcmd_temp = savedcmd;
3976 arch_spin_unlock(&trace_cmdline_lock);
3977 free_saved_cmdlines_buffer(savedcmd_temp);
3983 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
3984 size_t cnt, loff_t *ppos)
3989 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3993 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
3994 if (!val || val > PID_MAX_DEFAULT)
3997 ret = tracing_resize_saved_cmdlines((unsigned int)val);
4006 static const struct file_operations tracing_saved_cmdlines_size_fops = {
4007 .open = tracing_open_generic,
4008 .read = tracing_saved_cmdlines_size_read,
4009 .write = tracing_saved_cmdlines_size_write,
4012 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
4013 static union trace_enum_map_item *
4014 update_enum_map(union trace_enum_map_item *ptr)
4016 if (!ptr->map.enum_string) {
4017 if (ptr->tail.next) {
4018 ptr = ptr->tail.next;
4019 /* Set ptr to the next real item (skip head) */
4027 static void *enum_map_next(struct seq_file *m, void *v, loff_t *pos)
4029 union trace_enum_map_item *ptr = v;
4032 * Paranoid! If ptr points to end, we don't want to increment past it.
4033 * This really should never happen.
4035 ptr = update_enum_map(ptr);
4036 if (WARN_ON_ONCE(!ptr))
4043 ptr = update_enum_map(ptr);
4048 static void *enum_map_start(struct seq_file *m, loff_t *pos)
4050 union trace_enum_map_item *v;
4053 mutex_lock(&trace_enum_mutex);
4055 v = trace_enum_maps;
4059 while (v && l < *pos) {
4060 v = enum_map_next(m, v, &l);
4066 static void enum_map_stop(struct seq_file *m, void *v)
4068 mutex_unlock(&trace_enum_mutex);
4071 static int enum_map_show(struct seq_file *m, void *v)
4073 union trace_enum_map_item *ptr = v;
4075 seq_printf(m, "%s %ld (%s)\n",
4076 ptr->map.enum_string, ptr->map.enum_value,
4082 static const struct seq_operations tracing_enum_map_seq_ops = {
4083 .start = enum_map_start,
4084 .next = enum_map_next,
4085 .stop = enum_map_stop,
4086 .show = enum_map_show,
4089 static int tracing_enum_map_open(struct inode *inode, struct file *filp)
4091 if (tracing_disabled)
4094 return seq_open(filp, &tracing_enum_map_seq_ops);
4097 static const struct file_operations tracing_enum_map_fops = {
4098 .open = tracing_enum_map_open,
4100 .llseek = seq_lseek,
4101 .release = seq_release,
4104 static inline union trace_enum_map_item *
4105 trace_enum_jmp_to_tail(union trace_enum_map_item *ptr)
4107 /* Return tail of array given the head */
4108 return ptr + ptr->head.length + 1;
4112 trace_insert_enum_map_file(struct module *mod, struct trace_enum_map **start,
4115 struct trace_enum_map **stop;
4116 struct trace_enum_map **map;
4117 union trace_enum_map_item *map_array;
4118 union trace_enum_map_item *ptr;
4123 * The trace_enum_maps contains the map plus a head and tail item,
4124 * where the head holds the module and length of array, and the
4125 * tail holds a pointer to the next list.
4127 map_array = kmalloc(sizeof(*map_array) * (len + 2), GFP_KERNEL);
4129 pr_warning("Unable to allocate trace enum mapping\n");
4133 mutex_lock(&trace_enum_mutex);
4135 if (!trace_enum_maps)
4136 trace_enum_maps = map_array;
4138 ptr = trace_enum_maps;
4140 ptr = trace_enum_jmp_to_tail(ptr);
4141 if (!ptr->tail.next)
4143 ptr = ptr->tail.next;
4146 ptr->tail.next = map_array;
4148 map_array->head.mod = mod;
4149 map_array->head.length = len;
4152 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
4153 map_array->map = **map;
4156 memset(map_array, 0, sizeof(*map_array));
4158 mutex_unlock(&trace_enum_mutex);
4161 static void trace_create_enum_file(struct dentry *d_tracer)
4163 trace_create_file("enum_map", 0444, d_tracer,
4164 NULL, &tracing_enum_map_fops);
4167 #else /* CONFIG_TRACE_ENUM_MAP_FILE */
4168 static inline void trace_create_enum_file(struct dentry *d_tracer) { }
4169 static inline void trace_insert_enum_map_file(struct module *mod,
4170 struct trace_enum_map **start, int len) { }
4171 #endif /* !CONFIG_TRACE_ENUM_MAP_FILE */
4173 static void trace_insert_enum_map(struct module *mod,
4174 struct trace_enum_map **start, int len)
4176 struct trace_enum_map **map;
4183 trace_event_enum_update(map, len);
4185 trace_insert_enum_map_file(mod, start, len);
4189 tracing_set_trace_read(struct file *filp, char __user *ubuf,
4190 size_t cnt, loff_t *ppos)
4192 struct trace_array *tr = filp->private_data;
4193 char buf[MAX_TRACER_SIZE+2];
4196 mutex_lock(&trace_types_lock);
4197 r = sprintf(buf, "%s\n", tr->current_trace->name);
4198 mutex_unlock(&trace_types_lock);
4200 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4203 int tracer_init(struct tracer *t, struct trace_array *tr)
4205 tracing_reset_online_cpus(&tr->trace_buffer);
4209 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
4213 for_each_tracing_cpu(cpu)
4214 per_cpu_ptr(buf->data, cpu)->entries = val;
4217 #ifdef CONFIG_TRACER_MAX_TRACE
4218 /* resize @tr's buffer to the size of @size_tr's entries */
4219 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
4220 struct trace_buffer *size_buf, int cpu_id)
4224 if (cpu_id == RING_BUFFER_ALL_CPUS) {
4225 for_each_tracing_cpu(cpu) {
4226 ret = ring_buffer_resize(trace_buf->buffer,
4227 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
4230 per_cpu_ptr(trace_buf->data, cpu)->entries =
4231 per_cpu_ptr(size_buf->data, cpu)->entries;
4234 ret = ring_buffer_resize(trace_buf->buffer,
4235 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
4237 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
4238 per_cpu_ptr(size_buf->data, cpu_id)->entries;
4243 #endif /* CONFIG_TRACER_MAX_TRACE */
4245 static int __tracing_resize_ring_buffer(struct trace_array *tr,
4246 unsigned long size, int cpu)
4251 * If kernel or user changes the size of the ring buffer
4252 * we use the size that was given, and we can forget about
4253 * expanding it later.
4255 ring_buffer_expanded = true;
4257 /* May be called before buffers are initialized */
4258 if (!tr->trace_buffer.buffer)
4261 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
4265 #ifdef CONFIG_TRACER_MAX_TRACE
4266 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
4267 !tr->current_trace->use_max_tr)
4270 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
4272 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
4273 &tr->trace_buffer, cpu);
4276 * AARGH! We are left with different
4277 * size max buffer!!!!
4278 * The max buffer is our "snapshot" buffer.
4279 * When a tracer needs a snapshot (one of the
4280 * latency tracers), it swaps the max buffer
4281 * with the saved snap shot. We succeeded to
4282 * update the size of the main buffer, but failed to
4283 * update the size of the max buffer. But when we tried
4284 * to reset the main buffer to the original size, we
4285 * failed there too. This is very unlikely to
4286 * happen, but if it does, warn and kill all
4290 tracing_disabled = 1;
4295 if (cpu == RING_BUFFER_ALL_CPUS)
4296 set_buffer_entries(&tr->max_buffer, size);
4298 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
4301 #endif /* CONFIG_TRACER_MAX_TRACE */
4303 if (cpu == RING_BUFFER_ALL_CPUS)
4304 set_buffer_entries(&tr->trace_buffer, size);
4306 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
4311 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4312 unsigned long size, int cpu_id)
4316 mutex_lock(&trace_types_lock);
4318 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4319 /* make sure, this cpu is enabled in the mask */
4320 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4326 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4331 mutex_unlock(&trace_types_lock);
4338 * tracing_update_buffers - used by tracing facility to expand ring buffers
4340 * To save on memory when the tracing is never used on a system with it
4341 * configured in. The ring buffers are set to a minimum size. But once
4342 * a user starts to use the tracing facility, then they need to grow
4343 * to their default size.
4345 * This function is to be called when a tracer is about to be used.
4347 int tracing_update_buffers(void)
4351 mutex_lock(&trace_types_lock);
4352 if (!ring_buffer_expanded)
4353 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
4354 RING_BUFFER_ALL_CPUS);
4355 mutex_unlock(&trace_types_lock);
4360 struct trace_option_dentry;
4363 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
4366 * Used to clear out the tracer before deletion of an instance.
4367 * Must have trace_types_lock held.
4369 static void tracing_set_nop(struct trace_array *tr)
4371 if (tr->current_trace == &nop_trace)
4374 tr->current_trace->enabled--;
4376 if (tr->current_trace->reset)
4377 tr->current_trace->reset(tr);
4379 tr->current_trace = &nop_trace;
4382 static void add_tracer_options(struct trace_array *tr, struct tracer *t)
4384 /* Only enable if the directory has been created already. */
4388 create_trace_option_files(tr, t);
4391 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4394 #ifdef CONFIG_TRACER_MAX_TRACE
4399 mutex_lock(&trace_types_lock);
4401 if (!ring_buffer_expanded) {
4402 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4403 RING_BUFFER_ALL_CPUS);
4409 for (t = trace_types; t; t = t->next) {
4410 if (strcmp(t->name, buf) == 0)
4417 if (t == tr->current_trace)
4420 /* Some tracers are only allowed for the top level buffer */
4421 if (!trace_ok_for_array(t, tr)) {
4426 /* If trace pipe files are being read, we can't change the tracer */
4427 if (tr->current_trace->ref) {
4432 trace_branch_disable();
4434 tr->current_trace->enabled--;
4436 if (tr->current_trace->reset)
4437 tr->current_trace->reset(tr);
4439 /* Current trace needs to be nop_trace before synchronize_sched */
4440 tr->current_trace = &nop_trace;
4442 #ifdef CONFIG_TRACER_MAX_TRACE
4443 had_max_tr = tr->allocated_snapshot;
4445 if (had_max_tr && !t->use_max_tr) {
4447 * We need to make sure that the update_max_tr sees that
4448 * current_trace changed to nop_trace to keep it from
4449 * swapping the buffers after we resize it.
4450 * The update_max_tr is called from interrupts disabled
4451 * so a synchronized_sched() is sufficient.
4453 synchronize_sched();
4458 #ifdef CONFIG_TRACER_MAX_TRACE
4459 if (t->use_max_tr && !had_max_tr) {
4460 ret = alloc_snapshot(tr);
4467 ret = tracer_init(t, tr);
4472 tr->current_trace = t;
4473 tr->current_trace->enabled++;
4474 trace_branch_enable(tr);
4476 mutex_unlock(&trace_types_lock);
4482 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4483 size_t cnt, loff_t *ppos)
4485 struct trace_array *tr = filp->private_data;
4486 char buf[MAX_TRACER_SIZE+1];
4493 if (cnt > MAX_TRACER_SIZE)
4494 cnt = MAX_TRACER_SIZE;
4496 if (copy_from_user(&buf, ubuf, cnt))
4501 /* strip ending whitespace. */
4502 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4505 err = tracing_set_tracer(tr, buf);
4515 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
4516 size_t cnt, loff_t *ppos)
4521 r = snprintf(buf, sizeof(buf), "%ld\n",
4522 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
4523 if (r > sizeof(buf))
4525 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4529 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
4530 size_t cnt, loff_t *ppos)
4535 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4545 tracing_thresh_read(struct file *filp, char __user *ubuf,
4546 size_t cnt, loff_t *ppos)
4548 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
4552 tracing_thresh_write(struct file *filp, const char __user *ubuf,
4553 size_t cnt, loff_t *ppos)
4555 struct trace_array *tr = filp->private_data;
4558 mutex_lock(&trace_types_lock);
4559 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
4563 if (tr->current_trace->update_thresh) {
4564 ret = tr->current_trace->update_thresh(tr);
4571 mutex_unlock(&trace_types_lock);
4576 #ifdef CONFIG_TRACER_MAX_TRACE
4579 tracing_max_lat_read(struct file *filp, char __user *ubuf,
4580 size_t cnt, loff_t *ppos)
4582 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
4586 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4587 size_t cnt, loff_t *ppos)
4589 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
4594 static int tracing_open_pipe(struct inode *inode, struct file *filp)
4596 struct trace_array *tr = inode->i_private;
4597 struct trace_iterator *iter;
4600 if (tracing_disabled)
4603 if (trace_array_get(tr) < 0)
4606 mutex_lock(&trace_types_lock);
4608 /* create a buffer to store the information to pass to userspace */
4609 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4612 __trace_array_put(tr);
4616 trace_seq_init(&iter->seq);
4617 iter->trace = tr->current_trace;
4619 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4624 /* trace pipe does not show start of buffer */
4625 cpumask_setall(iter->started);
4627 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
4628 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4630 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4631 if (trace_clocks[tr->clock_id].in_ns)
4632 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4635 iter->trace_buffer = &tr->trace_buffer;
4636 iter->cpu_file = tracing_get_cpu(inode);
4637 mutex_init(&iter->mutex);
4638 filp->private_data = iter;
4640 if (iter->trace->pipe_open)
4641 iter->trace->pipe_open(iter);
4643 nonseekable_open(inode, filp);
4645 tr->current_trace->ref++;
4647 mutex_unlock(&trace_types_lock);
4652 __trace_array_put(tr);
4653 mutex_unlock(&trace_types_lock);
4657 static int tracing_release_pipe(struct inode *inode, struct file *file)
4659 struct trace_iterator *iter = file->private_data;
4660 struct trace_array *tr = inode->i_private;
4662 mutex_lock(&trace_types_lock);
4664 tr->current_trace->ref--;
4666 if (iter->trace->pipe_close)
4667 iter->trace->pipe_close(iter);
4669 mutex_unlock(&trace_types_lock);
4671 free_cpumask_var(iter->started);
4672 mutex_destroy(&iter->mutex);
4675 trace_array_put(tr);
4681 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
4683 struct trace_array *tr = iter->tr;
4685 /* Iterators are static, they should be filled or empty */
4686 if (trace_buffer_iter(iter, iter->cpu_file))
4687 return POLLIN | POLLRDNORM;
4689 if (tr->trace_flags & TRACE_ITER_BLOCK)
4691 * Always select as readable when in blocking mode
4693 return POLLIN | POLLRDNORM;
4695 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4700 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4702 struct trace_iterator *iter = filp->private_data;
4704 return trace_poll(iter, filp, poll_table);
4707 /* Must be called with iter->mutex held. */
4708 static int tracing_wait_pipe(struct file *filp)
4710 struct trace_iterator *iter = filp->private_data;
4713 while (trace_empty(iter)) {
4715 if ((filp->f_flags & O_NONBLOCK)) {
4720 * We block until we read something and tracing is disabled.
4721 * We still block if tracing is disabled, but we have never
4722 * read anything. This allows a user to cat this file, and
4723 * then enable tracing. But after we have read something,
4724 * we give an EOF when tracing is again disabled.
4726 * iter->pos will be 0 if we haven't read anything.
4728 if (!tracer_tracing_is_on(iter->tr) && iter->pos)
4731 mutex_unlock(&iter->mutex);
4733 ret = wait_on_pipe(iter, false);
4735 mutex_lock(&iter->mutex);
4748 tracing_read_pipe(struct file *filp, char __user *ubuf,
4749 size_t cnt, loff_t *ppos)
4751 struct trace_iterator *iter = filp->private_data;
4755 * Avoid more than one consumer on a single file descriptor
4756 * This is just a matter of traces coherency, the ring buffer itself
4759 mutex_lock(&iter->mutex);
4761 /* return any leftover data */
4762 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4766 trace_seq_init(&iter->seq);
4768 if (iter->trace->read) {
4769 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4775 sret = tracing_wait_pipe(filp);
4779 /* stop when tracing is finished */
4780 if (trace_empty(iter)) {
4785 if (cnt >= PAGE_SIZE)
4786 cnt = PAGE_SIZE - 1;
4788 /* reset all but tr, trace, and overruns */
4789 memset(&iter->seq, 0,
4790 sizeof(struct trace_iterator) -
4791 offsetof(struct trace_iterator, seq));
4792 cpumask_clear(iter->started);
4793 trace_seq_init(&iter->seq);
4796 trace_event_read_lock();
4797 trace_access_lock(iter->cpu_file);
4798 while (trace_find_next_entry_inc(iter) != NULL) {
4799 enum print_line_t ret;
4800 int save_len = iter->seq.seq.len;
4802 ret = print_trace_line(iter);
4803 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4804 /* don't print partial lines */
4805 iter->seq.seq.len = save_len;
4808 if (ret != TRACE_TYPE_NO_CONSUME)
4809 trace_consume(iter);
4811 if (trace_seq_used(&iter->seq) >= cnt)
4815 * Setting the full flag means we reached the trace_seq buffer
4816 * size and we should leave by partial output condition above.
4817 * One of the trace_seq_* functions is not used properly.
4819 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4822 trace_access_unlock(iter->cpu_file);
4823 trace_event_read_unlock();
4825 /* Now copy what we have to the user */
4826 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4827 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
4828 trace_seq_init(&iter->seq);
4831 * If there was nothing to send to user, in spite of consuming trace
4832 * entries, go back to wait for more entries.
4838 mutex_unlock(&iter->mutex);
4843 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4846 __free_page(spd->pages[idx]);
4849 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4851 .confirm = generic_pipe_buf_confirm,
4852 .release = generic_pipe_buf_release,
4853 .steal = generic_pipe_buf_steal,
4854 .get = generic_pipe_buf_get,
4858 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4864 /* Seq buffer is page-sized, exactly what we need. */
4866 save_len = iter->seq.seq.len;
4867 ret = print_trace_line(iter);
4869 if (trace_seq_has_overflowed(&iter->seq)) {
4870 iter->seq.seq.len = save_len;
4875 * This should not be hit, because it should only
4876 * be set if the iter->seq overflowed. But check it
4877 * anyway to be safe.
4879 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4880 iter->seq.seq.len = save_len;
4884 count = trace_seq_used(&iter->seq) - save_len;
4887 iter->seq.seq.len = save_len;
4891 if (ret != TRACE_TYPE_NO_CONSUME)
4892 trace_consume(iter);
4894 if (!trace_find_next_entry_inc(iter)) {
4904 static ssize_t tracing_splice_read_pipe(struct file *filp,
4906 struct pipe_inode_info *pipe,
4910 struct page *pages_def[PIPE_DEF_BUFFERS];
4911 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4912 struct trace_iterator *iter = filp->private_data;
4913 struct splice_pipe_desc spd = {
4915 .partial = partial_def,
4916 .nr_pages = 0, /* This gets updated below. */
4917 .nr_pages_max = PIPE_DEF_BUFFERS,
4919 .ops = &tracing_pipe_buf_ops,
4920 .spd_release = tracing_spd_release_pipe,
4926 if (splice_grow_spd(pipe, &spd))
4929 mutex_lock(&iter->mutex);
4931 if (iter->trace->splice_read) {
4932 ret = iter->trace->splice_read(iter, filp,
4933 ppos, pipe, len, flags);
4938 ret = tracing_wait_pipe(filp);
4942 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4947 trace_event_read_lock();
4948 trace_access_lock(iter->cpu_file);
4950 /* Fill as many pages as possible. */
4951 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
4952 spd.pages[i] = alloc_page(GFP_KERNEL);
4956 rem = tracing_fill_pipe_page(rem, iter);
4958 /* Copy the data into the page, so we can start over. */
4959 ret = trace_seq_to_buffer(&iter->seq,
4960 page_address(spd.pages[i]),
4961 trace_seq_used(&iter->seq));
4963 __free_page(spd.pages[i]);
4966 spd.partial[i].offset = 0;
4967 spd.partial[i].len = trace_seq_used(&iter->seq);
4969 trace_seq_init(&iter->seq);
4972 trace_access_unlock(iter->cpu_file);
4973 trace_event_read_unlock();
4974 mutex_unlock(&iter->mutex);
4979 ret = splice_to_pipe(pipe, &spd);
4983 splice_shrink_spd(&spd);
4987 mutex_unlock(&iter->mutex);
4992 tracing_entries_read(struct file *filp, char __user *ubuf,
4993 size_t cnt, loff_t *ppos)
4995 struct inode *inode = file_inode(filp);
4996 struct trace_array *tr = inode->i_private;
4997 int cpu = tracing_get_cpu(inode);
5002 mutex_lock(&trace_types_lock);
5004 if (cpu == RING_BUFFER_ALL_CPUS) {
5005 int cpu, buf_size_same;
5010 /* check if all cpu sizes are same */
5011 for_each_tracing_cpu(cpu) {
5012 /* fill in the size from first enabled cpu */
5014 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
5015 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
5021 if (buf_size_same) {
5022 if (!ring_buffer_expanded)
5023 r = sprintf(buf, "%lu (expanded: %lu)\n",
5025 trace_buf_size >> 10);
5027 r = sprintf(buf, "%lu\n", size >> 10);
5029 r = sprintf(buf, "X\n");
5031 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
5033 mutex_unlock(&trace_types_lock);
5035 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5040 tracing_entries_write(struct file *filp, const char __user *ubuf,
5041 size_t cnt, loff_t *ppos)
5043 struct inode *inode = file_inode(filp);
5044 struct trace_array *tr = inode->i_private;
5048 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5052 /* must have at least 1 entry */
5056 /* value is in KB */
5058 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
5068 tracing_total_entries_read(struct file *filp, char __user *ubuf,
5069 size_t cnt, loff_t *ppos)
5071 struct trace_array *tr = filp->private_data;
5074 unsigned long size = 0, expanded_size = 0;
5076 mutex_lock(&trace_types_lock);
5077 for_each_tracing_cpu(cpu) {
5078 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
5079 if (!ring_buffer_expanded)
5080 expanded_size += trace_buf_size >> 10;
5082 if (ring_buffer_expanded)
5083 r = sprintf(buf, "%lu\n", size);
5085 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
5086 mutex_unlock(&trace_types_lock);
5088 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5092 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
5093 size_t cnt, loff_t *ppos)
5096 * There is no need to read what the user has written, this function
5097 * is just to make sure that there is no error when "echo" is used
5106 tracing_free_buffer_release(struct inode *inode, struct file *filp)
5108 struct trace_array *tr = inode->i_private;
5110 /* disable tracing ? */
5111 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
5112 tracer_tracing_off(tr);
5113 /* resize the ring buffer to 0 */
5114 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
5116 trace_array_put(tr);
5122 tracing_mark_write(struct file *filp, const char __user *ubuf,
5123 size_t cnt, loff_t *fpos)
5125 unsigned long addr = (unsigned long)ubuf;
5126 struct trace_array *tr = filp->private_data;
5127 struct ring_buffer_event *event;
5128 struct ring_buffer *buffer;
5129 struct print_entry *entry;
5130 unsigned long irq_flags;
5131 struct page *pages[2];
5141 if (tracing_disabled)
5144 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
5147 if (cnt > TRACE_BUF_SIZE)
5148 cnt = TRACE_BUF_SIZE;
5151 * Userspace is injecting traces into the kernel trace buffer.
5152 * We want to be as non intrusive as possible.
5153 * To do so, we do not want to allocate any special buffers
5154 * or take any locks, but instead write the userspace data
5155 * straight into the ring buffer.
5157 * First we need to pin the userspace buffer into memory,
5158 * which, most likely it is, because it just referenced it.
5159 * But there's no guarantee that it is. By using get_user_pages_fast()
5160 * and kmap_atomic/kunmap_atomic() we can get access to the
5161 * pages directly. We then write the data directly into the
5164 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
5166 /* check if we cross pages */
5167 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
5170 offset = addr & (PAGE_SIZE - 1);
5173 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
5174 if (ret < nr_pages) {
5176 put_page(pages[ret]);
5181 for (i = 0; i < nr_pages; i++)
5182 map_page[i] = kmap_atomic(pages[i]);
5184 local_save_flags(irq_flags);
5185 size = sizeof(*entry) + cnt + 2; /* possible \n added */
5186 buffer = tr->trace_buffer.buffer;
5187 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
5188 irq_flags, preempt_count());
5190 /* Ring buffer disabled, return as if not open for write */
5195 entry = ring_buffer_event_data(event);
5196 entry->ip = _THIS_IP_;
5198 if (nr_pages == 2) {
5199 len = PAGE_SIZE - offset;
5200 memcpy(&entry->buf, map_page[0] + offset, len);
5201 memcpy(&entry->buf[len], map_page[1], cnt - len);
5203 memcpy(&entry->buf, map_page[0] + offset, cnt);
5205 if (entry->buf[cnt - 1] != '\n') {
5206 entry->buf[cnt] = '\n';
5207 entry->buf[cnt + 1] = '\0';
5209 entry->buf[cnt] = '\0';
5211 __buffer_unlock_commit(buffer, event);
5218 for (i = nr_pages - 1; i >= 0; i--) {
5219 kunmap_atomic(map_page[i]);
5226 static int tracing_clock_show(struct seq_file *m, void *v)
5228 struct trace_array *tr = m->private;
5231 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
5233 "%s%s%s%s", i ? " " : "",
5234 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
5235 i == tr->clock_id ? "]" : "");
5241 static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
5245 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
5246 if (strcmp(trace_clocks[i].name, clockstr) == 0)
5249 if (i == ARRAY_SIZE(trace_clocks))
5252 mutex_lock(&trace_types_lock);
5256 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
5259 * New clock may not be consistent with the previous clock.
5260 * Reset the buffer so that it doesn't have incomparable timestamps.
5262 tracing_reset_online_cpus(&tr->trace_buffer);
5264 #ifdef CONFIG_TRACER_MAX_TRACE
5265 if (tr->max_buffer.buffer)
5266 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
5267 tracing_reset_online_cpus(&tr->max_buffer);
5270 mutex_unlock(&trace_types_lock);
5275 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
5276 size_t cnt, loff_t *fpos)
5278 struct seq_file *m = filp->private_data;
5279 struct trace_array *tr = m->private;
5281 const char *clockstr;
5284 if (cnt >= sizeof(buf))
5287 if (copy_from_user(&buf, ubuf, cnt))
5292 clockstr = strstrip(buf);
5294 ret = tracing_set_clock(tr, clockstr);
5303 static int tracing_clock_open(struct inode *inode, struct file *file)
5305 struct trace_array *tr = inode->i_private;
5308 if (tracing_disabled)
5311 if (trace_array_get(tr))
5314 ret = single_open(file, tracing_clock_show, inode->i_private);
5316 trace_array_put(tr);
5321 struct ftrace_buffer_info {
5322 struct trace_iterator iter;
5327 #ifdef CONFIG_TRACER_SNAPSHOT
5328 static int tracing_snapshot_open(struct inode *inode, struct file *file)
5330 struct trace_array *tr = inode->i_private;
5331 struct trace_iterator *iter;
5335 if (trace_array_get(tr) < 0)
5338 if (file->f_mode & FMODE_READ) {
5339 iter = __tracing_open(inode, file, true);
5341 ret = PTR_ERR(iter);
5343 /* Writes still need the seq_file to hold the private data */
5345 m = kzalloc(sizeof(*m), GFP_KERNEL);
5348 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5356 iter->trace_buffer = &tr->max_buffer;
5357 iter->cpu_file = tracing_get_cpu(inode);
5359 file->private_data = m;
5363 trace_array_put(tr);
5369 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5372 struct seq_file *m = filp->private_data;
5373 struct trace_iterator *iter = m->private;
5374 struct trace_array *tr = iter->tr;
5378 ret = tracing_update_buffers();
5382 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5386 mutex_lock(&trace_types_lock);
5388 if (tr->current_trace->use_max_tr) {
5395 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5399 if (tr->allocated_snapshot)
5403 /* Only allow per-cpu swap if the ring buffer supports it */
5404 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5405 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5410 if (!tr->allocated_snapshot)
5411 ret = resize_buffer_duplicate_size(&tr->max_buffer,
5412 &tr->trace_buffer, iter->cpu_file);
5414 ret = alloc_snapshot(tr);
5419 local_irq_disable();
5420 /* Now, we're going to swap */
5421 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5422 update_max_tr(tr, current, smp_processor_id());
5424 update_max_tr_single(tr, current, iter->cpu_file);
5428 if (tr->allocated_snapshot) {
5429 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5430 tracing_reset_online_cpus(&tr->max_buffer);
5432 tracing_reset(&tr->max_buffer, iter->cpu_file);
5442 mutex_unlock(&trace_types_lock);
5446 static int tracing_snapshot_release(struct inode *inode, struct file *file)
5448 struct seq_file *m = file->private_data;
5451 ret = tracing_release(inode, file);
5453 if (file->f_mode & FMODE_READ)
5456 /* If write only, the seq_file is just a stub */
5464 static int tracing_buffers_open(struct inode *inode, struct file *filp);
5465 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5466 size_t count, loff_t *ppos);
5467 static int tracing_buffers_release(struct inode *inode, struct file *file);
5468 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5469 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5471 static int snapshot_raw_open(struct inode *inode, struct file *filp)
5473 struct ftrace_buffer_info *info;
5476 ret = tracing_buffers_open(inode, filp);
5480 info = filp->private_data;
5482 if (info->iter.trace->use_max_tr) {
5483 tracing_buffers_release(inode, filp);
5487 info->iter.snapshot = true;
5488 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5493 #endif /* CONFIG_TRACER_SNAPSHOT */
5496 static const struct file_operations tracing_thresh_fops = {
5497 .open = tracing_open_generic,
5498 .read = tracing_thresh_read,
5499 .write = tracing_thresh_write,
5500 .llseek = generic_file_llseek,
5503 #ifdef CONFIG_TRACER_MAX_TRACE
5504 static const struct file_operations tracing_max_lat_fops = {
5505 .open = tracing_open_generic,
5506 .read = tracing_max_lat_read,
5507 .write = tracing_max_lat_write,
5508 .llseek = generic_file_llseek,
5512 static const struct file_operations set_tracer_fops = {
5513 .open = tracing_open_generic,
5514 .read = tracing_set_trace_read,
5515 .write = tracing_set_trace_write,
5516 .llseek = generic_file_llseek,
5519 static const struct file_operations tracing_pipe_fops = {
5520 .open = tracing_open_pipe,
5521 .poll = tracing_poll_pipe,
5522 .read = tracing_read_pipe,
5523 .splice_read = tracing_splice_read_pipe,
5524 .release = tracing_release_pipe,
5525 .llseek = no_llseek,
5528 static const struct file_operations tracing_entries_fops = {
5529 .open = tracing_open_generic_tr,
5530 .read = tracing_entries_read,
5531 .write = tracing_entries_write,
5532 .llseek = generic_file_llseek,
5533 .release = tracing_release_generic_tr,
5536 static const struct file_operations tracing_total_entries_fops = {
5537 .open = tracing_open_generic_tr,
5538 .read = tracing_total_entries_read,
5539 .llseek = generic_file_llseek,
5540 .release = tracing_release_generic_tr,
5543 static const struct file_operations tracing_free_buffer_fops = {
5544 .open = tracing_open_generic_tr,
5545 .write = tracing_free_buffer_write,
5546 .release = tracing_free_buffer_release,
5549 static const struct file_operations tracing_mark_fops = {
5550 .open = tracing_open_generic_tr,
5551 .write = tracing_mark_write,
5552 .llseek = generic_file_llseek,
5553 .release = tracing_release_generic_tr,
5556 static const struct file_operations trace_clock_fops = {
5557 .open = tracing_clock_open,
5559 .llseek = seq_lseek,
5560 .release = tracing_single_release_tr,
5561 .write = tracing_clock_write,
5564 #ifdef CONFIG_TRACER_SNAPSHOT
5565 static const struct file_operations snapshot_fops = {
5566 .open = tracing_snapshot_open,
5568 .write = tracing_snapshot_write,
5569 .llseek = tracing_lseek,
5570 .release = tracing_snapshot_release,
5573 static const struct file_operations snapshot_raw_fops = {
5574 .open = snapshot_raw_open,
5575 .read = tracing_buffers_read,
5576 .release = tracing_buffers_release,
5577 .splice_read = tracing_buffers_splice_read,
5578 .llseek = no_llseek,
5581 #endif /* CONFIG_TRACER_SNAPSHOT */
5583 static int tracing_buffers_open(struct inode *inode, struct file *filp)
5585 struct trace_array *tr = inode->i_private;
5586 struct ftrace_buffer_info *info;
5589 if (tracing_disabled)
5592 if (trace_array_get(tr) < 0)
5595 info = kzalloc(sizeof(*info), GFP_KERNEL);
5597 trace_array_put(tr);
5601 mutex_lock(&trace_types_lock);
5604 info->iter.cpu_file = tracing_get_cpu(inode);
5605 info->iter.trace = tr->current_trace;
5606 info->iter.trace_buffer = &tr->trace_buffer;
5608 /* Force reading ring buffer for first read */
5609 info->read = (unsigned int)-1;
5611 filp->private_data = info;
5613 tr->current_trace->ref++;
5615 mutex_unlock(&trace_types_lock);
5617 ret = nonseekable_open(inode, filp);
5619 trace_array_put(tr);
5625 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5627 struct ftrace_buffer_info *info = filp->private_data;
5628 struct trace_iterator *iter = &info->iter;
5630 return trace_poll(iter, filp, poll_table);
5634 tracing_buffers_read(struct file *filp, char __user *ubuf,
5635 size_t count, loff_t *ppos)
5637 struct ftrace_buffer_info *info = filp->private_data;
5638 struct trace_iterator *iter = &info->iter;
5645 #ifdef CONFIG_TRACER_MAX_TRACE
5646 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5651 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5656 /* Do we have previous read data to read? */
5657 if (info->read < PAGE_SIZE)
5661 trace_access_lock(iter->cpu_file);
5662 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
5666 trace_access_unlock(iter->cpu_file);
5669 if (trace_empty(iter)) {
5670 if ((filp->f_flags & O_NONBLOCK))
5673 ret = wait_on_pipe(iter, false);
5684 size = PAGE_SIZE - info->read;
5688 ret = copy_to_user(ubuf, info->spare + info->read, size);
5700 static int tracing_buffers_release(struct inode *inode, struct file *file)
5702 struct ftrace_buffer_info *info = file->private_data;
5703 struct trace_iterator *iter = &info->iter;
5705 mutex_lock(&trace_types_lock);
5707 iter->tr->current_trace->ref--;
5709 __trace_array_put(iter->tr);
5712 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5715 mutex_unlock(&trace_types_lock);
5721 struct ring_buffer *buffer;
5726 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5727 struct pipe_buffer *buf)
5729 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5734 ring_buffer_free_read_page(ref->buffer, ref->page);
5739 static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5740 struct pipe_buffer *buf)
5742 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5744 if (ref->ref > INT_MAX/2)
5751 /* Pipe buffer operations for a buffer. */
5752 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5754 .confirm = generic_pipe_buf_confirm,
5755 .release = buffer_pipe_buf_release,
5756 .steal = generic_pipe_buf_steal,
5757 .get = buffer_pipe_buf_get,
5761 * Callback from splice_to_pipe(), if we need to release some pages
5762 * at the end of the spd in case we error'ed out in filling the pipe.
5764 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5766 struct buffer_ref *ref =
5767 (struct buffer_ref *)spd->partial[i].private;
5772 ring_buffer_free_read_page(ref->buffer, ref->page);
5774 spd->partial[i].private = 0;
5778 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5779 struct pipe_inode_info *pipe, size_t len,
5782 struct ftrace_buffer_info *info = file->private_data;
5783 struct trace_iterator *iter = &info->iter;
5784 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5785 struct page *pages_def[PIPE_DEF_BUFFERS];
5786 struct splice_pipe_desc spd = {
5788 .partial = partial_def,
5789 .nr_pages_max = PIPE_DEF_BUFFERS,
5791 .ops = &buffer_pipe_buf_ops,
5792 .spd_release = buffer_spd_release,
5794 struct buffer_ref *ref;
5798 #ifdef CONFIG_TRACER_MAX_TRACE
5799 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
5803 if (*ppos & (PAGE_SIZE - 1))
5806 if (len & (PAGE_SIZE - 1)) {
5807 if (len < PAGE_SIZE)
5812 if (splice_grow_spd(pipe, &spd))
5816 trace_access_lock(iter->cpu_file);
5817 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5819 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
5823 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5830 ref->buffer = iter->trace_buffer->buffer;
5831 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5838 r = ring_buffer_read_page(ref->buffer, &ref->page,
5839 len, iter->cpu_file, 1);
5841 ring_buffer_free_read_page(ref->buffer, ref->page);
5846 page = virt_to_page(ref->page);
5848 spd.pages[i] = page;
5849 spd.partial[i].len = PAGE_SIZE;
5850 spd.partial[i].offset = 0;
5851 spd.partial[i].private = (unsigned long)ref;
5855 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5858 trace_access_unlock(iter->cpu_file);
5861 /* did we read anything? */
5862 if (!spd.nr_pages) {
5867 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
5870 ret = wait_on_pipe(iter, true);
5877 ret = splice_to_pipe(pipe, &spd);
5879 splice_shrink_spd(&spd);
5884 static const struct file_operations tracing_buffers_fops = {
5885 .open = tracing_buffers_open,
5886 .read = tracing_buffers_read,
5887 .poll = tracing_buffers_poll,
5888 .release = tracing_buffers_release,
5889 .splice_read = tracing_buffers_splice_read,
5890 .llseek = no_llseek,
5894 tracing_stats_read(struct file *filp, char __user *ubuf,
5895 size_t count, loff_t *ppos)
5897 struct inode *inode = file_inode(filp);
5898 struct trace_array *tr = inode->i_private;
5899 struct trace_buffer *trace_buf = &tr->trace_buffer;
5900 int cpu = tracing_get_cpu(inode);
5901 struct trace_seq *s;
5903 unsigned long long t;
5904 unsigned long usec_rem;
5906 s = kmalloc(sizeof(*s), GFP_KERNEL);
5912 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5913 trace_seq_printf(s, "entries: %ld\n", cnt);
5915 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5916 trace_seq_printf(s, "overrun: %ld\n", cnt);
5918 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5919 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5921 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5922 trace_seq_printf(s, "bytes: %ld\n", cnt);
5924 if (trace_clocks[tr->clock_id].in_ns) {
5925 /* local or global for trace_clock */
5926 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5927 usec_rem = do_div(t, USEC_PER_SEC);
5928 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5931 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5932 usec_rem = do_div(t, USEC_PER_SEC);
5933 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5935 /* counter or tsc mode for trace_clock */
5936 trace_seq_printf(s, "oldest event ts: %llu\n",
5937 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5939 trace_seq_printf(s, "now ts: %llu\n",
5940 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5943 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5944 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5946 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5947 trace_seq_printf(s, "read events: %ld\n", cnt);
5949 count = simple_read_from_buffer(ubuf, count, ppos,
5950 s->buffer, trace_seq_used(s));
5957 static const struct file_operations tracing_stats_fops = {
5958 .open = tracing_open_generic_tr,
5959 .read = tracing_stats_read,
5960 .llseek = generic_file_llseek,
5961 .release = tracing_release_generic_tr,
5964 #ifdef CONFIG_DYNAMIC_FTRACE
5966 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5972 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5973 size_t cnt, loff_t *ppos)
5975 static char ftrace_dyn_info_buffer[1024];
5976 static DEFINE_MUTEX(dyn_info_mutex);
5977 unsigned long *p = filp->private_data;
5978 char *buf = ftrace_dyn_info_buffer;
5979 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5982 mutex_lock(&dyn_info_mutex);
5983 r = sprintf(buf, "%ld ", *p);
5985 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5988 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5990 mutex_unlock(&dyn_info_mutex);
5995 static const struct file_operations tracing_dyn_info_fops = {
5996 .open = tracing_open_generic,
5997 .read = tracing_read_dyn_info,
5998 .llseek = generic_file_llseek,
6000 #endif /* CONFIG_DYNAMIC_FTRACE */
6002 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
6004 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
6010 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
6012 unsigned long *count = (long *)data;
6024 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
6025 struct ftrace_probe_ops *ops, void *data)
6027 long count = (long)data;
6029 seq_printf(m, "%ps:", (void *)ip);
6031 seq_puts(m, "snapshot");
6034 seq_puts(m, ":unlimited\n");
6036 seq_printf(m, ":count=%ld\n", count);
6041 static struct ftrace_probe_ops snapshot_probe_ops = {
6042 .func = ftrace_snapshot,
6043 .print = ftrace_snapshot_print,
6046 static struct ftrace_probe_ops snapshot_count_probe_ops = {
6047 .func = ftrace_count_snapshot,
6048 .print = ftrace_snapshot_print,
6052 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
6053 char *glob, char *cmd, char *param, int enable)
6055 struct ftrace_probe_ops *ops;
6056 void *count = (void *)-1;
6060 /* hash funcs only work with set_ftrace_filter */
6064 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
6066 if (glob[0] == '!') {
6067 unregister_ftrace_function_probe_func(glob+1, ops);
6074 number = strsep(¶m, ":");
6076 if (!strlen(number))
6080 * We use the callback data field (which is a pointer)
6083 ret = kstrtoul(number, 0, (unsigned long *)&count);
6088 ret = alloc_snapshot(&global_trace);
6092 ret = register_ftrace_function_probe(glob, ops, count);
6095 return ret < 0 ? ret : 0;
6098 static struct ftrace_func_command ftrace_snapshot_cmd = {
6100 .func = ftrace_trace_snapshot_callback,
6103 static __init int register_snapshot_cmd(void)
6105 return register_ftrace_command(&ftrace_snapshot_cmd);
6108 static inline __init int register_snapshot_cmd(void) { return 0; }
6109 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
6111 static struct dentry *tracing_get_dentry(struct trace_array *tr)
6113 if (WARN_ON(!tr->dir))
6114 return ERR_PTR(-ENODEV);
6116 /* Top directory uses NULL as the parent */
6117 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
6120 /* All sub buffers have a descriptor */
6124 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
6126 struct dentry *d_tracer;
6129 return tr->percpu_dir;
6131 d_tracer = tracing_get_dentry(tr);
6132 if (IS_ERR(d_tracer))
6135 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
6137 WARN_ONCE(!tr->percpu_dir,
6138 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
6140 return tr->percpu_dir;
6143 static struct dentry *
6144 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
6145 void *data, long cpu, const struct file_operations *fops)
6147 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
6149 if (ret) /* See tracing_get_cpu() */
6150 d_inode(ret)->i_cdev = (void *)(cpu + 1);
6155 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
6157 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
6158 struct dentry *d_cpu;
6159 char cpu_dir[30]; /* 30 characters should be more than enough */
6164 snprintf(cpu_dir, 30, "cpu%ld", cpu);
6165 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
6167 pr_warning("Could not create tracefs '%s' entry\n", cpu_dir);
6171 /* per cpu trace_pipe */
6172 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
6173 tr, cpu, &tracing_pipe_fops);
6176 trace_create_cpu_file("trace", 0644, d_cpu,
6177 tr, cpu, &tracing_fops);
6179 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
6180 tr, cpu, &tracing_buffers_fops);
6182 trace_create_cpu_file("stats", 0444, d_cpu,
6183 tr, cpu, &tracing_stats_fops);
6185 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
6186 tr, cpu, &tracing_entries_fops);
6188 #ifdef CONFIG_TRACER_SNAPSHOT
6189 trace_create_cpu_file("snapshot", 0644, d_cpu,
6190 tr, cpu, &snapshot_fops);
6192 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
6193 tr, cpu, &snapshot_raw_fops);
6197 #ifdef CONFIG_FTRACE_SELFTEST
6198 /* Let selftest have access to static functions in this file */
6199 #include "trace_selftest.c"
6203 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
6206 struct trace_option_dentry *topt = filp->private_data;
6209 if (topt->flags->val & topt->opt->bit)
6214 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6218 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
6221 struct trace_option_dentry *topt = filp->private_data;
6225 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6229 if (val != 0 && val != 1)
6232 if (!!(topt->flags->val & topt->opt->bit) != val) {
6233 mutex_lock(&trace_types_lock);
6234 ret = __set_tracer_option(topt->tr, topt->flags,
6236 mutex_unlock(&trace_types_lock);
6247 static const struct file_operations trace_options_fops = {
6248 .open = tracing_open_generic,
6249 .read = trace_options_read,
6250 .write = trace_options_write,
6251 .llseek = generic_file_llseek,
6255 * In order to pass in both the trace_array descriptor as well as the index
6256 * to the flag that the trace option file represents, the trace_array
6257 * has a character array of trace_flags_index[], which holds the index
6258 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
6259 * The address of this character array is passed to the flag option file
6260 * read/write callbacks.
6262 * In order to extract both the index and the trace_array descriptor,
6263 * get_tr_index() uses the following algorithm.
6267 * As the pointer itself contains the address of the index (remember
6270 * Then to get the trace_array descriptor, by subtracting that index
6271 * from the ptr, we get to the start of the index itself.
6273 * ptr - idx == &index[0]
6275 * Then a simple container_of() from that pointer gets us to the
6276 * trace_array descriptor.
6278 static void get_tr_index(void *data, struct trace_array **ptr,
6279 unsigned int *pindex)
6281 *pindex = *(unsigned char *)data;
6283 *ptr = container_of(data - *pindex, struct trace_array,
6288 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
6291 void *tr_index = filp->private_data;
6292 struct trace_array *tr;
6296 get_tr_index(tr_index, &tr, &index);
6298 if (tr->trace_flags & (1 << index))
6303 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6307 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
6310 void *tr_index = filp->private_data;
6311 struct trace_array *tr;
6316 get_tr_index(tr_index, &tr, &index);
6318 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6322 if (val != 0 && val != 1)
6325 mutex_lock(&trace_types_lock);
6326 ret = set_tracer_flag(tr, 1 << index, val);
6327 mutex_unlock(&trace_types_lock);
6337 static const struct file_operations trace_options_core_fops = {
6338 .open = tracing_open_generic,
6339 .read = trace_options_core_read,
6340 .write = trace_options_core_write,
6341 .llseek = generic_file_llseek,
6344 struct dentry *trace_create_file(const char *name,
6346 struct dentry *parent,
6348 const struct file_operations *fops)
6352 ret = tracefs_create_file(name, mode, parent, data, fops);
6354 pr_warning("Could not create tracefs '%s' entry\n", name);
6360 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
6362 struct dentry *d_tracer;
6367 d_tracer = tracing_get_dentry(tr);
6368 if (IS_ERR(d_tracer))
6371 tr->options = tracefs_create_dir("options", d_tracer);
6373 pr_warning("Could not create tracefs directory 'options'\n");
6381 create_trace_option_file(struct trace_array *tr,
6382 struct trace_option_dentry *topt,
6383 struct tracer_flags *flags,
6384 struct tracer_opt *opt)
6386 struct dentry *t_options;
6388 t_options = trace_options_init_dentry(tr);
6392 topt->flags = flags;
6396 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
6397 &trace_options_fops);
6402 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
6404 struct trace_option_dentry *topts;
6405 struct trace_options *tr_topts;
6406 struct tracer_flags *flags;
6407 struct tracer_opt *opts;
6414 flags = tracer->flags;
6416 if (!flags || !flags->opts)
6420 * If this is an instance, only create flags for tracers
6421 * the instance may have.
6423 if (!trace_ok_for_array(tracer, tr))
6426 for (i = 0; i < tr->nr_topts; i++) {
6428 * Check if these flags have already been added.
6429 * Some tracers share flags.
6431 if (tr->topts[i].tracer->flags == tracer->flags)
6437 for (cnt = 0; opts[cnt].name; cnt++)
6440 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
6444 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
6451 tr->topts = tr_topts;
6452 tr->topts[tr->nr_topts].tracer = tracer;
6453 tr->topts[tr->nr_topts].topts = topts;
6456 for (cnt = 0; opts[cnt].name; cnt++) {
6457 create_trace_option_file(tr, &topts[cnt], flags,
6459 WARN_ONCE(topts[cnt].entry == NULL,
6460 "Failed to create trace option: %s",
6465 static struct dentry *
6466 create_trace_option_core_file(struct trace_array *tr,
6467 const char *option, long index)
6469 struct dentry *t_options;
6471 t_options = trace_options_init_dentry(tr);
6475 return trace_create_file(option, 0644, t_options,
6476 (void *)&tr->trace_flags_index[index],
6477 &trace_options_core_fops);
6480 static void create_trace_options_dir(struct trace_array *tr)
6482 struct dentry *t_options;
6483 bool top_level = tr == &global_trace;
6486 t_options = trace_options_init_dentry(tr);
6490 for (i = 0; trace_options[i]; i++) {
6492 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
6493 create_trace_option_core_file(tr, trace_options[i], i);
6498 rb_simple_read(struct file *filp, char __user *ubuf,
6499 size_t cnt, loff_t *ppos)
6501 struct trace_array *tr = filp->private_data;
6505 r = tracer_tracing_is_on(tr);
6506 r = sprintf(buf, "%d\n", r);
6508 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6512 rb_simple_write(struct file *filp, const char __user *ubuf,
6513 size_t cnt, loff_t *ppos)
6515 struct trace_array *tr = filp->private_data;
6516 struct ring_buffer *buffer = tr->trace_buffer.buffer;
6520 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6525 mutex_lock(&trace_types_lock);
6526 if (!!val == tracer_tracing_is_on(tr)) {
6527 val = 0; /* do nothing */
6529 tracer_tracing_on(tr);
6530 if (tr->current_trace->start)
6531 tr->current_trace->start(tr);
6533 tracer_tracing_off(tr);
6534 if (tr->current_trace->stop)
6535 tr->current_trace->stop(tr);
6537 mutex_unlock(&trace_types_lock);
6545 static const struct file_operations rb_simple_fops = {
6546 .open = tracing_open_generic_tr,
6547 .read = rb_simple_read,
6548 .write = rb_simple_write,
6549 .release = tracing_release_generic_tr,
6550 .llseek = default_llseek,
6553 struct dentry *trace_instance_dir;
6556 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
6559 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
6561 enum ring_buffer_flags rb_flags;
6563 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
6567 buf->buffer = ring_buffer_alloc(size, rb_flags);
6571 buf->data = alloc_percpu(struct trace_array_cpu);
6573 ring_buffer_free(buf->buffer);
6578 /* Allocate the first page for all buffers */
6579 set_buffer_entries(&tr->trace_buffer,
6580 ring_buffer_size(tr->trace_buffer.buffer, 0));
6585 static int allocate_trace_buffers(struct trace_array *tr, int size)
6589 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6593 #ifdef CONFIG_TRACER_MAX_TRACE
6594 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6595 allocate_snapshot ? size : 1);
6597 ring_buffer_free(tr->trace_buffer.buffer);
6598 tr->trace_buffer.buffer = NULL;
6599 free_percpu(tr->trace_buffer.data);
6600 tr->trace_buffer.data = NULL;
6603 tr->allocated_snapshot = allocate_snapshot;
6606 * Only the top level trace array gets its snapshot allocated
6607 * from the kernel command line.
6609 allocate_snapshot = false;
6613 * Because of some magic with the way alloc_percpu() works on
6614 * x86_64, we need to synchronize the pgd of all the tables,
6615 * otherwise the trace events that happen in x86_64 page fault
6616 * handlers can't cope with accessing the chance that a
6617 * alloc_percpu()'d memory might be touched in the page fault trace
6618 * event. Oh, and we need to audit all other alloc_percpu() and vmalloc()
6619 * calls in tracing, because something might get triggered within a
6620 * page fault trace event!
6622 vmalloc_sync_mappings();
6627 static void free_trace_buffer(struct trace_buffer *buf)
6630 ring_buffer_free(buf->buffer);
6632 free_percpu(buf->data);
6637 static void free_trace_buffers(struct trace_array *tr)
6642 free_trace_buffer(&tr->trace_buffer);
6644 #ifdef CONFIG_TRACER_MAX_TRACE
6645 free_trace_buffer(&tr->max_buffer);
6649 static void init_trace_flags_index(struct trace_array *tr)
6653 /* Used by the trace options files */
6654 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
6655 tr->trace_flags_index[i] = i;
6658 static void __update_tracer_options(struct trace_array *tr)
6662 for (t = trace_types; t; t = t->next)
6663 add_tracer_options(tr, t);
6666 static void update_tracer_options(struct trace_array *tr)
6668 mutex_lock(&trace_types_lock);
6669 __update_tracer_options(tr);
6670 mutex_unlock(&trace_types_lock);
6673 static int instance_mkdir(const char *name)
6675 struct trace_array *tr;
6678 mutex_lock(&trace_types_lock);
6681 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6682 if (tr->name && strcmp(tr->name, name) == 0)
6687 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6691 tr->name = kstrdup(name, GFP_KERNEL);
6695 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6698 tr->trace_flags = global_trace.trace_flags;
6700 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6702 raw_spin_lock_init(&tr->start_lock);
6704 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6706 tr->current_trace = &nop_trace;
6708 INIT_LIST_HEAD(&tr->systems);
6709 INIT_LIST_HEAD(&tr->events);
6711 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
6714 tr->dir = tracefs_create_dir(name, trace_instance_dir);
6718 ret = event_trace_add_tracer(tr->dir, tr);
6720 tracefs_remove_recursive(tr->dir);
6724 init_tracer_tracefs(tr, tr->dir);
6725 init_trace_flags_index(tr);
6726 __update_tracer_options(tr);
6728 list_add(&tr->list, &ftrace_trace_arrays);
6730 mutex_unlock(&trace_types_lock);
6735 free_trace_buffers(tr);
6736 free_cpumask_var(tr->tracing_cpumask);
6741 mutex_unlock(&trace_types_lock);
6747 static int instance_rmdir(const char *name)
6749 struct trace_array *tr;
6754 mutex_lock(&trace_types_lock);
6757 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6758 if (tr->name && strcmp(tr->name, name) == 0) {
6767 if (tr->ref || (tr->current_trace && tr->current_trace->ref))
6770 list_del(&tr->list);
6772 tracing_set_nop(tr);
6773 event_trace_del_tracer(tr);
6774 ftrace_destroy_function_files(tr);
6775 tracefs_remove_recursive(tr->dir);
6776 free_trace_buffers(tr);
6778 for (i = 0; i < tr->nr_topts; i++) {
6779 kfree(tr->topts[i].topts);
6783 free_cpumask_var(tr->tracing_cpumask);
6790 mutex_unlock(&trace_types_lock);
6795 static __init void create_trace_instances(struct dentry *d_tracer)
6797 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
6800 if (WARN_ON(!trace_instance_dir))
6805 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
6809 trace_create_file("available_tracers", 0444, d_tracer,
6810 tr, &show_traces_fops);
6812 trace_create_file("current_tracer", 0644, d_tracer,
6813 tr, &set_tracer_fops);
6815 trace_create_file("tracing_cpumask", 0644, d_tracer,
6816 tr, &tracing_cpumask_fops);
6818 trace_create_file("trace_options", 0644, d_tracer,
6819 tr, &tracing_iter_fops);
6821 trace_create_file("trace", 0644, d_tracer,
6824 trace_create_file("trace_pipe", 0444, d_tracer,
6825 tr, &tracing_pipe_fops);
6827 trace_create_file("buffer_size_kb", 0644, d_tracer,
6828 tr, &tracing_entries_fops);
6830 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6831 tr, &tracing_total_entries_fops);
6833 trace_create_file("free_buffer", 0200, d_tracer,
6834 tr, &tracing_free_buffer_fops);
6836 trace_create_file("trace_marker", 0220, d_tracer,
6837 tr, &tracing_mark_fops);
6839 trace_create_file("trace_clock", 0644, d_tracer, tr,
6842 trace_create_file("tracing_on", 0644, d_tracer,
6843 tr, &rb_simple_fops);
6845 create_trace_options_dir(tr);
6847 #ifdef CONFIG_TRACER_MAX_TRACE
6848 trace_create_file("tracing_max_latency", 0644, d_tracer,
6849 &tr->max_latency, &tracing_max_lat_fops);
6852 if (ftrace_create_function_files(tr, d_tracer))
6853 WARN(1, "Could not allocate function filter files");
6855 #ifdef CONFIG_TRACER_SNAPSHOT
6856 trace_create_file("snapshot", 0644, d_tracer,
6857 tr, &snapshot_fops);
6860 for_each_tracing_cpu(cpu)
6861 tracing_init_tracefs_percpu(tr, cpu);
6865 static struct vfsmount *trace_automount(void *ingore)
6867 struct vfsmount *mnt;
6868 struct file_system_type *type;
6871 * To maintain backward compatibility for tools that mount
6872 * debugfs to get to the tracing facility, tracefs is automatically
6873 * mounted to the debugfs/tracing directory.
6875 type = get_fs_type("tracefs");
6878 mnt = vfs_kern_mount(type, 0, "tracefs", NULL);
6879 put_filesystem(type);
6888 * tracing_init_dentry - initialize top level trace array
6890 * This is called when creating files or directories in the tracing
6891 * directory. It is called via fs_initcall() by any of the boot up code
6892 * and expects to return the dentry of the top level tracing directory.
6894 struct dentry *tracing_init_dentry(void)
6896 struct trace_array *tr = &global_trace;
6898 /* The top level trace array uses NULL as parent */
6902 if (WARN_ON(!tracefs_initialized()) ||
6903 (IS_ENABLED(CONFIG_DEBUG_FS) &&
6904 WARN_ON(!debugfs_initialized())))
6905 return ERR_PTR(-ENODEV);
6908 * As there may still be users that expect the tracing
6909 * files to exist in debugfs/tracing, we must automount
6910 * the tracefs file system there, so older tools still
6911 * work with the newer kerenl.
6913 tr->dir = debugfs_create_automount("tracing", NULL,
6914 trace_automount, NULL);
6916 pr_warn_once("Could not create debugfs directory 'tracing'\n");
6917 return ERR_PTR(-ENOMEM);
6923 extern struct trace_enum_map *__start_ftrace_enum_maps[];
6924 extern struct trace_enum_map *__stop_ftrace_enum_maps[];
6926 static void __init trace_enum_init(void)
6930 len = __stop_ftrace_enum_maps - __start_ftrace_enum_maps;
6931 trace_insert_enum_map(NULL, __start_ftrace_enum_maps, len);
6934 #ifdef CONFIG_MODULES
6935 static void trace_module_add_enums(struct module *mod)
6937 if (!mod->num_trace_enums)
6941 * Modules with bad taint do not have events created, do
6942 * not bother with enums either.
6944 if (trace_module_has_bad_taint(mod))
6947 trace_insert_enum_map(mod, mod->trace_enums, mod->num_trace_enums);
6950 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
6951 static void trace_module_remove_enums(struct module *mod)
6953 union trace_enum_map_item *map;
6954 union trace_enum_map_item **last = &trace_enum_maps;
6956 if (!mod->num_trace_enums)
6959 mutex_lock(&trace_enum_mutex);
6961 map = trace_enum_maps;
6964 if (map->head.mod == mod)
6966 map = trace_enum_jmp_to_tail(map);
6967 last = &map->tail.next;
6968 map = map->tail.next;
6973 *last = trace_enum_jmp_to_tail(map)->tail.next;
6976 mutex_unlock(&trace_enum_mutex);
6979 static inline void trace_module_remove_enums(struct module *mod) { }
6980 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
6982 static int trace_module_notify(struct notifier_block *self,
6983 unsigned long val, void *data)
6985 struct module *mod = data;
6988 case MODULE_STATE_COMING:
6989 trace_module_add_enums(mod);
6991 case MODULE_STATE_GOING:
6992 trace_module_remove_enums(mod);
6999 static struct notifier_block trace_module_nb = {
7000 .notifier_call = trace_module_notify,
7003 #endif /* CONFIG_MODULES */
7005 static __init int tracer_init_tracefs(void)
7007 struct dentry *d_tracer;
7009 trace_access_lock_init();
7011 d_tracer = tracing_init_dentry();
7012 if (IS_ERR(d_tracer))
7015 init_tracer_tracefs(&global_trace, d_tracer);
7017 trace_create_file("tracing_thresh", 0644, d_tracer,
7018 &global_trace, &tracing_thresh_fops);
7020 trace_create_file("README", 0444, d_tracer,
7021 NULL, &tracing_readme_fops);
7023 trace_create_file("saved_cmdlines", 0444, d_tracer,
7024 NULL, &tracing_saved_cmdlines_fops);
7026 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
7027 NULL, &tracing_saved_cmdlines_size_fops);
7031 trace_create_enum_file(d_tracer);
7033 #ifdef CONFIG_MODULES
7034 register_module_notifier(&trace_module_nb);
7037 #ifdef CONFIG_DYNAMIC_FTRACE
7038 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
7039 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
7042 create_trace_instances(d_tracer);
7044 update_tracer_options(&global_trace);
7049 static int trace_panic_handler(struct notifier_block *this,
7050 unsigned long event, void *unused)
7052 if (ftrace_dump_on_oops)
7053 ftrace_dump(ftrace_dump_on_oops);
7057 static struct notifier_block trace_panic_notifier = {
7058 .notifier_call = trace_panic_handler,
7060 .priority = 150 /* priority: INT_MAX >= x >= 0 */
7063 static int trace_die_handler(struct notifier_block *self,
7069 if (ftrace_dump_on_oops)
7070 ftrace_dump(ftrace_dump_on_oops);
7078 static struct notifier_block trace_die_notifier = {
7079 .notifier_call = trace_die_handler,
7084 * printk is set to max of 1024, we really don't need it that big.
7085 * Nothing should be printing 1000 characters anyway.
7087 #define TRACE_MAX_PRINT 1000
7090 * Define here KERN_TRACE so that we have one place to modify
7091 * it if we decide to change what log level the ftrace dump
7094 #define KERN_TRACE KERN_EMERG
7097 trace_printk_seq(struct trace_seq *s)
7099 /* Probably should print a warning here. */
7100 if (s->seq.len >= TRACE_MAX_PRINT)
7101 s->seq.len = TRACE_MAX_PRINT;
7104 * More paranoid code. Although the buffer size is set to
7105 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
7106 * an extra layer of protection.
7108 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
7109 s->seq.len = s->seq.size - 1;
7111 /* should be zero ended, but we are paranoid. */
7112 s->buffer[s->seq.len] = 0;
7114 printk(KERN_TRACE "%s", s->buffer);
7119 void trace_init_global_iter(struct trace_iterator *iter)
7121 iter->tr = &global_trace;
7122 iter->trace = iter->tr->current_trace;
7123 iter->cpu_file = RING_BUFFER_ALL_CPUS;
7124 iter->trace_buffer = &global_trace.trace_buffer;
7126 if (iter->trace && iter->trace->open)
7127 iter->trace->open(iter);
7129 /* Annotate start of buffers if we had overruns */
7130 if (ring_buffer_overruns(iter->trace_buffer->buffer))
7131 iter->iter_flags |= TRACE_FILE_ANNOTATE;
7133 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
7134 if (trace_clocks[iter->tr->clock_id].in_ns)
7135 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
7138 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
7140 /* use static because iter can be a bit big for the stack */
7141 static struct trace_iterator iter;
7142 static atomic_t dump_running;
7143 struct trace_array *tr = &global_trace;
7144 unsigned int old_userobj;
7145 unsigned long flags;
7148 /* Only allow one dump user at a time. */
7149 if (atomic_inc_return(&dump_running) != 1) {
7150 atomic_dec(&dump_running);
7155 * Always turn off tracing when we dump.
7156 * We don't need to show trace output of what happens
7157 * between multiple crashes.
7159 * If the user does a sysrq-z, then they can re-enable
7160 * tracing with echo 1 > tracing_on.
7164 local_irq_save(flags);
7166 /* Simulate the iterator */
7167 trace_init_global_iter(&iter);
7169 for_each_tracing_cpu(cpu) {
7170 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
7173 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
7175 /* don't look at user memory in panic mode */
7176 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
7178 switch (oops_dump_mode) {
7180 iter.cpu_file = RING_BUFFER_ALL_CPUS;
7183 iter.cpu_file = raw_smp_processor_id();
7188 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
7189 iter.cpu_file = RING_BUFFER_ALL_CPUS;
7192 printk(KERN_TRACE "Dumping ftrace buffer:\n");
7194 /* Did function tracer already get disabled? */
7195 if (ftrace_is_dead()) {
7196 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
7197 printk("# MAY BE MISSING FUNCTION EVENTS\n");
7201 * We need to stop all tracing on all CPUS to read the
7202 * the next buffer. This is a bit expensive, but is
7203 * not done often. We fill all what we can read,
7204 * and then release the locks again.
7207 while (!trace_empty(&iter)) {
7210 printk(KERN_TRACE "---------------------------------\n");
7214 trace_iterator_reset(&iter);
7215 iter.iter_flags |= TRACE_FILE_LAT_FMT;
7217 if (trace_find_next_entry_inc(&iter) != NULL) {
7220 ret = print_trace_line(&iter);
7221 if (ret != TRACE_TYPE_NO_CONSUME)
7222 trace_consume(&iter);
7224 touch_nmi_watchdog();
7226 trace_printk_seq(&iter.seq);
7230 printk(KERN_TRACE " (ftrace buffer empty)\n");
7232 printk(KERN_TRACE "---------------------------------\n");
7235 tr->trace_flags |= old_userobj;
7237 for_each_tracing_cpu(cpu) {
7238 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
7240 atomic_dec(&dump_running);
7241 local_irq_restore(flags);
7243 EXPORT_SYMBOL_GPL(ftrace_dump);
7245 __init static int tracer_alloc_buffers(void)
7251 * Make sure we don't accidently add more trace options
7252 * than we have bits for.
7254 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
7256 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
7259 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
7260 goto out_free_buffer_mask;
7262 /* Only allocate trace_printk buffers if a trace_printk exists */
7263 if (&__stop___trace_bprintk_fmt != &__start___trace_bprintk_fmt)
7264 /* Must be called before global_trace.buffer is allocated */
7265 trace_printk_init_buffers();
7267 /* To save memory, keep the ring buffer size to its minimum */
7268 if (ring_buffer_expanded)
7269 ring_buf_size = trace_buf_size;
7273 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
7274 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
7276 raw_spin_lock_init(&global_trace.start_lock);
7278 /* Used for event triggers */
7279 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
7281 goto out_free_cpumask;
7283 if (trace_create_savedcmd() < 0)
7284 goto out_free_temp_buffer;
7286 /* TODO: make the number of buffers hot pluggable with CPUS */
7287 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
7288 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
7290 goto out_free_savedcmd;
7293 if (global_trace.buffer_disabled)
7296 if (trace_boot_clock) {
7297 ret = tracing_set_clock(&global_trace, trace_boot_clock);
7299 pr_warning("Trace clock %s not defined, going back to default\n",
7304 * register_tracer() might reference current_trace, so it
7305 * needs to be set before we register anything. This is
7306 * just a bootstrap of current_trace anyway.
7308 global_trace.current_trace = &nop_trace;
7310 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7312 ftrace_init_global_array_ops(&global_trace);
7314 init_trace_flags_index(&global_trace);
7316 register_tracer(&nop_trace);
7318 /* All seems OK, enable tracing */
7319 tracing_disabled = 0;
7321 atomic_notifier_chain_register(&panic_notifier_list,
7322 &trace_panic_notifier);
7324 register_die_notifier(&trace_die_notifier);
7326 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
7328 INIT_LIST_HEAD(&global_trace.systems);
7329 INIT_LIST_HEAD(&global_trace.events);
7330 list_add(&global_trace.list, &ftrace_trace_arrays);
7332 apply_trace_boot_options();
7334 register_snapshot_cmd();
7339 free_saved_cmdlines_buffer(savedcmd);
7340 out_free_temp_buffer:
7341 ring_buffer_free(temp_buffer);
7343 free_cpumask_var(global_trace.tracing_cpumask);
7344 out_free_buffer_mask:
7345 free_cpumask_var(tracing_buffer_mask);
7350 void __init trace_init(void)
7352 if (tracepoint_printk) {
7353 tracepoint_print_iter =
7354 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
7355 if (WARN_ON(!tracepoint_print_iter))
7356 tracepoint_printk = 0;
7358 tracer_alloc_buffers();
7362 __init static int clear_boot_tracer(void)
7365 * The default tracer at boot buffer is an init section.
7366 * This function is called in lateinit. If we did not
7367 * find the boot tracer, then clear it out, to prevent
7368 * later registration from accessing the buffer that is
7369 * about to be freed.
7371 if (!default_bootup_tracer)
7374 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
7375 default_bootup_tracer);
7376 default_bootup_tracer = NULL;
7381 fs_initcall(tracer_init_tracefs);
7382 late_initcall(clear_boot_tracer);