2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/pagemap.h>
24 #include <linux/hardirq.h>
25 #include <linux/linkage.h>
26 #include <linux/uaccess.h>
27 #include <linux/kprobes.h>
28 #include <linux/ftrace.h>
29 #include <linux/module.h>
30 #include <linux/percpu.h>
31 #include <linux/splice.h>
32 #include <linux/kdebug.h>
33 #include <linux/string.h>
34 #include <linux/rwsem.h>
35 #include <linux/slab.h>
36 #include <linux/ctype.h>
37 #include <linux/init.h>
38 #include <linux/poll.h>
39 #include <linux/nmi.h>
41 #include <linux/sched/rt.h>
44 #include "trace_output.h"
47 * On boot up, the ring buffer is set to the minimum size, so that
48 * we do not waste memory on systems that are not using tracing.
50 bool ring_buffer_expanded;
53 * We need to change this state when a selftest is running.
54 * A selftest will lurk into the ring-buffer to count the
55 * entries inserted during the selftest although some concurrent
56 * insertions into the ring-buffer such as trace_printk could occurred
57 * at the same time, giving false positive or negative results.
59 static bool __read_mostly tracing_selftest_running;
62 * If a tracer is running, we do not want to run SELFTEST.
64 bool __read_mostly tracing_selftest_disabled;
66 /* For tracers that don't implement custom flags */
67 static struct tracer_opt dummy_tracer_opt[] = {
71 static struct tracer_flags dummy_tracer_flags = {
73 .opts = dummy_tracer_opt
77 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
83 * To prevent the comm cache from being overwritten when no
84 * tracing is active, only save the comm when a trace event
87 static DEFINE_PER_CPU(bool, trace_cmdline_save);
90 * Kill all tracing for good (never come back).
91 * It is initialized to 1 but will turn to zero if the initialization
92 * of the tracer is successful. But that is the only place that sets
95 static int tracing_disabled = 1;
97 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
99 cpumask_var_t __read_mostly tracing_buffer_mask;
102 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
104 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
105 * is set, then ftrace_dump is called. This will output the contents
106 * of the ftrace buffers to the console. This is very useful for
107 * capturing traces that lead to crashes and outputing it to a
110 * It is default off, but you can enable it with either specifying
111 * "ftrace_dump_on_oops" in the kernel command line, or setting
112 * /proc/sys/kernel/ftrace_dump_on_oops
113 * Set 1 if you want to dump buffers of all CPUs
114 * Set 2 if you want to dump the buffer of the CPU that triggered oops
117 enum ftrace_dump_mode ftrace_dump_on_oops;
119 /* When set, tracing will stop when a WARN*() is hit */
120 int __disable_trace_on_warning;
122 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
124 #define MAX_TRACER_SIZE 100
125 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
126 static char *default_bootup_tracer;
128 static bool allocate_snapshot;
130 static int __init set_cmdline_ftrace(char *str)
132 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
133 default_bootup_tracer = bootup_tracer_buf;
134 /* We are using ftrace early, expand it */
135 ring_buffer_expanded = true;
138 __setup("ftrace=", set_cmdline_ftrace);
140 static int __init set_ftrace_dump_on_oops(char *str)
142 if (*str++ != '=' || !*str) {
143 ftrace_dump_on_oops = DUMP_ALL;
147 if (!strcmp("orig_cpu", str)) {
148 ftrace_dump_on_oops = DUMP_ORIG;
154 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
156 static int __init stop_trace_on_warning(char *str)
158 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
159 __disable_trace_on_warning = 1;
162 __setup("traceoff_on_warning", stop_trace_on_warning);
164 static int __init boot_alloc_snapshot(char *str)
166 allocate_snapshot = true;
167 /* We also need the main ring buffer expanded */
168 ring_buffer_expanded = true;
171 __setup("alloc_snapshot", boot_alloc_snapshot);
174 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
175 static char *trace_boot_options __initdata;
177 static int __init set_trace_boot_options(char *str)
179 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
180 trace_boot_options = trace_boot_options_buf;
183 __setup("trace_options=", set_trace_boot_options);
185 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
186 static char *trace_boot_clock __initdata;
188 static int __init set_trace_boot_clock(char *str)
190 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
191 trace_boot_clock = trace_boot_clock_buf;
194 __setup("trace_clock=", set_trace_boot_clock);
197 unsigned long long ns2usecs(cycle_t nsec)
205 * The global_trace is the descriptor that holds the tracing
206 * buffers for the live tracing. For each CPU, it contains
207 * a link list of pages that will store trace entries. The
208 * page descriptor of the pages in the memory is used to hold
209 * the link list by linking the lru item in the page descriptor
210 * to each of the pages in the buffer per CPU.
212 * For each active CPU there is a data field that holds the
213 * pages for the buffer for that CPU. Each CPU has the same number
214 * of pages allocated for its buffer.
216 static struct trace_array global_trace;
218 LIST_HEAD(ftrace_trace_arrays);
220 int trace_array_get(struct trace_array *this_tr)
222 struct trace_array *tr;
225 mutex_lock(&trace_types_lock);
226 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
233 mutex_unlock(&trace_types_lock);
238 static void __trace_array_put(struct trace_array *this_tr)
240 WARN_ON(!this_tr->ref);
244 void trace_array_put(struct trace_array *this_tr)
246 mutex_lock(&trace_types_lock);
247 __trace_array_put(this_tr);
248 mutex_unlock(&trace_types_lock);
251 int filter_check_discard(struct ftrace_event_file *file, void *rec,
252 struct ring_buffer *buffer,
253 struct ring_buffer_event *event)
255 if (unlikely(file->flags & FTRACE_EVENT_FL_FILTERED) &&
256 !filter_match_preds(file->filter, rec)) {
257 ring_buffer_discard_commit(buffer, event);
263 EXPORT_SYMBOL_GPL(filter_check_discard);
265 int call_filter_check_discard(struct ftrace_event_call *call, void *rec,
266 struct ring_buffer *buffer,
267 struct ring_buffer_event *event)
269 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
270 !filter_match_preds(call->filter, rec)) {
271 ring_buffer_discard_commit(buffer, event);
277 EXPORT_SYMBOL_GPL(call_filter_check_discard);
279 static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
283 /* Early boot up does not have a buffer yet */
285 return trace_clock_local();
287 ts = ring_buffer_time_stamp(buf->buffer, cpu);
288 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
293 cycle_t ftrace_now(int cpu)
295 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
299 * tracing_is_enabled - Show if global_trace has been disabled
301 * Shows if the global trace has been enabled or not. It uses the
302 * mirror flag "buffer_disabled" to be used in fast paths such as for
303 * the irqsoff tracer. But it may be inaccurate due to races. If you
304 * need to know the accurate state, use tracing_is_on() which is a little
305 * slower, but accurate.
307 int tracing_is_enabled(void)
310 * For quick access (irqsoff uses this in fast path), just
311 * return the mirror variable of the state of the ring buffer.
312 * It's a little racy, but we don't really care.
315 return !global_trace.buffer_disabled;
319 * trace_buf_size is the size in bytes that is allocated
320 * for a buffer. Note, the number of bytes is always rounded
323 * This number is purposely set to a low number of 16384.
324 * If the dump on oops happens, it will be much appreciated
325 * to not have to wait for all that output. Anyway this can be
326 * boot time and run time configurable.
328 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
330 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
332 /* trace_types holds a link list of available tracers. */
333 static struct tracer *trace_types __read_mostly;
336 * trace_types_lock is used to protect the trace_types list.
338 DEFINE_MUTEX(trace_types_lock);
341 * serialize the access of the ring buffer
343 * ring buffer serializes readers, but it is low level protection.
344 * The validity of the events (which returns by ring_buffer_peek() ..etc)
345 * are not protected by ring buffer.
347 * The content of events may become garbage if we allow other process consumes
348 * these events concurrently:
349 * A) the page of the consumed events may become a normal page
350 * (not reader page) in ring buffer, and this page will be rewrited
351 * by events producer.
352 * B) The page of the consumed events may become a page for splice_read,
353 * and this page will be returned to system.
355 * These primitives allow multi process access to different cpu ring buffer
358 * These primitives don't distinguish read-only and read-consume access.
359 * Multi read-only access are also serialized.
363 static DECLARE_RWSEM(all_cpu_access_lock);
364 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
366 static inline void trace_access_lock(int cpu)
368 if (cpu == RING_BUFFER_ALL_CPUS) {
369 /* gain it for accessing the whole ring buffer. */
370 down_write(&all_cpu_access_lock);
372 /* gain it for accessing a cpu ring buffer. */
374 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
375 down_read(&all_cpu_access_lock);
377 /* Secondly block other access to this @cpu ring buffer. */
378 mutex_lock(&per_cpu(cpu_access_lock, cpu));
382 static inline void trace_access_unlock(int cpu)
384 if (cpu == RING_BUFFER_ALL_CPUS) {
385 up_write(&all_cpu_access_lock);
387 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
388 up_read(&all_cpu_access_lock);
392 static inline void trace_access_lock_init(void)
396 for_each_possible_cpu(cpu)
397 mutex_init(&per_cpu(cpu_access_lock, cpu));
402 static DEFINE_MUTEX(access_lock);
404 static inline void trace_access_lock(int cpu)
407 mutex_lock(&access_lock);
410 static inline void trace_access_unlock(int cpu)
413 mutex_unlock(&access_lock);
416 static inline void trace_access_lock_init(void)
422 /* trace_flags holds trace_options default values */
423 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
424 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
425 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
426 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
428 static void tracer_tracing_on(struct trace_array *tr)
430 if (tr->trace_buffer.buffer)
431 ring_buffer_record_on(tr->trace_buffer.buffer);
433 * This flag is looked at when buffers haven't been allocated
434 * yet, or by some tracers (like irqsoff), that just want to
435 * know if the ring buffer has been disabled, but it can handle
436 * races of where it gets disabled but we still do a record.
437 * As the check is in the fast path of the tracers, it is more
438 * important to be fast than accurate.
440 tr->buffer_disabled = 0;
441 /* Make the flag seen by readers */
446 * tracing_on - enable tracing buffers
448 * This function enables tracing buffers that may have been
449 * disabled with tracing_off.
451 void tracing_on(void)
453 tracer_tracing_on(&global_trace);
455 EXPORT_SYMBOL_GPL(tracing_on);
458 * __trace_puts - write a constant string into the trace buffer.
459 * @ip: The address of the caller
460 * @str: The constant string to write
461 * @size: The size of the string.
463 int __trace_puts(unsigned long ip, const char *str, int size)
465 struct ring_buffer_event *event;
466 struct ring_buffer *buffer;
467 struct print_entry *entry;
468 unsigned long irq_flags;
472 if (!(trace_flags & TRACE_ITER_PRINTK))
475 pc = preempt_count();
477 if (unlikely(tracing_selftest_running || tracing_disabled))
480 alloc = sizeof(*entry) + size + 2; /* possible \n added */
482 local_save_flags(irq_flags);
483 buffer = global_trace.trace_buffer.buffer;
484 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
489 entry = ring_buffer_event_data(event);
492 memcpy(&entry->buf, str, size);
494 /* Add a newline if necessary */
495 if (entry->buf[size - 1] != '\n') {
496 entry->buf[size] = '\n';
497 entry->buf[size + 1] = '\0';
499 entry->buf[size] = '\0';
501 __buffer_unlock_commit(buffer, event);
502 ftrace_trace_stack(buffer, irq_flags, 4, pc);
506 EXPORT_SYMBOL_GPL(__trace_puts);
509 * __trace_bputs - write the pointer to a constant string into trace buffer
510 * @ip: The address of the caller
511 * @str: The constant string to write to the buffer to
513 int __trace_bputs(unsigned long ip, const char *str)
515 struct ring_buffer_event *event;
516 struct ring_buffer *buffer;
517 struct bputs_entry *entry;
518 unsigned long irq_flags;
519 int size = sizeof(struct bputs_entry);
522 if (!(trace_flags & TRACE_ITER_PRINTK))
525 pc = preempt_count();
527 if (unlikely(tracing_selftest_running || tracing_disabled))
530 local_save_flags(irq_flags);
531 buffer = global_trace.trace_buffer.buffer;
532 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
537 entry = ring_buffer_event_data(event);
541 __buffer_unlock_commit(buffer, event);
542 ftrace_trace_stack(buffer, irq_flags, 4, pc);
546 EXPORT_SYMBOL_GPL(__trace_bputs);
548 #ifdef CONFIG_TRACER_SNAPSHOT
550 * trace_snapshot - take a snapshot of the current buffer.
552 * This causes a swap between the snapshot buffer and the current live
553 * tracing buffer. You can use this to take snapshots of the live
554 * trace when some condition is triggered, but continue to trace.
556 * Note, make sure to allocate the snapshot with either
557 * a tracing_snapshot_alloc(), or by doing it manually
558 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
560 * If the snapshot buffer is not allocated, it will stop tracing.
561 * Basically making a permanent snapshot.
563 void tracing_snapshot(void)
565 struct trace_array *tr = &global_trace;
566 struct tracer *tracer = tr->current_trace;
570 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
571 internal_trace_puts("*** snapshot is being ignored ***\n");
575 if (!tr->allocated_snapshot) {
576 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
577 internal_trace_puts("*** stopping trace here! ***\n");
582 /* Note, snapshot can not be used when the tracer uses it */
583 if (tracer->use_max_tr) {
584 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
585 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
589 local_irq_save(flags);
590 update_max_tr(tr, current, smp_processor_id());
591 local_irq_restore(flags);
593 EXPORT_SYMBOL_GPL(tracing_snapshot);
595 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
596 struct trace_buffer *size_buf, int cpu_id);
597 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
599 static int alloc_snapshot(struct trace_array *tr)
603 if (!tr->allocated_snapshot) {
605 /* allocate spare buffer */
606 ret = resize_buffer_duplicate_size(&tr->max_buffer,
607 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
611 tr->allocated_snapshot = true;
617 static void free_snapshot(struct trace_array *tr)
620 * We don't free the ring buffer. instead, resize it because
621 * The max_tr ring buffer has some state (e.g. ring->clock) and
622 * we want preserve it.
624 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
625 set_buffer_entries(&tr->max_buffer, 1);
626 tracing_reset_online_cpus(&tr->max_buffer);
627 tr->allocated_snapshot = false;
631 * tracing_alloc_snapshot - allocate snapshot buffer.
633 * This only allocates the snapshot buffer if it isn't already
634 * allocated - it doesn't also take a snapshot.
636 * This is meant to be used in cases where the snapshot buffer needs
637 * to be set up for events that can't sleep but need to be able to
638 * trigger a snapshot.
640 int tracing_alloc_snapshot(void)
642 struct trace_array *tr = &global_trace;
645 ret = alloc_snapshot(tr);
650 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
653 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
655 * This is similar to trace_snapshot(), but it will allocate the
656 * snapshot buffer if it isn't already allocated. Use this only
657 * where it is safe to sleep, as the allocation may sleep.
659 * This causes a swap between the snapshot buffer and the current live
660 * tracing buffer. You can use this to take snapshots of the live
661 * trace when some condition is triggered, but continue to trace.
663 void tracing_snapshot_alloc(void)
667 ret = tracing_alloc_snapshot();
673 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
675 void tracing_snapshot(void)
677 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
679 EXPORT_SYMBOL_GPL(tracing_snapshot);
680 int tracing_alloc_snapshot(void)
682 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
685 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
686 void tracing_snapshot_alloc(void)
691 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
692 #endif /* CONFIG_TRACER_SNAPSHOT */
694 static void tracer_tracing_off(struct trace_array *tr)
696 if (tr->trace_buffer.buffer)
697 ring_buffer_record_off(tr->trace_buffer.buffer);
699 * This flag is looked at when buffers haven't been allocated
700 * yet, or by some tracers (like irqsoff), that just want to
701 * know if the ring buffer has been disabled, but it can handle
702 * races of where it gets disabled but we still do a record.
703 * As the check is in the fast path of the tracers, it is more
704 * important to be fast than accurate.
706 tr->buffer_disabled = 1;
707 /* Make the flag seen by readers */
712 * tracing_off - turn off tracing buffers
714 * This function stops the tracing buffers from recording data.
715 * It does not disable any overhead the tracers themselves may
716 * be causing. This function simply causes all recording to
717 * the ring buffers to fail.
719 void tracing_off(void)
721 tracer_tracing_off(&global_trace);
723 EXPORT_SYMBOL_GPL(tracing_off);
725 void disable_trace_on_warning(void)
727 if (__disable_trace_on_warning)
732 * tracer_tracing_is_on - show real state of ring buffer enabled
733 * @tr : the trace array to know if ring buffer is enabled
735 * Shows real state of the ring buffer if it is enabled or not.
737 static int tracer_tracing_is_on(struct trace_array *tr)
739 if (tr->trace_buffer.buffer)
740 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
741 return !tr->buffer_disabled;
745 * tracing_is_on - show state of ring buffers enabled
747 int tracing_is_on(void)
749 return tracer_tracing_is_on(&global_trace);
751 EXPORT_SYMBOL_GPL(tracing_is_on);
753 static int __init set_buf_size(char *str)
755 unsigned long buf_size;
759 buf_size = memparse(str, &str);
760 /* nr_entries can not be zero */
763 trace_buf_size = buf_size;
766 __setup("trace_buf_size=", set_buf_size);
768 static int __init set_tracing_thresh(char *str)
770 unsigned long threshold;
775 ret = kstrtoul(str, 0, &threshold);
778 tracing_thresh = threshold * 1000;
781 __setup("tracing_thresh=", set_tracing_thresh);
783 unsigned long nsecs_to_usecs(unsigned long nsecs)
788 /* These must match the bit postions in trace_iterator_flags */
789 static const char *trace_options[] = {
822 int in_ns; /* is this clock in nanoseconds? */
824 { trace_clock_local, "local", 1 },
825 { trace_clock_global, "global", 1 },
826 { trace_clock_counter, "counter", 0 },
827 { trace_clock_jiffies, "uptime", 0 },
828 { trace_clock, "perf", 1 },
829 { ktime_get_mono_fast_ns, "mono", 1 },
834 * trace_parser_get_init - gets the buffer for trace parser
836 int trace_parser_get_init(struct trace_parser *parser, int size)
838 memset(parser, 0, sizeof(*parser));
840 parser->buffer = kmalloc(size, GFP_KERNEL);
849 * trace_parser_put - frees the buffer for trace parser
851 void trace_parser_put(struct trace_parser *parser)
853 kfree(parser->buffer);
857 * trace_get_user - reads the user input string separated by space
858 * (matched by isspace(ch))
860 * For each string found the 'struct trace_parser' is updated,
861 * and the function returns.
863 * Returns number of bytes read.
865 * See kernel/trace/trace.h for 'struct trace_parser' details.
867 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
868 size_t cnt, loff_t *ppos)
875 trace_parser_clear(parser);
877 ret = get_user(ch, ubuf++);
885 * The parser is not finished with the last write,
886 * continue reading the user input without skipping spaces.
889 /* skip white space */
890 while (cnt && isspace(ch)) {
891 ret = get_user(ch, ubuf++);
898 /* only spaces were written */
908 /* read the non-space input */
909 while (cnt && !isspace(ch)) {
910 if (parser->idx < parser->size - 1)
911 parser->buffer[parser->idx++] = ch;
916 ret = get_user(ch, ubuf++);
923 /* We either got finished input or we have to wait for another call. */
925 parser->buffer[parser->idx] = 0;
926 parser->cont = false;
927 } else if (parser->idx < parser->size - 1) {
929 parser->buffer[parser->idx++] = ch;
942 /* TODO add a seq_buf_to_buffer() */
943 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
947 if (trace_seq_used(s) <= s->seq.readpos)
950 len = trace_seq_used(s) - s->seq.readpos;
953 memcpy(buf, s->buffer + s->seq.readpos, cnt);
955 s->seq.readpos += cnt;
959 unsigned long __read_mostly tracing_thresh;
961 #ifdef CONFIG_TRACER_MAX_TRACE
963 * Copy the new maximum trace into the separate maximum-trace
964 * structure. (this way the maximum trace is permanently saved,
965 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
968 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
970 struct trace_buffer *trace_buf = &tr->trace_buffer;
971 struct trace_buffer *max_buf = &tr->max_buffer;
972 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
973 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
976 max_buf->time_start = data->preempt_timestamp;
978 max_data->saved_latency = tr->max_latency;
979 max_data->critical_start = data->critical_start;
980 max_data->critical_end = data->critical_end;
982 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
983 max_data->pid = tsk->pid;
985 * If tsk == current, then use current_uid(), as that does not use
986 * RCU. The irq tracer can be called out of RCU scope.
989 max_data->uid = current_uid();
991 max_data->uid = task_uid(tsk);
993 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
994 max_data->policy = tsk->policy;
995 max_data->rt_priority = tsk->rt_priority;
997 /* record this tasks comm */
998 tracing_record_cmdline(tsk);
1002 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1004 * @tsk: the task with the latency
1005 * @cpu: The cpu that initiated the trace.
1007 * Flip the buffers between the @tr and the max_tr and record information
1008 * about which task was the cause of this latency.
1011 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1013 struct ring_buffer *buf;
1018 WARN_ON_ONCE(!irqs_disabled());
1020 if (!tr->allocated_snapshot) {
1021 /* Only the nop tracer should hit this when disabling */
1022 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1026 arch_spin_lock(&tr->max_lock);
1028 buf = tr->trace_buffer.buffer;
1029 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1030 tr->max_buffer.buffer = buf;
1032 __update_max_tr(tr, tsk, cpu);
1033 arch_spin_unlock(&tr->max_lock);
1037 * update_max_tr_single - only copy one trace over, and reset the rest
1039 * @tsk - task with the latency
1040 * @cpu - the cpu of the buffer to copy.
1042 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1045 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1052 WARN_ON_ONCE(!irqs_disabled());
1053 if (!tr->allocated_snapshot) {
1054 /* Only the nop tracer should hit this when disabling */
1055 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1059 arch_spin_lock(&tr->max_lock);
1061 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1063 if (ret == -EBUSY) {
1065 * We failed to swap the buffer due to a commit taking
1066 * place on this CPU. We fail to record, but we reset
1067 * the max trace buffer (no one writes directly to it)
1068 * and flag that it failed.
1070 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1071 "Failed to swap buffers due to commit in progress\n");
1074 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1076 __update_max_tr(tr, tsk, cpu);
1077 arch_spin_unlock(&tr->max_lock);
1079 #endif /* CONFIG_TRACER_MAX_TRACE */
1081 static int wait_on_pipe(struct trace_iterator *iter, bool full)
1083 /* Iterators are static, they should be filled or empty */
1084 if (trace_buffer_iter(iter, iter->cpu_file))
1087 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1091 #ifdef CONFIG_FTRACE_STARTUP_TEST
1092 static int run_tracer_selftest(struct tracer *type)
1094 struct trace_array *tr = &global_trace;
1095 struct tracer *saved_tracer = tr->current_trace;
1098 if (!type->selftest || tracing_selftest_disabled)
1102 * Run a selftest on this tracer.
1103 * Here we reset the trace buffer, and set the current
1104 * tracer to be this tracer. The tracer can then run some
1105 * internal tracing to verify that everything is in order.
1106 * If we fail, we do not register this tracer.
1108 tracing_reset_online_cpus(&tr->trace_buffer);
1110 tr->current_trace = type;
1112 #ifdef CONFIG_TRACER_MAX_TRACE
1113 if (type->use_max_tr) {
1114 /* If we expanded the buffers, make sure the max is expanded too */
1115 if (ring_buffer_expanded)
1116 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1117 RING_BUFFER_ALL_CPUS);
1118 tr->allocated_snapshot = true;
1122 /* the test is responsible for initializing and enabling */
1123 pr_info("Testing tracer %s: ", type->name);
1124 ret = type->selftest(type, tr);
1125 /* the test is responsible for resetting too */
1126 tr->current_trace = saved_tracer;
1128 printk(KERN_CONT "FAILED!\n");
1129 /* Add the warning after printing 'FAILED' */
1133 /* Only reset on passing, to avoid touching corrupted buffers */
1134 tracing_reset_online_cpus(&tr->trace_buffer);
1136 #ifdef CONFIG_TRACER_MAX_TRACE
1137 if (type->use_max_tr) {
1138 tr->allocated_snapshot = false;
1140 /* Shrink the max buffer again */
1141 if (ring_buffer_expanded)
1142 ring_buffer_resize(tr->max_buffer.buffer, 1,
1143 RING_BUFFER_ALL_CPUS);
1147 printk(KERN_CONT "PASSED\n");
1151 static inline int run_tracer_selftest(struct tracer *type)
1155 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1158 * register_tracer - register a tracer with the ftrace system.
1159 * @type - the plugin for the tracer
1161 * Register a new plugin tracer.
1163 int register_tracer(struct tracer *type)
1169 pr_info("Tracer must have a name\n");
1173 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1174 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1178 mutex_lock(&trace_types_lock);
1180 tracing_selftest_running = true;
1182 for (t = trace_types; t; t = t->next) {
1183 if (strcmp(type->name, t->name) == 0) {
1185 pr_info("Tracer %s already registered\n",
1192 if (!type->set_flag)
1193 type->set_flag = &dummy_set_flag;
1195 type->flags = &dummy_tracer_flags;
1197 if (!type->flags->opts)
1198 type->flags->opts = dummy_tracer_opt;
1200 ret = run_tracer_selftest(type);
1204 type->next = trace_types;
1208 tracing_selftest_running = false;
1209 mutex_unlock(&trace_types_lock);
1211 if (ret || !default_bootup_tracer)
1214 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1217 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1218 /* Do we want this tracer to start on bootup? */
1219 tracing_set_tracer(&global_trace, type->name);
1220 default_bootup_tracer = NULL;
1221 /* disable other selftests, since this will break it. */
1222 tracing_selftest_disabled = true;
1223 #ifdef CONFIG_FTRACE_STARTUP_TEST
1224 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1232 void tracing_reset(struct trace_buffer *buf, int cpu)
1234 struct ring_buffer *buffer = buf->buffer;
1239 ring_buffer_record_disable(buffer);
1241 /* Make sure all commits have finished */
1242 synchronize_sched();
1243 ring_buffer_reset_cpu(buffer, cpu);
1245 ring_buffer_record_enable(buffer);
1248 void tracing_reset_online_cpus(struct trace_buffer *buf)
1250 struct ring_buffer *buffer = buf->buffer;
1256 ring_buffer_record_disable(buffer);
1258 /* Make sure all commits have finished */
1259 synchronize_sched();
1261 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1263 for_each_online_cpu(cpu)
1264 ring_buffer_reset_cpu(buffer, cpu);
1266 ring_buffer_record_enable(buffer);
1269 /* Must have trace_types_lock held */
1270 void tracing_reset_all_online_cpus(void)
1272 struct trace_array *tr;
1274 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1275 tracing_reset_online_cpus(&tr->trace_buffer);
1276 #ifdef CONFIG_TRACER_MAX_TRACE
1277 tracing_reset_online_cpus(&tr->max_buffer);
1282 #define SAVED_CMDLINES_DEFAULT 128
1283 #define NO_CMDLINE_MAP UINT_MAX
1284 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1285 struct saved_cmdlines_buffer {
1286 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1287 unsigned *map_cmdline_to_pid;
1288 unsigned cmdline_num;
1290 char *saved_cmdlines;
1292 static struct saved_cmdlines_buffer *savedcmd;
1294 /* temporary disable recording */
1295 static atomic_t trace_record_cmdline_disabled __read_mostly;
1297 static inline char *get_saved_cmdlines(int idx)
1299 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1302 static inline void set_cmdline(int idx, const char *cmdline)
1304 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1307 static int allocate_cmdlines_buffer(unsigned int val,
1308 struct saved_cmdlines_buffer *s)
1310 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1312 if (!s->map_cmdline_to_pid)
1315 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1316 if (!s->saved_cmdlines) {
1317 kfree(s->map_cmdline_to_pid);
1322 s->cmdline_num = val;
1323 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1324 sizeof(s->map_pid_to_cmdline));
1325 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1326 val * sizeof(*s->map_cmdline_to_pid));
1331 static int trace_create_savedcmd(void)
1335 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1339 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1349 int is_tracing_stopped(void)
1351 return global_trace.stop_count;
1355 * tracing_start - quick start of the tracer
1357 * If tracing is enabled but was stopped by tracing_stop,
1358 * this will start the tracer back up.
1360 void tracing_start(void)
1362 struct ring_buffer *buffer;
1363 unsigned long flags;
1365 if (tracing_disabled)
1368 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1369 if (--global_trace.stop_count) {
1370 if (global_trace.stop_count < 0) {
1371 /* Someone screwed up their debugging */
1373 global_trace.stop_count = 0;
1378 /* Prevent the buffers from switching */
1379 arch_spin_lock(&global_trace.max_lock);
1381 buffer = global_trace.trace_buffer.buffer;
1383 ring_buffer_record_enable(buffer);
1385 #ifdef CONFIG_TRACER_MAX_TRACE
1386 buffer = global_trace.max_buffer.buffer;
1388 ring_buffer_record_enable(buffer);
1391 arch_spin_unlock(&global_trace.max_lock);
1394 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1397 static void tracing_start_tr(struct trace_array *tr)
1399 struct ring_buffer *buffer;
1400 unsigned long flags;
1402 if (tracing_disabled)
1405 /* If global, we need to also start the max tracer */
1406 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1407 return tracing_start();
1409 raw_spin_lock_irqsave(&tr->start_lock, flags);
1411 if (--tr->stop_count) {
1412 if (tr->stop_count < 0) {
1413 /* Someone screwed up their debugging */
1420 buffer = tr->trace_buffer.buffer;
1422 ring_buffer_record_enable(buffer);
1425 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1429 * tracing_stop - quick stop of the tracer
1431 * Light weight way to stop tracing. Use in conjunction with
1434 void tracing_stop(void)
1436 struct ring_buffer *buffer;
1437 unsigned long flags;
1439 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1440 if (global_trace.stop_count++)
1443 /* Prevent the buffers from switching */
1444 arch_spin_lock(&global_trace.max_lock);
1446 buffer = global_trace.trace_buffer.buffer;
1448 ring_buffer_record_disable(buffer);
1450 #ifdef CONFIG_TRACER_MAX_TRACE
1451 buffer = global_trace.max_buffer.buffer;
1453 ring_buffer_record_disable(buffer);
1456 arch_spin_unlock(&global_trace.max_lock);
1459 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1462 static void tracing_stop_tr(struct trace_array *tr)
1464 struct ring_buffer *buffer;
1465 unsigned long flags;
1467 /* If global, we need to also stop the max tracer */
1468 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1469 return tracing_stop();
1471 raw_spin_lock_irqsave(&tr->start_lock, flags);
1472 if (tr->stop_count++)
1475 buffer = tr->trace_buffer.buffer;
1477 ring_buffer_record_disable(buffer);
1480 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1483 void trace_stop_cmdline_recording(void);
1485 static int trace_save_cmdline(struct task_struct *tsk)
1489 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1493 * It's not the end of the world if we don't get
1494 * the lock, but we also don't want to spin
1495 * nor do we want to disable interrupts,
1496 * so if we miss here, then better luck next time.
1498 if (!arch_spin_trylock(&trace_cmdline_lock))
1501 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
1502 if (idx == NO_CMDLINE_MAP) {
1503 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1506 * Check whether the cmdline buffer at idx has a pid
1507 * mapped. We are going to overwrite that entry so we
1508 * need to clear the map_pid_to_cmdline. Otherwise we
1509 * would read the new comm for the old pid.
1511 pid = savedcmd->map_cmdline_to_pid[idx];
1512 if (pid != NO_CMDLINE_MAP)
1513 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1515 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1516 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
1518 savedcmd->cmdline_idx = idx;
1521 set_cmdline(idx, tsk->comm);
1523 arch_spin_unlock(&trace_cmdline_lock);
1528 static void __trace_find_cmdline(int pid, char comm[])
1533 strcpy(comm, "<idle>");
1537 if (WARN_ON_ONCE(pid < 0)) {
1538 strcpy(comm, "<XXX>");
1542 if (pid > PID_MAX_DEFAULT) {
1543 strcpy(comm, "<...>");
1547 map = savedcmd->map_pid_to_cmdline[pid];
1548 if (map != NO_CMDLINE_MAP)
1549 strcpy(comm, get_saved_cmdlines(map));
1551 strcpy(comm, "<...>");
1554 void trace_find_cmdline(int pid, char comm[])
1557 arch_spin_lock(&trace_cmdline_lock);
1559 __trace_find_cmdline(pid, comm);
1561 arch_spin_unlock(&trace_cmdline_lock);
1565 void tracing_record_cmdline(struct task_struct *tsk)
1567 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1570 if (!__this_cpu_read(trace_cmdline_save))
1573 if (trace_save_cmdline(tsk))
1574 __this_cpu_write(trace_cmdline_save, false);
1578 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1581 struct task_struct *tsk = current;
1583 entry->preempt_count = pc & 0xff;
1584 entry->pid = (tsk) ? tsk->pid : 0;
1586 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1587 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1589 TRACE_FLAG_IRQS_NOSUPPORT |
1591 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1592 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1593 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1594 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1596 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1598 struct ring_buffer_event *
1599 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1602 unsigned long flags, int pc)
1604 struct ring_buffer_event *event;
1606 event = ring_buffer_lock_reserve(buffer, len);
1607 if (event != NULL) {
1608 struct trace_entry *ent = ring_buffer_event_data(event);
1610 tracing_generic_entry_update(ent, flags, pc);
1618 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1620 __this_cpu_write(trace_cmdline_save, true);
1621 ring_buffer_unlock_commit(buffer, event);
1625 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1626 struct ring_buffer_event *event,
1627 unsigned long flags, int pc)
1629 __buffer_unlock_commit(buffer, event);
1631 ftrace_trace_stack(buffer, flags, 6, pc);
1632 ftrace_trace_userstack(buffer, flags, pc);
1635 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1636 struct ring_buffer_event *event,
1637 unsigned long flags, int pc)
1639 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1641 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1643 static struct ring_buffer *temp_buffer;
1645 struct ring_buffer_event *
1646 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1647 struct ftrace_event_file *ftrace_file,
1648 int type, unsigned long len,
1649 unsigned long flags, int pc)
1651 struct ring_buffer_event *entry;
1653 *current_rb = ftrace_file->tr->trace_buffer.buffer;
1654 entry = trace_buffer_lock_reserve(*current_rb,
1655 type, len, flags, pc);
1657 * If tracing is off, but we have triggers enabled
1658 * we still need to look at the event data. Use the temp_buffer
1659 * to store the trace event for the tigger to use. It's recusive
1660 * safe and will not be recorded anywhere.
1662 if (!entry && ftrace_file->flags & FTRACE_EVENT_FL_TRIGGER_COND) {
1663 *current_rb = temp_buffer;
1664 entry = trace_buffer_lock_reserve(*current_rb,
1665 type, len, flags, pc);
1669 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1671 struct ring_buffer_event *
1672 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1673 int type, unsigned long len,
1674 unsigned long flags, int pc)
1676 *current_rb = global_trace.trace_buffer.buffer;
1677 return trace_buffer_lock_reserve(*current_rb,
1678 type, len, flags, pc);
1680 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1682 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1683 struct ring_buffer_event *event,
1684 unsigned long flags, int pc)
1686 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1688 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1690 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1691 struct ring_buffer_event *event,
1692 unsigned long flags, int pc,
1693 struct pt_regs *regs)
1695 __buffer_unlock_commit(buffer, event);
1697 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1698 ftrace_trace_userstack(buffer, flags, pc);
1700 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1702 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1703 struct ring_buffer_event *event)
1705 ring_buffer_discard_commit(buffer, event);
1707 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1710 trace_function(struct trace_array *tr,
1711 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1714 struct ftrace_event_call *call = &event_function;
1715 struct ring_buffer *buffer = tr->trace_buffer.buffer;
1716 struct ring_buffer_event *event;
1717 struct ftrace_entry *entry;
1719 /* If we are reading the ring buffer, don't trace */
1720 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1723 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1727 entry = ring_buffer_event_data(event);
1729 entry->parent_ip = parent_ip;
1731 if (!call_filter_check_discard(call, entry, buffer, event))
1732 __buffer_unlock_commit(buffer, event);
1735 #ifdef CONFIG_STACKTRACE
1737 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1738 struct ftrace_stack {
1739 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1742 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1743 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1745 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1746 unsigned long flags,
1747 int skip, int pc, struct pt_regs *regs)
1749 struct ftrace_event_call *call = &event_kernel_stack;
1750 struct ring_buffer_event *event;
1751 struct stack_entry *entry;
1752 struct stack_trace trace;
1754 int size = FTRACE_STACK_ENTRIES;
1756 trace.nr_entries = 0;
1760 * Since events can happen in NMIs there's no safe way to
1761 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1762 * or NMI comes in, it will just have to use the default
1763 * FTRACE_STACK_SIZE.
1765 preempt_disable_notrace();
1767 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1769 * We don't need any atomic variables, just a barrier.
1770 * If an interrupt comes in, we don't care, because it would
1771 * have exited and put the counter back to what we want.
1772 * We just need a barrier to keep gcc from moving things
1776 if (use_stack == 1) {
1777 trace.entries = this_cpu_ptr(ftrace_stack.calls);
1778 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1781 save_stack_trace_regs(regs, &trace);
1783 save_stack_trace(&trace);
1785 if (trace.nr_entries > size)
1786 size = trace.nr_entries;
1788 /* From now on, use_stack is a boolean */
1791 size *= sizeof(unsigned long);
1793 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1794 sizeof(*entry) + size, flags, pc);
1797 entry = ring_buffer_event_data(event);
1799 memset(&entry->caller, 0, size);
1802 memcpy(&entry->caller, trace.entries,
1803 trace.nr_entries * sizeof(unsigned long));
1805 trace.max_entries = FTRACE_STACK_ENTRIES;
1806 trace.entries = entry->caller;
1808 save_stack_trace_regs(regs, &trace);
1810 save_stack_trace(&trace);
1813 entry->size = trace.nr_entries;
1815 if (!call_filter_check_discard(call, entry, buffer, event))
1816 __buffer_unlock_commit(buffer, event);
1819 /* Again, don't let gcc optimize things here */
1821 __this_cpu_dec(ftrace_stack_reserve);
1822 preempt_enable_notrace();
1826 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1827 int skip, int pc, struct pt_regs *regs)
1829 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1832 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1835 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1838 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1841 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1844 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1847 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1851 * trace_dump_stack - record a stack back trace in the trace buffer
1852 * @skip: Number of functions to skip (helper handlers)
1854 void trace_dump_stack(int skip)
1856 unsigned long flags;
1858 if (tracing_disabled || tracing_selftest_running)
1861 local_save_flags(flags);
1864 * Skip 3 more, seems to get us at the caller of
1868 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1869 flags, skip, preempt_count(), NULL);
1872 static DEFINE_PER_CPU(int, user_stack_count);
1875 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1877 struct ftrace_event_call *call = &event_user_stack;
1878 struct ring_buffer_event *event;
1879 struct userstack_entry *entry;
1880 struct stack_trace trace;
1882 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1886 * NMIs can not handle page faults, even with fix ups.
1887 * The save user stack can (and often does) fault.
1889 if (unlikely(in_nmi()))
1893 * prevent recursion, since the user stack tracing may
1894 * trigger other kernel events.
1897 if (__this_cpu_read(user_stack_count))
1900 __this_cpu_inc(user_stack_count);
1902 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1903 sizeof(*entry), flags, pc);
1905 goto out_drop_count;
1906 entry = ring_buffer_event_data(event);
1908 entry->tgid = current->tgid;
1909 memset(&entry->caller, 0, sizeof(entry->caller));
1911 trace.nr_entries = 0;
1912 trace.max_entries = FTRACE_STACK_ENTRIES;
1914 trace.entries = entry->caller;
1916 save_stack_trace_user(&trace);
1917 if (!call_filter_check_discard(call, entry, buffer, event))
1918 __buffer_unlock_commit(buffer, event);
1921 __this_cpu_dec(user_stack_count);
1927 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1929 ftrace_trace_userstack(tr, flags, preempt_count());
1933 #endif /* CONFIG_STACKTRACE */
1935 /* created for use with alloc_percpu */
1936 struct trace_buffer_struct {
1937 char buffer[TRACE_BUF_SIZE];
1940 static struct trace_buffer_struct *trace_percpu_buffer;
1941 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1942 static struct trace_buffer_struct *trace_percpu_irq_buffer;
1943 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1946 * The buffer used is dependent on the context. There is a per cpu
1947 * buffer for normal context, softirq contex, hard irq context and
1948 * for NMI context. Thise allows for lockless recording.
1950 * Note, if the buffers failed to be allocated, then this returns NULL
1952 static char *get_trace_buf(void)
1954 struct trace_buffer_struct *percpu_buffer;
1957 * If we have allocated per cpu buffers, then we do not
1958 * need to do any locking.
1961 percpu_buffer = trace_percpu_nmi_buffer;
1963 percpu_buffer = trace_percpu_irq_buffer;
1964 else if (in_softirq())
1965 percpu_buffer = trace_percpu_sirq_buffer;
1967 percpu_buffer = trace_percpu_buffer;
1972 return this_cpu_ptr(&percpu_buffer->buffer[0]);
1975 static int alloc_percpu_trace_buffer(void)
1977 struct trace_buffer_struct *buffers;
1978 struct trace_buffer_struct *sirq_buffers;
1979 struct trace_buffer_struct *irq_buffers;
1980 struct trace_buffer_struct *nmi_buffers;
1982 buffers = alloc_percpu(struct trace_buffer_struct);
1986 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1990 irq_buffers = alloc_percpu(struct trace_buffer_struct);
1994 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1998 trace_percpu_buffer = buffers;
1999 trace_percpu_sirq_buffer = sirq_buffers;
2000 trace_percpu_irq_buffer = irq_buffers;
2001 trace_percpu_nmi_buffer = nmi_buffers;
2006 free_percpu(irq_buffers);
2008 free_percpu(sirq_buffers);
2010 free_percpu(buffers);
2012 WARN(1, "Could not allocate percpu trace_printk buffer");
2016 static int buffers_allocated;
2018 void trace_printk_init_buffers(void)
2020 if (buffers_allocated)
2023 if (alloc_percpu_trace_buffer())
2026 /* trace_printk() is for debug use only. Don't use it in production. */
2028 pr_warning("\n**********************************************************\n");
2029 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2030 pr_warning("** **\n");
2031 pr_warning("** trace_printk() being used. Allocating extra memory. **\n");
2032 pr_warning("** **\n");
2033 pr_warning("** This means that this is a DEBUG kernel and it is **\n");
2034 pr_warning("** unsafe for produciton use. **\n");
2035 pr_warning("** **\n");
2036 pr_warning("** If you see this message and you are not debugging **\n");
2037 pr_warning("** the kernel, report this immediately to your vendor! **\n");
2038 pr_warning("** **\n");
2039 pr_warning("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2040 pr_warning("**********************************************************\n");
2042 /* Expand the buffers to set size */
2043 tracing_update_buffers();
2045 buffers_allocated = 1;
2048 * trace_printk_init_buffers() can be called by modules.
2049 * If that happens, then we need to start cmdline recording
2050 * directly here. If the global_trace.buffer is already
2051 * allocated here, then this was called by module code.
2053 if (global_trace.trace_buffer.buffer)
2054 tracing_start_cmdline_record();
2057 void trace_printk_start_comm(void)
2059 /* Start tracing comms if trace printk is set */
2060 if (!buffers_allocated)
2062 tracing_start_cmdline_record();
2065 static void trace_printk_start_stop_comm(int enabled)
2067 if (!buffers_allocated)
2071 tracing_start_cmdline_record();
2073 tracing_stop_cmdline_record();
2077 * trace_vbprintk - write binary msg to tracing buffer
2080 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2082 struct ftrace_event_call *call = &event_bprint;
2083 struct ring_buffer_event *event;
2084 struct ring_buffer *buffer;
2085 struct trace_array *tr = &global_trace;
2086 struct bprint_entry *entry;
2087 unsigned long flags;
2089 int len = 0, size, pc;
2091 if (unlikely(tracing_selftest_running || tracing_disabled))
2094 /* Don't pollute graph traces with trace_vprintk internals */
2095 pause_graph_tracing();
2097 pc = preempt_count();
2098 preempt_disable_notrace();
2100 tbuffer = get_trace_buf();
2106 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2108 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2111 local_save_flags(flags);
2112 size = sizeof(*entry) + sizeof(u32) * len;
2113 buffer = tr->trace_buffer.buffer;
2114 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2118 entry = ring_buffer_event_data(event);
2122 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2123 if (!call_filter_check_discard(call, entry, buffer, event)) {
2124 __buffer_unlock_commit(buffer, event);
2125 ftrace_trace_stack(buffer, flags, 6, pc);
2129 preempt_enable_notrace();
2130 unpause_graph_tracing();
2134 EXPORT_SYMBOL_GPL(trace_vbprintk);
2137 __trace_array_vprintk(struct ring_buffer *buffer,
2138 unsigned long ip, const char *fmt, va_list args)
2140 struct ftrace_event_call *call = &event_print;
2141 struct ring_buffer_event *event;
2142 int len = 0, size, pc;
2143 struct print_entry *entry;
2144 unsigned long flags;
2147 if (tracing_disabled || tracing_selftest_running)
2150 /* Don't pollute graph traces with trace_vprintk internals */
2151 pause_graph_tracing();
2153 pc = preempt_count();
2154 preempt_disable_notrace();
2157 tbuffer = get_trace_buf();
2163 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2165 local_save_flags(flags);
2166 size = sizeof(*entry) + len + 1;
2167 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2171 entry = ring_buffer_event_data(event);
2174 memcpy(&entry->buf, tbuffer, len + 1);
2175 if (!call_filter_check_discard(call, entry, buffer, event)) {
2176 __buffer_unlock_commit(buffer, event);
2177 ftrace_trace_stack(buffer, flags, 6, pc);
2180 preempt_enable_notrace();
2181 unpause_graph_tracing();
2186 int trace_array_vprintk(struct trace_array *tr,
2187 unsigned long ip, const char *fmt, va_list args)
2189 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2192 int trace_array_printk(struct trace_array *tr,
2193 unsigned long ip, const char *fmt, ...)
2198 if (!(trace_flags & TRACE_ITER_PRINTK))
2202 ret = trace_array_vprintk(tr, ip, fmt, ap);
2207 int trace_array_printk_buf(struct ring_buffer *buffer,
2208 unsigned long ip, const char *fmt, ...)
2213 if (!(trace_flags & TRACE_ITER_PRINTK))
2217 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2222 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2224 return trace_array_vprintk(&global_trace, ip, fmt, args);
2226 EXPORT_SYMBOL_GPL(trace_vprintk);
2228 static void trace_iterator_increment(struct trace_iterator *iter)
2230 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2234 ring_buffer_read(buf_iter, NULL);
2237 static struct trace_entry *
2238 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2239 unsigned long *lost_events)
2241 struct ring_buffer_event *event;
2242 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2245 event = ring_buffer_iter_peek(buf_iter, ts);
2247 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2251 iter->ent_size = ring_buffer_event_length(event);
2252 return ring_buffer_event_data(event);
2258 static struct trace_entry *
2259 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2260 unsigned long *missing_events, u64 *ent_ts)
2262 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2263 struct trace_entry *ent, *next = NULL;
2264 unsigned long lost_events = 0, next_lost = 0;
2265 int cpu_file = iter->cpu_file;
2266 u64 next_ts = 0, ts;
2272 * If we are in a per_cpu trace file, don't bother by iterating over
2273 * all cpu and peek directly.
2275 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2276 if (ring_buffer_empty_cpu(buffer, cpu_file))
2278 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2280 *ent_cpu = cpu_file;
2285 for_each_tracing_cpu(cpu) {
2287 if (ring_buffer_empty_cpu(buffer, cpu))
2290 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2293 * Pick the entry with the smallest timestamp:
2295 if (ent && (!next || ts < next_ts)) {
2299 next_lost = lost_events;
2300 next_size = iter->ent_size;
2304 iter->ent_size = next_size;
2307 *ent_cpu = next_cpu;
2313 *missing_events = next_lost;
2318 /* Find the next real entry, without updating the iterator itself */
2319 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2320 int *ent_cpu, u64 *ent_ts)
2322 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2325 /* Find the next real entry, and increment the iterator to the next entry */
2326 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2328 iter->ent = __find_next_entry(iter, &iter->cpu,
2329 &iter->lost_events, &iter->ts);
2332 trace_iterator_increment(iter);
2334 return iter->ent ? iter : NULL;
2337 static void trace_consume(struct trace_iterator *iter)
2339 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2340 &iter->lost_events);
2343 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2345 struct trace_iterator *iter = m->private;
2349 WARN_ON_ONCE(iter->leftover);
2353 /* can't go backwards */
2358 ent = trace_find_next_entry_inc(iter);
2362 while (ent && iter->idx < i)
2363 ent = trace_find_next_entry_inc(iter);
2370 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2372 struct ring_buffer_event *event;
2373 struct ring_buffer_iter *buf_iter;
2374 unsigned long entries = 0;
2377 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2379 buf_iter = trace_buffer_iter(iter, cpu);
2383 ring_buffer_iter_reset(buf_iter);
2386 * We could have the case with the max latency tracers
2387 * that a reset never took place on a cpu. This is evident
2388 * by the timestamp being before the start of the buffer.
2390 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2391 if (ts >= iter->trace_buffer->time_start)
2394 ring_buffer_read(buf_iter, NULL);
2397 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2401 * The current tracer is copied to avoid a global locking
2404 static void *s_start(struct seq_file *m, loff_t *pos)
2406 struct trace_iterator *iter = m->private;
2407 struct trace_array *tr = iter->tr;
2408 int cpu_file = iter->cpu_file;
2414 * copy the tracer to avoid using a global lock all around.
2415 * iter->trace is a copy of current_trace, the pointer to the
2416 * name may be used instead of a strcmp(), as iter->trace->name
2417 * will point to the same string as current_trace->name.
2419 mutex_lock(&trace_types_lock);
2420 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2421 *iter->trace = *tr->current_trace;
2422 mutex_unlock(&trace_types_lock);
2424 #ifdef CONFIG_TRACER_MAX_TRACE
2425 if (iter->snapshot && iter->trace->use_max_tr)
2426 return ERR_PTR(-EBUSY);
2429 if (!iter->snapshot)
2430 atomic_inc(&trace_record_cmdline_disabled);
2432 if (*pos != iter->pos) {
2437 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2438 for_each_tracing_cpu(cpu)
2439 tracing_iter_reset(iter, cpu);
2441 tracing_iter_reset(iter, cpu_file);
2444 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2449 * If we overflowed the seq_file before, then we want
2450 * to just reuse the trace_seq buffer again.
2456 p = s_next(m, p, &l);
2460 trace_event_read_lock();
2461 trace_access_lock(cpu_file);
2465 static void s_stop(struct seq_file *m, void *p)
2467 struct trace_iterator *iter = m->private;
2469 #ifdef CONFIG_TRACER_MAX_TRACE
2470 if (iter->snapshot && iter->trace->use_max_tr)
2474 if (!iter->snapshot)
2475 atomic_dec(&trace_record_cmdline_disabled);
2477 trace_access_unlock(iter->cpu_file);
2478 trace_event_read_unlock();
2482 get_total_entries(struct trace_buffer *buf,
2483 unsigned long *total, unsigned long *entries)
2485 unsigned long count;
2491 for_each_tracing_cpu(cpu) {
2492 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2494 * If this buffer has skipped entries, then we hold all
2495 * entries for the trace and we need to ignore the
2496 * ones before the time stamp.
2498 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2499 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2500 /* total is the same as the entries */
2504 ring_buffer_overrun_cpu(buf->buffer, cpu);
2509 static void print_lat_help_header(struct seq_file *m)
2511 seq_puts(m, "# _------=> CPU# \n"
2512 "# / _-----=> irqs-off \n"
2513 "# | / _----=> need-resched \n"
2514 "# || / _---=> hardirq/softirq \n"
2515 "# ||| / _--=> preempt-depth \n"
2517 "# cmd pid ||||| time | caller \n"
2518 "# \\ / ||||| \\ | / \n");
2521 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2523 unsigned long total;
2524 unsigned long entries;
2526 get_total_entries(buf, &total, &entries);
2527 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2528 entries, total, num_online_cpus());
2532 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2534 print_event_info(buf, m);
2535 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2539 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2541 print_event_info(buf, m);
2542 seq_puts(m, "# _-----=> irqs-off\n"
2543 "# / _----=> need-resched\n"
2544 "# | / _---=> hardirq/softirq\n"
2545 "# || / _--=> preempt-depth\n"
2547 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
2548 "# | | | |||| | |\n");
2552 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2554 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2555 struct trace_buffer *buf = iter->trace_buffer;
2556 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2557 struct tracer *type = iter->trace;
2558 unsigned long entries;
2559 unsigned long total;
2560 const char *name = "preemption";
2564 get_total_entries(buf, &total, &entries);
2566 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2568 seq_puts(m, "# -----------------------------------"
2569 "---------------------------------\n");
2570 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2571 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2572 nsecs_to_usecs(data->saved_latency),
2576 #if defined(CONFIG_PREEMPT_NONE)
2578 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2580 #elif defined(CONFIG_PREEMPT)
2585 /* These are reserved for later use */
2588 seq_printf(m, " #P:%d)\n", num_online_cpus());
2592 seq_puts(m, "# -----------------\n");
2593 seq_printf(m, "# | task: %.16s-%d "
2594 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2595 data->comm, data->pid,
2596 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2597 data->policy, data->rt_priority);
2598 seq_puts(m, "# -----------------\n");
2600 if (data->critical_start) {
2601 seq_puts(m, "# => started at: ");
2602 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2603 trace_print_seq(m, &iter->seq);
2604 seq_puts(m, "\n# => ended at: ");
2605 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2606 trace_print_seq(m, &iter->seq);
2607 seq_puts(m, "\n#\n");
2613 static void test_cpu_buff_start(struct trace_iterator *iter)
2615 struct trace_seq *s = &iter->seq;
2617 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2620 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2623 if (cpumask_test_cpu(iter->cpu, iter->started))
2626 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2629 cpumask_set_cpu(iter->cpu, iter->started);
2631 /* Don't print started cpu buffer for the first entry of the trace */
2633 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2637 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2639 struct trace_seq *s = &iter->seq;
2640 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2641 struct trace_entry *entry;
2642 struct trace_event *event;
2646 test_cpu_buff_start(iter);
2648 event = ftrace_find_event(entry->type);
2650 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2651 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2652 trace_print_lat_context(iter);
2654 trace_print_context(iter);
2657 if (trace_seq_has_overflowed(s))
2658 return TRACE_TYPE_PARTIAL_LINE;
2661 return event->funcs->trace(iter, sym_flags, event);
2663 trace_seq_printf(s, "Unknown type %d\n", entry->type);
2665 return trace_handle_return(s);
2668 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2670 struct trace_seq *s = &iter->seq;
2671 struct trace_entry *entry;
2672 struct trace_event *event;
2676 if (trace_flags & TRACE_ITER_CONTEXT_INFO)
2677 trace_seq_printf(s, "%d %d %llu ",
2678 entry->pid, iter->cpu, iter->ts);
2680 if (trace_seq_has_overflowed(s))
2681 return TRACE_TYPE_PARTIAL_LINE;
2683 event = ftrace_find_event(entry->type);
2685 return event->funcs->raw(iter, 0, event);
2687 trace_seq_printf(s, "%d ?\n", entry->type);
2689 return trace_handle_return(s);
2692 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2694 struct trace_seq *s = &iter->seq;
2695 unsigned char newline = '\n';
2696 struct trace_entry *entry;
2697 struct trace_event *event;
2701 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2702 SEQ_PUT_HEX_FIELD(s, entry->pid);
2703 SEQ_PUT_HEX_FIELD(s, iter->cpu);
2704 SEQ_PUT_HEX_FIELD(s, iter->ts);
2705 if (trace_seq_has_overflowed(s))
2706 return TRACE_TYPE_PARTIAL_LINE;
2709 event = ftrace_find_event(entry->type);
2711 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2712 if (ret != TRACE_TYPE_HANDLED)
2716 SEQ_PUT_FIELD(s, newline);
2718 return trace_handle_return(s);
2721 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2723 struct trace_seq *s = &iter->seq;
2724 struct trace_entry *entry;
2725 struct trace_event *event;
2729 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2730 SEQ_PUT_FIELD(s, entry->pid);
2731 SEQ_PUT_FIELD(s, iter->cpu);
2732 SEQ_PUT_FIELD(s, iter->ts);
2733 if (trace_seq_has_overflowed(s))
2734 return TRACE_TYPE_PARTIAL_LINE;
2737 event = ftrace_find_event(entry->type);
2738 return event ? event->funcs->binary(iter, 0, event) :
2742 int trace_empty(struct trace_iterator *iter)
2744 struct ring_buffer_iter *buf_iter;
2747 /* If we are looking at one CPU buffer, only check that one */
2748 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2749 cpu = iter->cpu_file;
2750 buf_iter = trace_buffer_iter(iter, cpu);
2752 if (!ring_buffer_iter_empty(buf_iter))
2755 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2761 for_each_tracing_cpu(cpu) {
2762 buf_iter = trace_buffer_iter(iter, cpu);
2764 if (!ring_buffer_iter_empty(buf_iter))
2767 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2775 /* Called with trace_event_read_lock() held. */
2776 enum print_line_t print_trace_line(struct trace_iterator *iter)
2778 enum print_line_t ret;
2780 if (iter->lost_events) {
2781 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2782 iter->cpu, iter->lost_events);
2783 if (trace_seq_has_overflowed(&iter->seq))
2784 return TRACE_TYPE_PARTIAL_LINE;
2787 if (iter->trace && iter->trace->print_line) {
2788 ret = iter->trace->print_line(iter);
2789 if (ret != TRACE_TYPE_UNHANDLED)
2793 if (iter->ent->type == TRACE_BPUTS &&
2794 trace_flags & TRACE_ITER_PRINTK &&
2795 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2796 return trace_print_bputs_msg_only(iter);
2798 if (iter->ent->type == TRACE_BPRINT &&
2799 trace_flags & TRACE_ITER_PRINTK &&
2800 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2801 return trace_print_bprintk_msg_only(iter);
2803 if (iter->ent->type == TRACE_PRINT &&
2804 trace_flags & TRACE_ITER_PRINTK &&
2805 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2806 return trace_print_printk_msg_only(iter);
2808 if (trace_flags & TRACE_ITER_BIN)
2809 return print_bin_fmt(iter);
2811 if (trace_flags & TRACE_ITER_HEX)
2812 return print_hex_fmt(iter);
2814 if (trace_flags & TRACE_ITER_RAW)
2815 return print_raw_fmt(iter);
2817 return print_trace_fmt(iter);
2820 void trace_latency_header(struct seq_file *m)
2822 struct trace_iterator *iter = m->private;
2824 /* print nothing if the buffers are empty */
2825 if (trace_empty(iter))
2828 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2829 print_trace_header(m, iter);
2831 if (!(trace_flags & TRACE_ITER_VERBOSE))
2832 print_lat_help_header(m);
2835 void trace_default_header(struct seq_file *m)
2837 struct trace_iterator *iter = m->private;
2839 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2842 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2843 /* print nothing if the buffers are empty */
2844 if (trace_empty(iter))
2846 print_trace_header(m, iter);
2847 if (!(trace_flags & TRACE_ITER_VERBOSE))
2848 print_lat_help_header(m);
2850 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2851 if (trace_flags & TRACE_ITER_IRQ_INFO)
2852 print_func_help_header_irq(iter->trace_buffer, m);
2854 print_func_help_header(iter->trace_buffer, m);
2859 static void test_ftrace_alive(struct seq_file *m)
2861 if (!ftrace_is_dead())
2863 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
2864 "# MAY BE MISSING FUNCTION EVENTS\n");
2867 #ifdef CONFIG_TRACER_MAX_TRACE
2868 static void show_snapshot_main_help(struct seq_file *m)
2870 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
2871 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2872 "# Takes a snapshot of the main buffer.\n"
2873 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
2874 "# (Doesn't have to be '2' works with any number that\n"
2875 "# is not a '0' or '1')\n");
2878 static void show_snapshot_percpu_help(struct seq_file *m)
2880 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2881 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2882 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
2883 "# Takes a snapshot of the main buffer for this cpu.\n");
2885 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
2886 "# Must use main snapshot file to allocate.\n");
2888 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
2889 "# (Doesn't have to be '2' works with any number that\n"
2890 "# is not a '0' or '1')\n");
2893 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2895 if (iter->tr->allocated_snapshot)
2896 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
2898 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
2900 seq_puts(m, "# Snapshot commands:\n");
2901 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2902 show_snapshot_main_help(m);
2904 show_snapshot_percpu_help(m);
2907 /* Should never be called */
2908 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2911 static int s_show(struct seq_file *m, void *v)
2913 struct trace_iterator *iter = v;
2916 if (iter->ent == NULL) {
2918 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2920 test_ftrace_alive(m);
2922 if (iter->snapshot && trace_empty(iter))
2923 print_snapshot_help(m, iter);
2924 else if (iter->trace && iter->trace->print_header)
2925 iter->trace->print_header(m);
2927 trace_default_header(m);
2929 } else if (iter->leftover) {
2931 * If we filled the seq_file buffer earlier, we
2932 * want to just show it now.
2934 ret = trace_print_seq(m, &iter->seq);
2936 /* ret should this time be zero, but you never know */
2937 iter->leftover = ret;
2940 print_trace_line(iter);
2941 ret = trace_print_seq(m, &iter->seq);
2943 * If we overflow the seq_file buffer, then it will
2944 * ask us for this data again at start up.
2946 * ret is 0 if seq_file write succeeded.
2949 iter->leftover = ret;
2956 * Should be used after trace_array_get(), trace_types_lock
2957 * ensures that i_cdev was already initialized.
2959 static inline int tracing_get_cpu(struct inode *inode)
2961 if (inode->i_cdev) /* See trace_create_cpu_file() */
2962 return (long)inode->i_cdev - 1;
2963 return RING_BUFFER_ALL_CPUS;
2966 static const struct seq_operations tracer_seq_ops = {
2973 static struct trace_iterator *
2974 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
2976 struct trace_array *tr = inode->i_private;
2977 struct trace_iterator *iter;
2980 if (tracing_disabled)
2981 return ERR_PTR(-ENODEV);
2983 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2985 return ERR_PTR(-ENOMEM);
2987 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2989 if (!iter->buffer_iter)
2993 * We make a copy of the current tracer to avoid concurrent
2994 * changes on it while we are reading.
2996 mutex_lock(&trace_types_lock);
2997 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
3001 *iter->trace = *tr->current_trace;
3003 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3008 #ifdef CONFIG_TRACER_MAX_TRACE
3009 /* Currently only the top directory has a snapshot */
3010 if (tr->current_trace->print_max || snapshot)
3011 iter->trace_buffer = &tr->max_buffer;
3014 iter->trace_buffer = &tr->trace_buffer;
3015 iter->snapshot = snapshot;
3017 iter->cpu_file = tracing_get_cpu(inode);
3018 mutex_init(&iter->mutex);
3020 /* Notify the tracer early; before we stop tracing. */
3021 if (iter->trace && iter->trace->open)
3022 iter->trace->open(iter);
3024 /* Annotate start of buffers if we had overruns */
3025 if (ring_buffer_overruns(iter->trace_buffer->buffer))
3026 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3028 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3029 if (trace_clocks[tr->clock_id].in_ns)
3030 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3032 /* stop the trace while dumping if we are not opening "snapshot" */
3033 if (!iter->snapshot)
3034 tracing_stop_tr(tr);
3036 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3037 for_each_tracing_cpu(cpu) {
3038 iter->buffer_iter[cpu] =
3039 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3041 ring_buffer_read_prepare_sync();
3042 for_each_tracing_cpu(cpu) {
3043 ring_buffer_read_start(iter->buffer_iter[cpu]);
3044 tracing_iter_reset(iter, cpu);
3047 cpu = iter->cpu_file;
3048 iter->buffer_iter[cpu] =
3049 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3050 ring_buffer_read_prepare_sync();
3051 ring_buffer_read_start(iter->buffer_iter[cpu]);
3052 tracing_iter_reset(iter, cpu);
3055 mutex_unlock(&trace_types_lock);
3060 mutex_unlock(&trace_types_lock);
3062 kfree(iter->buffer_iter);
3064 seq_release_private(inode, file);
3065 return ERR_PTR(-ENOMEM);
3068 int tracing_open_generic(struct inode *inode, struct file *filp)
3070 if (tracing_disabled)
3073 filp->private_data = inode->i_private;
3077 bool tracing_is_disabled(void)
3079 return (tracing_disabled) ? true: false;
3083 * Open and update trace_array ref count.
3084 * Must have the current trace_array passed to it.
3086 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3088 struct trace_array *tr = inode->i_private;
3090 if (tracing_disabled)
3093 if (trace_array_get(tr) < 0)
3096 filp->private_data = inode->i_private;
3101 static int tracing_release(struct inode *inode, struct file *file)
3103 struct trace_array *tr = inode->i_private;
3104 struct seq_file *m = file->private_data;
3105 struct trace_iterator *iter;
3108 if (!(file->f_mode & FMODE_READ)) {
3109 trace_array_put(tr);
3113 /* Writes do not use seq_file */
3115 mutex_lock(&trace_types_lock);
3117 for_each_tracing_cpu(cpu) {
3118 if (iter->buffer_iter[cpu])
3119 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3122 if (iter->trace && iter->trace->close)
3123 iter->trace->close(iter);
3125 if (!iter->snapshot)
3126 /* reenable tracing if it was previously enabled */
3127 tracing_start_tr(tr);
3129 __trace_array_put(tr);
3131 mutex_unlock(&trace_types_lock);
3133 mutex_destroy(&iter->mutex);
3134 free_cpumask_var(iter->started);
3136 kfree(iter->buffer_iter);
3137 seq_release_private(inode, file);
3142 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3144 struct trace_array *tr = inode->i_private;
3146 trace_array_put(tr);
3150 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3152 struct trace_array *tr = inode->i_private;
3154 trace_array_put(tr);
3156 return single_release(inode, file);
3159 static int tracing_open(struct inode *inode, struct file *file)
3161 struct trace_array *tr = inode->i_private;
3162 struct trace_iterator *iter;
3165 if (trace_array_get(tr) < 0)
3168 /* If this file was open for write, then erase contents */
3169 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3170 int cpu = tracing_get_cpu(inode);
3172 if (cpu == RING_BUFFER_ALL_CPUS)
3173 tracing_reset_online_cpus(&tr->trace_buffer);
3175 tracing_reset(&tr->trace_buffer, cpu);
3178 if (file->f_mode & FMODE_READ) {
3179 iter = __tracing_open(inode, file, false);
3181 ret = PTR_ERR(iter);
3182 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
3183 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3187 trace_array_put(tr);
3193 * Some tracers are not suitable for instance buffers.
3194 * A tracer is always available for the global array (toplevel)
3195 * or if it explicitly states that it is.
3198 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3200 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3203 /* Find the next tracer that this trace array may use */
3204 static struct tracer *
3205 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3207 while (t && !trace_ok_for_array(t, tr))
3214 t_next(struct seq_file *m, void *v, loff_t *pos)
3216 struct trace_array *tr = m->private;
3217 struct tracer *t = v;
3222 t = get_tracer_for_array(tr, t->next);
3227 static void *t_start(struct seq_file *m, loff_t *pos)
3229 struct trace_array *tr = m->private;
3233 mutex_lock(&trace_types_lock);
3235 t = get_tracer_for_array(tr, trace_types);
3236 for (; t && l < *pos; t = t_next(m, t, &l))
3242 static void t_stop(struct seq_file *m, void *p)
3244 mutex_unlock(&trace_types_lock);
3247 static int t_show(struct seq_file *m, void *v)
3249 struct tracer *t = v;
3254 seq_puts(m, t->name);
3263 static const struct seq_operations show_traces_seq_ops = {
3270 static int show_traces_open(struct inode *inode, struct file *file)
3272 struct trace_array *tr = inode->i_private;
3276 if (tracing_disabled)
3279 ret = seq_open(file, &show_traces_seq_ops);
3283 m = file->private_data;
3290 tracing_write_stub(struct file *filp, const char __user *ubuf,
3291 size_t count, loff_t *ppos)
3296 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3300 if (file->f_mode & FMODE_READ)
3301 ret = seq_lseek(file, offset, whence);
3303 file->f_pos = ret = 0;
3308 static const struct file_operations tracing_fops = {
3309 .open = tracing_open,
3311 .write = tracing_write_stub,
3312 .llseek = tracing_lseek,
3313 .release = tracing_release,
3316 static const struct file_operations show_traces_fops = {
3317 .open = show_traces_open,
3319 .release = seq_release,
3320 .llseek = seq_lseek,
3324 * The tracer itself will not take this lock, but still we want
3325 * to provide a consistent cpumask to user-space:
3327 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3330 * Temporary storage for the character representation of the
3331 * CPU bitmask (and one more byte for the newline):
3333 static char mask_str[NR_CPUS + 1];
3336 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3337 size_t count, loff_t *ppos)
3339 struct trace_array *tr = file_inode(filp)->i_private;
3342 mutex_lock(&tracing_cpumask_update_lock);
3344 len = cpumask_scnprintf(mask_str, count, tr->tracing_cpumask);
3345 if (count - len < 2) {
3349 len += sprintf(mask_str + len, "\n");
3350 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3353 mutex_unlock(&tracing_cpumask_update_lock);
3359 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3360 size_t count, loff_t *ppos)
3362 struct trace_array *tr = file_inode(filp)->i_private;
3363 cpumask_var_t tracing_cpumask_new;
3366 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3369 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3373 mutex_lock(&tracing_cpumask_update_lock);
3375 local_irq_disable();
3376 arch_spin_lock(&tr->max_lock);
3377 for_each_tracing_cpu(cpu) {
3379 * Increase/decrease the disabled counter if we are
3380 * about to flip a bit in the cpumask:
3382 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3383 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3384 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3385 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3387 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3388 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3389 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3390 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3393 arch_spin_unlock(&tr->max_lock);
3396 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3398 mutex_unlock(&tracing_cpumask_update_lock);
3399 free_cpumask_var(tracing_cpumask_new);
3404 free_cpumask_var(tracing_cpumask_new);
3409 static const struct file_operations tracing_cpumask_fops = {
3410 .open = tracing_open_generic_tr,
3411 .read = tracing_cpumask_read,
3412 .write = tracing_cpumask_write,
3413 .release = tracing_release_generic_tr,
3414 .llseek = generic_file_llseek,
3417 static int tracing_trace_options_show(struct seq_file *m, void *v)
3419 struct tracer_opt *trace_opts;
3420 struct trace_array *tr = m->private;
3424 mutex_lock(&trace_types_lock);
3425 tracer_flags = tr->current_trace->flags->val;
3426 trace_opts = tr->current_trace->flags->opts;
3428 for (i = 0; trace_options[i]; i++) {
3429 if (trace_flags & (1 << i))
3430 seq_printf(m, "%s\n", trace_options[i]);
3432 seq_printf(m, "no%s\n", trace_options[i]);
3435 for (i = 0; trace_opts[i].name; i++) {
3436 if (tracer_flags & trace_opts[i].bit)
3437 seq_printf(m, "%s\n", trace_opts[i].name);
3439 seq_printf(m, "no%s\n", trace_opts[i].name);
3441 mutex_unlock(&trace_types_lock);
3446 static int __set_tracer_option(struct trace_array *tr,
3447 struct tracer_flags *tracer_flags,
3448 struct tracer_opt *opts, int neg)
3450 struct tracer *trace = tr->current_trace;
3453 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3458 tracer_flags->val &= ~opts->bit;
3460 tracer_flags->val |= opts->bit;
3464 /* Try to assign a tracer specific option */
3465 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3467 struct tracer *trace = tr->current_trace;
3468 struct tracer_flags *tracer_flags = trace->flags;
3469 struct tracer_opt *opts = NULL;
3472 for (i = 0; tracer_flags->opts[i].name; i++) {
3473 opts = &tracer_flags->opts[i];
3475 if (strcmp(cmp, opts->name) == 0)
3476 return __set_tracer_option(tr, trace->flags, opts, neg);
3482 /* Some tracers require overwrite to stay enabled */
3483 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3485 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3491 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3493 /* do nothing if flag is already set */
3494 if (!!(trace_flags & mask) == !!enabled)
3497 /* Give the tracer a chance to approve the change */
3498 if (tr->current_trace->flag_changed)
3499 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3503 trace_flags |= mask;
3505 trace_flags &= ~mask;
3507 if (mask == TRACE_ITER_RECORD_CMD)
3508 trace_event_enable_cmd_record(enabled);
3510 if (mask == TRACE_ITER_OVERWRITE) {
3511 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3512 #ifdef CONFIG_TRACER_MAX_TRACE
3513 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3517 if (mask == TRACE_ITER_PRINTK)
3518 trace_printk_start_stop_comm(enabled);
3523 static int trace_set_options(struct trace_array *tr, char *option)
3530 cmp = strstrip(option);
3532 if (strncmp(cmp, "no", 2) == 0) {
3537 mutex_lock(&trace_types_lock);
3539 for (i = 0; trace_options[i]; i++) {
3540 if (strcmp(cmp, trace_options[i]) == 0) {
3541 ret = set_tracer_flag(tr, 1 << i, !neg);
3546 /* If no option could be set, test the specific tracer options */
3547 if (!trace_options[i])
3548 ret = set_tracer_option(tr, cmp, neg);
3550 mutex_unlock(&trace_types_lock);
3556 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3557 size_t cnt, loff_t *ppos)
3559 struct seq_file *m = filp->private_data;
3560 struct trace_array *tr = m->private;
3564 if (cnt >= sizeof(buf))
3567 if (copy_from_user(&buf, ubuf, cnt))
3572 ret = trace_set_options(tr, buf);
3581 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3583 struct trace_array *tr = inode->i_private;
3586 if (tracing_disabled)
3589 if (trace_array_get(tr) < 0)
3592 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3594 trace_array_put(tr);
3599 static const struct file_operations tracing_iter_fops = {
3600 .open = tracing_trace_options_open,
3602 .llseek = seq_lseek,
3603 .release = tracing_single_release_tr,
3604 .write = tracing_trace_options_write,
3607 static const char readme_msg[] =
3608 "tracing mini-HOWTO:\n\n"
3609 "# echo 0 > tracing_on : quick way to disable tracing\n"
3610 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3611 " Important files:\n"
3612 " trace\t\t\t- The static contents of the buffer\n"
3613 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3614 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3615 " current_tracer\t- function and latency tracers\n"
3616 " available_tracers\t- list of configured tracers for current_tracer\n"
3617 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3618 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3619 " trace_clock\t\t-change the clock used to order events\n"
3620 " local: Per cpu clock but may not be synced across CPUs\n"
3621 " global: Synced across CPUs but slows tracing down.\n"
3622 " counter: Not a clock, but just an increment\n"
3623 " uptime: Jiffy counter from time of boot\n"
3624 " perf: Same clock that perf events use\n"
3625 #ifdef CONFIG_X86_64
3626 " x86-tsc: TSC cycle counter\n"
3628 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3629 " tracing_cpumask\t- Limit which CPUs to trace\n"
3630 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3631 "\t\t\t Remove sub-buffer with rmdir\n"
3632 " trace_options\t\t- Set format or modify how tracing happens\n"
3633 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3634 "\t\t\t option name\n"
3635 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
3636 #ifdef CONFIG_DYNAMIC_FTRACE
3637 "\n available_filter_functions - list of functions that can be filtered on\n"
3638 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3639 "\t\t\t functions\n"
3640 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3641 "\t modules: Can select a group via module\n"
3642 "\t Format: :mod:<module-name>\n"
3643 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3644 "\t triggers: a command to perform when function is hit\n"
3645 "\t Format: <function>:<trigger>[:count]\n"
3646 "\t trigger: traceon, traceoff\n"
3647 "\t\t enable_event:<system>:<event>\n"
3648 "\t\t disable_event:<system>:<event>\n"
3649 #ifdef CONFIG_STACKTRACE
3652 #ifdef CONFIG_TRACER_SNAPSHOT
3657 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3658 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3659 "\t The first one will disable tracing every time do_fault is hit\n"
3660 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3661 "\t The first time do trap is hit and it disables tracing, the\n"
3662 "\t counter will decrement to 2. If tracing is already disabled,\n"
3663 "\t the counter will not decrement. It only decrements when the\n"
3664 "\t trigger did work\n"
3665 "\t To remove trigger without count:\n"
3666 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3667 "\t To remove trigger with a count:\n"
3668 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3669 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3670 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3671 "\t modules: Can select a group via module command :mod:\n"
3672 "\t Does not accept triggers\n"
3673 #endif /* CONFIG_DYNAMIC_FTRACE */
3674 #ifdef CONFIG_FUNCTION_TRACER
3675 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3678 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3679 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3680 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
3681 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3683 #ifdef CONFIG_TRACER_SNAPSHOT
3684 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3685 "\t\t\t snapshot buffer. Read the contents for more\n"
3686 "\t\t\t information\n"
3688 #ifdef CONFIG_STACK_TRACER
3689 " stack_trace\t\t- Shows the max stack trace when active\n"
3690 " stack_max_size\t- Shows current max stack size that was traced\n"
3691 "\t\t\t Write into this file to reset the max size (trigger a\n"
3692 "\t\t\t new trace)\n"
3693 #ifdef CONFIG_DYNAMIC_FTRACE
3694 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3697 #endif /* CONFIG_STACK_TRACER */
3698 " events/\t\t- Directory containing all trace event subsystems:\n"
3699 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3700 " events/<system>/\t- Directory containing all trace events for <system>:\n"
3701 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3703 " filter\t\t- If set, only events passing filter are traced\n"
3704 " events/<system>/<event>/\t- Directory containing control files for\n"
3706 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3707 " filter\t\t- If set, only events passing filter are traced\n"
3708 " trigger\t\t- If set, a command to perform when event is hit\n"
3709 "\t Format: <trigger>[:count][if <filter>]\n"
3710 "\t trigger: traceon, traceoff\n"
3711 "\t enable_event:<system>:<event>\n"
3712 "\t disable_event:<system>:<event>\n"
3713 #ifdef CONFIG_STACKTRACE
3716 #ifdef CONFIG_TRACER_SNAPSHOT
3719 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3720 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3721 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3722 "\t events/block/block_unplug/trigger\n"
3723 "\t The first disables tracing every time block_unplug is hit.\n"
3724 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3725 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3726 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3727 "\t Like function triggers, the counter is only decremented if it\n"
3728 "\t enabled or disabled tracing.\n"
3729 "\t To remove a trigger without a count:\n"
3730 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3731 "\t To remove a trigger with a count:\n"
3732 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3733 "\t Filters can be ignored when removing a trigger.\n"
3737 tracing_readme_read(struct file *filp, char __user *ubuf,
3738 size_t cnt, loff_t *ppos)
3740 return simple_read_from_buffer(ubuf, cnt, ppos,
3741 readme_msg, strlen(readme_msg));
3744 static const struct file_operations tracing_readme_fops = {
3745 .open = tracing_open_generic,
3746 .read = tracing_readme_read,
3747 .llseek = generic_file_llseek,
3750 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
3752 unsigned int *ptr = v;
3754 if (*pos || m->count)
3759 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
3761 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
3770 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
3776 arch_spin_lock(&trace_cmdline_lock);
3778 v = &savedcmd->map_cmdline_to_pid[0];
3780 v = saved_cmdlines_next(m, v, &l);
3788 static void saved_cmdlines_stop(struct seq_file *m, void *v)
3790 arch_spin_unlock(&trace_cmdline_lock);
3794 static int saved_cmdlines_show(struct seq_file *m, void *v)
3796 char buf[TASK_COMM_LEN];
3797 unsigned int *pid = v;
3799 __trace_find_cmdline(*pid, buf);
3800 seq_printf(m, "%d %s\n", *pid, buf);
3804 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
3805 .start = saved_cmdlines_start,
3806 .next = saved_cmdlines_next,
3807 .stop = saved_cmdlines_stop,
3808 .show = saved_cmdlines_show,
3811 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
3813 if (tracing_disabled)
3816 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
3819 static const struct file_operations tracing_saved_cmdlines_fops = {
3820 .open = tracing_saved_cmdlines_open,
3822 .llseek = seq_lseek,
3823 .release = seq_release,
3827 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
3828 size_t cnt, loff_t *ppos)
3833 arch_spin_lock(&trace_cmdline_lock);
3834 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
3835 arch_spin_unlock(&trace_cmdline_lock);
3837 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3840 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
3842 kfree(s->saved_cmdlines);
3843 kfree(s->map_cmdline_to_pid);
3847 static int tracing_resize_saved_cmdlines(unsigned int val)
3849 struct saved_cmdlines_buffer *s, *savedcmd_temp;
3851 s = kmalloc(sizeof(*s), GFP_KERNEL);
3855 if (allocate_cmdlines_buffer(val, s) < 0) {
3860 arch_spin_lock(&trace_cmdline_lock);
3861 savedcmd_temp = savedcmd;
3863 arch_spin_unlock(&trace_cmdline_lock);
3864 free_saved_cmdlines_buffer(savedcmd_temp);
3870 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
3871 size_t cnt, loff_t *ppos)
3876 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3880 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
3881 if (!val || val > PID_MAX_DEFAULT)
3884 ret = tracing_resize_saved_cmdlines((unsigned int)val);
3893 static const struct file_operations tracing_saved_cmdlines_size_fops = {
3894 .open = tracing_open_generic,
3895 .read = tracing_saved_cmdlines_size_read,
3896 .write = tracing_saved_cmdlines_size_write,
3900 tracing_set_trace_read(struct file *filp, char __user *ubuf,
3901 size_t cnt, loff_t *ppos)
3903 struct trace_array *tr = filp->private_data;
3904 char buf[MAX_TRACER_SIZE+2];
3907 mutex_lock(&trace_types_lock);
3908 r = sprintf(buf, "%s\n", tr->current_trace->name);
3909 mutex_unlock(&trace_types_lock);
3911 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3914 int tracer_init(struct tracer *t, struct trace_array *tr)
3916 tracing_reset_online_cpus(&tr->trace_buffer);
3920 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
3924 for_each_tracing_cpu(cpu)
3925 per_cpu_ptr(buf->data, cpu)->entries = val;
3928 #ifdef CONFIG_TRACER_MAX_TRACE
3929 /* resize @tr's buffer to the size of @size_tr's entries */
3930 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3931 struct trace_buffer *size_buf, int cpu_id)
3935 if (cpu_id == RING_BUFFER_ALL_CPUS) {
3936 for_each_tracing_cpu(cpu) {
3937 ret = ring_buffer_resize(trace_buf->buffer,
3938 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
3941 per_cpu_ptr(trace_buf->data, cpu)->entries =
3942 per_cpu_ptr(size_buf->data, cpu)->entries;
3945 ret = ring_buffer_resize(trace_buf->buffer,
3946 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
3948 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3949 per_cpu_ptr(size_buf->data, cpu_id)->entries;
3954 #endif /* CONFIG_TRACER_MAX_TRACE */
3956 static int __tracing_resize_ring_buffer(struct trace_array *tr,
3957 unsigned long size, int cpu)
3962 * If kernel or user changes the size of the ring buffer
3963 * we use the size that was given, and we can forget about
3964 * expanding it later.
3966 ring_buffer_expanded = true;
3968 /* May be called before buffers are initialized */
3969 if (!tr->trace_buffer.buffer)
3972 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
3976 #ifdef CONFIG_TRACER_MAX_TRACE
3977 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3978 !tr->current_trace->use_max_tr)
3981 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
3983 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3984 &tr->trace_buffer, cpu);
3987 * AARGH! We are left with different
3988 * size max buffer!!!!
3989 * The max buffer is our "snapshot" buffer.
3990 * When a tracer needs a snapshot (one of the
3991 * latency tracers), it swaps the max buffer
3992 * with the saved snap shot. We succeeded to
3993 * update the size of the main buffer, but failed to
3994 * update the size of the max buffer. But when we tried
3995 * to reset the main buffer to the original size, we
3996 * failed there too. This is very unlikely to
3997 * happen, but if it does, warn and kill all
4001 tracing_disabled = 1;
4006 if (cpu == RING_BUFFER_ALL_CPUS)
4007 set_buffer_entries(&tr->max_buffer, size);
4009 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
4012 #endif /* CONFIG_TRACER_MAX_TRACE */
4014 if (cpu == RING_BUFFER_ALL_CPUS)
4015 set_buffer_entries(&tr->trace_buffer, size);
4017 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
4022 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4023 unsigned long size, int cpu_id)
4027 mutex_lock(&trace_types_lock);
4029 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4030 /* make sure, this cpu is enabled in the mask */
4031 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4037 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4042 mutex_unlock(&trace_types_lock);
4049 * tracing_update_buffers - used by tracing facility to expand ring buffers
4051 * To save on memory when the tracing is never used on a system with it
4052 * configured in. The ring buffers are set to a minimum size. But once
4053 * a user starts to use the tracing facility, then they need to grow
4054 * to their default size.
4056 * This function is to be called when a tracer is about to be used.
4058 int tracing_update_buffers(void)
4062 mutex_lock(&trace_types_lock);
4063 if (!ring_buffer_expanded)
4064 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
4065 RING_BUFFER_ALL_CPUS);
4066 mutex_unlock(&trace_types_lock);
4071 struct trace_option_dentry;
4073 static struct trace_option_dentry *
4074 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
4077 destroy_trace_option_files(struct trace_option_dentry *topts);
4080 * Used to clear out the tracer before deletion of an instance.
4081 * Must have trace_types_lock held.
4083 static void tracing_set_nop(struct trace_array *tr)
4085 if (tr->current_trace == &nop_trace)
4088 tr->current_trace->enabled--;
4090 if (tr->current_trace->reset)
4091 tr->current_trace->reset(tr);
4093 tr->current_trace = &nop_trace;
4096 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4098 static struct trace_option_dentry *topts;
4100 #ifdef CONFIG_TRACER_MAX_TRACE
4105 mutex_lock(&trace_types_lock);
4107 if (!ring_buffer_expanded) {
4108 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4109 RING_BUFFER_ALL_CPUS);
4115 for (t = trace_types; t; t = t->next) {
4116 if (strcmp(t->name, buf) == 0)
4123 if (t == tr->current_trace)
4126 /* Some tracers are only allowed for the top level buffer */
4127 if (!trace_ok_for_array(t, tr)) {
4132 trace_branch_disable();
4134 tr->current_trace->enabled--;
4136 if (tr->current_trace->reset)
4137 tr->current_trace->reset(tr);
4139 /* Current trace needs to be nop_trace before synchronize_sched */
4140 tr->current_trace = &nop_trace;
4142 #ifdef CONFIG_TRACER_MAX_TRACE
4143 had_max_tr = tr->allocated_snapshot;
4145 if (had_max_tr && !t->use_max_tr) {
4147 * We need to make sure that the update_max_tr sees that
4148 * current_trace changed to nop_trace to keep it from
4149 * swapping the buffers after we resize it.
4150 * The update_max_tr is called from interrupts disabled
4151 * so a synchronized_sched() is sufficient.
4153 synchronize_sched();
4157 /* Currently, only the top instance has options */
4158 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
4159 destroy_trace_option_files(topts);
4160 topts = create_trace_option_files(tr, t);
4163 #ifdef CONFIG_TRACER_MAX_TRACE
4164 if (t->use_max_tr && !had_max_tr) {
4165 ret = alloc_snapshot(tr);
4172 ret = tracer_init(t, tr);
4177 tr->current_trace = t;
4178 tr->current_trace->enabled++;
4179 trace_branch_enable(tr);
4181 mutex_unlock(&trace_types_lock);
4187 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4188 size_t cnt, loff_t *ppos)
4190 struct trace_array *tr = filp->private_data;
4191 char buf[MAX_TRACER_SIZE+1];
4198 if (cnt > MAX_TRACER_SIZE)
4199 cnt = MAX_TRACER_SIZE;
4201 if (copy_from_user(&buf, ubuf, cnt))
4206 /* strip ending whitespace. */
4207 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4210 err = tracing_set_tracer(tr, buf);
4220 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
4221 size_t cnt, loff_t *ppos)
4226 r = snprintf(buf, sizeof(buf), "%ld\n",
4227 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
4228 if (r > sizeof(buf))
4230 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4234 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
4235 size_t cnt, loff_t *ppos)
4240 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4250 tracing_thresh_read(struct file *filp, char __user *ubuf,
4251 size_t cnt, loff_t *ppos)
4253 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
4257 tracing_thresh_write(struct file *filp, const char __user *ubuf,
4258 size_t cnt, loff_t *ppos)
4260 struct trace_array *tr = filp->private_data;
4263 mutex_lock(&trace_types_lock);
4264 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
4268 if (tr->current_trace->update_thresh) {
4269 ret = tr->current_trace->update_thresh(tr);
4276 mutex_unlock(&trace_types_lock);
4282 tracing_max_lat_read(struct file *filp, char __user *ubuf,
4283 size_t cnt, loff_t *ppos)
4285 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
4289 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4290 size_t cnt, loff_t *ppos)
4292 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
4295 static int tracing_open_pipe(struct inode *inode, struct file *filp)
4297 struct trace_array *tr = inode->i_private;
4298 struct trace_iterator *iter;
4301 if (tracing_disabled)
4304 if (trace_array_get(tr) < 0)
4307 mutex_lock(&trace_types_lock);
4309 /* create a buffer to store the information to pass to userspace */
4310 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4313 __trace_array_put(tr);
4317 trace_seq_init(&iter->seq);
4320 * We make a copy of the current tracer to avoid concurrent
4321 * changes on it while we are reading.
4323 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
4328 *iter->trace = *tr->current_trace;
4330 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4335 /* trace pipe does not show start of buffer */
4336 cpumask_setall(iter->started);
4338 if (trace_flags & TRACE_ITER_LATENCY_FMT)
4339 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4341 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4342 if (trace_clocks[tr->clock_id].in_ns)
4343 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4346 iter->trace_buffer = &tr->trace_buffer;
4347 iter->cpu_file = tracing_get_cpu(inode);
4348 mutex_init(&iter->mutex);
4349 filp->private_data = iter;
4351 if (iter->trace->pipe_open)
4352 iter->trace->pipe_open(iter);
4354 nonseekable_open(inode, filp);
4356 mutex_unlock(&trace_types_lock);
4362 __trace_array_put(tr);
4363 mutex_unlock(&trace_types_lock);
4367 static int tracing_release_pipe(struct inode *inode, struct file *file)
4369 struct trace_iterator *iter = file->private_data;
4370 struct trace_array *tr = inode->i_private;
4372 mutex_lock(&trace_types_lock);
4374 if (iter->trace->pipe_close)
4375 iter->trace->pipe_close(iter);
4377 mutex_unlock(&trace_types_lock);
4379 free_cpumask_var(iter->started);
4380 mutex_destroy(&iter->mutex);
4384 trace_array_put(tr);
4390 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
4392 /* Iterators are static, they should be filled or empty */
4393 if (trace_buffer_iter(iter, iter->cpu_file))
4394 return POLLIN | POLLRDNORM;
4396 if (trace_flags & TRACE_ITER_BLOCK)
4398 * Always select as readable when in blocking mode
4400 return POLLIN | POLLRDNORM;
4402 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4407 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4409 struct trace_iterator *iter = filp->private_data;
4411 return trace_poll(iter, filp, poll_table);
4414 /* Must be called with trace_types_lock mutex held. */
4415 static int tracing_wait_pipe(struct file *filp)
4417 struct trace_iterator *iter = filp->private_data;
4420 while (trace_empty(iter)) {
4422 if ((filp->f_flags & O_NONBLOCK)) {
4427 * We block until we read something and tracing is disabled.
4428 * We still block if tracing is disabled, but we have never
4429 * read anything. This allows a user to cat this file, and
4430 * then enable tracing. But after we have read something,
4431 * we give an EOF when tracing is again disabled.
4433 * iter->pos will be 0 if we haven't read anything.
4435 if (!tracing_is_on() && iter->pos)
4438 mutex_unlock(&iter->mutex);
4440 ret = wait_on_pipe(iter, false);
4442 mutex_lock(&iter->mutex);
4455 tracing_read_pipe(struct file *filp, char __user *ubuf,
4456 size_t cnt, loff_t *ppos)
4458 struct trace_iterator *iter = filp->private_data;
4459 struct trace_array *tr = iter->tr;
4462 /* return any leftover data */
4463 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4467 trace_seq_init(&iter->seq);
4469 /* copy the tracer to avoid using a global lock all around */
4470 mutex_lock(&trace_types_lock);
4471 if (unlikely(iter->trace->name != tr->current_trace->name))
4472 *iter->trace = *tr->current_trace;
4473 mutex_unlock(&trace_types_lock);
4476 * Avoid more than one consumer on a single file descriptor
4477 * This is just a matter of traces coherency, the ring buffer itself
4480 mutex_lock(&iter->mutex);
4481 if (iter->trace->read) {
4482 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4488 sret = tracing_wait_pipe(filp);
4492 /* stop when tracing is finished */
4493 if (trace_empty(iter)) {
4498 if (cnt >= PAGE_SIZE)
4499 cnt = PAGE_SIZE - 1;
4501 /* reset all but tr, trace, and overruns */
4502 memset(&iter->seq, 0,
4503 sizeof(struct trace_iterator) -
4504 offsetof(struct trace_iterator, seq));
4505 cpumask_clear(iter->started);
4508 trace_event_read_lock();
4509 trace_access_lock(iter->cpu_file);
4510 while (trace_find_next_entry_inc(iter) != NULL) {
4511 enum print_line_t ret;
4512 int save_len = iter->seq.seq.len;
4514 ret = print_trace_line(iter);
4515 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4516 /* don't print partial lines */
4517 iter->seq.seq.len = save_len;
4520 if (ret != TRACE_TYPE_NO_CONSUME)
4521 trace_consume(iter);
4523 if (trace_seq_used(&iter->seq) >= cnt)
4527 * Setting the full flag means we reached the trace_seq buffer
4528 * size and we should leave by partial output condition above.
4529 * One of the trace_seq_* functions is not used properly.
4531 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4534 trace_access_unlock(iter->cpu_file);
4535 trace_event_read_unlock();
4537 /* Now copy what we have to the user */
4538 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4539 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
4540 trace_seq_init(&iter->seq);
4543 * If there was nothing to send to user, in spite of consuming trace
4544 * entries, go back to wait for more entries.
4550 mutex_unlock(&iter->mutex);
4555 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4558 __free_page(spd->pages[idx]);
4561 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4563 .confirm = generic_pipe_buf_confirm,
4564 .release = generic_pipe_buf_release,
4565 .steal = generic_pipe_buf_steal,
4566 .get = generic_pipe_buf_get,
4570 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4576 /* Seq buffer is page-sized, exactly what we need. */
4578 save_len = iter->seq.seq.len;
4579 ret = print_trace_line(iter);
4581 if (trace_seq_has_overflowed(&iter->seq)) {
4582 iter->seq.seq.len = save_len;
4587 * This should not be hit, because it should only
4588 * be set if the iter->seq overflowed. But check it
4589 * anyway to be safe.
4591 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4592 iter->seq.seq.len = save_len;
4596 count = trace_seq_used(&iter->seq) - save_len;
4599 iter->seq.seq.len = save_len;
4603 if (ret != TRACE_TYPE_NO_CONSUME)
4604 trace_consume(iter);
4606 if (!trace_find_next_entry_inc(iter)) {
4616 static ssize_t tracing_splice_read_pipe(struct file *filp,
4618 struct pipe_inode_info *pipe,
4622 struct page *pages_def[PIPE_DEF_BUFFERS];
4623 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4624 struct trace_iterator *iter = filp->private_data;
4625 struct splice_pipe_desc spd = {
4627 .partial = partial_def,
4628 .nr_pages = 0, /* This gets updated below. */
4629 .nr_pages_max = PIPE_DEF_BUFFERS,
4631 .ops = &tracing_pipe_buf_ops,
4632 .spd_release = tracing_spd_release_pipe,
4634 struct trace_array *tr = iter->tr;
4639 if (splice_grow_spd(pipe, &spd))
4642 /* copy the tracer to avoid using a global lock all around */
4643 mutex_lock(&trace_types_lock);
4644 if (unlikely(iter->trace->name != tr->current_trace->name))
4645 *iter->trace = *tr->current_trace;
4646 mutex_unlock(&trace_types_lock);
4648 mutex_lock(&iter->mutex);
4650 if (iter->trace->splice_read) {
4651 ret = iter->trace->splice_read(iter, filp,
4652 ppos, pipe, len, flags);
4657 ret = tracing_wait_pipe(filp);
4661 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4666 trace_event_read_lock();
4667 trace_access_lock(iter->cpu_file);
4669 /* Fill as many pages as possible. */
4670 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
4671 spd.pages[i] = alloc_page(GFP_KERNEL);
4675 rem = tracing_fill_pipe_page(rem, iter);
4677 /* Copy the data into the page, so we can start over. */
4678 ret = trace_seq_to_buffer(&iter->seq,
4679 page_address(spd.pages[i]),
4680 trace_seq_used(&iter->seq));
4682 __free_page(spd.pages[i]);
4685 spd.partial[i].offset = 0;
4686 spd.partial[i].len = trace_seq_used(&iter->seq);
4688 trace_seq_init(&iter->seq);
4691 trace_access_unlock(iter->cpu_file);
4692 trace_event_read_unlock();
4693 mutex_unlock(&iter->mutex);
4697 ret = splice_to_pipe(pipe, &spd);
4699 splice_shrink_spd(&spd);
4703 mutex_unlock(&iter->mutex);
4708 tracing_entries_read(struct file *filp, char __user *ubuf,
4709 size_t cnt, loff_t *ppos)
4711 struct inode *inode = file_inode(filp);
4712 struct trace_array *tr = inode->i_private;
4713 int cpu = tracing_get_cpu(inode);
4718 mutex_lock(&trace_types_lock);
4720 if (cpu == RING_BUFFER_ALL_CPUS) {
4721 int cpu, buf_size_same;
4726 /* check if all cpu sizes are same */
4727 for_each_tracing_cpu(cpu) {
4728 /* fill in the size from first enabled cpu */
4730 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4731 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4737 if (buf_size_same) {
4738 if (!ring_buffer_expanded)
4739 r = sprintf(buf, "%lu (expanded: %lu)\n",
4741 trace_buf_size >> 10);
4743 r = sprintf(buf, "%lu\n", size >> 10);
4745 r = sprintf(buf, "X\n");
4747 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
4749 mutex_unlock(&trace_types_lock);
4751 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4756 tracing_entries_write(struct file *filp, const char __user *ubuf,
4757 size_t cnt, loff_t *ppos)
4759 struct inode *inode = file_inode(filp);
4760 struct trace_array *tr = inode->i_private;
4764 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4768 /* must have at least 1 entry */
4772 /* value is in KB */
4774 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
4784 tracing_total_entries_read(struct file *filp, char __user *ubuf,
4785 size_t cnt, loff_t *ppos)
4787 struct trace_array *tr = filp->private_data;
4790 unsigned long size = 0, expanded_size = 0;
4792 mutex_lock(&trace_types_lock);
4793 for_each_tracing_cpu(cpu) {
4794 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
4795 if (!ring_buffer_expanded)
4796 expanded_size += trace_buf_size >> 10;
4798 if (ring_buffer_expanded)
4799 r = sprintf(buf, "%lu\n", size);
4801 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4802 mutex_unlock(&trace_types_lock);
4804 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4808 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4809 size_t cnt, loff_t *ppos)
4812 * There is no need to read what the user has written, this function
4813 * is just to make sure that there is no error when "echo" is used
4822 tracing_free_buffer_release(struct inode *inode, struct file *filp)
4824 struct trace_array *tr = inode->i_private;
4826 /* disable tracing ? */
4827 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4828 tracer_tracing_off(tr);
4829 /* resize the ring buffer to 0 */
4830 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4832 trace_array_put(tr);
4838 tracing_mark_write(struct file *filp, const char __user *ubuf,
4839 size_t cnt, loff_t *fpos)
4841 unsigned long addr = (unsigned long)ubuf;
4842 struct trace_array *tr = filp->private_data;
4843 struct ring_buffer_event *event;
4844 struct ring_buffer *buffer;
4845 struct print_entry *entry;
4846 unsigned long irq_flags;
4847 struct page *pages[2];
4857 if (tracing_disabled)
4860 if (!(trace_flags & TRACE_ITER_MARKERS))
4863 if (cnt > TRACE_BUF_SIZE)
4864 cnt = TRACE_BUF_SIZE;
4867 * Userspace is injecting traces into the kernel trace buffer.
4868 * We want to be as non intrusive as possible.
4869 * To do so, we do not want to allocate any special buffers
4870 * or take any locks, but instead write the userspace data
4871 * straight into the ring buffer.
4873 * First we need to pin the userspace buffer into memory,
4874 * which, most likely it is, because it just referenced it.
4875 * But there's no guarantee that it is. By using get_user_pages_fast()
4876 * and kmap_atomic/kunmap_atomic() we can get access to the
4877 * pages directly. We then write the data directly into the
4880 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4882 /* check if we cross pages */
4883 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4886 offset = addr & (PAGE_SIZE - 1);
4889 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4890 if (ret < nr_pages) {
4892 put_page(pages[ret]);
4897 for (i = 0; i < nr_pages; i++)
4898 map_page[i] = kmap_atomic(pages[i]);
4900 local_save_flags(irq_flags);
4901 size = sizeof(*entry) + cnt + 2; /* possible \n added */
4902 buffer = tr->trace_buffer.buffer;
4903 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4904 irq_flags, preempt_count());
4906 /* Ring buffer disabled, return as if not open for write */
4911 entry = ring_buffer_event_data(event);
4912 entry->ip = _THIS_IP_;
4914 if (nr_pages == 2) {
4915 len = PAGE_SIZE - offset;
4916 memcpy(&entry->buf, map_page[0] + offset, len);
4917 memcpy(&entry->buf[len], map_page[1], cnt - len);
4919 memcpy(&entry->buf, map_page[0] + offset, cnt);
4921 if (entry->buf[cnt - 1] != '\n') {
4922 entry->buf[cnt] = '\n';
4923 entry->buf[cnt + 1] = '\0';
4925 entry->buf[cnt] = '\0';
4927 __buffer_unlock_commit(buffer, event);
4934 for (i = 0; i < nr_pages; i++){
4935 kunmap_atomic(map_page[i]);
4942 static int tracing_clock_show(struct seq_file *m, void *v)
4944 struct trace_array *tr = m->private;
4947 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
4949 "%s%s%s%s", i ? " " : "",
4950 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4951 i == tr->clock_id ? "]" : "");
4957 static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
4961 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4962 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4965 if (i == ARRAY_SIZE(trace_clocks))
4968 mutex_lock(&trace_types_lock);
4972 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
4975 * New clock may not be consistent with the previous clock.
4976 * Reset the buffer so that it doesn't have incomparable timestamps.
4978 tracing_reset_online_cpus(&tr->trace_buffer);
4980 #ifdef CONFIG_TRACER_MAX_TRACE
4981 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4982 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4983 tracing_reset_online_cpus(&tr->max_buffer);
4986 mutex_unlock(&trace_types_lock);
4991 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4992 size_t cnt, loff_t *fpos)
4994 struct seq_file *m = filp->private_data;
4995 struct trace_array *tr = m->private;
4997 const char *clockstr;
5000 if (cnt >= sizeof(buf))
5003 if (copy_from_user(&buf, ubuf, cnt))
5008 clockstr = strstrip(buf);
5010 ret = tracing_set_clock(tr, clockstr);
5019 static int tracing_clock_open(struct inode *inode, struct file *file)
5021 struct trace_array *tr = inode->i_private;
5024 if (tracing_disabled)
5027 if (trace_array_get(tr))
5030 ret = single_open(file, tracing_clock_show, inode->i_private);
5032 trace_array_put(tr);
5037 struct ftrace_buffer_info {
5038 struct trace_iterator iter;
5043 #ifdef CONFIG_TRACER_SNAPSHOT
5044 static int tracing_snapshot_open(struct inode *inode, struct file *file)
5046 struct trace_array *tr = inode->i_private;
5047 struct trace_iterator *iter;
5051 if (trace_array_get(tr) < 0)
5054 if (file->f_mode & FMODE_READ) {
5055 iter = __tracing_open(inode, file, true);
5057 ret = PTR_ERR(iter);
5059 /* Writes still need the seq_file to hold the private data */
5061 m = kzalloc(sizeof(*m), GFP_KERNEL);
5064 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5072 iter->trace_buffer = &tr->max_buffer;
5073 iter->cpu_file = tracing_get_cpu(inode);
5075 file->private_data = m;
5079 trace_array_put(tr);
5085 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5088 struct seq_file *m = filp->private_data;
5089 struct trace_iterator *iter = m->private;
5090 struct trace_array *tr = iter->tr;
5094 ret = tracing_update_buffers();
5098 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5102 mutex_lock(&trace_types_lock);
5104 if (tr->current_trace->use_max_tr) {
5111 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5115 if (tr->allocated_snapshot)
5119 /* Only allow per-cpu swap if the ring buffer supports it */
5120 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5121 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5126 if (!tr->allocated_snapshot) {
5127 ret = alloc_snapshot(tr);
5131 local_irq_disable();
5132 /* Now, we're going to swap */
5133 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5134 update_max_tr(tr, current, smp_processor_id());
5136 update_max_tr_single(tr, current, iter->cpu_file);
5140 if (tr->allocated_snapshot) {
5141 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5142 tracing_reset_online_cpus(&tr->max_buffer);
5144 tracing_reset(&tr->max_buffer, iter->cpu_file);
5154 mutex_unlock(&trace_types_lock);
5158 static int tracing_snapshot_release(struct inode *inode, struct file *file)
5160 struct seq_file *m = file->private_data;
5163 ret = tracing_release(inode, file);
5165 if (file->f_mode & FMODE_READ)
5168 /* If write only, the seq_file is just a stub */
5176 static int tracing_buffers_open(struct inode *inode, struct file *filp);
5177 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5178 size_t count, loff_t *ppos);
5179 static int tracing_buffers_release(struct inode *inode, struct file *file);
5180 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5181 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5183 static int snapshot_raw_open(struct inode *inode, struct file *filp)
5185 struct ftrace_buffer_info *info;
5188 ret = tracing_buffers_open(inode, filp);
5192 info = filp->private_data;
5194 if (info->iter.trace->use_max_tr) {
5195 tracing_buffers_release(inode, filp);
5199 info->iter.snapshot = true;
5200 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5205 #endif /* CONFIG_TRACER_SNAPSHOT */
5208 static const struct file_operations tracing_thresh_fops = {
5209 .open = tracing_open_generic,
5210 .read = tracing_thresh_read,
5211 .write = tracing_thresh_write,
5212 .llseek = generic_file_llseek,
5215 static const struct file_operations tracing_max_lat_fops = {
5216 .open = tracing_open_generic,
5217 .read = tracing_max_lat_read,
5218 .write = tracing_max_lat_write,
5219 .llseek = generic_file_llseek,
5222 static const struct file_operations set_tracer_fops = {
5223 .open = tracing_open_generic,
5224 .read = tracing_set_trace_read,
5225 .write = tracing_set_trace_write,
5226 .llseek = generic_file_llseek,
5229 static const struct file_operations tracing_pipe_fops = {
5230 .open = tracing_open_pipe,
5231 .poll = tracing_poll_pipe,
5232 .read = tracing_read_pipe,
5233 .splice_read = tracing_splice_read_pipe,
5234 .release = tracing_release_pipe,
5235 .llseek = no_llseek,
5238 static const struct file_operations tracing_entries_fops = {
5239 .open = tracing_open_generic_tr,
5240 .read = tracing_entries_read,
5241 .write = tracing_entries_write,
5242 .llseek = generic_file_llseek,
5243 .release = tracing_release_generic_tr,
5246 static const struct file_operations tracing_total_entries_fops = {
5247 .open = tracing_open_generic_tr,
5248 .read = tracing_total_entries_read,
5249 .llseek = generic_file_llseek,
5250 .release = tracing_release_generic_tr,
5253 static const struct file_operations tracing_free_buffer_fops = {
5254 .open = tracing_open_generic_tr,
5255 .write = tracing_free_buffer_write,
5256 .release = tracing_free_buffer_release,
5259 static const struct file_operations tracing_mark_fops = {
5260 .open = tracing_open_generic_tr,
5261 .write = tracing_mark_write,
5262 .llseek = generic_file_llseek,
5263 .release = tracing_release_generic_tr,
5266 static const struct file_operations trace_clock_fops = {
5267 .open = tracing_clock_open,
5269 .llseek = seq_lseek,
5270 .release = tracing_single_release_tr,
5271 .write = tracing_clock_write,
5274 #ifdef CONFIG_TRACER_SNAPSHOT
5275 static const struct file_operations snapshot_fops = {
5276 .open = tracing_snapshot_open,
5278 .write = tracing_snapshot_write,
5279 .llseek = tracing_lseek,
5280 .release = tracing_snapshot_release,
5283 static const struct file_operations snapshot_raw_fops = {
5284 .open = snapshot_raw_open,
5285 .read = tracing_buffers_read,
5286 .release = tracing_buffers_release,
5287 .splice_read = tracing_buffers_splice_read,
5288 .llseek = no_llseek,
5291 #endif /* CONFIG_TRACER_SNAPSHOT */
5293 static int tracing_buffers_open(struct inode *inode, struct file *filp)
5295 struct trace_array *tr = inode->i_private;
5296 struct ftrace_buffer_info *info;
5299 if (tracing_disabled)
5302 if (trace_array_get(tr) < 0)
5305 info = kzalloc(sizeof(*info), GFP_KERNEL);
5307 trace_array_put(tr);
5311 mutex_lock(&trace_types_lock);
5314 info->iter.cpu_file = tracing_get_cpu(inode);
5315 info->iter.trace = tr->current_trace;
5316 info->iter.trace_buffer = &tr->trace_buffer;
5318 /* Force reading ring buffer for first read */
5319 info->read = (unsigned int)-1;
5321 filp->private_data = info;
5323 mutex_unlock(&trace_types_lock);
5325 ret = nonseekable_open(inode, filp);
5327 trace_array_put(tr);
5333 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5335 struct ftrace_buffer_info *info = filp->private_data;
5336 struct trace_iterator *iter = &info->iter;
5338 return trace_poll(iter, filp, poll_table);
5342 tracing_buffers_read(struct file *filp, char __user *ubuf,
5343 size_t count, loff_t *ppos)
5345 struct ftrace_buffer_info *info = filp->private_data;
5346 struct trace_iterator *iter = &info->iter;
5353 mutex_lock(&trace_types_lock);
5355 #ifdef CONFIG_TRACER_MAX_TRACE
5356 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5363 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5369 /* Do we have previous read data to read? */
5370 if (info->read < PAGE_SIZE)
5374 trace_access_lock(iter->cpu_file);
5375 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
5379 trace_access_unlock(iter->cpu_file);
5382 if (trace_empty(iter)) {
5383 if ((filp->f_flags & O_NONBLOCK)) {
5387 mutex_unlock(&trace_types_lock);
5388 ret = wait_on_pipe(iter, false);
5389 mutex_lock(&trace_types_lock);
5402 size = PAGE_SIZE - info->read;
5406 ret = copy_to_user(ubuf, info->spare + info->read, size);
5417 mutex_unlock(&trace_types_lock);
5422 static int tracing_buffers_release(struct inode *inode, struct file *file)
5424 struct ftrace_buffer_info *info = file->private_data;
5425 struct trace_iterator *iter = &info->iter;
5427 mutex_lock(&trace_types_lock);
5429 __trace_array_put(iter->tr);
5432 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5435 mutex_unlock(&trace_types_lock);
5441 struct ring_buffer *buffer;
5446 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5447 struct pipe_buffer *buf)
5449 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5454 ring_buffer_free_read_page(ref->buffer, ref->page);
5459 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5460 struct pipe_buffer *buf)
5462 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5467 /* Pipe buffer operations for a buffer. */
5468 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5470 .confirm = generic_pipe_buf_confirm,
5471 .release = buffer_pipe_buf_release,
5472 .steal = generic_pipe_buf_steal,
5473 .get = buffer_pipe_buf_get,
5477 * Callback from splice_to_pipe(), if we need to release some pages
5478 * at the end of the spd in case we error'ed out in filling the pipe.
5480 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5482 struct buffer_ref *ref =
5483 (struct buffer_ref *)spd->partial[i].private;
5488 ring_buffer_free_read_page(ref->buffer, ref->page);
5490 spd->partial[i].private = 0;
5494 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5495 struct pipe_inode_info *pipe, size_t len,
5498 struct ftrace_buffer_info *info = file->private_data;
5499 struct trace_iterator *iter = &info->iter;
5500 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5501 struct page *pages_def[PIPE_DEF_BUFFERS];
5502 struct splice_pipe_desc spd = {
5504 .partial = partial_def,
5505 .nr_pages_max = PIPE_DEF_BUFFERS,
5507 .ops = &buffer_pipe_buf_ops,
5508 .spd_release = buffer_spd_release,
5510 struct buffer_ref *ref;
5511 int entries, size, i;
5514 mutex_lock(&trace_types_lock);
5516 #ifdef CONFIG_TRACER_MAX_TRACE
5517 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5523 if (splice_grow_spd(pipe, &spd)) {
5528 if (*ppos & (PAGE_SIZE - 1)) {
5533 if (len & (PAGE_SIZE - 1)) {
5534 if (len < PAGE_SIZE) {
5542 trace_access_lock(iter->cpu_file);
5543 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5545 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
5549 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5556 ref->buffer = iter->trace_buffer->buffer;
5557 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5564 r = ring_buffer_read_page(ref->buffer, &ref->page,
5565 len, iter->cpu_file, 1);
5567 ring_buffer_free_read_page(ref->buffer, ref->page);
5573 * zero out any left over data, this is going to
5576 size = ring_buffer_page_len(ref->page);
5577 if (size < PAGE_SIZE)
5578 memset(ref->page + size, 0, PAGE_SIZE - size);
5580 page = virt_to_page(ref->page);
5582 spd.pages[i] = page;
5583 spd.partial[i].len = PAGE_SIZE;
5584 spd.partial[i].offset = 0;
5585 spd.partial[i].private = (unsigned long)ref;
5589 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5592 trace_access_unlock(iter->cpu_file);
5595 /* did we read anything? */
5596 if (!spd.nr_pages) {
5600 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
5604 mutex_unlock(&trace_types_lock);
5605 ret = wait_on_pipe(iter, true);
5606 mutex_lock(&trace_types_lock);
5613 ret = splice_to_pipe(pipe, &spd);
5614 splice_shrink_spd(&spd);
5616 mutex_unlock(&trace_types_lock);
5621 static const struct file_operations tracing_buffers_fops = {
5622 .open = tracing_buffers_open,
5623 .read = tracing_buffers_read,
5624 .poll = tracing_buffers_poll,
5625 .release = tracing_buffers_release,
5626 .splice_read = tracing_buffers_splice_read,
5627 .llseek = no_llseek,
5631 tracing_stats_read(struct file *filp, char __user *ubuf,
5632 size_t count, loff_t *ppos)
5634 struct inode *inode = file_inode(filp);
5635 struct trace_array *tr = inode->i_private;
5636 struct trace_buffer *trace_buf = &tr->trace_buffer;
5637 int cpu = tracing_get_cpu(inode);
5638 struct trace_seq *s;
5640 unsigned long long t;
5641 unsigned long usec_rem;
5643 s = kmalloc(sizeof(*s), GFP_KERNEL);
5649 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5650 trace_seq_printf(s, "entries: %ld\n", cnt);
5652 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5653 trace_seq_printf(s, "overrun: %ld\n", cnt);
5655 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5656 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5658 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5659 trace_seq_printf(s, "bytes: %ld\n", cnt);
5661 if (trace_clocks[tr->clock_id].in_ns) {
5662 /* local or global for trace_clock */
5663 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5664 usec_rem = do_div(t, USEC_PER_SEC);
5665 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5668 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5669 usec_rem = do_div(t, USEC_PER_SEC);
5670 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5672 /* counter or tsc mode for trace_clock */
5673 trace_seq_printf(s, "oldest event ts: %llu\n",
5674 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5676 trace_seq_printf(s, "now ts: %llu\n",
5677 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5680 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5681 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5683 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5684 trace_seq_printf(s, "read events: %ld\n", cnt);
5686 count = simple_read_from_buffer(ubuf, count, ppos,
5687 s->buffer, trace_seq_used(s));
5694 static const struct file_operations tracing_stats_fops = {
5695 .open = tracing_open_generic_tr,
5696 .read = tracing_stats_read,
5697 .llseek = generic_file_llseek,
5698 .release = tracing_release_generic_tr,
5701 #ifdef CONFIG_DYNAMIC_FTRACE
5703 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5709 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5710 size_t cnt, loff_t *ppos)
5712 static char ftrace_dyn_info_buffer[1024];
5713 static DEFINE_MUTEX(dyn_info_mutex);
5714 unsigned long *p = filp->private_data;
5715 char *buf = ftrace_dyn_info_buffer;
5716 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5719 mutex_lock(&dyn_info_mutex);
5720 r = sprintf(buf, "%ld ", *p);
5722 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5725 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5727 mutex_unlock(&dyn_info_mutex);
5732 static const struct file_operations tracing_dyn_info_fops = {
5733 .open = tracing_open_generic,
5734 .read = tracing_read_dyn_info,
5735 .llseek = generic_file_llseek,
5737 #endif /* CONFIG_DYNAMIC_FTRACE */
5739 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5741 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5747 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5749 unsigned long *count = (long *)data;
5761 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5762 struct ftrace_probe_ops *ops, void *data)
5764 long count = (long)data;
5766 seq_printf(m, "%ps:", (void *)ip);
5768 seq_puts(m, "snapshot");
5771 seq_puts(m, ":unlimited\n");
5773 seq_printf(m, ":count=%ld\n", count);
5778 static struct ftrace_probe_ops snapshot_probe_ops = {
5779 .func = ftrace_snapshot,
5780 .print = ftrace_snapshot_print,
5783 static struct ftrace_probe_ops snapshot_count_probe_ops = {
5784 .func = ftrace_count_snapshot,
5785 .print = ftrace_snapshot_print,
5789 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5790 char *glob, char *cmd, char *param, int enable)
5792 struct ftrace_probe_ops *ops;
5793 void *count = (void *)-1;
5797 /* hash funcs only work with set_ftrace_filter */
5801 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
5803 if (glob[0] == '!') {
5804 unregister_ftrace_function_probe_func(glob+1, ops);
5811 number = strsep(¶m, ":");
5813 if (!strlen(number))
5817 * We use the callback data field (which is a pointer)
5820 ret = kstrtoul(number, 0, (unsigned long *)&count);
5825 ret = register_ftrace_function_probe(glob, ops, count);
5828 alloc_snapshot(&global_trace);
5830 return ret < 0 ? ret : 0;
5833 static struct ftrace_func_command ftrace_snapshot_cmd = {
5835 .func = ftrace_trace_snapshot_callback,
5838 static __init int register_snapshot_cmd(void)
5840 return register_ftrace_command(&ftrace_snapshot_cmd);
5843 static inline __init int register_snapshot_cmd(void) { return 0; }
5844 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
5846 struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
5851 if (!debugfs_initialized())
5854 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5855 tr->dir = debugfs_create_dir("tracing", NULL);
5858 pr_warn_once("Could not create debugfs directory 'tracing'\n");
5863 struct dentry *tracing_init_dentry(void)
5865 return tracing_init_dentry_tr(&global_trace);
5868 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
5870 struct dentry *d_tracer;
5873 return tr->percpu_dir;
5875 d_tracer = tracing_init_dentry_tr(tr);
5879 tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
5881 WARN_ONCE(!tr->percpu_dir,
5882 "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
5884 return tr->percpu_dir;
5887 static struct dentry *
5888 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
5889 void *data, long cpu, const struct file_operations *fops)
5891 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
5893 if (ret) /* See tracing_get_cpu() */
5894 ret->d_inode->i_cdev = (void *)(cpu + 1);
5899 tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
5901 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5902 struct dentry *d_cpu;
5903 char cpu_dir[30]; /* 30 characters should be more than enough */
5908 snprintf(cpu_dir, 30, "cpu%ld", cpu);
5909 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5911 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5915 /* per cpu trace_pipe */
5916 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
5917 tr, cpu, &tracing_pipe_fops);
5920 trace_create_cpu_file("trace", 0644, d_cpu,
5921 tr, cpu, &tracing_fops);
5923 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
5924 tr, cpu, &tracing_buffers_fops);
5926 trace_create_cpu_file("stats", 0444, d_cpu,
5927 tr, cpu, &tracing_stats_fops);
5929 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
5930 tr, cpu, &tracing_entries_fops);
5932 #ifdef CONFIG_TRACER_SNAPSHOT
5933 trace_create_cpu_file("snapshot", 0644, d_cpu,
5934 tr, cpu, &snapshot_fops);
5936 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
5937 tr, cpu, &snapshot_raw_fops);
5941 #ifdef CONFIG_FTRACE_SELFTEST
5942 /* Let selftest have access to static functions in this file */
5943 #include "trace_selftest.c"
5946 struct trace_option_dentry {
5947 struct tracer_opt *opt;
5948 struct tracer_flags *flags;
5949 struct trace_array *tr;
5950 struct dentry *entry;
5954 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5957 struct trace_option_dentry *topt = filp->private_data;
5960 if (topt->flags->val & topt->opt->bit)
5965 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5969 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5972 struct trace_option_dentry *topt = filp->private_data;
5976 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5980 if (val != 0 && val != 1)
5983 if (!!(topt->flags->val & topt->opt->bit) != val) {
5984 mutex_lock(&trace_types_lock);
5985 ret = __set_tracer_option(topt->tr, topt->flags,
5987 mutex_unlock(&trace_types_lock);
5998 static const struct file_operations trace_options_fops = {
5999 .open = tracing_open_generic,
6000 .read = trace_options_read,
6001 .write = trace_options_write,
6002 .llseek = generic_file_llseek,
6006 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
6009 long index = (long)filp->private_data;
6012 if (trace_flags & (1 << index))
6017 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6021 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
6024 struct trace_array *tr = &global_trace;
6025 long index = (long)filp->private_data;
6029 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6033 if (val != 0 && val != 1)
6036 mutex_lock(&trace_types_lock);
6037 ret = set_tracer_flag(tr, 1 << index, val);
6038 mutex_unlock(&trace_types_lock);
6048 static const struct file_operations trace_options_core_fops = {
6049 .open = tracing_open_generic,
6050 .read = trace_options_core_read,
6051 .write = trace_options_core_write,
6052 .llseek = generic_file_llseek,
6055 struct dentry *trace_create_file(const char *name,
6057 struct dentry *parent,
6059 const struct file_operations *fops)
6063 ret = debugfs_create_file(name, mode, parent, data, fops);
6065 pr_warning("Could not create debugfs '%s' entry\n", name);
6071 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
6073 struct dentry *d_tracer;
6078 d_tracer = tracing_init_dentry_tr(tr);
6082 tr->options = debugfs_create_dir("options", d_tracer);
6084 pr_warning("Could not create debugfs directory 'options'\n");
6092 create_trace_option_file(struct trace_array *tr,
6093 struct trace_option_dentry *topt,
6094 struct tracer_flags *flags,
6095 struct tracer_opt *opt)
6097 struct dentry *t_options;
6099 t_options = trace_options_init_dentry(tr);
6103 topt->flags = flags;
6107 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
6108 &trace_options_fops);
6112 static struct trace_option_dentry *
6113 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
6115 struct trace_option_dentry *topts;
6116 struct tracer_flags *flags;
6117 struct tracer_opt *opts;
6123 flags = tracer->flags;
6125 if (!flags || !flags->opts)
6130 for (cnt = 0; opts[cnt].name; cnt++)
6133 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
6137 for (cnt = 0; opts[cnt].name; cnt++)
6138 create_trace_option_file(tr, &topts[cnt], flags,
6145 destroy_trace_option_files(struct trace_option_dentry *topts)
6152 for (cnt = 0; topts[cnt].opt; cnt++)
6153 debugfs_remove(topts[cnt].entry);
6158 static struct dentry *
6159 create_trace_option_core_file(struct trace_array *tr,
6160 const char *option, long index)
6162 struct dentry *t_options;
6164 t_options = trace_options_init_dentry(tr);
6168 return trace_create_file(option, 0644, t_options, (void *)index,
6169 &trace_options_core_fops);
6172 static __init void create_trace_options_dir(struct trace_array *tr)
6174 struct dentry *t_options;
6177 t_options = trace_options_init_dentry(tr);
6181 for (i = 0; trace_options[i]; i++)
6182 create_trace_option_core_file(tr, trace_options[i], i);
6186 rb_simple_read(struct file *filp, char __user *ubuf,
6187 size_t cnt, loff_t *ppos)
6189 struct trace_array *tr = filp->private_data;
6193 r = tracer_tracing_is_on(tr);
6194 r = sprintf(buf, "%d\n", r);
6196 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6200 rb_simple_write(struct file *filp, const char __user *ubuf,
6201 size_t cnt, loff_t *ppos)
6203 struct trace_array *tr = filp->private_data;
6204 struct ring_buffer *buffer = tr->trace_buffer.buffer;
6208 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6213 mutex_lock(&trace_types_lock);
6215 tracer_tracing_on(tr);
6216 if (tr->current_trace->start)
6217 tr->current_trace->start(tr);
6219 tracer_tracing_off(tr);
6220 if (tr->current_trace->stop)
6221 tr->current_trace->stop(tr);
6223 mutex_unlock(&trace_types_lock);
6231 static const struct file_operations rb_simple_fops = {
6232 .open = tracing_open_generic_tr,
6233 .read = rb_simple_read,
6234 .write = rb_simple_write,
6235 .release = tracing_release_generic_tr,
6236 .llseek = default_llseek,
6239 struct dentry *trace_instance_dir;
6242 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
6245 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
6247 enum ring_buffer_flags rb_flags;
6249 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
6253 buf->buffer = ring_buffer_alloc(size, rb_flags);
6257 buf->data = alloc_percpu(struct trace_array_cpu);
6259 ring_buffer_free(buf->buffer);
6263 /* Allocate the first page for all buffers */
6264 set_buffer_entries(&tr->trace_buffer,
6265 ring_buffer_size(tr->trace_buffer.buffer, 0));
6270 static int allocate_trace_buffers(struct trace_array *tr, int size)
6274 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6278 #ifdef CONFIG_TRACER_MAX_TRACE
6279 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6280 allocate_snapshot ? size : 1);
6282 ring_buffer_free(tr->trace_buffer.buffer);
6283 free_percpu(tr->trace_buffer.data);
6286 tr->allocated_snapshot = allocate_snapshot;
6289 * Only the top level trace array gets its snapshot allocated
6290 * from the kernel command line.
6292 allocate_snapshot = false;
6297 static void free_trace_buffer(struct trace_buffer *buf)
6300 ring_buffer_free(buf->buffer);
6302 free_percpu(buf->data);
6307 static void free_trace_buffers(struct trace_array *tr)
6312 free_trace_buffer(&tr->trace_buffer);
6314 #ifdef CONFIG_TRACER_MAX_TRACE
6315 free_trace_buffer(&tr->max_buffer);
6319 static int new_instance_create(const char *name)
6321 struct trace_array *tr;
6324 mutex_lock(&trace_types_lock);
6327 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6328 if (tr->name && strcmp(tr->name, name) == 0)
6333 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6337 tr->name = kstrdup(name, GFP_KERNEL);
6341 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6344 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6346 raw_spin_lock_init(&tr->start_lock);
6348 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6350 tr->current_trace = &nop_trace;
6352 INIT_LIST_HEAD(&tr->systems);
6353 INIT_LIST_HEAD(&tr->events);
6355 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
6358 tr->dir = debugfs_create_dir(name, trace_instance_dir);
6362 ret = event_trace_add_tracer(tr->dir, tr);
6364 debugfs_remove_recursive(tr->dir);
6368 init_tracer_debugfs(tr, tr->dir);
6370 list_add(&tr->list, &ftrace_trace_arrays);
6372 mutex_unlock(&trace_types_lock);
6377 free_trace_buffers(tr);
6378 free_cpumask_var(tr->tracing_cpumask);
6383 mutex_unlock(&trace_types_lock);
6389 static int instance_delete(const char *name)
6391 struct trace_array *tr;
6395 mutex_lock(&trace_types_lock);
6398 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6399 if (tr->name && strcmp(tr->name, name) == 0) {
6411 list_del(&tr->list);
6413 tracing_set_nop(tr);
6414 event_trace_del_tracer(tr);
6415 ftrace_destroy_function_files(tr);
6416 debugfs_remove_recursive(tr->dir);
6417 free_trace_buffers(tr);
6425 mutex_unlock(&trace_types_lock);
6430 static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
6432 struct dentry *parent;
6435 /* Paranoid: Make sure the parent is the "instances" directory */
6436 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias);
6437 if (WARN_ON_ONCE(parent != trace_instance_dir))
6441 * The inode mutex is locked, but debugfs_create_dir() will also
6442 * take the mutex. As the instances directory can not be destroyed
6443 * or changed in any other way, it is safe to unlock it, and
6444 * let the dentry try. If two users try to make the same dir at
6445 * the same time, then the new_instance_create() will determine the
6448 mutex_unlock(&inode->i_mutex);
6450 ret = new_instance_create(dentry->d_iname);
6452 mutex_lock(&inode->i_mutex);
6457 static int instance_rmdir(struct inode *inode, struct dentry *dentry)
6459 struct dentry *parent;
6462 /* Paranoid: Make sure the parent is the "instances" directory */
6463 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias);
6464 if (WARN_ON_ONCE(parent != trace_instance_dir))
6467 /* The caller did a dget() on dentry */
6468 mutex_unlock(&dentry->d_inode->i_mutex);
6471 * The inode mutex is locked, but debugfs_create_dir() will also
6472 * take the mutex. As the instances directory can not be destroyed
6473 * or changed in any other way, it is safe to unlock it, and
6474 * let the dentry try. If two users try to make the same dir at
6475 * the same time, then the instance_delete() will determine the
6478 mutex_unlock(&inode->i_mutex);
6480 ret = instance_delete(dentry->d_iname);
6482 mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
6483 mutex_lock(&dentry->d_inode->i_mutex);
6488 static const struct inode_operations instance_dir_inode_operations = {
6489 .lookup = simple_lookup,
6490 .mkdir = instance_mkdir,
6491 .rmdir = instance_rmdir,
6494 static __init void create_trace_instances(struct dentry *d_tracer)
6496 trace_instance_dir = debugfs_create_dir("instances", d_tracer);
6497 if (WARN_ON(!trace_instance_dir))
6500 /* Hijack the dir inode operations, to allow mkdir */
6501 trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
6505 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
6509 trace_create_file("available_tracers", 0444, d_tracer,
6510 tr, &show_traces_fops);
6512 trace_create_file("current_tracer", 0644, d_tracer,
6513 tr, &set_tracer_fops);
6515 trace_create_file("tracing_cpumask", 0644, d_tracer,
6516 tr, &tracing_cpumask_fops);
6518 trace_create_file("trace_options", 0644, d_tracer,
6519 tr, &tracing_iter_fops);
6521 trace_create_file("trace", 0644, d_tracer,
6524 trace_create_file("trace_pipe", 0444, d_tracer,
6525 tr, &tracing_pipe_fops);
6527 trace_create_file("buffer_size_kb", 0644, d_tracer,
6528 tr, &tracing_entries_fops);
6530 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6531 tr, &tracing_total_entries_fops);
6533 trace_create_file("free_buffer", 0200, d_tracer,
6534 tr, &tracing_free_buffer_fops);
6536 trace_create_file("trace_marker", 0220, d_tracer,
6537 tr, &tracing_mark_fops);
6539 trace_create_file("trace_clock", 0644, d_tracer, tr,
6542 trace_create_file("tracing_on", 0644, d_tracer,
6543 tr, &rb_simple_fops);
6545 #ifdef CONFIG_TRACER_MAX_TRACE
6546 trace_create_file("tracing_max_latency", 0644, d_tracer,
6547 &tr->max_latency, &tracing_max_lat_fops);
6550 if (ftrace_create_function_files(tr, d_tracer))
6551 WARN(1, "Could not allocate function filter files");
6553 #ifdef CONFIG_TRACER_SNAPSHOT
6554 trace_create_file("snapshot", 0644, d_tracer,
6555 tr, &snapshot_fops);
6558 for_each_tracing_cpu(cpu)
6559 tracing_init_debugfs_percpu(tr, cpu);
6563 static __init int tracer_init_debugfs(void)
6565 struct dentry *d_tracer;
6567 trace_access_lock_init();
6569 d_tracer = tracing_init_dentry();
6573 init_tracer_debugfs(&global_trace, d_tracer);
6575 trace_create_file("tracing_thresh", 0644, d_tracer,
6576 &global_trace, &tracing_thresh_fops);
6578 trace_create_file("README", 0444, d_tracer,
6579 NULL, &tracing_readme_fops);
6581 trace_create_file("saved_cmdlines", 0444, d_tracer,
6582 NULL, &tracing_saved_cmdlines_fops);
6584 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
6585 NULL, &tracing_saved_cmdlines_size_fops);
6587 #ifdef CONFIG_DYNAMIC_FTRACE
6588 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6589 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
6592 create_trace_instances(d_tracer);
6594 create_trace_options_dir(&global_trace);
6599 static int trace_panic_handler(struct notifier_block *this,
6600 unsigned long event, void *unused)
6602 if (ftrace_dump_on_oops)
6603 ftrace_dump(ftrace_dump_on_oops);
6607 static struct notifier_block trace_panic_notifier = {
6608 .notifier_call = trace_panic_handler,
6610 .priority = 150 /* priority: INT_MAX >= x >= 0 */
6613 static int trace_die_handler(struct notifier_block *self,
6619 if (ftrace_dump_on_oops)
6620 ftrace_dump(ftrace_dump_on_oops);
6628 static struct notifier_block trace_die_notifier = {
6629 .notifier_call = trace_die_handler,
6634 * printk is set to max of 1024, we really don't need it that big.
6635 * Nothing should be printing 1000 characters anyway.
6637 #define TRACE_MAX_PRINT 1000
6640 * Define here KERN_TRACE so that we have one place to modify
6641 * it if we decide to change what log level the ftrace dump
6644 #define KERN_TRACE KERN_EMERG
6647 trace_printk_seq(struct trace_seq *s)
6649 /* Probably should print a warning here. */
6650 if (s->seq.len >= TRACE_MAX_PRINT)
6651 s->seq.len = TRACE_MAX_PRINT;
6654 * More paranoid code. Although the buffer size is set to
6655 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
6656 * an extra layer of protection.
6658 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
6659 s->seq.len = s->seq.size - 1;
6661 /* should be zero ended, but we are paranoid. */
6662 s->buffer[s->seq.len] = 0;
6664 printk(KERN_TRACE "%s", s->buffer);
6669 void trace_init_global_iter(struct trace_iterator *iter)
6671 iter->tr = &global_trace;
6672 iter->trace = iter->tr->current_trace;
6673 iter->cpu_file = RING_BUFFER_ALL_CPUS;
6674 iter->trace_buffer = &global_trace.trace_buffer;
6676 if (iter->trace && iter->trace->open)
6677 iter->trace->open(iter);
6679 /* Annotate start of buffers if we had overruns */
6680 if (ring_buffer_overruns(iter->trace_buffer->buffer))
6681 iter->iter_flags |= TRACE_FILE_ANNOTATE;
6683 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6684 if (trace_clocks[iter->tr->clock_id].in_ns)
6685 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6688 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6690 /* use static because iter can be a bit big for the stack */
6691 static struct trace_iterator iter;
6692 static atomic_t dump_running;
6693 unsigned int old_userobj;
6694 unsigned long flags;
6697 /* Only allow one dump user at a time. */
6698 if (atomic_inc_return(&dump_running) != 1) {
6699 atomic_dec(&dump_running);
6704 * Always turn off tracing when we dump.
6705 * We don't need to show trace output of what happens
6706 * between multiple crashes.
6708 * If the user does a sysrq-z, then they can re-enable
6709 * tracing with echo 1 > tracing_on.
6713 local_irq_save(flags);
6715 /* Simulate the iterator */
6716 trace_init_global_iter(&iter);
6718 for_each_tracing_cpu(cpu) {
6719 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
6722 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6724 /* don't look at user memory in panic mode */
6725 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6727 switch (oops_dump_mode) {
6729 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6732 iter.cpu_file = raw_smp_processor_id();
6737 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
6738 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6741 printk(KERN_TRACE "Dumping ftrace buffer:\n");
6743 /* Did function tracer already get disabled? */
6744 if (ftrace_is_dead()) {
6745 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6746 printk("# MAY BE MISSING FUNCTION EVENTS\n");
6750 * We need to stop all tracing on all CPUS to read the
6751 * the next buffer. This is a bit expensive, but is
6752 * not done often. We fill all what we can read,
6753 * and then release the locks again.
6756 while (!trace_empty(&iter)) {
6759 printk(KERN_TRACE "---------------------------------\n");
6763 /* reset all but tr, trace, and overruns */
6764 memset(&iter.seq, 0,
6765 sizeof(struct trace_iterator) -
6766 offsetof(struct trace_iterator, seq));
6767 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6770 if (trace_find_next_entry_inc(&iter) != NULL) {
6773 ret = print_trace_line(&iter);
6774 if (ret != TRACE_TYPE_NO_CONSUME)
6775 trace_consume(&iter);
6777 touch_nmi_watchdog();
6779 trace_printk_seq(&iter.seq);
6783 printk(KERN_TRACE " (ftrace buffer empty)\n");
6785 printk(KERN_TRACE "---------------------------------\n");
6788 trace_flags |= old_userobj;
6790 for_each_tracing_cpu(cpu) {
6791 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
6793 atomic_dec(&dump_running);
6794 local_irq_restore(flags);
6796 EXPORT_SYMBOL_GPL(ftrace_dump);
6798 __init static int tracer_alloc_buffers(void)
6804 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6807 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
6808 goto out_free_buffer_mask;
6810 /* Only allocate trace_printk buffers if a trace_printk exists */
6811 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
6812 /* Must be called before global_trace.buffer is allocated */
6813 trace_printk_init_buffers();
6815 /* To save memory, keep the ring buffer size to its minimum */
6816 if (ring_buffer_expanded)
6817 ring_buf_size = trace_buf_size;
6821 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6822 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
6824 raw_spin_lock_init(&global_trace.start_lock);
6826 /* Used for event triggers */
6827 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
6829 goto out_free_cpumask;
6831 if (trace_create_savedcmd() < 0)
6832 goto out_free_temp_buffer;
6834 /* TODO: make the number of buffers hot pluggable with CPUS */
6835 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
6836 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6838 goto out_free_savedcmd;
6841 if (global_trace.buffer_disabled)
6844 if (trace_boot_clock) {
6845 ret = tracing_set_clock(&global_trace, trace_boot_clock);
6847 pr_warning("Trace clock %s not defined, going back to default\n",
6852 * register_tracer() might reference current_trace, so it
6853 * needs to be set before we register anything. This is
6854 * just a bootstrap of current_trace anyway.
6856 global_trace.current_trace = &nop_trace;
6858 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
6860 ftrace_init_global_array_ops(&global_trace);
6862 register_tracer(&nop_trace);
6864 /* All seems OK, enable tracing */
6865 tracing_disabled = 0;
6867 atomic_notifier_chain_register(&panic_notifier_list,
6868 &trace_panic_notifier);
6870 register_die_notifier(&trace_die_notifier);
6872 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6874 INIT_LIST_HEAD(&global_trace.systems);
6875 INIT_LIST_HEAD(&global_trace.events);
6876 list_add(&global_trace.list, &ftrace_trace_arrays);
6878 while (trace_boot_options) {
6881 option = strsep(&trace_boot_options, ",");
6882 trace_set_options(&global_trace, option);
6885 register_snapshot_cmd();
6890 free_saved_cmdlines_buffer(savedcmd);
6891 out_free_temp_buffer:
6892 ring_buffer_free(temp_buffer);
6894 free_cpumask_var(global_trace.tracing_cpumask);
6895 out_free_buffer_mask:
6896 free_cpumask_var(tracing_buffer_mask);
6901 __init static int clear_boot_tracer(void)
6904 * The default tracer at boot buffer is an init section.
6905 * This function is called in lateinit. If we did not
6906 * find the boot tracer, then clear it out, to prevent
6907 * later registration from accessing the buffer that is
6908 * about to be freed.
6910 if (!default_bootup_tracer)
6913 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6914 default_bootup_tracer);
6915 default_bootup_tracer = NULL;
6920 early_initcall(tracer_alloc_buffers);
6921 fs_initcall(tracer_init_debugfs);
6922 late_initcall(clear_boot_tracer);