2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/pagemap.h>
24 #include <linux/hardirq.h>
25 #include <linux/linkage.h>
26 #include <linux/uaccess.h>
27 #include <linux/kprobes.h>
28 #include <linux/ftrace.h>
29 #include <linux/module.h>
30 #include <linux/percpu.h>
31 #include <linux/splice.h>
32 #include <linux/kdebug.h>
33 #include <linux/string.h>
34 #include <linux/rwsem.h>
35 #include <linux/slab.h>
36 #include <linux/ctype.h>
37 #include <linux/init.h>
38 #include <linux/poll.h>
39 #include <linux/nmi.h>
41 #include <linux/sched/rt.h>
44 #include "trace_output.h"
47 * On boot up, the ring buffer is set to the minimum size, so that
48 * we do not waste memory on systems that are not using tracing.
50 bool ring_buffer_expanded;
53 * We need to change this state when a selftest is running.
54 * A selftest will lurk into the ring-buffer to count the
55 * entries inserted during the selftest although some concurrent
56 * insertions into the ring-buffer such as trace_printk could occurred
57 * at the same time, giving false positive or negative results.
59 static bool __read_mostly tracing_selftest_running;
62 * If a tracer is running, we do not want to run SELFTEST.
64 bool __read_mostly tracing_selftest_disabled;
66 /* For tracers that don't implement custom flags */
67 static struct tracer_opt dummy_tracer_opt[] = {
71 static struct tracer_flags dummy_tracer_flags = {
73 .opts = dummy_tracer_opt
77 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
83 * To prevent the comm cache from being overwritten when no
84 * tracing is active, only save the comm when a trace event
87 static DEFINE_PER_CPU(bool, trace_cmdline_save);
90 * Kill all tracing for good (never come back).
91 * It is initialized to 1 but will turn to zero if the initialization
92 * of the tracer is successful. But that is the only place that sets
95 static int tracing_disabled = 1;
97 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
99 cpumask_var_t __read_mostly tracing_buffer_mask;
102 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
104 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
105 * is set, then ftrace_dump is called. This will output the contents
106 * of the ftrace buffers to the console. This is very useful for
107 * capturing traces that lead to crashes and outputing it to a
110 * It is default off, but you can enable it with either specifying
111 * "ftrace_dump_on_oops" in the kernel command line, or setting
112 * /proc/sys/kernel/ftrace_dump_on_oops
113 * Set 1 if you want to dump buffers of all CPUs
114 * Set 2 if you want to dump the buffer of the CPU that triggered oops
117 enum ftrace_dump_mode ftrace_dump_on_oops;
119 /* When set, tracing will stop when a WARN*() is hit */
120 int __disable_trace_on_warning;
122 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
124 #define MAX_TRACER_SIZE 100
125 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
126 static char *default_bootup_tracer;
128 static bool allocate_snapshot;
130 static int __init set_cmdline_ftrace(char *str)
132 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
133 default_bootup_tracer = bootup_tracer_buf;
134 /* We are using ftrace early, expand it */
135 ring_buffer_expanded = true;
138 __setup("ftrace=", set_cmdline_ftrace);
140 static int __init set_ftrace_dump_on_oops(char *str)
142 if (*str++ != '=' || !*str) {
143 ftrace_dump_on_oops = DUMP_ALL;
147 if (!strcmp("orig_cpu", str)) {
148 ftrace_dump_on_oops = DUMP_ORIG;
154 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
156 static int __init stop_trace_on_warning(char *str)
158 __disable_trace_on_warning = 1;
161 __setup("traceoff_on_warning=", stop_trace_on_warning);
163 static int __init boot_alloc_snapshot(char *str)
165 allocate_snapshot = true;
166 /* We also need the main ring buffer expanded */
167 ring_buffer_expanded = true;
170 __setup("alloc_snapshot", boot_alloc_snapshot);
173 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
174 static char *trace_boot_options __initdata;
176 static int __init set_trace_boot_options(char *str)
178 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
179 trace_boot_options = trace_boot_options_buf;
182 __setup("trace_options=", set_trace_boot_options);
184 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
185 static char *trace_boot_clock __initdata;
187 static int __init set_trace_boot_clock(char *str)
189 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
190 trace_boot_clock = trace_boot_clock_buf;
193 __setup("trace_clock=", set_trace_boot_clock);
196 unsigned long long ns2usecs(cycle_t nsec)
204 * The global_trace is the descriptor that holds the tracing
205 * buffers for the live tracing. For each CPU, it contains
206 * a link list of pages that will store trace entries. The
207 * page descriptor of the pages in the memory is used to hold
208 * the link list by linking the lru item in the page descriptor
209 * to each of the pages in the buffer per CPU.
211 * For each active CPU there is a data field that holds the
212 * pages for the buffer for that CPU. Each CPU has the same number
213 * of pages allocated for its buffer.
215 static struct trace_array global_trace;
217 LIST_HEAD(ftrace_trace_arrays);
219 int trace_array_get(struct trace_array *this_tr)
221 struct trace_array *tr;
224 mutex_lock(&trace_types_lock);
225 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
232 mutex_unlock(&trace_types_lock);
237 static void __trace_array_put(struct trace_array *this_tr)
239 WARN_ON(!this_tr->ref);
243 void trace_array_put(struct trace_array *this_tr)
245 mutex_lock(&trace_types_lock);
246 __trace_array_put(this_tr);
247 mutex_unlock(&trace_types_lock);
250 int filter_check_discard(struct ftrace_event_file *file, void *rec,
251 struct ring_buffer *buffer,
252 struct ring_buffer_event *event)
254 if (unlikely(file->flags & FTRACE_EVENT_FL_FILTERED) &&
255 !filter_match_preds(file->filter, rec)) {
256 ring_buffer_discard_commit(buffer, event);
262 EXPORT_SYMBOL_GPL(filter_check_discard);
264 int call_filter_check_discard(struct ftrace_event_call *call, void *rec,
265 struct ring_buffer *buffer,
266 struct ring_buffer_event *event)
268 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
269 !filter_match_preds(call->filter, rec)) {
270 ring_buffer_discard_commit(buffer, event);
276 EXPORT_SYMBOL_GPL(call_filter_check_discard);
278 cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
282 /* Early boot up does not have a buffer yet */
284 return trace_clock_local();
286 ts = ring_buffer_time_stamp(buf->buffer, cpu);
287 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
292 cycle_t ftrace_now(int cpu)
294 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
298 * tracing_is_enabled - Show if global_trace has been disabled
300 * Shows if the global trace has been enabled or not. It uses the
301 * mirror flag "buffer_disabled" to be used in fast paths such as for
302 * the irqsoff tracer. But it may be inaccurate due to races. If you
303 * need to know the accurate state, use tracing_is_on() which is a little
304 * slower, but accurate.
306 int tracing_is_enabled(void)
309 * For quick access (irqsoff uses this in fast path), just
310 * return the mirror variable of the state of the ring buffer.
311 * It's a little racy, but we don't really care.
314 return !global_trace.buffer_disabled;
318 * trace_buf_size is the size in bytes that is allocated
319 * for a buffer. Note, the number of bytes is always rounded
322 * This number is purposely set to a low number of 16384.
323 * If the dump on oops happens, it will be much appreciated
324 * to not have to wait for all that output. Anyway this can be
325 * boot time and run time configurable.
327 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
329 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
331 /* trace_types holds a link list of available tracers. */
332 static struct tracer *trace_types __read_mostly;
335 * trace_types_lock is used to protect the trace_types list.
337 DEFINE_MUTEX(trace_types_lock);
340 * serialize the access of the ring buffer
342 * ring buffer serializes readers, but it is low level protection.
343 * The validity of the events (which returns by ring_buffer_peek() ..etc)
344 * are not protected by ring buffer.
346 * The content of events may become garbage if we allow other process consumes
347 * these events concurrently:
348 * A) the page of the consumed events may become a normal page
349 * (not reader page) in ring buffer, and this page will be rewrited
350 * by events producer.
351 * B) The page of the consumed events may become a page for splice_read,
352 * and this page will be returned to system.
354 * These primitives allow multi process access to different cpu ring buffer
357 * These primitives don't distinguish read-only and read-consume access.
358 * Multi read-only access are also serialized.
362 static DECLARE_RWSEM(all_cpu_access_lock);
363 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
365 static inline void trace_access_lock(int cpu)
367 if (cpu == RING_BUFFER_ALL_CPUS) {
368 /* gain it for accessing the whole ring buffer. */
369 down_write(&all_cpu_access_lock);
371 /* gain it for accessing a cpu ring buffer. */
373 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
374 down_read(&all_cpu_access_lock);
376 /* Secondly block other access to this @cpu ring buffer. */
377 mutex_lock(&per_cpu(cpu_access_lock, cpu));
381 static inline void trace_access_unlock(int cpu)
383 if (cpu == RING_BUFFER_ALL_CPUS) {
384 up_write(&all_cpu_access_lock);
386 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
387 up_read(&all_cpu_access_lock);
391 static inline void trace_access_lock_init(void)
395 for_each_possible_cpu(cpu)
396 mutex_init(&per_cpu(cpu_access_lock, cpu));
401 static DEFINE_MUTEX(access_lock);
403 static inline void trace_access_lock(int cpu)
406 mutex_lock(&access_lock);
409 static inline void trace_access_unlock(int cpu)
412 mutex_unlock(&access_lock);
415 static inline void trace_access_lock_init(void)
421 /* trace_flags holds trace_options default values */
422 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
423 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
424 TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
425 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
427 static void tracer_tracing_on(struct trace_array *tr)
429 if (tr->trace_buffer.buffer)
430 ring_buffer_record_on(tr->trace_buffer.buffer);
432 * This flag is looked at when buffers haven't been allocated
433 * yet, or by some tracers (like irqsoff), that just want to
434 * know if the ring buffer has been disabled, but it can handle
435 * races of where it gets disabled but we still do a record.
436 * As the check is in the fast path of the tracers, it is more
437 * important to be fast than accurate.
439 tr->buffer_disabled = 0;
440 /* Make the flag seen by readers */
445 * tracing_on - enable tracing buffers
447 * This function enables tracing buffers that may have been
448 * disabled with tracing_off.
450 void tracing_on(void)
452 tracer_tracing_on(&global_trace);
454 EXPORT_SYMBOL_GPL(tracing_on);
457 * __trace_puts - write a constant string into the trace buffer.
458 * @ip: The address of the caller
459 * @str: The constant string to write
460 * @size: The size of the string.
462 int __trace_puts(unsigned long ip, const char *str, int size)
464 struct ring_buffer_event *event;
465 struct ring_buffer *buffer;
466 struct print_entry *entry;
467 unsigned long irq_flags;
470 if (unlikely(tracing_selftest_running || tracing_disabled))
473 alloc = sizeof(*entry) + size + 2; /* possible \n added */
475 local_save_flags(irq_flags);
476 buffer = global_trace.trace_buffer.buffer;
477 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
478 irq_flags, preempt_count());
482 entry = ring_buffer_event_data(event);
485 memcpy(&entry->buf, str, size);
487 /* Add a newline if necessary */
488 if (entry->buf[size - 1] != '\n') {
489 entry->buf[size] = '\n';
490 entry->buf[size + 1] = '\0';
492 entry->buf[size] = '\0';
494 __buffer_unlock_commit(buffer, event);
498 EXPORT_SYMBOL_GPL(__trace_puts);
501 * __trace_bputs - write the pointer to a constant string into trace buffer
502 * @ip: The address of the caller
503 * @str: The constant string to write to the buffer to
505 int __trace_bputs(unsigned long ip, const char *str)
507 struct ring_buffer_event *event;
508 struct ring_buffer *buffer;
509 struct bputs_entry *entry;
510 unsigned long irq_flags;
511 int size = sizeof(struct bputs_entry);
513 if (unlikely(tracing_selftest_running || tracing_disabled))
516 local_save_flags(irq_flags);
517 buffer = global_trace.trace_buffer.buffer;
518 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
519 irq_flags, preempt_count());
523 entry = ring_buffer_event_data(event);
527 __buffer_unlock_commit(buffer, event);
531 EXPORT_SYMBOL_GPL(__trace_bputs);
533 #ifdef CONFIG_TRACER_SNAPSHOT
535 * trace_snapshot - take a snapshot of the current buffer.
537 * This causes a swap between the snapshot buffer and the current live
538 * tracing buffer. You can use this to take snapshots of the live
539 * trace when some condition is triggered, but continue to trace.
541 * Note, make sure to allocate the snapshot with either
542 * a tracing_snapshot_alloc(), or by doing it manually
543 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
545 * If the snapshot buffer is not allocated, it will stop tracing.
546 * Basically making a permanent snapshot.
548 void tracing_snapshot(void)
550 struct trace_array *tr = &global_trace;
551 struct tracer *tracer = tr->current_trace;
555 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
556 internal_trace_puts("*** snapshot is being ignored ***\n");
560 if (!tr->allocated_snapshot) {
561 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
562 internal_trace_puts("*** stopping trace here! ***\n");
567 /* Note, snapshot can not be used when the tracer uses it */
568 if (tracer->use_max_tr) {
569 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
570 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
574 local_irq_save(flags);
575 update_max_tr(tr, current, smp_processor_id());
576 local_irq_restore(flags);
578 EXPORT_SYMBOL_GPL(tracing_snapshot);
580 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
581 struct trace_buffer *size_buf, int cpu_id);
582 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
584 static int alloc_snapshot(struct trace_array *tr)
588 if (!tr->allocated_snapshot) {
590 /* allocate spare buffer */
591 ret = resize_buffer_duplicate_size(&tr->max_buffer,
592 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
596 tr->allocated_snapshot = true;
602 void free_snapshot(struct trace_array *tr)
605 * We don't free the ring buffer. instead, resize it because
606 * The max_tr ring buffer has some state (e.g. ring->clock) and
607 * we want preserve it.
609 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
610 set_buffer_entries(&tr->max_buffer, 1);
611 tracing_reset_online_cpus(&tr->max_buffer);
612 tr->allocated_snapshot = false;
616 * tracing_alloc_snapshot - allocate snapshot buffer.
618 * This only allocates the snapshot buffer if it isn't already
619 * allocated - it doesn't also take a snapshot.
621 * This is meant to be used in cases where the snapshot buffer needs
622 * to be set up for events that can't sleep but need to be able to
623 * trigger a snapshot.
625 int tracing_alloc_snapshot(void)
627 struct trace_array *tr = &global_trace;
630 ret = alloc_snapshot(tr);
635 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
638 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
640 * This is similar to trace_snapshot(), but it will allocate the
641 * snapshot buffer if it isn't already allocated. Use this only
642 * where it is safe to sleep, as the allocation may sleep.
644 * This causes a swap between the snapshot buffer and the current live
645 * tracing buffer. You can use this to take snapshots of the live
646 * trace when some condition is triggered, but continue to trace.
648 void tracing_snapshot_alloc(void)
652 ret = tracing_alloc_snapshot();
658 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
660 void tracing_snapshot(void)
662 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
664 EXPORT_SYMBOL_GPL(tracing_snapshot);
665 int tracing_alloc_snapshot(void)
667 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
670 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
671 void tracing_snapshot_alloc(void)
676 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
677 #endif /* CONFIG_TRACER_SNAPSHOT */
679 static void tracer_tracing_off(struct trace_array *tr)
681 if (tr->trace_buffer.buffer)
682 ring_buffer_record_off(tr->trace_buffer.buffer);
684 * This flag is looked at when buffers haven't been allocated
685 * yet, or by some tracers (like irqsoff), that just want to
686 * know if the ring buffer has been disabled, but it can handle
687 * races of where it gets disabled but we still do a record.
688 * As the check is in the fast path of the tracers, it is more
689 * important to be fast than accurate.
691 tr->buffer_disabled = 1;
692 /* Make the flag seen by readers */
697 * tracing_off - turn off tracing buffers
699 * This function stops the tracing buffers from recording data.
700 * It does not disable any overhead the tracers themselves may
701 * be causing. This function simply causes all recording to
702 * the ring buffers to fail.
704 void tracing_off(void)
706 tracer_tracing_off(&global_trace);
708 EXPORT_SYMBOL_GPL(tracing_off);
710 void disable_trace_on_warning(void)
712 if (__disable_trace_on_warning)
717 * tracer_tracing_is_on - show real state of ring buffer enabled
718 * @tr : the trace array to know if ring buffer is enabled
720 * Shows real state of the ring buffer if it is enabled or not.
722 static int tracer_tracing_is_on(struct trace_array *tr)
724 if (tr->trace_buffer.buffer)
725 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
726 return !tr->buffer_disabled;
730 * tracing_is_on - show state of ring buffers enabled
732 int tracing_is_on(void)
734 return tracer_tracing_is_on(&global_trace);
736 EXPORT_SYMBOL_GPL(tracing_is_on);
738 static int __init set_buf_size(char *str)
740 unsigned long buf_size;
744 buf_size = memparse(str, &str);
745 /* nr_entries can not be zero */
748 trace_buf_size = buf_size;
751 __setup("trace_buf_size=", set_buf_size);
753 static int __init set_tracing_thresh(char *str)
755 unsigned long threshold;
760 ret = kstrtoul(str, 0, &threshold);
763 tracing_thresh = threshold * 1000;
766 __setup("tracing_thresh=", set_tracing_thresh);
768 unsigned long nsecs_to_usecs(unsigned long nsecs)
773 /* These must match the bit postions in trace_iterator_flags */
774 static const char *trace_options[] = {
807 int in_ns; /* is this clock in nanoseconds? */
809 { trace_clock_local, "local", 1 },
810 { trace_clock_global, "global", 1 },
811 { trace_clock_counter, "counter", 0 },
812 { trace_clock_jiffies, "uptime", 1 },
813 { trace_clock, "perf", 1 },
818 * trace_parser_get_init - gets the buffer for trace parser
820 int trace_parser_get_init(struct trace_parser *parser, int size)
822 memset(parser, 0, sizeof(*parser));
824 parser->buffer = kmalloc(size, GFP_KERNEL);
833 * trace_parser_put - frees the buffer for trace parser
835 void trace_parser_put(struct trace_parser *parser)
837 kfree(parser->buffer);
841 * trace_get_user - reads the user input string separated by space
842 * (matched by isspace(ch))
844 * For each string found the 'struct trace_parser' is updated,
845 * and the function returns.
847 * Returns number of bytes read.
849 * See kernel/trace/trace.h for 'struct trace_parser' details.
851 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
852 size_t cnt, loff_t *ppos)
859 trace_parser_clear(parser);
861 ret = get_user(ch, ubuf++);
869 * The parser is not finished with the last write,
870 * continue reading the user input without skipping spaces.
873 /* skip white space */
874 while (cnt && isspace(ch)) {
875 ret = get_user(ch, ubuf++);
882 /* only spaces were written */
892 /* read the non-space input */
893 while (cnt && !isspace(ch)) {
894 if (parser->idx < parser->size - 1)
895 parser->buffer[parser->idx++] = ch;
900 ret = get_user(ch, ubuf++);
907 /* We either got finished input or we have to wait for another call. */
909 parser->buffer[parser->idx] = 0;
910 parser->cont = false;
911 } else if (parser->idx < parser->size - 1) {
913 parser->buffer[parser->idx++] = ch;
926 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
934 if (s->len <= s->readpos)
937 len = s->len - s->readpos;
940 ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
950 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
954 if (s->len <= s->readpos)
957 len = s->len - s->readpos;
960 memcpy(buf, s->buffer + s->readpos, cnt);
967 * ftrace_max_lock is used to protect the swapping of buffers
968 * when taking a max snapshot. The buffers themselves are
969 * protected by per_cpu spinlocks. But the action of the swap
970 * needs its own lock.
972 * This is defined as a arch_spinlock_t in order to help
973 * with performance when lockdep debugging is enabled.
975 * It is also used in other places outside the update_max_tr
976 * so it needs to be defined outside of the
977 * CONFIG_TRACER_MAX_TRACE.
979 static arch_spinlock_t ftrace_max_lock =
980 (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
982 unsigned long __read_mostly tracing_thresh;
984 #ifdef CONFIG_TRACER_MAX_TRACE
985 unsigned long __read_mostly tracing_max_latency;
988 * Copy the new maximum trace into the separate maximum-trace
989 * structure. (this way the maximum trace is permanently saved,
990 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
993 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
995 struct trace_buffer *trace_buf = &tr->trace_buffer;
996 struct trace_buffer *max_buf = &tr->max_buffer;
997 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
998 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1001 max_buf->time_start = data->preempt_timestamp;
1003 max_data->saved_latency = tracing_max_latency;
1004 max_data->critical_start = data->critical_start;
1005 max_data->critical_end = data->critical_end;
1007 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1008 max_data->pid = tsk->pid;
1010 * If tsk == current, then use current_uid(), as that does not use
1011 * RCU. The irq tracer can be called out of RCU scope.
1014 max_data->uid = current_uid();
1016 max_data->uid = task_uid(tsk);
1018 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1019 max_data->policy = tsk->policy;
1020 max_data->rt_priority = tsk->rt_priority;
1022 /* record this tasks comm */
1023 tracing_record_cmdline(tsk);
1027 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1029 * @tsk: the task with the latency
1030 * @cpu: The cpu that initiated the trace.
1032 * Flip the buffers between the @tr and the max_tr and record information
1033 * about which task was the cause of this latency.
1036 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1038 struct ring_buffer *buf;
1043 WARN_ON_ONCE(!irqs_disabled());
1045 if (!tr->allocated_snapshot) {
1046 /* Only the nop tracer should hit this when disabling */
1047 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1051 arch_spin_lock(&ftrace_max_lock);
1053 buf = tr->trace_buffer.buffer;
1054 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1055 tr->max_buffer.buffer = buf;
1057 __update_max_tr(tr, tsk, cpu);
1058 arch_spin_unlock(&ftrace_max_lock);
1062 * update_max_tr_single - only copy one trace over, and reset the rest
1064 * @tsk - task with the latency
1065 * @cpu - the cpu of the buffer to copy.
1067 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1070 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1077 WARN_ON_ONCE(!irqs_disabled());
1078 if (!tr->allocated_snapshot) {
1079 /* Only the nop tracer should hit this when disabling */
1080 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1084 arch_spin_lock(&ftrace_max_lock);
1086 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1088 if (ret == -EBUSY) {
1090 * We failed to swap the buffer due to a commit taking
1091 * place on this CPU. We fail to record, but we reset
1092 * the max trace buffer (no one writes directly to it)
1093 * and flag that it failed.
1095 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1096 "Failed to swap buffers due to commit in progress\n");
1099 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1101 __update_max_tr(tr, tsk, cpu);
1102 arch_spin_unlock(&ftrace_max_lock);
1104 #endif /* CONFIG_TRACER_MAX_TRACE */
1106 static void default_wait_pipe(struct trace_iterator *iter)
1108 /* Iterators are static, they should be filled or empty */
1109 if (trace_buffer_iter(iter, iter->cpu_file))
1112 ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
1115 #ifdef CONFIG_FTRACE_STARTUP_TEST
1116 static int run_tracer_selftest(struct tracer *type)
1118 struct trace_array *tr = &global_trace;
1119 struct tracer *saved_tracer = tr->current_trace;
1122 if (!type->selftest || tracing_selftest_disabled)
1126 * Run a selftest on this tracer.
1127 * Here we reset the trace buffer, and set the current
1128 * tracer to be this tracer. The tracer can then run some
1129 * internal tracing to verify that everything is in order.
1130 * If we fail, we do not register this tracer.
1132 tracing_reset_online_cpus(&tr->trace_buffer);
1134 tr->current_trace = type;
1136 #ifdef CONFIG_TRACER_MAX_TRACE
1137 if (type->use_max_tr) {
1138 /* If we expanded the buffers, make sure the max is expanded too */
1139 if (ring_buffer_expanded)
1140 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1141 RING_BUFFER_ALL_CPUS);
1142 tr->allocated_snapshot = true;
1146 /* the test is responsible for initializing and enabling */
1147 pr_info("Testing tracer %s: ", type->name);
1148 ret = type->selftest(type, tr);
1149 /* the test is responsible for resetting too */
1150 tr->current_trace = saved_tracer;
1152 printk(KERN_CONT "FAILED!\n");
1153 /* Add the warning after printing 'FAILED' */
1157 /* Only reset on passing, to avoid touching corrupted buffers */
1158 tracing_reset_online_cpus(&tr->trace_buffer);
1160 #ifdef CONFIG_TRACER_MAX_TRACE
1161 if (type->use_max_tr) {
1162 tr->allocated_snapshot = false;
1164 /* Shrink the max buffer again */
1165 if (ring_buffer_expanded)
1166 ring_buffer_resize(tr->max_buffer.buffer, 1,
1167 RING_BUFFER_ALL_CPUS);
1171 printk(KERN_CONT "PASSED\n");
1175 static inline int run_tracer_selftest(struct tracer *type)
1179 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1182 * register_tracer - register a tracer with the ftrace system.
1183 * @type - the plugin for the tracer
1185 * Register a new plugin tracer.
1187 int register_tracer(struct tracer *type)
1193 pr_info("Tracer must have a name\n");
1197 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1198 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1202 mutex_lock(&trace_types_lock);
1204 tracing_selftest_running = true;
1206 for (t = trace_types; t; t = t->next) {
1207 if (strcmp(type->name, t->name) == 0) {
1209 pr_info("Tracer %s already registered\n",
1216 if (!type->set_flag)
1217 type->set_flag = &dummy_set_flag;
1219 type->flags = &dummy_tracer_flags;
1221 if (!type->flags->opts)
1222 type->flags->opts = dummy_tracer_opt;
1223 if (!type->wait_pipe)
1224 type->wait_pipe = default_wait_pipe;
1226 ret = run_tracer_selftest(type);
1230 type->next = trace_types;
1234 tracing_selftest_running = false;
1235 mutex_unlock(&trace_types_lock);
1237 if (ret || !default_bootup_tracer)
1240 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1243 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1244 /* Do we want this tracer to start on bootup? */
1245 tracing_set_tracer(&global_trace, type->name);
1246 default_bootup_tracer = NULL;
1247 /* disable other selftests, since this will break it. */
1248 tracing_selftest_disabled = true;
1249 #ifdef CONFIG_FTRACE_STARTUP_TEST
1250 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1258 void tracing_reset(struct trace_buffer *buf, int cpu)
1260 struct ring_buffer *buffer = buf->buffer;
1265 ring_buffer_record_disable(buffer);
1267 /* Make sure all commits have finished */
1268 synchronize_sched();
1269 ring_buffer_reset_cpu(buffer, cpu);
1271 ring_buffer_record_enable(buffer);
1274 void tracing_reset_online_cpus(struct trace_buffer *buf)
1276 struct ring_buffer *buffer = buf->buffer;
1282 ring_buffer_record_disable(buffer);
1284 /* Make sure all commits have finished */
1285 synchronize_sched();
1287 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1289 for_each_online_cpu(cpu)
1290 ring_buffer_reset_cpu(buffer, cpu);
1292 ring_buffer_record_enable(buffer);
1295 /* Must have trace_types_lock held */
1296 void tracing_reset_all_online_cpus(void)
1298 struct trace_array *tr;
1300 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1301 tracing_reset_online_cpus(&tr->trace_buffer);
1302 #ifdef CONFIG_TRACER_MAX_TRACE
1303 tracing_reset_online_cpus(&tr->max_buffer);
1308 #define SAVED_CMDLINES 128
1309 #define NO_CMDLINE_MAP UINT_MAX
1310 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1311 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
1312 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
1313 static int cmdline_idx;
1314 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1316 /* temporary disable recording */
1317 static atomic_t trace_record_cmdline_disabled __read_mostly;
1319 static void trace_init_cmdlines(void)
1321 memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
1322 memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
1326 int is_tracing_stopped(void)
1328 return global_trace.stop_count;
1332 * tracing_start - quick start of the tracer
1334 * If tracing is enabled but was stopped by tracing_stop,
1335 * this will start the tracer back up.
1337 void tracing_start(void)
1339 struct ring_buffer *buffer;
1340 unsigned long flags;
1342 if (tracing_disabled)
1345 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1346 if (--global_trace.stop_count) {
1347 if (global_trace.stop_count < 0) {
1348 /* Someone screwed up their debugging */
1350 global_trace.stop_count = 0;
1355 /* Prevent the buffers from switching */
1356 arch_spin_lock(&ftrace_max_lock);
1358 buffer = global_trace.trace_buffer.buffer;
1360 ring_buffer_record_enable(buffer);
1362 #ifdef CONFIG_TRACER_MAX_TRACE
1363 buffer = global_trace.max_buffer.buffer;
1365 ring_buffer_record_enable(buffer);
1368 arch_spin_unlock(&ftrace_max_lock);
1372 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1375 static void tracing_start_tr(struct trace_array *tr)
1377 struct ring_buffer *buffer;
1378 unsigned long flags;
1380 if (tracing_disabled)
1383 /* If global, we need to also start the max tracer */
1384 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1385 return tracing_start();
1387 raw_spin_lock_irqsave(&tr->start_lock, flags);
1389 if (--tr->stop_count) {
1390 if (tr->stop_count < 0) {
1391 /* Someone screwed up their debugging */
1398 buffer = tr->trace_buffer.buffer;
1400 ring_buffer_record_enable(buffer);
1403 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1407 * tracing_stop - quick stop of the tracer
1409 * Light weight way to stop tracing. Use in conjunction with
1412 void tracing_stop(void)
1414 struct ring_buffer *buffer;
1415 unsigned long flags;
1418 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1419 if (global_trace.stop_count++)
1422 /* Prevent the buffers from switching */
1423 arch_spin_lock(&ftrace_max_lock);
1425 buffer = global_trace.trace_buffer.buffer;
1427 ring_buffer_record_disable(buffer);
1429 #ifdef CONFIG_TRACER_MAX_TRACE
1430 buffer = global_trace.max_buffer.buffer;
1432 ring_buffer_record_disable(buffer);
1435 arch_spin_unlock(&ftrace_max_lock);
1438 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1441 static void tracing_stop_tr(struct trace_array *tr)
1443 struct ring_buffer *buffer;
1444 unsigned long flags;
1446 /* If global, we need to also stop the max tracer */
1447 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1448 return tracing_stop();
1450 raw_spin_lock_irqsave(&tr->start_lock, flags);
1451 if (tr->stop_count++)
1454 buffer = tr->trace_buffer.buffer;
1456 ring_buffer_record_disable(buffer);
1459 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1462 void trace_stop_cmdline_recording(void);
1464 static void trace_save_cmdline(struct task_struct *tsk)
1468 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1472 * It's not the end of the world if we don't get
1473 * the lock, but we also don't want to spin
1474 * nor do we want to disable interrupts,
1475 * so if we miss here, then better luck next time.
1477 if (!arch_spin_trylock(&trace_cmdline_lock))
1480 idx = map_pid_to_cmdline[tsk->pid];
1481 if (idx == NO_CMDLINE_MAP) {
1482 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1485 * Check whether the cmdline buffer at idx has a pid
1486 * mapped. We are going to overwrite that entry so we
1487 * need to clear the map_pid_to_cmdline. Otherwise we
1488 * would read the new comm for the old pid.
1490 pid = map_cmdline_to_pid[idx];
1491 if (pid != NO_CMDLINE_MAP)
1492 map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1494 map_cmdline_to_pid[idx] = tsk->pid;
1495 map_pid_to_cmdline[tsk->pid] = idx;
1500 memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1502 arch_spin_unlock(&trace_cmdline_lock);
1505 void trace_find_cmdline(int pid, char comm[])
1510 strcpy(comm, "<idle>");
1514 if (WARN_ON_ONCE(pid < 0)) {
1515 strcpy(comm, "<XXX>");
1519 if (pid > PID_MAX_DEFAULT) {
1520 strcpy(comm, "<...>");
1525 arch_spin_lock(&trace_cmdline_lock);
1526 map = map_pid_to_cmdline[pid];
1527 if (map != NO_CMDLINE_MAP)
1528 strcpy(comm, saved_cmdlines[map]);
1530 strcpy(comm, "<...>");
1532 arch_spin_unlock(&trace_cmdline_lock);
1536 void tracing_record_cmdline(struct task_struct *tsk)
1538 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1541 if (!__this_cpu_read(trace_cmdline_save))
1544 __this_cpu_write(trace_cmdline_save, false);
1546 trace_save_cmdline(tsk);
1550 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1553 struct task_struct *tsk = current;
1555 entry->preempt_count = pc & 0xff;
1556 entry->pid = (tsk) ? tsk->pid : 0;
1558 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1559 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1561 TRACE_FLAG_IRQS_NOSUPPORT |
1563 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1564 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1565 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1566 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1568 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1570 struct ring_buffer_event *
1571 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1574 unsigned long flags, int pc)
1576 struct ring_buffer_event *event;
1578 event = ring_buffer_lock_reserve(buffer, len);
1579 if (event != NULL) {
1580 struct trace_entry *ent = ring_buffer_event_data(event);
1582 tracing_generic_entry_update(ent, flags, pc);
1590 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1592 __this_cpu_write(trace_cmdline_save, true);
1593 ring_buffer_unlock_commit(buffer, event);
1597 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1598 struct ring_buffer_event *event,
1599 unsigned long flags, int pc)
1601 __buffer_unlock_commit(buffer, event);
1603 ftrace_trace_stack(buffer, flags, 6, pc);
1604 ftrace_trace_userstack(buffer, flags, pc);
1607 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1608 struct ring_buffer_event *event,
1609 unsigned long flags, int pc)
1611 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1613 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1615 static struct ring_buffer *temp_buffer;
1617 struct ring_buffer_event *
1618 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1619 struct ftrace_event_file *ftrace_file,
1620 int type, unsigned long len,
1621 unsigned long flags, int pc)
1623 struct ring_buffer_event *entry;
1625 *current_rb = ftrace_file->tr->trace_buffer.buffer;
1626 entry = trace_buffer_lock_reserve(*current_rb,
1627 type, len, flags, pc);
1629 * If tracing is off, but we have triggers enabled
1630 * we still need to look at the event data. Use the temp_buffer
1631 * to store the trace event for the tigger to use. It's recusive
1632 * safe and will not be recorded anywhere.
1634 if (!entry && ftrace_file->flags & FTRACE_EVENT_FL_TRIGGER_COND) {
1635 *current_rb = temp_buffer;
1636 entry = trace_buffer_lock_reserve(*current_rb,
1637 type, len, flags, pc);
1641 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1643 struct ring_buffer_event *
1644 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1645 int type, unsigned long len,
1646 unsigned long flags, int pc)
1648 *current_rb = global_trace.trace_buffer.buffer;
1649 return trace_buffer_lock_reserve(*current_rb,
1650 type, len, flags, pc);
1652 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1654 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1655 struct ring_buffer_event *event,
1656 unsigned long flags, int pc)
1658 __trace_buffer_unlock_commit(buffer, event, flags, pc);
1660 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1662 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1663 struct ring_buffer_event *event,
1664 unsigned long flags, int pc,
1665 struct pt_regs *regs)
1667 __buffer_unlock_commit(buffer, event);
1669 ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1670 ftrace_trace_userstack(buffer, flags, pc);
1672 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1674 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1675 struct ring_buffer_event *event)
1677 ring_buffer_discard_commit(buffer, event);
1679 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1682 trace_function(struct trace_array *tr,
1683 unsigned long ip, unsigned long parent_ip, unsigned long flags,
1686 struct ftrace_event_call *call = &event_function;
1687 struct ring_buffer *buffer = tr->trace_buffer.buffer;
1688 struct ring_buffer_event *event;
1689 struct ftrace_entry *entry;
1691 /* If we are reading the ring buffer, don't trace */
1692 if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1695 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1699 entry = ring_buffer_event_data(event);
1701 entry->parent_ip = parent_ip;
1703 if (!call_filter_check_discard(call, entry, buffer, event))
1704 __buffer_unlock_commit(buffer, event);
1707 #ifdef CONFIG_STACKTRACE
1709 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1710 struct ftrace_stack {
1711 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
1714 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1715 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1717 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1718 unsigned long flags,
1719 int skip, int pc, struct pt_regs *regs)
1721 struct ftrace_event_call *call = &event_kernel_stack;
1722 struct ring_buffer_event *event;
1723 struct stack_entry *entry;
1724 struct stack_trace trace;
1726 int size = FTRACE_STACK_ENTRIES;
1728 trace.nr_entries = 0;
1732 * Since events can happen in NMIs there's no safe way to
1733 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1734 * or NMI comes in, it will just have to use the default
1735 * FTRACE_STACK_SIZE.
1737 preempt_disable_notrace();
1739 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1741 * We don't need any atomic variables, just a barrier.
1742 * If an interrupt comes in, we don't care, because it would
1743 * have exited and put the counter back to what we want.
1744 * We just need a barrier to keep gcc from moving things
1748 if (use_stack == 1) {
1749 trace.entries = &__get_cpu_var(ftrace_stack).calls[0];
1750 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
1753 save_stack_trace_regs(regs, &trace);
1755 save_stack_trace(&trace);
1757 if (trace.nr_entries > size)
1758 size = trace.nr_entries;
1760 /* From now on, use_stack is a boolean */
1763 size *= sizeof(unsigned long);
1765 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1766 sizeof(*entry) + size, flags, pc);
1769 entry = ring_buffer_event_data(event);
1771 memset(&entry->caller, 0, size);
1774 memcpy(&entry->caller, trace.entries,
1775 trace.nr_entries * sizeof(unsigned long));
1777 trace.max_entries = FTRACE_STACK_ENTRIES;
1778 trace.entries = entry->caller;
1780 save_stack_trace_regs(regs, &trace);
1782 save_stack_trace(&trace);
1785 entry->size = trace.nr_entries;
1787 if (!call_filter_check_discard(call, entry, buffer, event))
1788 __buffer_unlock_commit(buffer, event);
1791 /* Again, don't let gcc optimize things here */
1793 __this_cpu_dec(ftrace_stack_reserve);
1794 preempt_enable_notrace();
1798 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1799 int skip, int pc, struct pt_regs *regs)
1801 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1804 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1807 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1810 if (!(trace_flags & TRACE_ITER_STACKTRACE))
1813 __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1816 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1819 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1823 * trace_dump_stack - record a stack back trace in the trace buffer
1824 * @skip: Number of functions to skip (helper handlers)
1826 void trace_dump_stack(int skip)
1828 unsigned long flags;
1830 if (tracing_disabled || tracing_selftest_running)
1833 local_save_flags(flags);
1836 * Skip 3 more, seems to get us at the caller of
1840 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1841 flags, skip, preempt_count(), NULL);
1844 static DEFINE_PER_CPU(int, user_stack_count);
1847 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1849 struct ftrace_event_call *call = &event_user_stack;
1850 struct ring_buffer_event *event;
1851 struct userstack_entry *entry;
1852 struct stack_trace trace;
1854 if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1858 * NMIs can not handle page faults, even with fix ups.
1859 * The save user stack can (and often does) fault.
1861 if (unlikely(in_nmi()))
1865 * prevent recursion, since the user stack tracing may
1866 * trigger other kernel events.
1869 if (__this_cpu_read(user_stack_count))
1872 __this_cpu_inc(user_stack_count);
1874 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1875 sizeof(*entry), flags, pc);
1877 goto out_drop_count;
1878 entry = ring_buffer_event_data(event);
1880 entry->tgid = current->tgid;
1881 memset(&entry->caller, 0, sizeof(entry->caller));
1883 trace.nr_entries = 0;
1884 trace.max_entries = FTRACE_STACK_ENTRIES;
1886 trace.entries = entry->caller;
1888 save_stack_trace_user(&trace);
1889 if (!call_filter_check_discard(call, entry, buffer, event))
1890 __buffer_unlock_commit(buffer, event);
1893 __this_cpu_dec(user_stack_count);
1899 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1901 ftrace_trace_userstack(tr, flags, preempt_count());
1905 #endif /* CONFIG_STACKTRACE */
1907 /* created for use with alloc_percpu */
1908 struct trace_buffer_struct {
1909 char buffer[TRACE_BUF_SIZE];
1912 static struct trace_buffer_struct *trace_percpu_buffer;
1913 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1914 static struct trace_buffer_struct *trace_percpu_irq_buffer;
1915 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1918 * The buffer used is dependent on the context. There is a per cpu
1919 * buffer for normal context, softirq contex, hard irq context and
1920 * for NMI context. Thise allows for lockless recording.
1922 * Note, if the buffers failed to be allocated, then this returns NULL
1924 static char *get_trace_buf(void)
1926 struct trace_buffer_struct *percpu_buffer;
1929 * If we have allocated per cpu buffers, then we do not
1930 * need to do any locking.
1933 percpu_buffer = trace_percpu_nmi_buffer;
1935 percpu_buffer = trace_percpu_irq_buffer;
1936 else if (in_softirq())
1937 percpu_buffer = trace_percpu_sirq_buffer;
1939 percpu_buffer = trace_percpu_buffer;
1944 return this_cpu_ptr(&percpu_buffer->buffer[0]);
1947 static int alloc_percpu_trace_buffer(void)
1949 struct trace_buffer_struct *buffers;
1950 struct trace_buffer_struct *sirq_buffers;
1951 struct trace_buffer_struct *irq_buffers;
1952 struct trace_buffer_struct *nmi_buffers;
1954 buffers = alloc_percpu(struct trace_buffer_struct);
1958 sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1962 irq_buffers = alloc_percpu(struct trace_buffer_struct);
1966 nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1970 trace_percpu_buffer = buffers;
1971 trace_percpu_sirq_buffer = sirq_buffers;
1972 trace_percpu_irq_buffer = irq_buffers;
1973 trace_percpu_nmi_buffer = nmi_buffers;
1978 free_percpu(irq_buffers);
1980 free_percpu(sirq_buffers);
1982 free_percpu(buffers);
1984 WARN(1, "Could not allocate percpu trace_printk buffer");
1988 static int buffers_allocated;
1990 void trace_printk_init_buffers(void)
1992 if (buffers_allocated)
1995 if (alloc_percpu_trace_buffer())
1998 pr_info("ftrace: Allocated trace_printk buffers\n");
2000 /* Expand the buffers to set size */
2001 tracing_update_buffers();
2003 buffers_allocated = 1;
2006 * trace_printk_init_buffers() can be called by modules.
2007 * If that happens, then we need to start cmdline recording
2008 * directly here. If the global_trace.buffer is already
2009 * allocated here, then this was called by module code.
2011 if (global_trace.trace_buffer.buffer)
2012 tracing_start_cmdline_record();
2015 void trace_printk_start_comm(void)
2017 /* Start tracing comms if trace printk is set */
2018 if (!buffers_allocated)
2020 tracing_start_cmdline_record();
2023 static void trace_printk_start_stop_comm(int enabled)
2025 if (!buffers_allocated)
2029 tracing_start_cmdline_record();
2031 tracing_stop_cmdline_record();
2035 * trace_vbprintk - write binary msg to tracing buffer
2038 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2040 struct ftrace_event_call *call = &event_bprint;
2041 struct ring_buffer_event *event;
2042 struct ring_buffer *buffer;
2043 struct trace_array *tr = &global_trace;
2044 struct bprint_entry *entry;
2045 unsigned long flags;
2047 int len = 0, size, pc;
2049 if (unlikely(tracing_selftest_running || tracing_disabled))
2052 /* Don't pollute graph traces with trace_vprintk internals */
2053 pause_graph_tracing();
2055 pc = preempt_count();
2056 preempt_disable_notrace();
2058 tbuffer = get_trace_buf();
2064 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2066 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2069 local_save_flags(flags);
2070 size = sizeof(*entry) + sizeof(u32) * len;
2071 buffer = tr->trace_buffer.buffer;
2072 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2076 entry = ring_buffer_event_data(event);
2080 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2081 if (!call_filter_check_discard(call, entry, buffer, event)) {
2082 __buffer_unlock_commit(buffer, event);
2083 ftrace_trace_stack(buffer, flags, 6, pc);
2087 preempt_enable_notrace();
2088 unpause_graph_tracing();
2092 EXPORT_SYMBOL_GPL(trace_vbprintk);
2095 __trace_array_vprintk(struct ring_buffer *buffer,
2096 unsigned long ip, const char *fmt, va_list args)
2098 struct ftrace_event_call *call = &event_print;
2099 struct ring_buffer_event *event;
2100 int len = 0, size, pc;
2101 struct print_entry *entry;
2102 unsigned long flags;
2105 if (tracing_disabled || tracing_selftest_running)
2108 /* Don't pollute graph traces with trace_vprintk internals */
2109 pause_graph_tracing();
2111 pc = preempt_count();
2112 preempt_disable_notrace();
2115 tbuffer = get_trace_buf();
2121 len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2122 if (len > TRACE_BUF_SIZE)
2125 local_save_flags(flags);
2126 size = sizeof(*entry) + len + 1;
2127 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2131 entry = ring_buffer_event_data(event);
2134 memcpy(&entry->buf, tbuffer, len);
2135 entry->buf[len] = '\0';
2136 if (!call_filter_check_discard(call, entry, buffer, event)) {
2137 __buffer_unlock_commit(buffer, event);
2138 ftrace_trace_stack(buffer, flags, 6, pc);
2141 preempt_enable_notrace();
2142 unpause_graph_tracing();
2147 int trace_array_vprintk(struct trace_array *tr,
2148 unsigned long ip, const char *fmt, va_list args)
2150 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2153 int trace_array_printk(struct trace_array *tr,
2154 unsigned long ip, const char *fmt, ...)
2159 if (!(trace_flags & TRACE_ITER_PRINTK))
2163 ret = trace_array_vprintk(tr, ip, fmt, ap);
2168 int trace_array_printk_buf(struct ring_buffer *buffer,
2169 unsigned long ip, const char *fmt, ...)
2174 if (!(trace_flags & TRACE_ITER_PRINTK))
2178 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2183 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2185 return trace_array_vprintk(&global_trace, ip, fmt, args);
2187 EXPORT_SYMBOL_GPL(trace_vprintk);
2189 static void trace_iterator_increment(struct trace_iterator *iter)
2191 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2195 ring_buffer_read(buf_iter, NULL);
2198 static struct trace_entry *
2199 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2200 unsigned long *lost_events)
2202 struct ring_buffer_event *event;
2203 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2206 event = ring_buffer_iter_peek(buf_iter, ts);
2208 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2212 iter->ent_size = ring_buffer_event_length(event);
2213 return ring_buffer_event_data(event);
2219 static struct trace_entry *
2220 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2221 unsigned long *missing_events, u64 *ent_ts)
2223 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2224 struct trace_entry *ent, *next = NULL;
2225 unsigned long lost_events = 0, next_lost = 0;
2226 int cpu_file = iter->cpu_file;
2227 u64 next_ts = 0, ts;
2233 * If we are in a per_cpu trace file, don't bother by iterating over
2234 * all cpu and peek directly.
2236 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2237 if (ring_buffer_empty_cpu(buffer, cpu_file))
2239 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2241 *ent_cpu = cpu_file;
2246 for_each_tracing_cpu(cpu) {
2248 if (ring_buffer_empty_cpu(buffer, cpu))
2251 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2254 * Pick the entry with the smallest timestamp:
2256 if (ent && (!next || ts < next_ts)) {
2260 next_lost = lost_events;
2261 next_size = iter->ent_size;
2265 iter->ent_size = next_size;
2268 *ent_cpu = next_cpu;
2274 *missing_events = next_lost;
2279 /* Find the next real entry, without updating the iterator itself */
2280 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2281 int *ent_cpu, u64 *ent_ts)
2283 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2286 /* Find the next real entry, and increment the iterator to the next entry */
2287 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2289 iter->ent = __find_next_entry(iter, &iter->cpu,
2290 &iter->lost_events, &iter->ts);
2293 trace_iterator_increment(iter);
2295 return iter->ent ? iter : NULL;
2298 static void trace_consume(struct trace_iterator *iter)
2300 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2301 &iter->lost_events);
2304 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2306 struct trace_iterator *iter = m->private;
2310 WARN_ON_ONCE(iter->leftover);
2314 /* can't go backwards */
2319 ent = trace_find_next_entry_inc(iter);
2323 while (ent && iter->idx < i)
2324 ent = trace_find_next_entry_inc(iter);
2331 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2333 struct ring_buffer_event *event;
2334 struct ring_buffer_iter *buf_iter;
2335 unsigned long entries = 0;
2338 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2340 buf_iter = trace_buffer_iter(iter, cpu);
2344 ring_buffer_iter_reset(buf_iter);
2347 * We could have the case with the max latency tracers
2348 * that a reset never took place on a cpu. This is evident
2349 * by the timestamp being before the start of the buffer.
2351 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2352 if (ts >= iter->trace_buffer->time_start)
2355 ring_buffer_read(buf_iter, NULL);
2358 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2362 * The current tracer is copied to avoid a global locking
2365 static void *s_start(struct seq_file *m, loff_t *pos)
2367 struct trace_iterator *iter = m->private;
2368 struct trace_array *tr = iter->tr;
2369 int cpu_file = iter->cpu_file;
2375 * copy the tracer to avoid using a global lock all around.
2376 * iter->trace is a copy of current_trace, the pointer to the
2377 * name may be used instead of a strcmp(), as iter->trace->name
2378 * will point to the same string as current_trace->name.
2380 mutex_lock(&trace_types_lock);
2381 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2382 *iter->trace = *tr->current_trace;
2383 mutex_unlock(&trace_types_lock);
2385 #ifdef CONFIG_TRACER_MAX_TRACE
2386 if (iter->snapshot && iter->trace->use_max_tr)
2387 return ERR_PTR(-EBUSY);
2390 if (!iter->snapshot)
2391 atomic_inc(&trace_record_cmdline_disabled);
2393 if (*pos != iter->pos) {
2398 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2399 for_each_tracing_cpu(cpu)
2400 tracing_iter_reset(iter, cpu);
2402 tracing_iter_reset(iter, cpu_file);
2405 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2410 * If we overflowed the seq_file before, then we want
2411 * to just reuse the trace_seq buffer again.
2417 p = s_next(m, p, &l);
2421 trace_event_read_lock();
2422 trace_access_lock(cpu_file);
2426 static void s_stop(struct seq_file *m, void *p)
2428 struct trace_iterator *iter = m->private;
2430 #ifdef CONFIG_TRACER_MAX_TRACE
2431 if (iter->snapshot && iter->trace->use_max_tr)
2435 if (!iter->snapshot)
2436 atomic_dec(&trace_record_cmdline_disabled);
2438 trace_access_unlock(iter->cpu_file);
2439 trace_event_read_unlock();
2443 get_total_entries(struct trace_buffer *buf,
2444 unsigned long *total, unsigned long *entries)
2446 unsigned long count;
2452 for_each_tracing_cpu(cpu) {
2453 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2455 * If this buffer has skipped entries, then we hold all
2456 * entries for the trace and we need to ignore the
2457 * ones before the time stamp.
2459 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2460 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2461 /* total is the same as the entries */
2465 ring_buffer_overrun_cpu(buf->buffer, cpu);
2470 static void print_lat_help_header(struct seq_file *m)
2472 seq_puts(m, "# _------=> CPU# \n");
2473 seq_puts(m, "# / _-----=> irqs-off \n");
2474 seq_puts(m, "# | / _----=> need-resched \n");
2475 seq_puts(m, "# || / _---=> hardirq/softirq \n");
2476 seq_puts(m, "# ||| / _--=> preempt-depth \n");
2477 seq_puts(m, "# |||| / delay \n");
2478 seq_puts(m, "# cmd pid ||||| time | caller \n");
2479 seq_puts(m, "# \\ / ||||| \\ | / \n");
2482 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2484 unsigned long total;
2485 unsigned long entries;
2487 get_total_entries(buf, &total, &entries);
2488 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2489 entries, total, num_online_cpus());
2493 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2495 print_event_info(buf, m);
2496 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n");
2497 seq_puts(m, "# | | | | |\n");
2500 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2502 print_event_info(buf, m);
2503 seq_puts(m, "# _-----=> irqs-off\n");
2504 seq_puts(m, "# / _----=> need-resched\n");
2505 seq_puts(m, "# | / _---=> hardirq/softirq\n");
2506 seq_puts(m, "# || / _--=> preempt-depth\n");
2507 seq_puts(m, "# ||| / delay\n");
2508 seq_puts(m, "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n");
2509 seq_puts(m, "# | | | |||| | |\n");
2513 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2515 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2516 struct trace_buffer *buf = iter->trace_buffer;
2517 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2518 struct tracer *type = iter->trace;
2519 unsigned long entries;
2520 unsigned long total;
2521 const char *name = "preemption";
2525 get_total_entries(buf, &total, &entries);
2527 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2529 seq_puts(m, "# -----------------------------------"
2530 "---------------------------------\n");
2531 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2532 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2533 nsecs_to_usecs(data->saved_latency),
2537 #if defined(CONFIG_PREEMPT_NONE)
2539 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2541 #elif defined(CONFIG_PREEMPT)
2546 /* These are reserved for later use */
2549 seq_printf(m, " #P:%d)\n", num_online_cpus());
2553 seq_puts(m, "# -----------------\n");
2554 seq_printf(m, "# | task: %.16s-%d "
2555 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2556 data->comm, data->pid,
2557 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2558 data->policy, data->rt_priority);
2559 seq_puts(m, "# -----------------\n");
2561 if (data->critical_start) {
2562 seq_puts(m, "# => started at: ");
2563 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2564 trace_print_seq(m, &iter->seq);
2565 seq_puts(m, "\n# => ended at: ");
2566 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2567 trace_print_seq(m, &iter->seq);
2568 seq_puts(m, "\n#\n");
2574 static void test_cpu_buff_start(struct trace_iterator *iter)
2576 struct trace_seq *s = &iter->seq;
2578 if (!(trace_flags & TRACE_ITER_ANNOTATE))
2581 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2584 if (cpumask_test_cpu(iter->cpu, iter->started))
2587 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2590 cpumask_set_cpu(iter->cpu, iter->started);
2592 /* Don't print started cpu buffer for the first entry of the trace */
2594 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2598 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2600 struct trace_seq *s = &iter->seq;
2601 unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2602 struct trace_entry *entry;
2603 struct trace_event *event;
2607 test_cpu_buff_start(iter);
2609 event = ftrace_find_event(entry->type);
2611 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2612 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2613 if (!trace_print_lat_context(iter))
2616 if (!trace_print_context(iter))
2622 return event->funcs->trace(iter, sym_flags, event);
2624 if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2627 return TRACE_TYPE_HANDLED;
2629 return TRACE_TYPE_PARTIAL_LINE;
2632 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2634 struct trace_seq *s = &iter->seq;
2635 struct trace_entry *entry;
2636 struct trace_event *event;
2640 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2641 if (!trace_seq_printf(s, "%d %d %llu ",
2642 entry->pid, iter->cpu, iter->ts))
2646 event = ftrace_find_event(entry->type);
2648 return event->funcs->raw(iter, 0, event);
2650 if (!trace_seq_printf(s, "%d ?\n", entry->type))
2653 return TRACE_TYPE_HANDLED;
2655 return TRACE_TYPE_PARTIAL_LINE;
2658 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2660 struct trace_seq *s = &iter->seq;
2661 unsigned char newline = '\n';
2662 struct trace_entry *entry;
2663 struct trace_event *event;
2667 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2668 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2669 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2670 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2673 event = ftrace_find_event(entry->type);
2675 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2676 if (ret != TRACE_TYPE_HANDLED)
2680 SEQ_PUT_FIELD_RET(s, newline);
2682 return TRACE_TYPE_HANDLED;
2685 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2687 struct trace_seq *s = &iter->seq;
2688 struct trace_entry *entry;
2689 struct trace_event *event;
2693 if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2694 SEQ_PUT_FIELD_RET(s, entry->pid);
2695 SEQ_PUT_FIELD_RET(s, iter->cpu);
2696 SEQ_PUT_FIELD_RET(s, iter->ts);
2699 event = ftrace_find_event(entry->type);
2700 return event ? event->funcs->binary(iter, 0, event) :
2704 int trace_empty(struct trace_iterator *iter)
2706 struct ring_buffer_iter *buf_iter;
2709 /* If we are looking at one CPU buffer, only check that one */
2710 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2711 cpu = iter->cpu_file;
2712 buf_iter = trace_buffer_iter(iter, cpu);
2714 if (!ring_buffer_iter_empty(buf_iter))
2717 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2723 for_each_tracing_cpu(cpu) {
2724 buf_iter = trace_buffer_iter(iter, cpu);
2726 if (!ring_buffer_iter_empty(buf_iter))
2729 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2737 /* Called with trace_event_read_lock() held. */
2738 enum print_line_t print_trace_line(struct trace_iterator *iter)
2740 enum print_line_t ret;
2742 if (iter->lost_events &&
2743 !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2744 iter->cpu, iter->lost_events))
2745 return TRACE_TYPE_PARTIAL_LINE;
2747 if (iter->trace && iter->trace->print_line) {
2748 ret = iter->trace->print_line(iter);
2749 if (ret != TRACE_TYPE_UNHANDLED)
2753 if (iter->ent->type == TRACE_BPUTS &&
2754 trace_flags & TRACE_ITER_PRINTK &&
2755 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2756 return trace_print_bputs_msg_only(iter);
2758 if (iter->ent->type == TRACE_BPRINT &&
2759 trace_flags & TRACE_ITER_PRINTK &&
2760 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2761 return trace_print_bprintk_msg_only(iter);
2763 if (iter->ent->type == TRACE_PRINT &&
2764 trace_flags & TRACE_ITER_PRINTK &&
2765 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2766 return trace_print_printk_msg_only(iter);
2768 if (trace_flags & TRACE_ITER_BIN)
2769 return print_bin_fmt(iter);
2771 if (trace_flags & TRACE_ITER_HEX)
2772 return print_hex_fmt(iter);
2774 if (trace_flags & TRACE_ITER_RAW)
2775 return print_raw_fmt(iter);
2777 return print_trace_fmt(iter);
2780 void trace_latency_header(struct seq_file *m)
2782 struct trace_iterator *iter = m->private;
2784 /* print nothing if the buffers are empty */
2785 if (trace_empty(iter))
2788 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2789 print_trace_header(m, iter);
2791 if (!(trace_flags & TRACE_ITER_VERBOSE))
2792 print_lat_help_header(m);
2795 void trace_default_header(struct seq_file *m)
2797 struct trace_iterator *iter = m->private;
2799 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2802 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2803 /* print nothing if the buffers are empty */
2804 if (trace_empty(iter))
2806 print_trace_header(m, iter);
2807 if (!(trace_flags & TRACE_ITER_VERBOSE))
2808 print_lat_help_header(m);
2810 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2811 if (trace_flags & TRACE_ITER_IRQ_INFO)
2812 print_func_help_header_irq(iter->trace_buffer, m);
2814 print_func_help_header(iter->trace_buffer, m);
2819 static void test_ftrace_alive(struct seq_file *m)
2821 if (!ftrace_is_dead())
2823 seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2824 seq_printf(m, "# MAY BE MISSING FUNCTION EVENTS\n");
2827 #ifdef CONFIG_TRACER_MAX_TRACE
2828 static void show_snapshot_main_help(struct seq_file *m)
2830 seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2831 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2832 seq_printf(m, "# Takes a snapshot of the main buffer.\n");
2833 seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n");
2834 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2835 seq_printf(m, "# is not a '0' or '1')\n");
2838 static void show_snapshot_percpu_help(struct seq_file *m)
2840 seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2841 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2842 seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2843 seq_printf(m, "# Takes a snapshot of the main buffer for this cpu.\n");
2845 seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
2846 seq_printf(m, "# Must use main snapshot file to allocate.\n");
2848 seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
2849 seq_printf(m, "# (Doesn't have to be '2' works with any number that\n");
2850 seq_printf(m, "# is not a '0' or '1')\n");
2853 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2855 if (iter->tr->allocated_snapshot)
2856 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2858 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2860 seq_printf(m, "# Snapshot commands:\n");
2861 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2862 show_snapshot_main_help(m);
2864 show_snapshot_percpu_help(m);
2867 /* Should never be called */
2868 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2871 static int s_show(struct seq_file *m, void *v)
2873 struct trace_iterator *iter = v;
2876 if (iter->ent == NULL) {
2878 seq_printf(m, "# tracer: %s\n", iter->trace->name);
2880 test_ftrace_alive(m);
2882 if (iter->snapshot && trace_empty(iter))
2883 print_snapshot_help(m, iter);
2884 else if (iter->trace && iter->trace->print_header)
2885 iter->trace->print_header(m);
2887 trace_default_header(m);
2889 } else if (iter->leftover) {
2891 * If we filled the seq_file buffer earlier, we
2892 * want to just show it now.
2894 ret = trace_print_seq(m, &iter->seq);
2896 /* ret should this time be zero, but you never know */
2897 iter->leftover = ret;
2900 print_trace_line(iter);
2901 ret = trace_print_seq(m, &iter->seq);
2903 * If we overflow the seq_file buffer, then it will
2904 * ask us for this data again at start up.
2906 * ret is 0 if seq_file write succeeded.
2909 iter->leftover = ret;
2916 * Should be used after trace_array_get(), trace_types_lock
2917 * ensures that i_cdev was already initialized.
2919 static inline int tracing_get_cpu(struct inode *inode)
2921 if (inode->i_cdev) /* See trace_create_cpu_file() */
2922 return (long)inode->i_cdev - 1;
2923 return RING_BUFFER_ALL_CPUS;
2926 static const struct seq_operations tracer_seq_ops = {
2933 static struct trace_iterator *
2934 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
2936 struct trace_array *tr = inode->i_private;
2937 struct trace_iterator *iter;
2940 if (tracing_disabled)
2941 return ERR_PTR(-ENODEV);
2943 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2945 return ERR_PTR(-ENOMEM);
2947 iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2949 if (!iter->buffer_iter)
2953 * We make a copy of the current tracer to avoid concurrent
2954 * changes on it while we are reading.
2956 mutex_lock(&trace_types_lock);
2957 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2961 *iter->trace = *tr->current_trace;
2963 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2968 #ifdef CONFIG_TRACER_MAX_TRACE
2969 /* Currently only the top directory has a snapshot */
2970 if (tr->current_trace->print_max || snapshot)
2971 iter->trace_buffer = &tr->max_buffer;
2974 iter->trace_buffer = &tr->trace_buffer;
2975 iter->snapshot = snapshot;
2977 iter->cpu_file = tracing_get_cpu(inode);
2978 mutex_init(&iter->mutex);
2980 /* Notify the tracer early; before we stop tracing. */
2981 if (iter->trace && iter->trace->open)
2982 iter->trace->open(iter);
2984 /* Annotate start of buffers if we had overruns */
2985 if (ring_buffer_overruns(iter->trace_buffer->buffer))
2986 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2988 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
2989 if (trace_clocks[tr->clock_id].in_ns)
2990 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
2992 /* stop the trace while dumping if we are not opening "snapshot" */
2993 if (!iter->snapshot)
2994 tracing_stop_tr(tr);
2996 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
2997 for_each_tracing_cpu(cpu) {
2998 iter->buffer_iter[cpu] =
2999 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3001 ring_buffer_read_prepare_sync();
3002 for_each_tracing_cpu(cpu) {
3003 ring_buffer_read_start(iter->buffer_iter[cpu]);
3004 tracing_iter_reset(iter, cpu);
3007 cpu = iter->cpu_file;
3008 iter->buffer_iter[cpu] =
3009 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
3010 ring_buffer_read_prepare_sync();
3011 ring_buffer_read_start(iter->buffer_iter[cpu]);
3012 tracing_iter_reset(iter, cpu);
3015 mutex_unlock(&trace_types_lock);
3020 mutex_unlock(&trace_types_lock);
3022 kfree(iter->buffer_iter);
3024 seq_release_private(inode, file);
3025 return ERR_PTR(-ENOMEM);
3028 int tracing_open_generic(struct inode *inode, struct file *filp)
3030 if (tracing_disabled)
3033 filp->private_data = inode->i_private;
3037 bool tracing_is_disabled(void)
3039 return (tracing_disabled) ? true: false;
3043 * Open and update trace_array ref count.
3044 * Must have the current trace_array passed to it.
3046 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3048 struct trace_array *tr = inode->i_private;
3050 if (tracing_disabled)
3053 if (trace_array_get(tr) < 0)
3056 filp->private_data = inode->i_private;
3061 static int tracing_release(struct inode *inode, struct file *file)
3063 struct trace_array *tr = inode->i_private;
3064 struct seq_file *m = file->private_data;
3065 struct trace_iterator *iter;
3068 if (!(file->f_mode & FMODE_READ)) {
3069 trace_array_put(tr);
3073 /* Writes do not use seq_file */
3075 mutex_lock(&trace_types_lock);
3077 for_each_tracing_cpu(cpu) {
3078 if (iter->buffer_iter[cpu])
3079 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3082 if (iter->trace && iter->trace->close)
3083 iter->trace->close(iter);
3085 if (!iter->snapshot)
3086 /* reenable tracing if it was previously enabled */
3087 tracing_start_tr(tr);
3089 __trace_array_put(tr);
3091 mutex_unlock(&trace_types_lock);
3093 mutex_destroy(&iter->mutex);
3094 free_cpumask_var(iter->started);
3096 kfree(iter->buffer_iter);
3097 seq_release_private(inode, file);
3102 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3104 struct trace_array *tr = inode->i_private;
3106 trace_array_put(tr);
3110 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3112 struct trace_array *tr = inode->i_private;
3114 trace_array_put(tr);
3116 return single_release(inode, file);
3119 static int tracing_open(struct inode *inode, struct file *file)
3121 struct trace_array *tr = inode->i_private;
3122 struct trace_iterator *iter;
3125 if (trace_array_get(tr) < 0)
3128 /* If this file was open for write, then erase contents */
3129 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3130 int cpu = tracing_get_cpu(inode);
3132 if (cpu == RING_BUFFER_ALL_CPUS)
3133 tracing_reset_online_cpus(&tr->trace_buffer);
3135 tracing_reset(&tr->trace_buffer, cpu);
3138 if (file->f_mode & FMODE_READ) {
3139 iter = __tracing_open(inode, file, false);
3141 ret = PTR_ERR(iter);
3142 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
3143 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3147 trace_array_put(tr);
3153 * Some tracers are not suitable for instance buffers.
3154 * A tracer is always available for the global array (toplevel)
3155 * or if it explicitly states that it is.
3158 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3160 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3163 /* Find the next tracer that this trace array may use */
3164 static struct tracer *
3165 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3167 while (t && !trace_ok_for_array(t, tr))
3174 t_next(struct seq_file *m, void *v, loff_t *pos)
3176 struct trace_array *tr = m->private;
3177 struct tracer *t = v;
3182 t = get_tracer_for_array(tr, t->next);
3187 static void *t_start(struct seq_file *m, loff_t *pos)
3189 struct trace_array *tr = m->private;
3193 mutex_lock(&trace_types_lock);
3195 t = get_tracer_for_array(tr, trace_types);
3196 for (; t && l < *pos; t = t_next(m, t, &l))
3202 static void t_stop(struct seq_file *m, void *p)
3204 mutex_unlock(&trace_types_lock);
3207 static int t_show(struct seq_file *m, void *v)
3209 struct tracer *t = v;
3214 seq_printf(m, "%s", t->name);
3223 static const struct seq_operations show_traces_seq_ops = {
3230 static int show_traces_open(struct inode *inode, struct file *file)
3232 struct trace_array *tr = inode->i_private;
3236 if (tracing_disabled)
3239 ret = seq_open(file, &show_traces_seq_ops);
3243 m = file->private_data;
3250 tracing_write_stub(struct file *filp, const char __user *ubuf,
3251 size_t count, loff_t *ppos)
3256 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3260 if (file->f_mode & FMODE_READ)
3261 ret = seq_lseek(file, offset, whence);
3263 file->f_pos = ret = 0;
3268 static const struct file_operations tracing_fops = {
3269 .open = tracing_open,
3271 .write = tracing_write_stub,
3272 .llseek = tracing_lseek,
3273 .release = tracing_release,
3276 static const struct file_operations show_traces_fops = {
3277 .open = show_traces_open,
3279 .release = seq_release,
3280 .llseek = seq_lseek,
3284 * The tracer itself will not take this lock, but still we want
3285 * to provide a consistent cpumask to user-space:
3287 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3290 * Temporary storage for the character representation of the
3291 * CPU bitmask (and one more byte for the newline):
3293 static char mask_str[NR_CPUS + 1];
3296 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3297 size_t count, loff_t *ppos)
3299 struct trace_array *tr = file_inode(filp)->i_private;
3302 mutex_lock(&tracing_cpumask_update_lock);
3304 len = cpumask_scnprintf(mask_str, count, tr->tracing_cpumask);
3305 if (count - len < 2) {
3309 len += sprintf(mask_str + len, "\n");
3310 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3313 mutex_unlock(&tracing_cpumask_update_lock);
3319 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3320 size_t count, loff_t *ppos)
3322 struct trace_array *tr = file_inode(filp)->i_private;
3323 cpumask_var_t tracing_cpumask_new;
3326 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3329 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3333 mutex_lock(&tracing_cpumask_update_lock);
3335 local_irq_disable();
3336 arch_spin_lock(&ftrace_max_lock);
3337 for_each_tracing_cpu(cpu) {
3339 * Increase/decrease the disabled counter if we are
3340 * about to flip a bit in the cpumask:
3342 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3343 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3344 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3345 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3347 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3348 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3349 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3350 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3353 arch_spin_unlock(&ftrace_max_lock);
3356 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3358 mutex_unlock(&tracing_cpumask_update_lock);
3359 free_cpumask_var(tracing_cpumask_new);
3364 free_cpumask_var(tracing_cpumask_new);
3369 static const struct file_operations tracing_cpumask_fops = {
3370 .open = tracing_open_generic_tr,
3371 .read = tracing_cpumask_read,
3372 .write = tracing_cpumask_write,
3373 .release = tracing_release_generic_tr,
3374 .llseek = generic_file_llseek,
3377 static int tracing_trace_options_show(struct seq_file *m, void *v)
3379 struct tracer_opt *trace_opts;
3380 struct trace_array *tr = m->private;
3384 mutex_lock(&trace_types_lock);
3385 tracer_flags = tr->current_trace->flags->val;
3386 trace_opts = tr->current_trace->flags->opts;
3388 for (i = 0; trace_options[i]; i++) {
3389 if (trace_flags & (1 << i))
3390 seq_printf(m, "%s\n", trace_options[i]);
3392 seq_printf(m, "no%s\n", trace_options[i]);
3395 for (i = 0; trace_opts[i].name; i++) {
3396 if (tracer_flags & trace_opts[i].bit)
3397 seq_printf(m, "%s\n", trace_opts[i].name);
3399 seq_printf(m, "no%s\n", trace_opts[i].name);
3401 mutex_unlock(&trace_types_lock);
3406 static int __set_tracer_option(struct trace_array *tr,
3407 struct tracer_flags *tracer_flags,
3408 struct tracer_opt *opts, int neg)
3410 struct tracer *trace = tr->current_trace;
3413 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3418 tracer_flags->val &= ~opts->bit;
3420 tracer_flags->val |= opts->bit;
3424 /* Try to assign a tracer specific option */
3425 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3427 struct tracer *trace = tr->current_trace;
3428 struct tracer_flags *tracer_flags = trace->flags;
3429 struct tracer_opt *opts = NULL;
3432 for (i = 0; tracer_flags->opts[i].name; i++) {
3433 opts = &tracer_flags->opts[i];
3435 if (strcmp(cmp, opts->name) == 0)
3436 return __set_tracer_option(tr, trace->flags, opts, neg);
3442 /* Some tracers require overwrite to stay enabled */
3443 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3445 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3451 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3453 /* do nothing if flag is already set */
3454 if (!!(trace_flags & mask) == !!enabled)
3457 /* Give the tracer a chance to approve the change */
3458 if (tr->current_trace->flag_changed)
3459 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3463 trace_flags |= mask;
3465 trace_flags &= ~mask;
3467 if (mask == TRACE_ITER_RECORD_CMD)
3468 trace_event_enable_cmd_record(enabled);
3470 if (mask == TRACE_ITER_OVERWRITE) {
3471 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3472 #ifdef CONFIG_TRACER_MAX_TRACE
3473 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3477 if (mask == TRACE_ITER_PRINTK)
3478 trace_printk_start_stop_comm(enabled);
3483 static int trace_set_options(struct trace_array *tr, char *option)
3490 cmp = strstrip(option);
3492 if (strncmp(cmp, "no", 2) == 0) {
3497 mutex_lock(&trace_types_lock);
3499 for (i = 0; trace_options[i]; i++) {
3500 if (strcmp(cmp, trace_options[i]) == 0) {
3501 ret = set_tracer_flag(tr, 1 << i, !neg);
3506 /* If no option could be set, test the specific tracer options */
3507 if (!trace_options[i])
3508 ret = set_tracer_option(tr, cmp, neg);
3510 mutex_unlock(&trace_types_lock);
3516 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3517 size_t cnt, loff_t *ppos)
3519 struct seq_file *m = filp->private_data;
3520 struct trace_array *tr = m->private;
3524 if (cnt >= sizeof(buf))
3527 if (copy_from_user(&buf, ubuf, cnt))
3532 ret = trace_set_options(tr, buf);
3541 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3543 struct trace_array *tr = inode->i_private;
3546 if (tracing_disabled)
3549 if (trace_array_get(tr) < 0)
3552 ret = single_open(file, tracing_trace_options_show, inode->i_private);
3554 trace_array_put(tr);
3559 static const struct file_operations tracing_iter_fops = {
3560 .open = tracing_trace_options_open,
3562 .llseek = seq_lseek,
3563 .release = tracing_single_release_tr,
3564 .write = tracing_trace_options_write,
3567 static const char readme_msg[] =
3568 "tracing mini-HOWTO:\n\n"
3569 "# echo 0 > tracing_on : quick way to disable tracing\n"
3570 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3571 " Important files:\n"
3572 " trace\t\t\t- The static contents of the buffer\n"
3573 "\t\t\t To clear the buffer write into this file: echo > trace\n"
3574 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3575 " current_tracer\t- function and latency tracers\n"
3576 " available_tracers\t- list of configured tracers for current_tracer\n"
3577 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
3578 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
3579 " trace_clock\t\t-change the clock used to order events\n"
3580 " local: Per cpu clock but may not be synced across CPUs\n"
3581 " global: Synced across CPUs but slows tracing down.\n"
3582 " counter: Not a clock, but just an increment\n"
3583 " uptime: Jiffy counter from time of boot\n"
3584 " perf: Same clock that perf events use\n"
3585 #ifdef CONFIG_X86_64
3586 " x86-tsc: TSC cycle counter\n"
3588 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3589 " tracing_cpumask\t- Limit which CPUs to trace\n"
3590 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3591 "\t\t\t Remove sub-buffer with rmdir\n"
3592 " trace_options\t\t- Set format or modify how tracing happens\n"
3593 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
3594 "\t\t\t option name\n"
3595 #ifdef CONFIG_DYNAMIC_FTRACE
3596 "\n available_filter_functions - list of functions that can be filtered on\n"
3597 " set_ftrace_filter\t- echo function name in here to only trace these\n"
3598 "\t\t\t functions\n"
3599 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3600 "\t modules: Can select a group via module\n"
3601 "\t Format: :mod:<module-name>\n"
3602 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
3603 "\t triggers: a command to perform when function is hit\n"
3604 "\t Format: <function>:<trigger>[:count]\n"
3605 "\t trigger: traceon, traceoff\n"
3606 "\t\t enable_event:<system>:<event>\n"
3607 "\t\t disable_event:<system>:<event>\n"
3608 #ifdef CONFIG_STACKTRACE
3611 #ifdef CONFIG_TRACER_SNAPSHOT
3614 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
3615 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
3616 "\t The first one will disable tracing every time do_fault is hit\n"
3617 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
3618 "\t The first time do trap is hit and it disables tracing, the\n"
3619 "\t counter will decrement to 2. If tracing is already disabled,\n"
3620 "\t the counter will not decrement. It only decrements when the\n"
3621 "\t trigger did work\n"
3622 "\t To remove trigger without count:\n"
3623 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
3624 "\t To remove trigger with a count:\n"
3625 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3626 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
3627 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3628 "\t modules: Can select a group via module command :mod:\n"
3629 "\t Does not accept triggers\n"
3630 #endif /* CONFIG_DYNAMIC_FTRACE */
3631 #ifdef CONFIG_FUNCTION_TRACER
3632 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
3635 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3636 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3637 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3639 #ifdef CONFIG_TRACER_SNAPSHOT
3640 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
3641 "\t\t\t snapshot buffer. Read the contents for more\n"
3642 "\t\t\t information\n"
3644 #ifdef CONFIG_STACK_TRACER
3645 " stack_trace\t\t- Shows the max stack trace when active\n"
3646 " stack_max_size\t- Shows current max stack size that was traced\n"
3647 "\t\t\t Write into this file to reset the max size (trigger a\n"
3648 "\t\t\t new trace)\n"
3649 #ifdef CONFIG_DYNAMIC_FTRACE
3650 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
3653 #endif /* CONFIG_STACK_TRACER */
3654 " events/\t\t- Directory containing all trace event subsystems:\n"
3655 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
3656 " events/<system>/\t- Directory containing all trace events for <system>:\n"
3657 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
3659 " filter\t\t- If set, only events passing filter are traced\n"
3660 " events/<system>/<event>/\t- Directory containing control files for\n"
3662 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
3663 " filter\t\t- If set, only events passing filter are traced\n"
3664 " trigger\t\t- If set, a command to perform when event is hit\n"
3665 "\t Format: <trigger>[:count][if <filter>]\n"
3666 "\t trigger: traceon, traceoff\n"
3667 "\t enable_event:<system>:<event>\n"
3668 "\t disable_event:<system>:<event>\n"
3669 #ifdef CONFIG_STACKTRACE
3672 #ifdef CONFIG_TRACER_SNAPSHOT
3675 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
3676 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
3677 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
3678 "\t events/block/block_unplug/trigger\n"
3679 "\t The first disables tracing every time block_unplug is hit.\n"
3680 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
3681 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
3682 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
3683 "\t Like function triggers, the counter is only decremented if it\n"
3684 "\t enabled or disabled tracing.\n"
3685 "\t To remove a trigger without a count:\n"
3686 "\t echo '!<trigger> > <system>/<event>/trigger\n"
3687 "\t To remove a trigger with a count:\n"
3688 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
3689 "\t Filters can be ignored when removing a trigger.\n"
3693 tracing_readme_read(struct file *filp, char __user *ubuf,
3694 size_t cnt, loff_t *ppos)
3696 return simple_read_from_buffer(ubuf, cnt, ppos,
3697 readme_msg, strlen(readme_msg));
3700 static const struct file_operations tracing_readme_fops = {
3701 .open = tracing_open_generic,
3702 .read = tracing_readme_read,
3703 .llseek = generic_file_llseek,
3707 tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
3708 size_t cnt, loff_t *ppos)
3717 file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
3721 buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
3729 for (i = 0; i < SAVED_CMDLINES; i++) {
3732 pid = map_cmdline_to_pid[i];
3733 if (pid == -1 || pid == NO_CMDLINE_MAP)
3736 trace_find_cmdline(pid, buf_comm);
3737 r = sprintf(buf, "%d %s\n", pid, buf_comm);
3742 len = simple_read_from_buffer(ubuf, cnt, ppos,
3751 static const struct file_operations tracing_saved_cmdlines_fops = {
3752 .open = tracing_open_generic,
3753 .read = tracing_saved_cmdlines_read,
3754 .llseek = generic_file_llseek,
3758 tracing_set_trace_read(struct file *filp, char __user *ubuf,
3759 size_t cnt, loff_t *ppos)
3761 struct trace_array *tr = filp->private_data;
3762 char buf[MAX_TRACER_SIZE+2];
3765 mutex_lock(&trace_types_lock);
3766 r = sprintf(buf, "%s\n", tr->current_trace->name);
3767 mutex_unlock(&trace_types_lock);
3769 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3772 int tracer_init(struct tracer *t, struct trace_array *tr)
3774 tracing_reset_online_cpus(&tr->trace_buffer);
3778 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
3782 for_each_tracing_cpu(cpu)
3783 per_cpu_ptr(buf->data, cpu)->entries = val;
3786 #ifdef CONFIG_TRACER_MAX_TRACE
3787 /* resize @tr's buffer to the size of @size_tr's entries */
3788 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3789 struct trace_buffer *size_buf, int cpu_id)
3793 if (cpu_id == RING_BUFFER_ALL_CPUS) {
3794 for_each_tracing_cpu(cpu) {
3795 ret = ring_buffer_resize(trace_buf->buffer,
3796 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
3799 per_cpu_ptr(trace_buf->data, cpu)->entries =
3800 per_cpu_ptr(size_buf->data, cpu)->entries;
3803 ret = ring_buffer_resize(trace_buf->buffer,
3804 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
3806 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3807 per_cpu_ptr(size_buf->data, cpu_id)->entries;
3812 #endif /* CONFIG_TRACER_MAX_TRACE */
3814 static int __tracing_resize_ring_buffer(struct trace_array *tr,
3815 unsigned long size, int cpu)
3820 * If kernel or user changes the size of the ring buffer
3821 * we use the size that was given, and we can forget about
3822 * expanding it later.
3824 ring_buffer_expanded = true;
3826 /* May be called before buffers are initialized */
3827 if (!tr->trace_buffer.buffer)
3830 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
3834 #ifdef CONFIG_TRACER_MAX_TRACE
3835 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3836 !tr->current_trace->use_max_tr)
3839 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
3841 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3842 &tr->trace_buffer, cpu);
3845 * AARGH! We are left with different
3846 * size max buffer!!!!
3847 * The max buffer is our "snapshot" buffer.
3848 * When a tracer needs a snapshot (one of the
3849 * latency tracers), it swaps the max buffer
3850 * with the saved snap shot. We succeeded to
3851 * update the size of the main buffer, but failed to
3852 * update the size of the max buffer. But when we tried
3853 * to reset the main buffer to the original size, we
3854 * failed there too. This is very unlikely to
3855 * happen, but if it does, warn and kill all
3859 tracing_disabled = 1;
3864 if (cpu == RING_BUFFER_ALL_CPUS)
3865 set_buffer_entries(&tr->max_buffer, size);
3867 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
3870 #endif /* CONFIG_TRACER_MAX_TRACE */
3872 if (cpu == RING_BUFFER_ALL_CPUS)
3873 set_buffer_entries(&tr->trace_buffer, size);
3875 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
3880 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
3881 unsigned long size, int cpu_id)
3885 mutex_lock(&trace_types_lock);
3887 if (cpu_id != RING_BUFFER_ALL_CPUS) {
3888 /* make sure, this cpu is enabled in the mask */
3889 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3895 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
3900 mutex_unlock(&trace_types_lock);
3907 * tracing_update_buffers - used by tracing facility to expand ring buffers
3909 * To save on memory when the tracing is never used on a system with it
3910 * configured in. The ring buffers are set to a minimum size. But once
3911 * a user starts to use the tracing facility, then they need to grow
3912 * to their default size.
3914 * This function is to be called when a tracer is about to be used.
3916 int tracing_update_buffers(void)
3920 mutex_lock(&trace_types_lock);
3921 if (!ring_buffer_expanded)
3922 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
3923 RING_BUFFER_ALL_CPUS);
3924 mutex_unlock(&trace_types_lock);
3929 struct trace_option_dentry;
3931 static struct trace_option_dentry *
3932 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
3935 destroy_trace_option_files(struct trace_option_dentry *topts);
3938 * Used to clear out the tracer before deletion of an instance.
3939 * Must have trace_types_lock held.
3941 static void tracing_set_nop(struct trace_array *tr)
3943 if (tr->current_trace == &nop_trace)
3946 tr->current_trace->enabled--;
3948 if (tr->current_trace->reset)
3949 tr->current_trace->reset(tr);
3951 tr->current_trace = &nop_trace;
3954 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
3956 static struct trace_option_dentry *topts;
3958 #ifdef CONFIG_TRACER_MAX_TRACE
3963 mutex_lock(&trace_types_lock);
3965 if (!ring_buffer_expanded) {
3966 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
3967 RING_BUFFER_ALL_CPUS);
3973 for (t = trace_types; t; t = t->next) {
3974 if (strcmp(t->name, buf) == 0)
3981 if (t == tr->current_trace)
3984 /* Some tracers are only allowed for the top level buffer */
3985 if (!trace_ok_for_array(t, tr)) {
3990 trace_branch_disable();
3992 tr->current_trace->enabled--;
3994 if (tr->current_trace->reset)
3995 tr->current_trace->reset(tr);
3997 /* Current trace needs to be nop_trace before synchronize_sched */
3998 tr->current_trace = &nop_trace;
4000 #ifdef CONFIG_TRACER_MAX_TRACE
4001 had_max_tr = tr->allocated_snapshot;
4003 if (had_max_tr && !t->use_max_tr) {
4005 * We need to make sure that the update_max_tr sees that
4006 * current_trace changed to nop_trace to keep it from
4007 * swapping the buffers after we resize it.
4008 * The update_max_tr is called from interrupts disabled
4009 * so a synchronized_sched() is sufficient.
4011 synchronize_sched();
4015 /* Currently, only the top instance has options */
4016 if (tr->flags & TRACE_ARRAY_FL_GLOBAL) {
4017 destroy_trace_option_files(topts);
4018 topts = create_trace_option_files(tr, t);
4021 #ifdef CONFIG_TRACER_MAX_TRACE
4022 if (t->use_max_tr && !had_max_tr) {
4023 ret = alloc_snapshot(tr);
4030 ret = tracer_init(t, tr);
4035 tr->current_trace = t;
4036 tr->current_trace->enabled++;
4037 trace_branch_enable(tr);
4039 mutex_unlock(&trace_types_lock);
4045 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4046 size_t cnt, loff_t *ppos)
4048 struct trace_array *tr = filp->private_data;
4049 char buf[MAX_TRACER_SIZE+1];
4056 if (cnt > MAX_TRACER_SIZE)
4057 cnt = MAX_TRACER_SIZE;
4059 if (copy_from_user(&buf, ubuf, cnt))
4064 /* strip ending whitespace. */
4065 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4068 err = tracing_set_tracer(tr, buf);
4078 tracing_max_lat_read(struct file *filp, char __user *ubuf,
4079 size_t cnt, loff_t *ppos)
4081 unsigned long *ptr = filp->private_data;
4085 r = snprintf(buf, sizeof(buf), "%ld\n",
4086 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
4087 if (r > sizeof(buf))
4089 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4093 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
4094 size_t cnt, loff_t *ppos)
4096 unsigned long *ptr = filp->private_data;
4100 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4109 static int tracing_open_pipe(struct inode *inode, struct file *filp)
4111 struct trace_array *tr = inode->i_private;
4112 struct trace_iterator *iter;
4115 if (tracing_disabled)
4118 if (trace_array_get(tr) < 0)
4121 mutex_lock(&trace_types_lock);
4123 /* create a buffer to store the information to pass to userspace */
4124 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4127 __trace_array_put(tr);
4132 * We make a copy of the current tracer to avoid concurrent
4133 * changes on it while we are reading.
4135 iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
4140 *iter->trace = *tr->current_trace;
4142 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
4147 /* trace pipe does not show start of buffer */
4148 cpumask_setall(iter->started);
4150 if (trace_flags & TRACE_ITER_LATENCY_FMT)
4151 iter->iter_flags |= TRACE_FILE_LAT_FMT;
4153 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
4154 if (trace_clocks[tr->clock_id].in_ns)
4155 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
4158 iter->trace_buffer = &tr->trace_buffer;
4159 iter->cpu_file = tracing_get_cpu(inode);
4160 mutex_init(&iter->mutex);
4161 filp->private_data = iter;
4163 if (iter->trace->pipe_open)
4164 iter->trace->pipe_open(iter);
4166 nonseekable_open(inode, filp);
4168 mutex_unlock(&trace_types_lock);
4174 __trace_array_put(tr);
4175 mutex_unlock(&trace_types_lock);
4179 static int tracing_release_pipe(struct inode *inode, struct file *file)
4181 struct trace_iterator *iter = file->private_data;
4182 struct trace_array *tr = inode->i_private;
4184 mutex_lock(&trace_types_lock);
4186 if (iter->trace->pipe_close)
4187 iter->trace->pipe_close(iter);
4189 mutex_unlock(&trace_types_lock);
4191 free_cpumask_var(iter->started);
4192 mutex_destroy(&iter->mutex);
4196 trace_array_put(tr);
4202 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
4204 /* Iterators are static, they should be filled or empty */
4205 if (trace_buffer_iter(iter, iter->cpu_file))
4206 return POLLIN | POLLRDNORM;
4208 if (trace_flags & TRACE_ITER_BLOCK)
4210 * Always select as readable when in blocking mode
4212 return POLLIN | POLLRDNORM;
4214 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
4219 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
4221 struct trace_iterator *iter = filp->private_data;
4223 return trace_poll(iter, filp, poll_table);
4227 * This is a make-shift waitqueue.
4228 * A tracer might use this callback on some rare cases:
4230 * 1) the current tracer might hold the runqueue lock when it wakes up
4231 * a reader, hence a deadlock (sched, function, and function graph tracers)
4232 * 2) the function tracers, trace all functions, we don't want
4233 * the overhead of calling wake_up and friends
4234 * (and tracing them too)
4236 * Anyway, this is really very primitive wakeup.
4238 void poll_wait_pipe(struct trace_iterator *iter)
4240 set_current_state(TASK_INTERRUPTIBLE);
4241 /* sleep for 100 msecs, and try again. */
4242 schedule_timeout(HZ / 10);
4245 /* Must be called with trace_types_lock mutex held. */
4246 static int tracing_wait_pipe(struct file *filp)
4248 struct trace_iterator *iter = filp->private_data;
4250 while (trace_empty(iter)) {
4252 if ((filp->f_flags & O_NONBLOCK)) {
4256 mutex_unlock(&iter->mutex);
4258 iter->trace->wait_pipe(iter);
4260 mutex_lock(&iter->mutex);
4262 if (signal_pending(current))
4266 * We block until we read something and tracing is disabled.
4267 * We still block if tracing is disabled, but we have never
4268 * read anything. This allows a user to cat this file, and
4269 * then enable tracing. But after we have read something,
4270 * we give an EOF when tracing is again disabled.
4272 * iter->pos will be 0 if we haven't read anything.
4274 if (!tracing_is_on() && iter->pos)
4285 tracing_read_pipe(struct file *filp, char __user *ubuf,
4286 size_t cnt, loff_t *ppos)
4288 struct trace_iterator *iter = filp->private_data;
4289 struct trace_array *tr = iter->tr;
4292 /* return any leftover data */
4293 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4297 trace_seq_init(&iter->seq);
4299 /* copy the tracer to avoid using a global lock all around */
4300 mutex_lock(&trace_types_lock);
4301 if (unlikely(iter->trace->name != tr->current_trace->name))
4302 *iter->trace = *tr->current_trace;
4303 mutex_unlock(&trace_types_lock);
4306 * Avoid more than one consumer on a single file descriptor
4307 * This is just a matter of traces coherency, the ring buffer itself
4310 mutex_lock(&iter->mutex);
4311 if (iter->trace->read) {
4312 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
4318 sret = tracing_wait_pipe(filp);
4322 /* stop when tracing is finished */
4323 if (trace_empty(iter)) {
4328 if (cnt >= PAGE_SIZE)
4329 cnt = PAGE_SIZE - 1;
4331 /* reset all but tr, trace, and overruns */
4332 memset(&iter->seq, 0,
4333 sizeof(struct trace_iterator) -
4334 offsetof(struct trace_iterator, seq));
4335 cpumask_clear(iter->started);
4338 trace_event_read_lock();
4339 trace_access_lock(iter->cpu_file);
4340 while (trace_find_next_entry_inc(iter) != NULL) {
4341 enum print_line_t ret;
4342 int len = iter->seq.len;
4344 ret = print_trace_line(iter);
4345 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4346 /* don't print partial lines */
4347 iter->seq.len = len;
4350 if (ret != TRACE_TYPE_NO_CONSUME)
4351 trace_consume(iter);
4353 if (iter->seq.len >= cnt)
4357 * Setting the full flag means we reached the trace_seq buffer
4358 * size and we should leave by partial output condition above.
4359 * One of the trace_seq_* functions is not used properly.
4361 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4364 trace_access_unlock(iter->cpu_file);
4365 trace_event_read_unlock();
4367 /* Now copy what we have to the user */
4368 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4369 if (iter->seq.readpos >= iter->seq.len)
4370 trace_seq_init(&iter->seq);
4373 * If there was nothing to send to user, in spite of consuming trace
4374 * entries, go back to wait for more entries.
4380 mutex_unlock(&iter->mutex);
4385 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4388 __free_page(spd->pages[idx]);
4391 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4393 .map = generic_pipe_buf_map,
4394 .unmap = generic_pipe_buf_unmap,
4395 .confirm = generic_pipe_buf_confirm,
4396 .release = generic_pipe_buf_release,
4397 .steal = generic_pipe_buf_steal,
4398 .get = generic_pipe_buf_get,
4402 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4407 /* Seq buffer is page-sized, exactly what we need. */
4409 count = iter->seq.len;
4410 ret = print_trace_line(iter);
4411 count = iter->seq.len - count;
4414 iter->seq.len -= count;
4417 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4418 iter->seq.len -= count;
4422 if (ret != TRACE_TYPE_NO_CONSUME)
4423 trace_consume(iter);
4425 if (!trace_find_next_entry_inc(iter)) {
4435 static ssize_t tracing_splice_read_pipe(struct file *filp,
4437 struct pipe_inode_info *pipe,
4441 struct page *pages_def[PIPE_DEF_BUFFERS];
4442 struct partial_page partial_def[PIPE_DEF_BUFFERS];
4443 struct trace_iterator *iter = filp->private_data;
4444 struct splice_pipe_desc spd = {
4446 .partial = partial_def,
4447 .nr_pages = 0, /* This gets updated below. */
4448 .nr_pages_max = PIPE_DEF_BUFFERS,
4450 .ops = &tracing_pipe_buf_ops,
4451 .spd_release = tracing_spd_release_pipe,
4453 struct trace_array *tr = iter->tr;
4458 if (splice_grow_spd(pipe, &spd))
4461 /* copy the tracer to avoid using a global lock all around */
4462 mutex_lock(&trace_types_lock);
4463 if (unlikely(iter->trace->name != tr->current_trace->name))
4464 *iter->trace = *tr->current_trace;
4465 mutex_unlock(&trace_types_lock);
4467 mutex_lock(&iter->mutex);
4469 if (iter->trace->splice_read) {
4470 ret = iter->trace->splice_read(iter, filp,
4471 ppos, pipe, len, flags);
4476 ret = tracing_wait_pipe(filp);
4480 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4485 trace_event_read_lock();
4486 trace_access_lock(iter->cpu_file);
4488 /* Fill as many pages as possible. */
4489 for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
4490 spd.pages[i] = alloc_page(GFP_KERNEL);
4494 rem = tracing_fill_pipe_page(rem, iter);
4496 /* Copy the data into the page, so we can start over. */
4497 ret = trace_seq_to_buffer(&iter->seq,
4498 page_address(spd.pages[i]),
4501 __free_page(spd.pages[i]);
4504 spd.partial[i].offset = 0;
4505 spd.partial[i].len = iter->seq.len;
4507 trace_seq_init(&iter->seq);
4510 trace_access_unlock(iter->cpu_file);
4511 trace_event_read_unlock();
4512 mutex_unlock(&iter->mutex);
4516 ret = splice_to_pipe(pipe, &spd);
4518 splice_shrink_spd(&spd);
4522 mutex_unlock(&iter->mutex);
4527 tracing_entries_read(struct file *filp, char __user *ubuf,
4528 size_t cnt, loff_t *ppos)
4530 struct inode *inode = file_inode(filp);
4531 struct trace_array *tr = inode->i_private;
4532 int cpu = tracing_get_cpu(inode);
4537 mutex_lock(&trace_types_lock);
4539 if (cpu == RING_BUFFER_ALL_CPUS) {
4540 int cpu, buf_size_same;
4545 /* check if all cpu sizes are same */
4546 for_each_tracing_cpu(cpu) {
4547 /* fill in the size from first enabled cpu */
4549 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4550 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4556 if (buf_size_same) {
4557 if (!ring_buffer_expanded)
4558 r = sprintf(buf, "%lu (expanded: %lu)\n",
4560 trace_buf_size >> 10);
4562 r = sprintf(buf, "%lu\n", size >> 10);
4564 r = sprintf(buf, "X\n");
4566 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
4568 mutex_unlock(&trace_types_lock);
4570 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4575 tracing_entries_write(struct file *filp, const char __user *ubuf,
4576 size_t cnt, loff_t *ppos)
4578 struct inode *inode = file_inode(filp);
4579 struct trace_array *tr = inode->i_private;
4583 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4587 /* must have at least 1 entry */
4591 /* value is in KB */
4593 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
4603 tracing_total_entries_read(struct file *filp, char __user *ubuf,
4604 size_t cnt, loff_t *ppos)
4606 struct trace_array *tr = filp->private_data;
4609 unsigned long size = 0, expanded_size = 0;
4611 mutex_lock(&trace_types_lock);
4612 for_each_tracing_cpu(cpu) {
4613 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
4614 if (!ring_buffer_expanded)
4615 expanded_size += trace_buf_size >> 10;
4617 if (ring_buffer_expanded)
4618 r = sprintf(buf, "%lu\n", size);
4620 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4621 mutex_unlock(&trace_types_lock);
4623 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4627 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4628 size_t cnt, loff_t *ppos)
4631 * There is no need to read what the user has written, this function
4632 * is just to make sure that there is no error when "echo" is used
4641 tracing_free_buffer_release(struct inode *inode, struct file *filp)
4643 struct trace_array *tr = inode->i_private;
4645 /* disable tracing ? */
4646 if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4647 tracer_tracing_off(tr);
4648 /* resize the ring buffer to 0 */
4649 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4651 trace_array_put(tr);
4657 tracing_mark_write(struct file *filp, const char __user *ubuf,
4658 size_t cnt, loff_t *fpos)
4660 unsigned long addr = (unsigned long)ubuf;
4661 struct trace_array *tr = filp->private_data;
4662 struct ring_buffer_event *event;
4663 struct ring_buffer *buffer;
4664 struct print_entry *entry;
4665 unsigned long irq_flags;
4666 struct page *pages[2];
4676 if (tracing_disabled)
4679 if (!(trace_flags & TRACE_ITER_MARKERS))
4682 if (cnt > TRACE_BUF_SIZE)
4683 cnt = TRACE_BUF_SIZE;
4686 * Userspace is injecting traces into the kernel trace buffer.
4687 * We want to be as non intrusive as possible.
4688 * To do so, we do not want to allocate any special buffers
4689 * or take any locks, but instead write the userspace data
4690 * straight into the ring buffer.
4692 * First we need to pin the userspace buffer into memory,
4693 * which, most likely it is, because it just referenced it.
4694 * But there's no guarantee that it is. By using get_user_pages_fast()
4695 * and kmap_atomic/kunmap_atomic() we can get access to the
4696 * pages directly. We then write the data directly into the
4699 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4701 /* check if we cross pages */
4702 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4705 offset = addr & (PAGE_SIZE - 1);
4708 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4709 if (ret < nr_pages) {
4711 put_page(pages[ret]);
4716 for (i = 0; i < nr_pages; i++)
4717 map_page[i] = kmap_atomic(pages[i]);
4719 local_save_flags(irq_flags);
4720 size = sizeof(*entry) + cnt + 2; /* possible \n added */
4721 buffer = tr->trace_buffer.buffer;
4722 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4723 irq_flags, preempt_count());
4725 /* Ring buffer disabled, return as if not open for write */
4730 entry = ring_buffer_event_data(event);
4731 entry->ip = _THIS_IP_;
4733 if (nr_pages == 2) {
4734 len = PAGE_SIZE - offset;
4735 memcpy(&entry->buf, map_page[0] + offset, len);
4736 memcpy(&entry->buf[len], map_page[1], cnt - len);
4738 memcpy(&entry->buf, map_page[0] + offset, cnt);
4740 if (entry->buf[cnt - 1] != '\n') {
4741 entry->buf[cnt] = '\n';
4742 entry->buf[cnt + 1] = '\0';
4744 entry->buf[cnt] = '\0';
4746 __buffer_unlock_commit(buffer, event);
4753 for (i = 0; i < nr_pages; i++){
4754 kunmap_atomic(map_page[i]);
4761 static int tracing_clock_show(struct seq_file *m, void *v)
4763 struct trace_array *tr = m->private;
4766 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
4768 "%s%s%s%s", i ? " " : "",
4769 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4770 i == tr->clock_id ? "]" : "");
4776 static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
4780 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4781 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4784 if (i == ARRAY_SIZE(trace_clocks))
4787 mutex_lock(&trace_types_lock);
4791 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
4794 * New clock may not be consistent with the previous clock.
4795 * Reset the buffer so that it doesn't have incomparable timestamps.
4797 tracing_reset_online_cpus(&tr->trace_buffer);
4799 #ifdef CONFIG_TRACER_MAX_TRACE
4800 if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4801 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4802 tracing_reset_online_cpus(&tr->max_buffer);
4805 mutex_unlock(&trace_types_lock);
4810 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4811 size_t cnt, loff_t *fpos)
4813 struct seq_file *m = filp->private_data;
4814 struct trace_array *tr = m->private;
4816 const char *clockstr;
4819 if (cnt >= sizeof(buf))
4822 if (copy_from_user(&buf, ubuf, cnt))
4827 clockstr = strstrip(buf);
4829 ret = tracing_set_clock(tr, clockstr);
4838 static int tracing_clock_open(struct inode *inode, struct file *file)
4840 struct trace_array *tr = inode->i_private;
4843 if (tracing_disabled)
4846 if (trace_array_get(tr))
4849 ret = single_open(file, tracing_clock_show, inode->i_private);
4851 trace_array_put(tr);
4856 struct ftrace_buffer_info {
4857 struct trace_iterator iter;
4862 #ifdef CONFIG_TRACER_SNAPSHOT
4863 static int tracing_snapshot_open(struct inode *inode, struct file *file)
4865 struct trace_array *tr = inode->i_private;
4866 struct trace_iterator *iter;
4870 if (trace_array_get(tr) < 0)
4873 if (file->f_mode & FMODE_READ) {
4874 iter = __tracing_open(inode, file, true);
4876 ret = PTR_ERR(iter);
4878 /* Writes still need the seq_file to hold the private data */
4880 m = kzalloc(sizeof(*m), GFP_KERNEL);
4883 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4891 iter->trace_buffer = &tr->max_buffer;
4892 iter->cpu_file = tracing_get_cpu(inode);
4894 file->private_data = m;
4898 trace_array_put(tr);
4904 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
4907 struct seq_file *m = filp->private_data;
4908 struct trace_iterator *iter = m->private;
4909 struct trace_array *tr = iter->tr;
4913 ret = tracing_update_buffers();
4917 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4921 mutex_lock(&trace_types_lock);
4923 if (tr->current_trace->use_max_tr) {
4930 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4934 if (tr->allocated_snapshot)
4938 /* Only allow per-cpu swap if the ring buffer supports it */
4939 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
4940 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4945 if (!tr->allocated_snapshot) {
4946 ret = alloc_snapshot(tr);
4950 local_irq_disable();
4951 /* Now, we're going to swap */
4952 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4953 update_max_tr(tr, current, smp_processor_id());
4955 update_max_tr_single(tr, current, iter->cpu_file);
4959 if (tr->allocated_snapshot) {
4960 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4961 tracing_reset_online_cpus(&tr->max_buffer);
4963 tracing_reset(&tr->max_buffer, iter->cpu_file);
4973 mutex_unlock(&trace_types_lock);
4977 static int tracing_snapshot_release(struct inode *inode, struct file *file)
4979 struct seq_file *m = file->private_data;
4982 ret = tracing_release(inode, file);
4984 if (file->f_mode & FMODE_READ)
4987 /* If write only, the seq_file is just a stub */
4995 static int tracing_buffers_open(struct inode *inode, struct file *filp);
4996 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
4997 size_t count, loff_t *ppos);
4998 static int tracing_buffers_release(struct inode *inode, struct file *file);
4999 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5000 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5002 static int snapshot_raw_open(struct inode *inode, struct file *filp)
5004 struct ftrace_buffer_info *info;
5007 ret = tracing_buffers_open(inode, filp);
5011 info = filp->private_data;
5013 if (info->iter.trace->use_max_tr) {
5014 tracing_buffers_release(inode, filp);
5018 info->iter.snapshot = true;
5019 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5024 #endif /* CONFIG_TRACER_SNAPSHOT */
5027 static const struct file_operations tracing_max_lat_fops = {
5028 .open = tracing_open_generic,
5029 .read = tracing_max_lat_read,
5030 .write = tracing_max_lat_write,
5031 .llseek = generic_file_llseek,
5034 static const struct file_operations set_tracer_fops = {
5035 .open = tracing_open_generic,
5036 .read = tracing_set_trace_read,
5037 .write = tracing_set_trace_write,
5038 .llseek = generic_file_llseek,
5041 static const struct file_operations tracing_pipe_fops = {
5042 .open = tracing_open_pipe,
5043 .poll = tracing_poll_pipe,
5044 .read = tracing_read_pipe,
5045 .splice_read = tracing_splice_read_pipe,
5046 .release = tracing_release_pipe,
5047 .llseek = no_llseek,
5050 static const struct file_operations tracing_entries_fops = {
5051 .open = tracing_open_generic_tr,
5052 .read = tracing_entries_read,
5053 .write = tracing_entries_write,
5054 .llseek = generic_file_llseek,
5055 .release = tracing_release_generic_tr,
5058 static const struct file_operations tracing_total_entries_fops = {
5059 .open = tracing_open_generic_tr,
5060 .read = tracing_total_entries_read,
5061 .llseek = generic_file_llseek,
5062 .release = tracing_release_generic_tr,
5065 static const struct file_operations tracing_free_buffer_fops = {
5066 .open = tracing_open_generic_tr,
5067 .write = tracing_free_buffer_write,
5068 .release = tracing_free_buffer_release,
5071 static const struct file_operations tracing_mark_fops = {
5072 .open = tracing_open_generic_tr,
5073 .write = tracing_mark_write,
5074 .llseek = generic_file_llseek,
5075 .release = tracing_release_generic_tr,
5078 static const struct file_operations trace_clock_fops = {
5079 .open = tracing_clock_open,
5081 .llseek = seq_lseek,
5082 .release = tracing_single_release_tr,
5083 .write = tracing_clock_write,
5086 #ifdef CONFIG_TRACER_SNAPSHOT
5087 static const struct file_operations snapshot_fops = {
5088 .open = tracing_snapshot_open,
5090 .write = tracing_snapshot_write,
5091 .llseek = tracing_lseek,
5092 .release = tracing_snapshot_release,
5095 static const struct file_operations snapshot_raw_fops = {
5096 .open = snapshot_raw_open,
5097 .read = tracing_buffers_read,
5098 .release = tracing_buffers_release,
5099 .splice_read = tracing_buffers_splice_read,
5100 .llseek = no_llseek,
5103 #endif /* CONFIG_TRACER_SNAPSHOT */
5105 static int tracing_buffers_open(struct inode *inode, struct file *filp)
5107 struct trace_array *tr = inode->i_private;
5108 struct ftrace_buffer_info *info;
5111 if (tracing_disabled)
5114 if (trace_array_get(tr) < 0)
5117 info = kzalloc(sizeof(*info), GFP_KERNEL);
5119 trace_array_put(tr);
5123 mutex_lock(&trace_types_lock);
5126 info->iter.cpu_file = tracing_get_cpu(inode);
5127 info->iter.trace = tr->current_trace;
5128 info->iter.trace_buffer = &tr->trace_buffer;
5130 /* Force reading ring buffer for first read */
5131 info->read = (unsigned int)-1;
5133 filp->private_data = info;
5135 mutex_unlock(&trace_types_lock);
5137 ret = nonseekable_open(inode, filp);
5139 trace_array_put(tr);
5145 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
5147 struct ftrace_buffer_info *info = filp->private_data;
5148 struct trace_iterator *iter = &info->iter;
5150 return trace_poll(iter, filp, poll_table);
5154 tracing_buffers_read(struct file *filp, char __user *ubuf,
5155 size_t count, loff_t *ppos)
5157 struct ftrace_buffer_info *info = filp->private_data;
5158 struct trace_iterator *iter = &info->iter;
5165 mutex_lock(&trace_types_lock);
5167 #ifdef CONFIG_TRACER_MAX_TRACE
5168 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5175 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
5181 /* Do we have previous read data to read? */
5182 if (info->read < PAGE_SIZE)
5186 trace_access_lock(iter->cpu_file);
5187 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
5191 trace_access_unlock(iter->cpu_file);
5194 if (trace_empty(iter)) {
5195 if ((filp->f_flags & O_NONBLOCK)) {
5199 mutex_unlock(&trace_types_lock);
5200 iter->trace->wait_pipe(iter);
5201 mutex_lock(&trace_types_lock);
5202 if (signal_pending(current)) {
5214 size = PAGE_SIZE - info->read;
5218 ret = copy_to_user(ubuf, info->spare + info->read, size);
5229 mutex_unlock(&trace_types_lock);
5234 static int tracing_buffers_release(struct inode *inode, struct file *file)
5236 struct ftrace_buffer_info *info = file->private_data;
5237 struct trace_iterator *iter = &info->iter;
5239 mutex_lock(&trace_types_lock);
5241 __trace_array_put(iter->tr);
5244 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
5247 mutex_unlock(&trace_types_lock);
5253 struct ring_buffer *buffer;
5258 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
5259 struct pipe_buffer *buf)
5261 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5266 ring_buffer_free_read_page(ref->buffer, ref->page);
5271 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
5272 struct pipe_buffer *buf)
5274 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
5279 /* Pipe buffer operations for a buffer. */
5280 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
5282 .map = generic_pipe_buf_map,
5283 .unmap = generic_pipe_buf_unmap,
5284 .confirm = generic_pipe_buf_confirm,
5285 .release = buffer_pipe_buf_release,
5286 .steal = generic_pipe_buf_steal,
5287 .get = buffer_pipe_buf_get,
5291 * Callback from splice_to_pipe(), if we need to release some pages
5292 * at the end of the spd in case we error'ed out in filling the pipe.
5294 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
5296 struct buffer_ref *ref =
5297 (struct buffer_ref *)spd->partial[i].private;
5302 ring_buffer_free_read_page(ref->buffer, ref->page);
5304 spd->partial[i].private = 0;
5308 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5309 struct pipe_inode_info *pipe, size_t len,
5312 struct ftrace_buffer_info *info = file->private_data;
5313 struct trace_iterator *iter = &info->iter;
5314 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5315 struct page *pages_def[PIPE_DEF_BUFFERS];
5316 struct splice_pipe_desc spd = {
5318 .partial = partial_def,
5319 .nr_pages_max = PIPE_DEF_BUFFERS,
5321 .ops = &buffer_pipe_buf_ops,
5322 .spd_release = buffer_spd_release,
5324 struct buffer_ref *ref;
5325 int entries, size, i;
5328 mutex_lock(&trace_types_lock);
5330 #ifdef CONFIG_TRACER_MAX_TRACE
5331 if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
5337 if (splice_grow_spd(pipe, &spd)) {
5342 if (*ppos & (PAGE_SIZE - 1)) {
5347 if (len & (PAGE_SIZE - 1)) {
5348 if (len < PAGE_SIZE) {
5356 trace_access_lock(iter->cpu_file);
5357 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5359 for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
5363 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
5368 ref->buffer = iter->trace_buffer->buffer;
5369 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
5375 r = ring_buffer_read_page(ref->buffer, &ref->page,
5376 len, iter->cpu_file, 1);
5378 ring_buffer_free_read_page(ref->buffer, ref->page);
5384 * zero out any left over data, this is going to
5387 size = ring_buffer_page_len(ref->page);
5388 if (size < PAGE_SIZE)
5389 memset(ref->page + size, 0, PAGE_SIZE - size);
5391 page = virt_to_page(ref->page);
5393 spd.pages[i] = page;
5394 spd.partial[i].len = PAGE_SIZE;
5395 spd.partial[i].offset = 0;
5396 spd.partial[i].private = (unsigned long)ref;
5400 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5403 trace_access_unlock(iter->cpu_file);
5406 /* did we read anything? */
5407 if (!spd.nr_pages) {
5408 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
5412 mutex_unlock(&trace_types_lock);
5413 iter->trace->wait_pipe(iter);
5414 mutex_lock(&trace_types_lock);
5415 if (signal_pending(current)) {
5422 ret = splice_to_pipe(pipe, &spd);
5423 splice_shrink_spd(&spd);
5425 mutex_unlock(&trace_types_lock);
5430 static const struct file_operations tracing_buffers_fops = {
5431 .open = tracing_buffers_open,
5432 .read = tracing_buffers_read,
5433 .poll = tracing_buffers_poll,
5434 .release = tracing_buffers_release,
5435 .splice_read = tracing_buffers_splice_read,
5436 .llseek = no_llseek,
5440 tracing_stats_read(struct file *filp, char __user *ubuf,
5441 size_t count, loff_t *ppos)
5443 struct inode *inode = file_inode(filp);
5444 struct trace_array *tr = inode->i_private;
5445 struct trace_buffer *trace_buf = &tr->trace_buffer;
5446 int cpu = tracing_get_cpu(inode);
5447 struct trace_seq *s;
5449 unsigned long long t;
5450 unsigned long usec_rem;
5452 s = kmalloc(sizeof(*s), GFP_KERNEL);
5458 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5459 trace_seq_printf(s, "entries: %ld\n", cnt);
5461 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5462 trace_seq_printf(s, "overrun: %ld\n", cnt);
5464 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5465 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5467 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5468 trace_seq_printf(s, "bytes: %ld\n", cnt);
5470 if (trace_clocks[tr->clock_id].in_ns) {
5471 /* local or global for trace_clock */
5472 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5473 usec_rem = do_div(t, USEC_PER_SEC);
5474 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5477 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5478 usec_rem = do_div(t, USEC_PER_SEC);
5479 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5481 /* counter or tsc mode for trace_clock */
5482 trace_seq_printf(s, "oldest event ts: %llu\n",
5483 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5485 trace_seq_printf(s, "now ts: %llu\n",
5486 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5489 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5490 trace_seq_printf(s, "dropped events: %ld\n", cnt);
5492 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5493 trace_seq_printf(s, "read events: %ld\n", cnt);
5495 count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5502 static const struct file_operations tracing_stats_fops = {
5503 .open = tracing_open_generic_tr,
5504 .read = tracing_stats_read,
5505 .llseek = generic_file_llseek,
5506 .release = tracing_release_generic_tr,
5509 #ifdef CONFIG_DYNAMIC_FTRACE
5511 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5517 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5518 size_t cnt, loff_t *ppos)
5520 static char ftrace_dyn_info_buffer[1024];
5521 static DEFINE_MUTEX(dyn_info_mutex);
5522 unsigned long *p = filp->private_data;
5523 char *buf = ftrace_dyn_info_buffer;
5524 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5527 mutex_lock(&dyn_info_mutex);
5528 r = sprintf(buf, "%ld ", *p);
5530 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5533 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5535 mutex_unlock(&dyn_info_mutex);
5540 static const struct file_operations tracing_dyn_info_fops = {
5541 .open = tracing_open_generic,
5542 .read = tracing_read_dyn_info,
5543 .llseek = generic_file_llseek,
5545 #endif /* CONFIG_DYNAMIC_FTRACE */
5547 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5549 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5555 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5557 unsigned long *count = (long *)data;
5569 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5570 struct ftrace_probe_ops *ops, void *data)
5572 long count = (long)data;
5574 seq_printf(m, "%ps:", (void *)ip);
5576 seq_printf(m, "snapshot");
5579 seq_printf(m, ":unlimited\n");
5581 seq_printf(m, ":count=%ld\n", count);
5586 static struct ftrace_probe_ops snapshot_probe_ops = {
5587 .func = ftrace_snapshot,
5588 .print = ftrace_snapshot_print,
5591 static struct ftrace_probe_ops snapshot_count_probe_ops = {
5592 .func = ftrace_count_snapshot,
5593 .print = ftrace_snapshot_print,
5597 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5598 char *glob, char *cmd, char *param, int enable)
5600 struct ftrace_probe_ops *ops;
5601 void *count = (void *)-1;
5605 /* hash funcs only work with set_ftrace_filter */
5609 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
5611 if (glob[0] == '!') {
5612 unregister_ftrace_function_probe_func(glob+1, ops);
5619 number = strsep(¶m, ":");
5621 if (!strlen(number))
5625 * We use the callback data field (which is a pointer)
5628 ret = kstrtoul(number, 0, (unsigned long *)&count);
5633 ret = register_ftrace_function_probe(glob, ops, count);
5636 alloc_snapshot(&global_trace);
5638 return ret < 0 ? ret : 0;
5641 static struct ftrace_func_command ftrace_snapshot_cmd = {
5643 .func = ftrace_trace_snapshot_callback,
5646 static __init int register_snapshot_cmd(void)
5648 return register_ftrace_command(&ftrace_snapshot_cmd);
5651 static inline __init int register_snapshot_cmd(void) { return 0; }
5652 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
5654 struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
5659 if (!debugfs_initialized())
5662 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5663 tr->dir = debugfs_create_dir("tracing", NULL);
5666 pr_warn_once("Could not create debugfs directory 'tracing'\n");
5671 struct dentry *tracing_init_dentry(void)
5673 return tracing_init_dentry_tr(&global_trace);
5676 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
5678 struct dentry *d_tracer;
5681 return tr->percpu_dir;
5683 d_tracer = tracing_init_dentry_tr(tr);
5687 tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
5689 WARN_ONCE(!tr->percpu_dir,
5690 "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
5692 return tr->percpu_dir;
5695 static struct dentry *
5696 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
5697 void *data, long cpu, const struct file_operations *fops)
5699 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
5701 if (ret) /* See tracing_get_cpu() */
5702 ret->d_inode->i_cdev = (void *)(cpu + 1);
5707 tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
5709 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5710 struct dentry *d_cpu;
5711 char cpu_dir[30]; /* 30 characters should be more than enough */
5716 snprintf(cpu_dir, 30, "cpu%ld", cpu);
5717 d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5719 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5723 /* per cpu trace_pipe */
5724 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
5725 tr, cpu, &tracing_pipe_fops);
5728 trace_create_cpu_file("trace", 0644, d_cpu,
5729 tr, cpu, &tracing_fops);
5731 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
5732 tr, cpu, &tracing_buffers_fops);
5734 trace_create_cpu_file("stats", 0444, d_cpu,
5735 tr, cpu, &tracing_stats_fops);
5737 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
5738 tr, cpu, &tracing_entries_fops);
5740 #ifdef CONFIG_TRACER_SNAPSHOT
5741 trace_create_cpu_file("snapshot", 0644, d_cpu,
5742 tr, cpu, &snapshot_fops);
5744 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
5745 tr, cpu, &snapshot_raw_fops);
5749 #ifdef CONFIG_FTRACE_SELFTEST
5750 /* Let selftest have access to static functions in this file */
5751 #include "trace_selftest.c"
5754 struct trace_option_dentry {
5755 struct tracer_opt *opt;
5756 struct tracer_flags *flags;
5757 struct trace_array *tr;
5758 struct dentry *entry;
5762 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5765 struct trace_option_dentry *topt = filp->private_data;
5768 if (topt->flags->val & topt->opt->bit)
5773 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5777 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5780 struct trace_option_dentry *topt = filp->private_data;
5784 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5788 if (val != 0 && val != 1)
5791 if (!!(topt->flags->val & topt->opt->bit) != val) {
5792 mutex_lock(&trace_types_lock);
5793 ret = __set_tracer_option(topt->tr, topt->flags,
5795 mutex_unlock(&trace_types_lock);
5806 static const struct file_operations trace_options_fops = {
5807 .open = tracing_open_generic,
5808 .read = trace_options_read,
5809 .write = trace_options_write,
5810 .llseek = generic_file_llseek,
5814 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5817 long index = (long)filp->private_data;
5820 if (trace_flags & (1 << index))
5825 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5829 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
5832 struct trace_array *tr = &global_trace;
5833 long index = (long)filp->private_data;
5837 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5841 if (val != 0 && val != 1)
5844 mutex_lock(&trace_types_lock);
5845 ret = set_tracer_flag(tr, 1 << index, val);
5846 mutex_unlock(&trace_types_lock);
5856 static const struct file_operations trace_options_core_fops = {
5857 .open = tracing_open_generic,
5858 .read = trace_options_core_read,
5859 .write = trace_options_core_write,
5860 .llseek = generic_file_llseek,
5863 struct dentry *trace_create_file(const char *name,
5865 struct dentry *parent,
5867 const struct file_operations *fops)
5871 ret = debugfs_create_file(name, mode, parent, data, fops);
5873 pr_warning("Could not create debugfs '%s' entry\n", name);
5879 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
5881 struct dentry *d_tracer;
5886 d_tracer = tracing_init_dentry_tr(tr);
5890 tr->options = debugfs_create_dir("options", d_tracer);
5892 pr_warning("Could not create debugfs directory 'options'\n");
5900 create_trace_option_file(struct trace_array *tr,
5901 struct trace_option_dentry *topt,
5902 struct tracer_flags *flags,
5903 struct tracer_opt *opt)
5905 struct dentry *t_options;
5907 t_options = trace_options_init_dentry(tr);
5911 topt->flags = flags;
5915 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
5916 &trace_options_fops);
5920 static struct trace_option_dentry *
5921 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
5923 struct trace_option_dentry *topts;
5924 struct tracer_flags *flags;
5925 struct tracer_opt *opts;
5931 flags = tracer->flags;
5933 if (!flags || !flags->opts)
5938 for (cnt = 0; opts[cnt].name; cnt++)
5941 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
5945 for (cnt = 0; opts[cnt].name; cnt++)
5946 create_trace_option_file(tr, &topts[cnt], flags,
5953 destroy_trace_option_files(struct trace_option_dentry *topts)
5960 for (cnt = 0; topts[cnt].opt; cnt++) {
5961 if (topts[cnt].entry)
5962 debugfs_remove(topts[cnt].entry);
5968 static struct dentry *
5969 create_trace_option_core_file(struct trace_array *tr,
5970 const char *option, long index)
5972 struct dentry *t_options;
5974 t_options = trace_options_init_dentry(tr);
5978 return trace_create_file(option, 0644, t_options, (void *)index,
5979 &trace_options_core_fops);
5982 static __init void create_trace_options_dir(struct trace_array *tr)
5984 struct dentry *t_options;
5987 t_options = trace_options_init_dentry(tr);
5991 for (i = 0; trace_options[i]; i++)
5992 create_trace_option_core_file(tr, trace_options[i], i);
5996 rb_simple_read(struct file *filp, char __user *ubuf,
5997 size_t cnt, loff_t *ppos)
5999 struct trace_array *tr = filp->private_data;
6003 r = tracer_tracing_is_on(tr);
6004 r = sprintf(buf, "%d\n", r);
6006 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6010 rb_simple_write(struct file *filp, const char __user *ubuf,
6011 size_t cnt, loff_t *ppos)
6013 struct trace_array *tr = filp->private_data;
6014 struct ring_buffer *buffer = tr->trace_buffer.buffer;
6018 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6023 mutex_lock(&trace_types_lock);
6025 tracer_tracing_on(tr);
6026 if (tr->current_trace->start)
6027 tr->current_trace->start(tr);
6029 tracer_tracing_off(tr);
6030 if (tr->current_trace->stop)
6031 tr->current_trace->stop(tr);
6033 mutex_unlock(&trace_types_lock);
6041 static const struct file_operations rb_simple_fops = {
6042 .open = tracing_open_generic_tr,
6043 .read = rb_simple_read,
6044 .write = rb_simple_write,
6045 .release = tracing_release_generic_tr,
6046 .llseek = default_llseek,
6049 struct dentry *trace_instance_dir;
6052 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
6055 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
6057 enum ring_buffer_flags rb_flags;
6059 rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
6063 buf->buffer = ring_buffer_alloc(size, rb_flags);
6067 buf->data = alloc_percpu(struct trace_array_cpu);
6069 ring_buffer_free(buf->buffer);
6073 /* Allocate the first page for all buffers */
6074 set_buffer_entries(&tr->trace_buffer,
6075 ring_buffer_size(tr->trace_buffer.buffer, 0));
6080 static int allocate_trace_buffers(struct trace_array *tr, int size)
6084 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6088 #ifdef CONFIG_TRACER_MAX_TRACE
6089 ret = allocate_trace_buffer(tr, &tr->max_buffer,
6090 allocate_snapshot ? size : 1);
6092 ring_buffer_free(tr->trace_buffer.buffer);
6093 free_percpu(tr->trace_buffer.data);
6096 tr->allocated_snapshot = allocate_snapshot;
6099 * Only the top level trace array gets its snapshot allocated
6100 * from the kernel command line.
6102 allocate_snapshot = false;
6107 static int new_instance_create(const char *name)
6109 struct trace_array *tr;
6112 mutex_lock(&trace_types_lock);
6115 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6116 if (tr->name && strcmp(tr->name, name) == 0)
6121 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
6125 tr->name = kstrdup(name, GFP_KERNEL);
6129 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
6132 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
6134 raw_spin_lock_init(&tr->start_lock);
6136 tr->current_trace = &nop_trace;
6138 INIT_LIST_HEAD(&tr->systems);
6139 INIT_LIST_HEAD(&tr->events);
6141 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
6144 tr->dir = debugfs_create_dir(name, trace_instance_dir);
6148 ret = event_trace_add_tracer(tr->dir, tr);
6150 debugfs_remove_recursive(tr->dir);
6154 init_tracer_debugfs(tr, tr->dir);
6156 list_add(&tr->list, &ftrace_trace_arrays);
6158 mutex_unlock(&trace_types_lock);
6163 if (tr->trace_buffer.buffer)
6164 ring_buffer_free(tr->trace_buffer.buffer);
6165 free_cpumask_var(tr->tracing_cpumask);
6170 mutex_unlock(&trace_types_lock);
6176 static int instance_delete(const char *name)
6178 struct trace_array *tr;
6182 mutex_lock(&trace_types_lock);
6185 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
6186 if (tr->name && strcmp(tr->name, name) == 0) {
6198 list_del(&tr->list);
6200 tracing_set_nop(tr);
6201 event_trace_del_tracer(tr);
6202 ftrace_destroy_function_files(tr);
6203 debugfs_remove_recursive(tr->dir);
6204 free_percpu(tr->trace_buffer.data);
6205 ring_buffer_free(tr->trace_buffer.buffer);
6213 mutex_unlock(&trace_types_lock);
6218 static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
6220 struct dentry *parent;
6223 /* Paranoid: Make sure the parent is the "instances" directory */
6224 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6225 if (WARN_ON_ONCE(parent != trace_instance_dir))
6229 * The inode mutex is locked, but debugfs_create_dir() will also
6230 * take the mutex. As the instances directory can not be destroyed
6231 * or changed in any other way, it is safe to unlock it, and
6232 * let the dentry try. If two users try to make the same dir at
6233 * the same time, then the new_instance_create() will determine the
6236 mutex_unlock(&inode->i_mutex);
6238 ret = new_instance_create(dentry->d_iname);
6240 mutex_lock(&inode->i_mutex);
6245 static int instance_rmdir(struct inode *inode, struct dentry *dentry)
6247 struct dentry *parent;
6250 /* Paranoid: Make sure the parent is the "instances" directory */
6251 parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
6252 if (WARN_ON_ONCE(parent != trace_instance_dir))
6255 /* The caller did a dget() on dentry */
6256 mutex_unlock(&dentry->d_inode->i_mutex);
6259 * The inode mutex is locked, but debugfs_create_dir() will also
6260 * take the mutex. As the instances directory can not be destroyed
6261 * or changed in any other way, it is safe to unlock it, and
6262 * let the dentry try. If two users try to make the same dir at
6263 * the same time, then the instance_delete() will determine the
6266 mutex_unlock(&inode->i_mutex);
6268 ret = instance_delete(dentry->d_iname);
6270 mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
6271 mutex_lock(&dentry->d_inode->i_mutex);
6276 static const struct inode_operations instance_dir_inode_operations = {
6277 .lookup = simple_lookup,
6278 .mkdir = instance_mkdir,
6279 .rmdir = instance_rmdir,
6282 static __init void create_trace_instances(struct dentry *d_tracer)
6284 trace_instance_dir = debugfs_create_dir("instances", d_tracer);
6285 if (WARN_ON(!trace_instance_dir))
6288 /* Hijack the dir inode operations, to allow mkdir */
6289 trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
6293 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
6297 trace_create_file("available_tracers", 0444, d_tracer,
6298 tr, &show_traces_fops);
6300 trace_create_file("current_tracer", 0644, d_tracer,
6301 tr, &set_tracer_fops);
6303 trace_create_file("tracing_cpumask", 0644, d_tracer,
6304 tr, &tracing_cpumask_fops);
6306 trace_create_file("trace_options", 0644, d_tracer,
6307 tr, &tracing_iter_fops);
6309 trace_create_file("trace", 0644, d_tracer,
6312 trace_create_file("trace_pipe", 0444, d_tracer,
6313 tr, &tracing_pipe_fops);
6315 trace_create_file("buffer_size_kb", 0644, d_tracer,
6316 tr, &tracing_entries_fops);
6318 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
6319 tr, &tracing_total_entries_fops);
6321 trace_create_file("free_buffer", 0200, d_tracer,
6322 tr, &tracing_free_buffer_fops);
6324 trace_create_file("trace_marker", 0220, d_tracer,
6325 tr, &tracing_mark_fops);
6327 trace_create_file("trace_clock", 0644, d_tracer, tr,
6330 trace_create_file("tracing_on", 0644, d_tracer,
6331 tr, &rb_simple_fops);
6333 if (ftrace_create_function_files(tr, d_tracer))
6334 WARN(1, "Could not allocate function filter files");
6336 #ifdef CONFIG_TRACER_SNAPSHOT
6337 trace_create_file("snapshot", 0644, d_tracer,
6338 tr, &snapshot_fops);
6341 for_each_tracing_cpu(cpu)
6342 tracing_init_debugfs_percpu(tr, cpu);
6346 static __init int tracer_init_debugfs(void)
6348 struct dentry *d_tracer;
6350 trace_access_lock_init();
6352 d_tracer = tracing_init_dentry();
6356 init_tracer_debugfs(&global_trace, d_tracer);
6358 #ifdef CONFIG_TRACER_MAX_TRACE
6359 trace_create_file("tracing_max_latency", 0644, d_tracer,
6360 &tracing_max_latency, &tracing_max_lat_fops);
6363 trace_create_file("tracing_thresh", 0644, d_tracer,
6364 &tracing_thresh, &tracing_max_lat_fops);
6366 trace_create_file("README", 0444, d_tracer,
6367 NULL, &tracing_readme_fops);
6369 trace_create_file("saved_cmdlines", 0444, d_tracer,
6370 NULL, &tracing_saved_cmdlines_fops);
6372 #ifdef CONFIG_DYNAMIC_FTRACE
6373 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
6374 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
6377 create_trace_instances(d_tracer);
6379 create_trace_options_dir(&global_trace);
6384 static int trace_panic_handler(struct notifier_block *this,
6385 unsigned long event, void *unused)
6387 if (ftrace_dump_on_oops)
6388 ftrace_dump(ftrace_dump_on_oops);
6392 static struct notifier_block trace_panic_notifier = {
6393 .notifier_call = trace_panic_handler,
6395 .priority = 150 /* priority: INT_MAX >= x >= 0 */
6398 static int trace_die_handler(struct notifier_block *self,
6404 if (ftrace_dump_on_oops)
6405 ftrace_dump(ftrace_dump_on_oops);
6413 static struct notifier_block trace_die_notifier = {
6414 .notifier_call = trace_die_handler,
6419 * printk is set to max of 1024, we really don't need it that big.
6420 * Nothing should be printing 1000 characters anyway.
6422 #define TRACE_MAX_PRINT 1000
6425 * Define here KERN_TRACE so that we have one place to modify
6426 * it if we decide to change what log level the ftrace dump
6429 #define KERN_TRACE KERN_EMERG
6432 trace_printk_seq(struct trace_seq *s)
6434 /* Probably should print a warning here. */
6435 if (s->len >= TRACE_MAX_PRINT)
6436 s->len = TRACE_MAX_PRINT;
6438 /* should be zero ended, but we are paranoid. */
6439 s->buffer[s->len] = 0;
6441 printk(KERN_TRACE "%s", s->buffer);
6446 void trace_init_global_iter(struct trace_iterator *iter)
6448 iter->tr = &global_trace;
6449 iter->trace = iter->tr->current_trace;
6450 iter->cpu_file = RING_BUFFER_ALL_CPUS;
6451 iter->trace_buffer = &global_trace.trace_buffer;
6453 if (iter->trace && iter->trace->open)
6454 iter->trace->open(iter);
6456 /* Annotate start of buffers if we had overruns */
6457 if (ring_buffer_overruns(iter->trace_buffer->buffer))
6458 iter->iter_flags |= TRACE_FILE_ANNOTATE;
6460 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
6461 if (trace_clocks[iter->tr->clock_id].in_ns)
6462 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
6465 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6467 /* use static because iter can be a bit big for the stack */
6468 static struct trace_iterator iter;
6469 static atomic_t dump_running;
6470 unsigned int old_userobj;
6471 unsigned long flags;
6474 /* Only allow one dump user at a time. */
6475 if (atomic_inc_return(&dump_running) != 1) {
6476 atomic_dec(&dump_running);
6481 * Always turn off tracing when we dump.
6482 * We don't need to show trace output of what happens
6483 * between multiple crashes.
6485 * If the user does a sysrq-z, then they can re-enable
6486 * tracing with echo 1 > tracing_on.
6490 local_irq_save(flags);
6492 /* Simulate the iterator */
6493 trace_init_global_iter(&iter);
6495 for_each_tracing_cpu(cpu) {
6496 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
6499 old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6501 /* don't look at user memory in panic mode */
6502 trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6504 switch (oops_dump_mode) {
6506 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6509 iter.cpu_file = raw_smp_processor_id();
6514 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
6515 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6518 printk(KERN_TRACE "Dumping ftrace buffer:\n");
6520 /* Did function tracer already get disabled? */
6521 if (ftrace_is_dead()) {
6522 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6523 printk("# MAY BE MISSING FUNCTION EVENTS\n");
6527 * We need to stop all tracing on all CPUS to read the
6528 * the next buffer. This is a bit expensive, but is
6529 * not done often. We fill all what we can read,
6530 * and then release the locks again.
6533 while (!trace_empty(&iter)) {
6536 printk(KERN_TRACE "---------------------------------\n");
6540 /* reset all but tr, trace, and overruns */
6541 memset(&iter.seq, 0,
6542 sizeof(struct trace_iterator) -
6543 offsetof(struct trace_iterator, seq));
6544 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6547 if (trace_find_next_entry_inc(&iter) != NULL) {
6550 ret = print_trace_line(&iter);
6551 if (ret != TRACE_TYPE_NO_CONSUME)
6552 trace_consume(&iter);
6554 touch_nmi_watchdog();
6556 trace_printk_seq(&iter.seq);
6560 printk(KERN_TRACE " (ftrace buffer empty)\n");
6562 printk(KERN_TRACE "---------------------------------\n");
6565 trace_flags |= old_userobj;
6567 for_each_tracing_cpu(cpu) {
6568 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
6570 atomic_dec(&dump_running);
6571 local_irq_restore(flags);
6573 EXPORT_SYMBOL_GPL(ftrace_dump);
6575 __init static int tracer_alloc_buffers(void)
6581 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6584 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
6585 goto out_free_buffer_mask;
6587 /* Only allocate trace_printk buffers if a trace_printk exists */
6588 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
6589 /* Must be called before global_trace.buffer is allocated */
6590 trace_printk_init_buffers();
6592 /* To save memory, keep the ring buffer size to its minimum */
6593 if (ring_buffer_expanded)
6594 ring_buf_size = trace_buf_size;
6598 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6599 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
6601 raw_spin_lock_init(&global_trace.start_lock);
6603 /* Used for event triggers */
6604 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
6606 goto out_free_cpumask;
6608 /* TODO: make the number of buffers hot pluggable with CPUS */
6609 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
6610 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6612 goto out_free_temp_buffer;
6615 if (global_trace.buffer_disabled)
6618 trace_init_cmdlines();
6620 if (trace_boot_clock) {
6621 ret = tracing_set_clock(&global_trace, trace_boot_clock);
6623 pr_warning("Trace clock %s not defined, going back to default\n",
6628 * register_tracer() might reference current_trace, so it
6629 * needs to be set before we register anything. This is
6630 * just a bootstrap of current_trace anyway.
6632 global_trace.current_trace = &nop_trace;
6634 register_tracer(&nop_trace);
6636 /* All seems OK, enable tracing */
6637 tracing_disabled = 0;
6639 atomic_notifier_chain_register(&panic_notifier_list,
6640 &trace_panic_notifier);
6642 register_die_notifier(&trace_die_notifier);
6644 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6646 INIT_LIST_HEAD(&global_trace.systems);
6647 INIT_LIST_HEAD(&global_trace.events);
6648 list_add(&global_trace.list, &ftrace_trace_arrays);
6650 while (trace_boot_options) {
6653 option = strsep(&trace_boot_options, ",");
6654 trace_set_options(&global_trace, option);
6657 register_snapshot_cmd();
6661 out_free_temp_buffer:
6662 ring_buffer_free(temp_buffer);
6664 free_percpu(global_trace.trace_buffer.data);
6665 #ifdef CONFIG_TRACER_MAX_TRACE
6666 free_percpu(global_trace.max_buffer.data);
6668 free_cpumask_var(global_trace.tracing_cpumask);
6669 out_free_buffer_mask:
6670 free_cpumask_var(tracing_buffer_mask);
6675 __init static int clear_boot_tracer(void)
6678 * The default tracer at boot buffer is an init section.
6679 * This function is called in lateinit. If we did not
6680 * find the boot tracer, then clear it out, to prevent
6681 * later registration from accessing the buffer that is
6682 * about to be freed.
6684 if (!default_bootup_tracer)
6687 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6688 default_bootup_tracer);
6689 default_bootup_tracer = NULL;
6694 early_initcall(tracer_alloc_buffers);
6695 fs_initcall(tracer_init_debugfs);
6696 late_initcall(clear_boot_tracer);