Merge tag 'fixes-3.10-3' of git://git.infradead.org/users/jcooper/linux into fixes
[firefly-linux-kernel-4.4.55.git] / kernel / trace / trace.c
1 /*
2  * ring buffer based function tracer
3  *
4  * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5  * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
6  *
7  * Originally taken from the RT patch by:
8  *    Arnaldo Carvalho de Melo <acme@redhat.com>
9  *
10  * Based on code from the latency_tracer, that is:
11  *  Copyright (C) 2004-2006 Ingo Molnar
12  *  Copyright (C) 2004 Nadia Yvette Chambers
13  */
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/pagemap.h>
24 #include <linux/hardirq.h>
25 #include <linux/linkage.h>
26 #include <linux/uaccess.h>
27 #include <linux/kprobes.h>
28 #include <linux/ftrace.h>
29 #include <linux/module.h>
30 #include <linux/percpu.h>
31 #include <linux/splice.h>
32 #include <linux/kdebug.h>
33 #include <linux/string.h>
34 #include <linux/rwsem.h>
35 #include <linux/slab.h>
36 #include <linux/ctype.h>
37 #include <linux/init.h>
38 #include <linux/poll.h>
39 #include <linux/nmi.h>
40 #include <linux/fs.h>
41 #include <linux/sched/rt.h>
42
43 #include "trace.h"
44 #include "trace_output.h"
45
46 /*
47  * On boot up, the ring buffer is set to the minimum size, so that
48  * we do not waste memory on systems that are not using tracing.
49  */
50 bool ring_buffer_expanded;
51
52 /*
53  * We need to change this state when a selftest is running.
54  * A selftest will lurk into the ring-buffer to count the
55  * entries inserted during the selftest although some concurrent
56  * insertions into the ring-buffer such as trace_printk could occurred
57  * at the same time, giving false positive or negative results.
58  */
59 static bool __read_mostly tracing_selftest_running;
60
61 /*
62  * If a tracer is running, we do not want to run SELFTEST.
63  */
64 bool __read_mostly tracing_selftest_disabled;
65
66 /* For tracers that don't implement custom flags */
67 static struct tracer_opt dummy_tracer_opt[] = {
68         { }
69 };
70
71 static struct tracer_flags dummy_tracer_flags = {
72         .val = 0,
73         .opts = dummy_tracer_opt
74 };
75
76 static int dummy_set_flag(u32 old_flags, u32 bit, int set)
77 {
78         return 0;
79 }
80
81 /*
82  * To prevent the comm cache from being overwritten when no
83  * tracing is active, only save the comm when a trace event
84  * occurred.
85  */
86 static DEFINE_PER_CPU(bool, trace_cmdline_save);
87
88 /*
89  * Kill all tracing for good (never come back).
90  * It is initialized to 1 but will turn to zero if the initialization
91  * of the tracer is successful. But that is the only place that sets
92  * this back to zero.
93  */
94 static int tracing_disabled = 1;
95
96 DEFINE_PER_CPU(int, ftrace_cpu_disabled);
97
98 cpumask_var_t __read_mostly     tracing_buffer_mask;
99
100 /*
101  * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
102  *
103  * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104  * is set, then ftrace_dump is called. This will output the contents
105  * of the ftrace buffers to the console.  This is very useful for
106  * capturing traces that lead to crashes and outputing it to a
107  * serial console.
108  *
109  * It is default off, but you can enable it with either specifying
110  * "ftrace_dump_on_oops" in the kernel command line, or setting
111  * /proc/sys/kernel/ftrace_dump_on_oops
112  * Set 1 if you want to dump buffers of all CPUs
113  * Set 2 if you want to dump the buffer of the CPU that triggered oops
114  */
115
116 enum ftrace_dump_mode ftrace_dump_on_oops;
117
118 static int tracing_set_tracer(const char *buf);
119
120 #define MAX_TRACER_SIZE         100
121 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
122 static char *default_bootup_tracer;
123
124 static bool allocate_snapshot;
125
126 static int __init set_cmdline_ftrace(char *str)
127 {
128         strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
129         default_bootup_tracer = bootup_tracer_buf;
130         /* We are using ftrace early, expand it */
131         ring_buffer_expanded = true;
132         return 1;
133 }
134 __setup("ftrace=", set_cmdline_ftrace);
135
136 static int __init set_ftrace_dump_on_oops(char *str)
137 {
138         if (*str++ != '=' || !*str) {
139                 ftrace_dump_on_oops = DUMP_ALL;
140                 return 1;
141         }
142
143         if (!strcmp("orig_cpu", str)) {
144                 ftrace_dump_on_oops = DUMP_ORIG;
145                 return 1;
146         }
147
148         return 0;
149 }
150 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
151
152 static int __init boot_alloc_snapshot(char *str)
153 {
154         allocate_snapshot = true;
155         /* We also need the main ring buffer expanded */
156         ring_buffer_expanded = true;
157         return 1;
158 }
159 __setup("alloc_snapshot", boot_alloc_snapshot);
160
161
162 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
163 static char *trace_boot_options __initdata;
164
165 static int __init set_trace_boot_options(char *str)
166 {
167         strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
168         trace_boot_options = trace_boot_options_buf;
169         return 0;
170 }
171 __setup("trace_options=", set_trace_boot_options);
172
173 unsigned long long ns2usecs(cycle_t nsec)
174 {
175         nsec += 500;
176         do_div(nsec, 1000);
177         return nsec;
178 }
179
180 /*
181  * The global_trace is the descriptor that holds the tracing
182  * buffers for the live tracing. For each CPU, it contains
183  * a link list of pages that will store trace entries. The
184  * page descriptor of the pages in the memory is used to hold
185  * the link list by linking the lru item in the page descriptor
186  * to each of the pages in the buffer per CPU.
187  *
188  * For each active CPU there is a data field that holds the
189  * pages for the buffer for that CPU. Each CPU has the same number
190  * of pages allocated for its buffer.
191  */
192 static struct trace_array       global_trace;
193
194 LIST_HEAD(ftrace_trace_arrays);
195
196 int filter_current_check_discard(struct ring_buffer *buffer,
197                                  struct ftrace_event_call *call, void *rec,
198                                  struct ring_buffer_event *event)
199 {
200         return filter_check_discard(call, rec, buffer, event);
201 }
202 EXPORT_SYMBOL_GPL(filter_current_check_discard);
203
204 cycle_t ftrace_now(int cpu)
205 {
206         u64 ts;
207
208         /* Early boot up does not have a buffer yet */
209         if (!global_trace.trace_buffer.buffer)
210                 return trace_clock_local();
211
212         ts = ring_buffer_time_stamp(global_trace.trace_buffer.buffer, cpu);
213         ring_buffer_normalize_time_stamp(global_trace.trace_buffer.buffer, cpu, &ts);
214
215         return ts;
216 }
217
218 int tracing_is_enabled(void)
219 {
220         return tracing_is_on();
221 }
222
223 /*
224  * trace_buf_size is the size in bytes that is allocated
225  * for a buffer. Note, the number of bytes is always rounded
226  * to page size.
227  *
228  * This number is purposely set to a low number of 16384.
229  * If the dump on oops happens, it will be much appreciated
230  * to not have to wait for all that output. Anyway this can be
231  * boot time and run time configurable.
232  */
233 #define TRACE_BUF_SIZE_DEFAULT  1441792UL /* 16384 * 88 (sizeof(entry)) */
234
235 static unsigned long            trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
236
237 /* trace_types holds a link list of available tracers. */
238 static struct tracer            *trace_types __read_mostly;
239
240 /*
241  * trace_types_lock is used to protect the trace_types list.
242  */
243 static DEFINE_MUTEX(trace_types_lock);
244
245 /*
246  * serialize the access of the ring buffer
247  *
248  * ring buffer serializes readers, but it is low level protection.
249  * The validity of the events (which returns by ring_buffer_peek() ..etc)
250  * are not protected by ring buffer.
251  *
252  * The content of events may become garbage if we allow other process consumes
253  * these events concurrently:
254  *   A) the page of the consumed events may become a normal page
255  *      (not reader page) in ring buffer, and this page will be rewrited
256  *      by events producer.
257  *   B) The page of the consumed events may become a page for splice_read,
258  *      and this page will be returned to system.
259  *
260  * These primitives allow multi process access to different cpu ring buffer
261  * concurrently.
262  *
263  * These primitives don't distinguish read-only and read-consume access.
264  * Multi read-only access are also serialized.
265  */
266
267 #ifdef CONFIG_SMP
268 static DECLARE_RWSEM(all_cpu_access_lock);
269 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
270
271 static inline void trace_access_lock(int cpu)
272 {
273         if (cpu == RING_BUFFER_ALL_CPUS) {
274                 /* gain it for accessing the whole ring buffer. */
275                 down_write(&all_cpu_access_lock);
276         } else {
277                 /* gain it for accessing a cpu ring buffer. */
278
279                 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
280                 down_read(&all_cpu_access_lock);
281
282                 /* Secondly block other access to this @cpu ring buffer. */
283                 mutex_lock(&per_cpu(cpu_access_lock, cpu));
284         }
285 }
286
287 static inline void trace_access_unlock(int cpu)
288 {
289         if (cpu == RING_BUFFER_ALL_CPUS) {
290                 up_write(&all_cpu_access_lock);
291         } else {
292                 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
293                 up_read(&all_cpu_access_lock);
294         }
295 }
296
297 static inline void trace_access_lock_init(void)
298 {
299         int cpu;
300
301         for_each_possible_cpu(cpu)
302                 mutex_init(&per_cpu(cpu_access_lock, cpu));
303 }
304
305 #else
306
307 static DEFINE_MUTEX(access_lock);
308
309 static inline void trace_access_lock(int cpu)
310 {
311         (void)cpu;
312         mutex_lock(&access_lock);
313 }
314
315 static inline void trace_access_unlock(int cpu)
316 {
317         (void)cpu;
318         mutex_unlock(&access_lock);
319 }
320
321 static inline void trace_access_lock_init(void)
322 {
323 }
324
325 #endif
326
327 /* trace_flags holds trace_options default values */
328 unsigned long trace_flags = TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK |
329         TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | TRACE_ITER_SLEEP_TIME |
330         TRACE_ITER_GRAPH_TIME | TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE |
331         TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS | TRACE_ITER_FUNCTION;
332
333 /**
334  * tracing_on - enable tracing buffers
335  *
336  * This function enables tracing buffers that may have been
337  * disabled with tracing_off.
338  */
339 void tracing_on(void)
340 {
341         if (global_trace.trace_buffer.buffer)
342                 ring_buffer_record_on(global_trace.trace_buffer.buffer);
343         /*
344          * This flag is only looked at when buffers haven't been
345          * allocated yet. We don't really care about the race
346          * between setting this flag and actually turning
347          * on the buffer.
348          */
349         global_trace.buffer_disabled = 0;
350 }
351 EXPORT_SYMBOL_GPL(tracing_on);
352
353 /**
354  * __trace_puts - write a constant string into the trace buffer.
355  * @ip:    The address of the caller
356  * @str:   The constant string to write
357  * @size:  The size of the string.
358  */
359 int __trace_puts(unsigned long ip, const char *str, int size)
360 {
361         struct ring_buffer_event *event;
362         struct ring_buffer *buffer;
363         struct print_entry *entry;
364         unsigned long irq_flags;
365         int alloc;
366
367         alloc = sizeof(*entry) + size + 2; /* possible \n added */
368
369         local_save_flags(irq_flags);
370         buffer = global_trace.trace_buffer.buffer;
371         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc, 
372                                           irq_flags, preempt_count());
373         if (!event)
374                 return 0;
375
376         entry = ring_buffer_event_data(event);
377         entry->ip = ip;
378
379         memcpy(&entry->buf, str, size);
380
381         /* Add a newline if necessary */
382         if (entry->buf[size - 1] != '\n') {
383                 entry->buf[size] = '\n';
384                 entry->buf[size + 1] = '\0';
385         } else
386                 entry->buf[size] = '\0';
387
388         __buffer_unlock_commit(buffer, event);
389
390         return size;
391 }
392 EXPORT_SYMBOL_GPL(__trace_puts);
393
394 /**
395  * __trace_bputs - write the pointer to a constant string into trace buffer
396  * @ip:    The address of the caller
397  * @str:   The constant string to write to the buffer to
398  */
399 int __trace_bputs(unsigned long ip, const char *str)
400 {
401         struct ring_buffer_event *event;
402         struct ring_buffer *buffer;
403         struct bputs_entry *entry;
404         unsigned long irq_flags;
405         int size = sizeof(struct bputs_entry);
406
407         local_save_flags(irq_flags);
408         buffer = global_trace.trace_buffer.buffer;
409         event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
410                                           irq_flags, preempt_count());
411         if (!event)
412                 return 0;
413
414         entry = ring_buffer_event_data(event);
415         entry->ip                       = ip;
416         entry->str                      = str;
417
418         __buffer_unlock_commit(buffer, event);
419
420         return 1;
421 }
422 EXPORT_SYMBOL_GPL(__trace_bputs);
423
424 #ifdef CONFIG_TRACER_SNAPSHOT
425 /**
426  * trace_snapshot - take a snapshot of the current buffer.
427  *
428  * This causes a swap between the snapshot buffer and the current live
429  * tracing buffer. You can use this to take snapshots of the live
430  * trace when some condition is triggered, but continue to trace.
431  *
432  * Note, make sure to allocate the snapshot with either
433  * a tracing_snapshot_alloc(), or by doing it manually
434  * with: echo 1 > /sys/kernel/debug/tracing/snapshot
435  *
436  * If the snapshot buffer is not allocated, it will stop tracing.
437  * Basically making a permanent snapshot.
438  */
439 void tracing_snapshot(void)
440 {
441         struct trace_array *tr = &global_trace;
442         struct tracer *tracer = tr->current_trace;
443         unsigned long flags;
444
445         if (in_nmi()) {
446                 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
447                 internal_trace_puts("*** snapshot is being ignored        ***\n");
448                 return;
449         }
450
451         if (!tr->allocated_snapshot) {
452                 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
453                 internal_trace_puts("*** stopping trace here!   ***\n");
454                 tracing_off();
455                 return;
456         }
457
458         /* Note, snapshot can not be used when the tracer uses it */
459         if (tracer->use_max_tr) {
460                 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
461                 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
462                 return;
463         }
464
465         local_irq_save(flags);
466         update_max_tr(tr, current, smp_processor_id());
467         local_irq_restore(flags);
468 }
469 EXPORT_SYMBOL_GPL(tracing_snapshot);
470
471 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
472                                         struct trace_buffer *size_buf, int cpu_id);
473 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
474
475 static int alloc_snapshot(struct trace_array *tr)
476 {
477         int ret;
478
479         if (!tr->allocated_snapshot) {
480
481                 /* allocate spare buffer */
482                 ret = resize_buffer_duplicate_size(&tr->max_buffer,
483                                    &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
484                 if (ret < 0)
485                         return ret;
486
487                 tr->allocated_snapshot = true;
488         }
489
490         return 0;
491 }
492
493 void free_snapshot(struct trace_array *tr)
494 {
495         /*
496          * We don't free the ring buffer. instead, resize it because
497          * The max_tr ring buffer has some state (e.g. ring->clock) and
498          * we want preserve it.
499          */
500         ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
501         set_buffer_entries(&tr->max_buffer, 1);
502         tracing_reset_online_cpus(&tr->max_buffer);
503         tr->allocated_snapshot = false;
504 }
505
506 /**
507  * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
508  *
509  * This is similar to trace_snapshot(), but it will allocate the
510  * snapshot buffer if it isn't already allocated. Use this only
511  * where it is safe to sleep, as the allocation may sleep.
512  *
513  * This causes a swap between the snapshot buffer and the current live
514  * tracing buffer. You can use this to take snapshots of the live
515  * trace when some condition is triggered, but continue to trace.
516  */
517 void tracing_snapshot_alloc(void)
518 {
519         struct trace_array *tr = &global_trace;
520         int ret;
521
522         ret = alloc_snapshot(tr);
523         if (WARN_ON(ret < 0))
524                 return;
525
526         tracing_snapshot();
527 }
528 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
529 #else
530 void tracing_snapshot(void)
531 {
532         WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
533 }
534 EXPORT_SYMBOL_GPL(tracing_snapshot);
535 void tracing_snapshot_alloc(void)
536 {
537         /* Give warning */
538         tracing_snapshot();
539 }
540 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
541 #endif /* CONFIG_TRACER_SNAPSHOT */
542
543 /**
544  * tracing_off - turn off tracing buffers
545  *
546  * This function stops the tracing buffers from recording data.
547  * It does not disable any overhead the tracers themselves may
548  * be causing. This function simply causes all recording to
549  * the ring buffers to fail.
550  */
551 void tracing_off(void)
552 {
553         if (global_trace.trace_buffer.buffer)
554                 ring_buffer_record_off(global_trace.trace_buffer.buffer);
555         /*
556          * This flag is only looked at when buffers haven't been
557          * allocated yet. We don't really care about the race
558          * between setting this flag and actually turning
559          * on the buffer.
560          */
561         global_trace.buffer_disabled = 1;
562 }
563 EXPORT_SYMBOL_GPL(tracing_off);
564
565 /**
566  * tracing_is_on - show state of ring buffers enabled
567  */
568 int tracing_is_on(void)
569 {
570         if (global_trace.trace_buffer.buffer)
571                 return ring_buffer_record_is_on(global_trace.trace_buffer.buffer);
572         return !global_trace.buffer_disabled;
573 }
574 EXPORT_SYMBOL_GPL(tracing_is_on);
575
576 static int __init set_buf_size(char *str)
577 {
578         unsigned long buf_size;
579
580         if (!str)
581                 return 0;
582         buf_size = memparse(str, &str);
583         /* nr_entries can not be zero */
584         if (buf_size == 0)
585                 return 0;
586         trace_buf_size = buf_size;
587         return 1;
588 }
589 __setup("trace_buf_size=", set_buf_size);
590
591 static int __init set_tracing_thresh(char *str)
592 {
593         unsigned long threshold;
594         int ret;
595
596         if (!str)
597                 return 0;
598         ret = kstrtoul(str, 0, &threshold);
599         if (ret < 0)
600                 return 0;
601         tracing_thresh = threshold * 1000;
602         return 1;
603 }
604 __setup("tracing_thresh=", set_tracing_thresh);
605
606 unsigned long nsecs_to_usecs(unsigned long nsecs)
607 {
608         return nsecs / 1000;
609 }
610
611 /* These must match the bit postions in trace_iterator_flags */
612 static const char *trace_options[] = {
613         "print-parent",
614         "sym-offset",
615         "sym-addr",
616         "verbose",
617         "raw",
618         "hex",
619         "bin",
620         "block",
621         "stacktrace",
622         "trace_printk",
623         "ftrace_preempt",
624         "branch",
625         "annotate",
626         "userstacktrace",
627         "sym-userobj",
628         "printk-msg-only",
629         "context-info",
630         "latency-format",
631         "sleep-time",
632         "graph-time",
633         "record-cmd",
634         "overwrite",
635         "disable_on_free",
636         "irq-info",
637         "markers",
638         "function-trace",
639         NULL
640 };
641
642 static struct {
643         u64 (*func)(void);
644         const char *name;
645         int in_ns;              /* is this clock in nanoseconds? */
646 } trace_clocks[] = {
647         { trace_clock_local,    "local",        1 },
648         { trace_clock_global,   "global",       1 },
649         { trace_clock_counter,  "counter",      0 },
650         { trace_clock_jiffies,  "uptime",       1 },
651         { trace_clock,          "perf",         1 },
652         ARCH_TRACE_CLOCKS
653 };
654
655 int trace_clock_id;
656
657 /*
658  * trace_parser_get_init - gets the buffer for trace parser
659  */
660 int trace_parser_get_init(struct trace_parser *parser, int size)
661 {
662         memset(parser, 0, sizeof(*parser));
663
664         parser->buffer = kmalloc(size, GFP_KERNEL);
665         if (!parser->buffer)
666                 return 1;
667
668         parser->size = size;
669         return 0;
670 }
671
672 /*
673  * trace_parser_put - frees the buffer for trace parser
674  */
675 void trace_parser_put(struct trace_parser *parser)
676 {
677         kfree(parser->buffer);
678 }
679
680 /*
681  * trace_get_user - reads the user input string separated by  space
682  * (matched by isspace(ch))
683  *
684  * For each string found the 'struct trace_parser' is updated,
685  * and the function returns.
686  *
687  * Returns number of bytes read.
688  *
689  * See kernel/trace/trace.h for 'struct trace_parser' details.
690  */
691 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
692         size_t cnt, loff_t *ppos)
693 {
694         char ch;
695         size_t read = 0;
696         ssize_t ret;
697
698         if (!*ppos)
699                 trace_parser_clear(parser);
700
701         ret = get_user(ch, ubuf++);
702         if (ret)
703                 goto out;
704
705         read++;
706         cnt--;
707
708         /*
709          * The parser is not finished with the last write,
710          * continue reading the user input without skipping spaces.
711          */
712         if (!parser->cont) {
713                 /* skip white space */
714                 while (cnt && isspace(ch)) {
715                         ret = get_user(ch, ubuf++);
716                         if (ret)
717                                 goto out;
718                         read++;
719                         cnt--;
720                 }
721
722                 /* only spaces were written */
723                 if (isspace(ch)) {
724                         *ppos += read;
725                         ret = read;
726                         goto out;
727                 }
728
729                 parser->idx = 0;
730         }
731
732         /* read the non-space input */
733         while (cnt && !isspace(ch)) {
734                 if (parser->idx < parser->size - 1)
735                         parser->buffer[parser->idx++] = ch;
736                 else {
737                         ret = -EINVAL;
738                         goto out;
739                 }
740                 ret = get_user(ch, ubuf++);
741                 if (ret)
742                         goto out;
743                 read++;
744                 cnt--;
745         }
746
747         /* We either got finished input or we have to wait for another call. */
748         if (isspace(ch)) {
749                 parser->buffer[parser->idx] = 0;
750                 parser->cont = false;
751         } else {
752                 parser->cont = true;
753                 parser->buffer[parser->idx++] = ch;
754         }
755
756         *ppos += read;
757         ret = read;
758
759 out:
760         return ret;
761 }
762
763 ssize_t trace_seq_to_user(struct trace_seq *s, char __user *ubuf, size_t cnt)
764 {
765         int len;
766         int ret;
767
768         if (!cnt)
769                 return 0;
770
771         if (s->len <= s->readpos)
772                 return -EBUSY;
773
774         len = s->len - s->readpos;
775         if (cnt > len)
776                 cnt = len;
777         ret = copy_to_user(ubuf, s->buffer + s->readpos, cnt);
778         if (ret == cnt)
779                 return -EFAULT;
780
781         cnt -= ret;
782
783         s->readpos += cnt;
784         return cnt;
785 }
786
787 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
788 {
789         int len;
790
791         if (s->len <= s->readpos)
792                 return -EBUSY;
793
794         len = s->len - s->readpos;
795         if (cnt > len)
796                 cnt = len;
797         memcpy(buf, s->buffer + s->readpos, cnt);
798
799         s->readpos += cnt;
800         return cnt;
801 }
802
803 /*
804  * ftrace_max_lock is used to protect the swapping of buffers
805  * when taking a max snapshot. The buffers themselves are
806  * protected by per_cpu spinlocks. But the action of the swap
807  * needs its own lock.
808  *
809  * This is defined as a arch_spinlock_t in order to help
810  * with performance when lockdep debugging is enabled.
811  *
812  * It is also used in other places outside the update_max_tr
813  * so it needs to be defined outside of the
814  * CONFIG_TRACER_MAX_TRACE.
815  */
816 static arch_spinlock_t ftrace_max_lock =
817         (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
818
819 unsigned long __read_mostly     tracing_thresh;
820
821 #ifdef CONFIG_TRACER_MAX_TRACE
822 unsigned long __read_mostly     tracing_max_latency;
823
824 /*
825  * Copy the new maximum trace into the separate maximum-trace
826  * structure. (this way the maximum trace is permanently saved,
827  * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
828  */
829 static void
830 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
831 {
832         struct trace_buffer *trace_buf = &tr->trace_buffer;
833         struct trace_buffer *max_buf = &tr->max_buffer;
834         struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
835         struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
836
837         max_buf->cpu = cpu;
838         max_buf->time_start = data->preempt_timestamp;
839
840         max_data->saved_latency = tracing_max_latency;
841         max_data->critical_start = data->critical_start;
842         max_data->critical_end = data->critical_end;
843
844         memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
845         max_data->pid = tsk->pid;
846         max_data->uid = task_uid(tsk);
847         max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
848         max_data->policy = tsk->policy;
849         max_data->rt_priority = tsk->rt_priority;
850
851         /* record this tasks comm */
852         tracing_record_cmdline(tsk);
853 }
854
855 /**
856  * update_max_tr - snapshot all trace buffers from global_trace to max_tr
857  * @tr: tracer
858  * @tsk: the task with the latency
859  * @cpu: The cpu that initiated the trace.
860  *
861  * Flip the buffers between the @tr and the max_tr and record information
862  * about which task was the cause of this latency.
863  */
864 void
865 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
866 {
867         struct ring_buffer *buf;
868
869         if (tr->stop_count)
870                 return;
871
872         WARN_ON_ONCE(!irqs_disabled());
873
874         if (!tr->allocated_snapshot) {
875                 /* Only the nop tracer should hit this when disabling */
876                 WARN_ON_ONCE(tr->current_trace != &nop_trace);
877                 return;
878         }
879
880         arch_spin_lock(&ftrace_max_lock);
881
882         buf = tr->trace_buffer.buffer;
883         tr->trace_buffer.buffer = tr->max_buffer.buffer;
884         tr->max_buffer.buffer = buf;
885
886         __update_max_tr(tr, tsk, cpu);
887         arch_spin_unlock(&ftrace_max_lock);
888 }
889
890 /**
891  * update_max_tr_single - only copy one trace over, and reset the rest
892  * @tr - tracer
893  * @tsk - task with the latency
894  * @cpu - the cpu of the buffer to copy.
895  *
896  * Flip the trace of a single CPU buffer between the @tr and the max_tr.
897  */
898 void
899 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
900 {
901         int ret;
902
903         if (tr->stop_count)
904                 return;
905
906         WARN_ON_ONCE(!irqs_disabled());
907         if (!tr->allocated_snapshot) {
908                 /* Only the nop tracer should hit this when disabling */
909                 WARN_ON_ONCE(tr->current_trace != &nop_trace);
910                 return;
911         }
912
913         arch_spin_lock(&ftrace_max_lock);
914
915         ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
916
917         if (ret == -EBUSY) {
918                 /*
919                  * We failed to swap the buffer due to a commit taking
920                  * place on this CPU. We fail to record, but we reset
921                  * the max trace buffer (no one writes directly to it)
922                  * and flag that it failed.
923                  */
924                 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
925                         "Failed to swap buffers due to commit in progress\n");
926         }
927
928         WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
929
930         __update_max_tr(tr, tsk, cpu);
931         arch_spin_unlock(&ftrace_max_lock);
932 }
933 #endif /* CONFIG_TRACER_MAX_TRACE */
934
935 static void default_wait_pipe(struct trace_iterator *iter)
936 {
937         /* Iterators are static, they should be filled or empty */
938         if (trace_buffer_iter(iter, iter->cpu_file))
939                 return;
940
941         ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file);
942 }
943
944 #ifdef CONFIG_FTRACE_STARTUP_TEST
945 static int run_tracer_selftest(struct tracer *type)
946 {
947         struct trace_array *tr = &global_trace;
948         struct tracer *saved_tracer = tr->current_trace;
949         int ret;
950
951         if (!type->selftest || tracing_selftest_disabled)
952                 return 0;
953
954         /*
955          * Run a selftest on this tracer.
956          * Here we reset the trace buffer, and set the current
957          * tracer to be this tracer. The tracer can then run some
958          * internal tracing to verify that everything is in order.
959          * If we fail, we do not register this tracer.
960          */
961         tracing_reset_online_cpus(&tr->trace_buffer);
962
963         tr->current_trace = type;
964
965 #ifdef CONFIG_TRACER_MAX_TRACE
966         if (type->use_max_tr) {
967                 /* If we expanded the buffers, make sure the max is expanded too */
968                 if (ring_buffer_expanded)
969                         ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
970                                            RING_BUFFER_ALL_CPUS);
971                 tr->allocated_snapshot = true;
972         }
973 #endif
974
975         /* the test is responsible for initializing and enabling */
976         pr_info("Testing tracer %s: ", type->name);
977         ret = type->selftest(type, tr);
978         /* the test is responsible for resetting too */
979         tr->current_trace = saved_tracer;
980         if (ret) {
981                 printk(KERN_CONT "FAILED!\n");
982                 /* Add the warning after printing 'FAILED' */
983                 WARN_ON(1);
984                 return -1;
985         }
986         /* Only reset on passing, to avoid touching corrupted buffers */
987         tracing_reset_online_cpus(&tr->trace_buffer);
988
989 #ifdef CONFIG_TRACER_MAX_TRACE
990         if (type->use_max_tr) {
991                 tr->allocated_snapshot = false;
992
993                 /* Shrink the max buffer again */
994                 if (ring_buffer_expanded)
995                         ring_buffer_resize(tr->max_buffer.buffer, 1,
996                                            RING_BUFFER_ALL_CPUS);
997         }
998 #endif
999
1000         printk(KERN_CONT "PASSED\n");
1001         return 0;
1002 }
1003 #else
1004 static inline int run_tracer_selftest(struct tracer *type)
1005 {
1006         return 0;
1007 }
1008 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1009
1010 /**
1011  * register_tracer - register a tracer with the ftrace system.
1012  * @type - the plugin for the tracer
1013  *
1014  * Register a new plugin tracer.
1015  */
1016 int register_tracer(struct tracer *type)
1017 {
1018         struct tracer *t;
1019         int ret = 0;
1020
1021         if (!type->name) {
1022                 pr_info("Tracer must have a name\n");
1023                 return -1;
1024         }
1025
1026         if (strlen(type->name) >= MAX_TRACER_SIZE) {
1027                 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1028                 return -1;
1029         }
1030
1031         mutex_lock(&trace_types_lock);
1032
1033         tracing_selftest_running = true;
1034
1035         for (t = trace_types; t; t = t->next) {
1036                 if (strcmp(type->name, t->name) == 0) {
1037                         /* already found */
1038                         pr_info("Tracer %s already registered\n",
1039                                 type->name);
1040                         ret = -1;
1041                         goto out;
1042                 }
1043         }
1044
1045         if (!type->set_flag)
1046                 type->set_flag = &dummy_set_flag;
1047         if (!type->flags)
1048                 type->flags = &dummy_tracer_flags;
1049         else
1050                 if (!type->flags->opts)
1051                         type->flags->opts = dummy_tracer_opt;
1052         if (!type->wait_pipe)
1053                 type->wait_pipe = default_wait_pipe;
1054
1055         ret = run_tracer_selftest(type);
1056         if (ret < 0)
1057                 goto out;
1058
1059         type->next = trace_types;
1060         trace_types = type;
1061
1062  out:
1063         tracing_selftest_running = false;
1064         mutex_unlock(&trace_types_lock);
1065
1066         if (ret || !default_bootup_tracer)
1067                 goto out_unlock;
1068
1069         if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1070                 goto out_unlock;
1071
1072         printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1073         /* Do we want this tracer to start on bootup? */
1074         tracing_set_tracer(type->name);
1075         default_bootup_tracer = NULL;
1076         /* disable other selftests, since this will break it. */
1077         tracing_selftest_disabled = true;
1078 #ifdef CONFIG_FTRACE_STARTUP_TEST
1079         printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1080                type->name);
1081 #endif
1082
1083  out_unlock:
1084         return ret;
1085 }
1086
1087 void tracing_reset(struct trace_buffer *buf, int cpu)
1088 {
1089         struct ring_buffer *buffer = buf->buffer;
1090
1091         if (!buffer)
1092                 return;
1093
1094         ring_buffer_record_disable(buffer);
1095
1096         /* Make sure all commits have finished */
1097         synchronize_sched();
1098         ring_buffer_reset_cpu(buffer, cpu);
1099
1100         ring_buffer_record_enable(buffer);
1101 }
1102
1103 void tracing_reset_online_cpus(struct trace_buffer *buf)
1104 {
1105         struct ring_buffer *buffer = buf->buffer;
1106         int cpu;
1107
1108         if (!buffer)
1109                 return;
1110
1111         ring_buffer_record_disable(buffer);
1112
1113         /* Make sure all commits have finished */
1114         synchronize_sched();
1115
1116         buf->time_start = ftrace_now(buf->cpu);
1117
1118         for_each_online_cpu(cpu)
1119                 ring_buffer_reset_cpu(buffer, cpu);
1120
1121         ring_buffer_record_enable(buffer);
1122 }
1123
1124 void tracing_reset_current(int cpu)
1125 {
1126         tracing_reset(&global_trace.trace_buffer, cpu);
1127 }
1128
1129 void tracing_reset_all_online_cpus(void)
1130 {
1131         struct trace_array *tr;
1132
1133         mutex_lock(&trace_types_lock);
1134         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1135                 tracing_reset_online_cpus(&tr->trace_buffer);
1136 #ifdef CONFIG_TRACER_MAX_TRACE
1137                 tracing_reset_online_cpus(&tr->max_buffer);
1138 #endif
1139         }
1140         mutex_unlock(&trace_types_lock);
1141 }
1142
1143 #define SAVED_CMDLINES 128
1144 #define NO_CMDLINE_MAP UINT_MAX
1145 static unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1146 static unsigned map_cmdline_to_pid[SAVED_CMDLINES];
1147 static char saved_cmdlines[SAVED_CMDLINES][TASK_COMM_LEN];
1148 static int cmdline_idx;
1149 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1150
1151 /* temporary disable recording */
1152 static atomic_t trace_record_cmdline_disabled __read_mostly;
1153
1154 static void trace_init_cmdlines(void)
1155 {
1156         memset(&map_pid_to_cmdline, NO_CMDLINE_MAP, sizeof(map_pid_to_cmdline));
1157         memset(&map_cmdline_to_pid, NO_CMDLINE_MAP, sizeof(map_cmdline_to_pid));
1158         cmdline_idx = 0;
1159 }
1160
1161 int is_tracing_stopped(void)
1162 {
1163         return global_trace.stop_count;
1164 }
1165
1166 /**
1167  * ftrace_off_permanent - disable all ftrace code permanently
1168  *
1169  * This should only be called when a serious anomally has
1170  * been detected.  This will turn off the function tracing,
1171  * ring buffers, and other tracing utilites. It takes no
1172  * locks and can be called from any context.
1173  */
1174 void ftrace_off_permanent(void)
1175 {
1176         tracing_disabled = 1;
1177         ftrace_stop();
1178         tracing_off_permanent();
1179 }
1180
1181 /**
1182  * tracing_start - quick start of the tracer
1183  *
1184  * If tracing is enabled but was stopped by tracing_stop,
1185  * this will start the tracer back up.
1186  */
1187 void tracing_start(void)
1188 {
1189         struct ring_buffer *buffer;
1190         unsigned long flags;
1191
1192         if (tracing_disabled)
1193                 return;
1194
1195         raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1196         if (--global_trace.stop_count) {
1197                 if (global_trace.stop_count < 0) {
1198                         /* Someone screwed up their debugging */
1199                         WARN_ON_ONCE(1);
1200                         global_trace.stop_count = 0;
1201                 }
1202                 goto out;
1203         }
1204
1205         /* Prevent the buffers from switching */
1206         arch_spin_lock(&ftrace_max_lock);
1207
1208         buffer = global_trace.trace_buffer.buffer;
1209         if (buffer)
1210                 ring_buffer_record_enable(buffer);
1211
1212 #ifdef CONFIG_TRACER_MAX_TRACE
1213         buffer = global_trace.max_buffer.buffer;
1214         if (buffer)
1215                 ring_buffer_record_enable(buffer);
1216 #endif
1217
1218         arch_spin_unlock(&ftrace_max_lock);
1219
1220         ftrace_start();
1221  out:
1222         raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1223 }
1224
1225 static void tracing_start_tr(struct trace_array *tr)
1226 {
1227         struct ring_buffer *buffer;
1228         unsigned long flags;
1229
1230         if (tracing_disabled)
1231                 return;
1232
1233         /* If global, we need to also start the max tracer */
1234         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1235                 return tracing_start();
1236
1237         raw_spin_lock_irqsave(&tr->start_lock, flags);
1238
1239         if (--tr->stop_count) {
1240                 if (tr->stop_count < 0) {
1241                         /* Someone screwed up their debugging */
1242                         WARN_ON_ONCE(1);
1243                         tr->stop_count = 0;
1244                 }
1245                 goto out;
1246         }
1247
1248         buffer = tr->trace_buffer.buffer;
1249         if (buffer)
1250                 ring_buffer_record_enable(buffer);
1251
1252  out:
1253         raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1254 }
1255
1256 /**
1257  * tracing_stop - quick stop of the tracer
1258  *
1259  * Light weight way to stop tracing. Use in conjunction with
1260  * tracing_start.
1261  */
1262 void tracing_stop(void)
1263 {
1264         struct ring_buffer *buffer;
1265         unsigned long flags;
1266
1267         ftrace_stop();
1268         raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1269         if (global_trace.stop_count++)
1270                 goto out;
1271
1272         /* Prevent the buffers from switching */
1273         arch_spin_lock(&ftrace_max_lock);
1274
1275         buffer = global_trace.trace_buffer.buffer;
1276         if (buffer)
1277                 ring_buffer_record_disable(buffer);
1278
1279 #ifdef CONFIG_TRACER_MAX_TRACE
1280         buffer = global_trace.max_buffer.buffer;
1281         if (buffer)
1282                 ring_buffer_record_disable(buffer);
1283 #endif
1284
1285         arch_spin_unlock(&ftrace_max_lock);
1286
1287  out:
1288         raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1289 }
1290
1291 static void tracing_stop_tr(struct trace_array *tr)
1292 {
1293         struct ring_buffer *buffer;
1294         unsigned long flags;
1295
1296         /* If global, we need to also stop the max tracer */
1297         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1298                 return tracing_stop();
1299
1300         raw_spin_lock_irqsave(&tr->start_lock, flags);
1301         if (tr->stop_count++)
1302                 goto out;
1303
1304         buffer = tr->trace_buffer.buffer;
1305         if (buffer)
1306                 ring_buffer_record_disable(buffer);
1307
1308  out:
1309         raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1310 }
1311
1312 void trace_stop_cmdline_recording(void);
1313
1314 static void trace_save_cmdline(struct task_struct *tsk)
1315 {
1316         unsigned pid, idx;
1317
1318         if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1319                 return;
1320
1321         /*
1322          * It's not the end of the world if we don't get
1323          * the lock, but we also don't want to spin
1324          * nor do we want to disable interrupts,
1325          * so if we miss here, then better luck next time.
1326          */
1327         if (!arch_spin_trylock(&trace_cmdline_lock))
1328                 return;
1329
1330         idx = map_pid_to_cmdline[tsk->pid];
1331         if (idx == NO_CMDLINE_MAP) {
1332                 idx = (cmdline_idx + 1) % SAVED_CMDLINES;
1333
1334                 /*
1335                  * Check whether the cmdline buffer at idx has a pid
1336                  * mapped. We are going to overwrite that entry so we
1337                  * need to clear the map_pid_to_cmdline. Otherwise we
1338                  * would read the new comm for the old pid.
1339                  */
1340                 pid = map_cmdline_to_pid[idx];
1341                 if (pid != NO_CMDLINE_MAP)
1342                         map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1343
1344                 map_cmdline_to_pid[idx] = tsk->pid;
1345                 map_pid_to_cmdline[tsk->pid] = idx;
1346
1347                 cmdline_idx = idx;
1348         }
1349
1350         memcpy(&saved_cmdlines[idx], tsk->comm, TASK_COMM_LEN);
1351
1352         arch_spin_unlock(&trace_cmdline_lock);
1353 }
1354
1355 void trace_find_cmdline(int pid, char comm[])
1356 {
1357         unsigned map;
1358
1359         if (!pid) {
1360                 strcpy(comm, "<idle>");
1361                 return;
1362         }
1363
1364         if (WARN_ON_ONCE(pid < 0)) {
1365                 strcpy(comm, "<XXX>");
1366                 return;
1367         }
1368
1369         if (pid > PID_MAX_DEFAULT) {
1370                 strcpy(comm, "<...>");
1371                 return;
1372         }
1373
1374         preempt_disable();
1375         arch_spin_lock(&trace_cmdline_lock);
1376         map = map_pid_to_cmdline[pid];
1377         if (map != NO_CMDLINE_MAP)
1378                 strcpy(comm, saved_cmdlines[map]);
1379         else
1380                 strcpy(comm, "<...>");
1381
1382         arch_spin_unlock(&trace_cmdline_lock);
1383         preempt_enable();
1384 }
1385
1386 void tracing_record_cmdline(struct task_struct *tsk)
1387 {
1388         if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1389                 return;
1390
1391         if (!__this_cpu_read(trace_cmdline_save))
1392                 return;
1393
1394         __this_cpu_write(trace_cmdline_save, false);
1395
1396         trace_save_cmdline(tsk);
1397 }
1398
1399 void
1400 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1401                              int pc)
1402 {
1403         struct task_struct *tsk = current;
1404
1405         entry->preempt_count            = pc & 0xff;
1406         entry->pid                      = (tsk) ? tsk->pid : 0;
1407         entry->flags =
1408 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1409                 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1410 #else
1411                 TRACE_FLAG_IRQS_NOSUPPORT |
1412 #endif
1413                 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1414                 ((pc & SOFTIRQ_MASK) ? TRACE_FLAG_SOFTIRQ : 0) |
1415                 (need_resched() ? TRACE_FLAG_NEED_RESCHED : 0);
1416 }
1417 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1418
1419 struct ring_buffer_event *
1420 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1421                           int type,
1422                           unsigned long len,
1423                           unsigned long flags, int pc)
1424 {
1425         struct ring_buffer_event *event;
1426
1427         event = ring_buffer_lock_reserve(buffer, len);
1428         if (event != NULL) {
1429                 struct trace_entry *ent = ring_buffer_event_data(event);
1430
1431                 tracing_generic_entry_update(ent, flags, pc);
1432                 ent->type = type;
1433         }
1434
1435         return event;
1436 }
1437
1438 void
1439 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
1440 {
1441         __this_cpu_write(trace_cmdline_save, true);
1442         ring_buffer_unlock_commit(buffer, event);
1443 }
1444
1445 static inline void
1446 __trace_buffer_unlock_commit(struct ring_buffer *buffer,
1447                              struct ring_buffer_event *event,
1448                              unsigned long flags, int pc)
1449 {
1450         __buffer_unlock_commit(buffer, event);
1451
1452         ftrace_trace_stack(buffer, flags, 6, pc);
1453         ftrace_trace_userstack(buffer, flags, pc);
1454 }
1455
1456 void trace_buffer_unlock_commit(struct ring_buffer *buffer,
1457                                 struct ring_buffer_event *event,
1458                                 unsigned long flags, int pc)
1459 {
1460         __trace_buffer_unlock_commit(buffer, event, flags, pc);
1461 }
1462 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit);
1463
1464 struct ring_buffer_event *
1465 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
1466                           struct ftrace_event_file *ftrace_file,
1467                           int type, unsigned long len,
1468                           unsigned long flags, int pc)
1469 {
1470         *current_rb = ftrace_file->tr->trace_buffer.buffer;
1471         return trace_buffer_lock_reserve(*current_rb,
1472                                          type, len, flags, pc);
1473 }
1474 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
1475
1476 struct ring_buffer_event *
1477 trace_current_buffer_lock_reserve(struct ring_buffer **current_rb,
1478                                   int type, unsigned long len,
1479                                   unsigned long flags, int pc)
1480 {
1481         *current_rb = global_trace.trace_buffer.buffer;
1482         return trace_buffer_lock_reserve(*current_rb,
1483                                          type, len, flags, pc);
1484 }
1485 EXPORT_SYMBOL_GPL(trace_current_buffer_lock_reserve);
1486
1487 void trace_current_buffer_unlock_commit(struct ring_buffer *buffer,
1488                                         struct ring_buffer_event *event,
1489                                         unsigned long flags, int pc)
1490 {
1491         __trace_buffer_unlock_commit(buffer, event, flags, pc);
1492 }
1493 EXPORT_SYMBOL_GPL(trace_current_buffer_unlock_commit);
1494
1495 void trace_buffer_unlock_commit_regs(struct ring_buffer *buffer,
1496                                      struct ring_buffer_event *event,
1497                                      unsigned long flags, int pc,
1498                                      struct pt_regs *regs)
1499 {
1500         __buffer_unlock_commit(buffer, event);
1501
1502         ftrace_trace_stack_regs(buffer, flags, 0, pc, regs);
1503         ftrace_trace_userstack(buffer, flags, pc);
1504 }
1505 EXPORT_SYMBOL_GPL(trace_buffer_unlock_commit_regs);
1506
1507 void trace_current_buffer_discard_commit(struct ring_buffer *buffer,
1508                                          struct ring_buffer_event *event)
1509 {
1510         ring_buffer_discard_commit(buffer, event);
1511 }
1512 EXPORT_SYMBOL_GPL(trace_current_buffer_discard_commit);
1513
1514 void
1515 trace_function(struct trace_array *tr,
1516                unsigned long ip, unsigned long parent_ip, unsigned long flags,
1517                int pc)
1518 {
1519         struct ftrace_event_call *call = &event_function;
1520         struct ring_buffer *buffer = tr->trace_buffer.buffer;
1521         struct ring_buffer_event *event;
1522         struct ftrace_entry *entry;
1523
1524         /* If we are reading the ring buffer, don't trace */
1525         if (unlikely(__this_cpu_read(ftrace_cpu_disabled)))
1526                 return;
1527
1528         event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
1529                                           flags, pc);
1530         if (!event)
1531                 return;
1532         entry   = ring_buffer_event_data(event);
1533         entry->ip                       = ip;
1534         entry->parent_ip                = parent_ip;
1535
1536         if (!filter_check_discard(call, entry, buffer, event))
1537                 __buffer_unlock_commit(buffer, event);
1538 }
1539
1540 void
1541 ftrace(struct trace_array *tr, struct trace_array_cpu *data,
1542        unsigned long ip, unsigned long parent_ip, unsigned long flags,
1543        int pc)
1544 {
1545         if (likely(!atomic_read(&data->disabled)))
1546                 trace_function(tr, ip, parent_ip, flags, pc);
1547 }
1548
1549 #ifdef CONFIG_STACKTRACE
1550
1551 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
1552 struct ftrace_stack {
1553         unsigned long           calls[FTRACE_STACK_MAX_ENTRIES];
1554 };
1555
1556 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
1557 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
1558
1559 static void __ftrace_trace_stack(struct ring_buffer *buffer,
1560                                  unsigned long flags,
1561                                  int skip, int pc, struct pt_regs *regs)
1562 {
1563         struct ftrace_event_call *call = &event_kernel_stack;
1564         struct ring_buffer_event *event;
1565         struct stack_entry *entry;
1566         struct stack_trace trace;
1567         int use_stack;
1568         int size = FTRACE_STACK_ENTRIES;
1569
1570         trace.nr_entries        = 0;
1571         trace.skip              = skip;
1572
1573         /*
1574          * Since events can happen in NMIs there's no safe way to
1575          * use the per cpu ftrace_stacks. We reserve it and if an interrupt
1576          * or NMI comes in, it will just have to use the default
1577          * FTRACE_STACK_SIZE.
1578          */
1579         preempt_disable_notrace();
1580
1581         use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
1582         /*
1583          * We don't need any atomic variables, just a barrier.
1584          * If an interrupt comes in, we don't care, because it would
1585          * have exited and put the counter back to what we want.
1586          * We just need a barrier to keep gcc from moving things
1587          * around.
1588          */
1589         barrier();
1590         if (use_stack == 1) {
1591                 trace.entries           = &__get_cpu_var(ftrace_stack).calls[0];
1592                 trace.max_entries       = FTRACE_STACK_MAX_ENTRIES;
1593
1594                 if (regs)
1595                         save_stack_trace_regs(regs, &trace);
1596                 else
1597                         save_stack_trace(&trace);
1598
1599                 if (trace.nr_entries > size)
1600                         size = trace.nr_entries;
1601         } else
1602                 /* From now on, use_stack is a boolean */
1603                 use_stack = 0;
1604
1605         size *= sizeof(unsigned long);
1606
1607         event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
1608                                           sizeof(*entry) + size, flags, pc);
1609         if (!event)
1610                 goto out;
1611         entry = ring_buffer_event_data(event);
1612
1613         memset(&entry->caller, 0, size);
1614
1615         if (use_stack)
1616                 memcpy(&entry->caller, trace.entries,
1617                        trace.nr_entries * sizeof(unsigned long));
1618         else {
1619                 trace.max_entries       = FTRACE_STACK_ENTRIES;
1620                 trace.entries           = entry->caller;
1621                 if (regs)
1622                         save_stack_trace_regs(regs, &trace);
1623                 else
1624                         save_stack_trace(&trace);
1625         }
1626
1627         entry->size = trace.nr_entries;
1628
1629         if (!filter_check_discard(call, entry, buffer, event))
1630                 __buffer_unlock_commit(buffer, event);
1631
1632  out:
1633         /* Again, don't let gcc optimize things here */
1634         barrier();
1635         __this_cpu_dec(ftrace_stack_reserve);
1636         preempt_enable_notrace();
1637
1638 }
1639
1640 void ftrace_trace_stack_regs(struct ring_buffer *buffer, unsigned long flags,
1641                              int skip, int pc, struct pt_regs *regs)
1642 {
1643         if (!(trace_flags & TRACE_ITER_STACKTRACE))
1644                 return;
1645
1646         __ftrace_trace_stack(buffer, flags, skip, pc, regs);
1647 }
1648
1649 void ftrace_trace_stack(struct ring_buffer *buffer, unsigned long flags,
1650                         int skip, int pc)
1651 {
1652         if (!(trace_flags & TRACE_ITER_STACKTRACE))
1653                 return;
1654
1655         __ftrace_trace_stack(buffer, flags, skip, pc, NULL);
1656 }
1657
1658 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
1659                    int pc)
1660 {
1661         __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
1662 }
1663
1664 /**
1665  * trace_dump_stack - record a stack back trace in the trace buffer
1666  * @skip: Number of functions to skip (helper handlers)
1667  */
1668 void trace_dump_stack(int skip)
1669 {
1670         unsigned long flags;
1671
1672         if (tracing_disabled || tracing_selftest_running)
1673                 return;
1674
1675         local_save_flags(flags);
1676
1677         /*
1678          * Skip 3 more, seems to get us at the caller of
1679          * this function.
1680          */
1681         skip += 3;
1682         __ftrace_trace_stack(global_trace.trace_buffer.buffer,
1683                              flags, skip, preempt_count(), NULL);
1684 }
1685
1686 static DEFINE_PER_CPU(int, user_stack_count);
1687
1688 void
1689 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
1690 {
1691         struct ftrace_event_call *call = &event_user_stack;
1692         struct ring_buffer_event *event;
1693         struct userstack_entry *entry;
1694         struct stack_trace trace;
1695
1696         if (!(trace_flags & TRACE_ITER_USERSTACKTRACE))
1697                 return;
1698
1699         /*
1700          * NMIs can not handle page faults, even with fix ups.
1701          * The save user stack can (and often does) fault.
1702          */
1703         if (unlikely(in_nmi()))
1704                 return;
1705
1706         /*
1707          * prevent recursion, since the user stack tracing may
1708          * trigger other kernel events.
1709          */
1710         preempt_disable();
1711         if (__this_cpu_read(user_stack_count))
1712                 goto out;
1713
1714         __this_cpu_inc(user_stack_count);
1715
1716         event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
1717                                           sizeof(*entry), flags, pc);
1718         if (!event)
1719                 goto out_drop_count;
1720         entry   = ring_buffer_event_data(event);
1721
1722         entry->tgid             = current->tgid;
1723         memset(&entry->caller, 0, sizeof(entry->caller));
1724
1725         trace.nr_entries        = 0;
1726         trace.max_entries       = FTRACE_STACK_ENTRIES;
1727         trace.skip              = 0;
1728         trace.entries           = entry->caller;
1729
1730         save_stack_trace_user(&trace);
1731         if (!filter_check_discard(call, entry, buffer, event))
1732                 __buffer_unlock_commit(buffer, event);
1733
1734  out_drop_count:
1735         __this_cpu_dec(user_stack_count);
1736  out:
1737         preempt_enable();
1738 }
1739
1740 #ifdef UNUSED
1741 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
1742 {
1743         ftrace_trace_userstack(tr, flags, preempt_count());
1744 }
1745 #endif /* UNUSED */
1746
1747 #endif /* CONFIG_STACKTRACE */
1748
1749 /* created for use with alloc_percpu */
1750 struct trace_buffer_struct {
1751         char buffer[TRACE_BUF_SIZE];
1752 };
1753
1754 static struct trace_buffer_struct *trace_percpu_buffer;
1755 static struct trace_buffer_struct *trace_percpu_sirq_buffer;
1756 static struct trace_buffer_struct *trace_percpu_irq_buffer;
1757 static struct trace_buffer_struct *trace_percpu_nmi_buffer;
1758
1759 /*
1760  * The buffer used is dependent on the context. There is a per cpu
1761  * buffer for normal context, softirq contex, hard irq context and
1762  * for NMI context. Thise allows for lockless recording.
1763  *
1764  * Note, if the buffers failed to be allocated, then this returns NULL
1765  */
1766 static char *get_trace_buf(void)
1767 {
1768         struct trace_buffer_struct *percpu_buffer;
1769
1770         /*
1771          * If we have allocated per cpu buffers, then we do not
1772          * need to do any locking.
1773          */
1774         if (in_nmi())
1775                 percpu_buffer = trace_percpu_nmi_buffer;
1776         else if (in_irq())
1777                 percpu_buffer = trace_percpu_irq_buffer;
1778         else if (in_softirq())
1779                 percpu_buffer = trace_percpu_sirq_buffer;
1780         else
1781                 percpu_buffer = trace_percpu_buffer;
1782
1783         if (!percpu_buffer)
1784                 return NULL;
1785
1786         return this_cpu_ptr(&percpu_buffer->buffer[0]);
1787 }
1788
1789 static int alloc_percpu_trace_buffer(void)
1790 {
1791         struct trace_buffer_struct *buffers;
1792         struct trace_buffer_struct *sirq_buffers;
1793         struct trace_buffer_struct *irq_buffers;
1794         struct trace_buffer_struct *nmi_buffers;
1795
1796         buffers = alloc_percpu(struct trace_buffer_struct);
1797         if (!buffers)
1798                 goto err_warn;
1799
1800         sirq_buffers = alloc_percpu(struct trace_buffer_struct);
1801         if (!sirq_buffers)
1802                 goto err_sirq;
1803
1804         irq_buffers = alloc_percpu(struct trace_buffer_struct);
1805         if (!irq_buffers)
1806                 goto err_irq;
1807
1808         nmi_buffers = alloc_percpu(struct trace_buffer_struct);
1809         if (!nmi_buffers)
1810                 goto err_nmi;
1811
1812         trace_percpu_buffer = buffers;
1813         trace_percpu_sirq_buffer = sirq_buffers;
1814         trace_percpu_irq_buffer = irq_buffers;
1815         trace_percpu_nmi_buffer = nmi_buffers;
1816
1817         return 0;
1818
1819  err_nmi:
1820         free_percpu(irq_buffers);
1821  err_irq:
1822         free_percpu(sirq_buffers);
1823  err_sirq:
1824         free_percpu(buffers);
1825  err_warn:
1826         WARN(1, "Could not allocate percpu trace_printk buffer");
1827         return -ENOMEM;
1828 }
1829
1830 static int buffers_allocated;
1831
1832 void trace_printk_init_buffers(void)
1833 {
1834         if (buffers_allocated)
1835                 return;
1836
1837         if (alloc_percpu_trace_buffer())
1838                 return;
1839
1840         pr_info("ftrace: Allocated trace_printk buffers\n");
1841
1842         /* Expand the buffers to set size */
1843         tracing_update_buffers();
1844
1845         buffers_allocated = 1;
1846
1847         /*
1848          * trace_printk_init_buffers() can be called by modules.
1849          * If that happens, then we need to start cmdline recording
1850          * directly here. If the global_trace.buffer is already
1851          * allocated here, then this was called by module code.
1852          */
1853         if (global_trace.trace_buffer.buffer)
1854                 tracing_start_cmdline_record();
1855 }
1856
1857 void trace_printk_start_comm(void)
1858 {
1859         /* Start tracing comms if trace printk is set */
1860         if (!buffers_allocated)
1861                 return;
1862         tracing_start_cmdline_record();
1863 }
1864
1865 static void trace_printk_start_stop_comm(int enabled)
1866 {
1867         if (!buffers_allocated)
1868                 return;
1869
1870         if (enabled)
1871                 tracing_start_cmdline_record();
1872         else
1873                 tracing_stop_cmdline_record();
1874 }
1875
1876 /**
1877  * trace_vbprintk - write binary msg to tracing buffer
1878  *
1879  */
1880 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
1881 {
1882         struct ftrace_event_call *call = &event_bprint;
1883         struct ring_buffer_event *event;
1884         struct ring_buffer *buffer;
1885         struct trace_array *tr = &global_trace;
1886         struct bprint_entry *entry;
1887         unsigned long flags;
1888         char *tbuffer;
1889         int len = 0, size, pc;
1890
1891         if (unlikely(tracing_selftest_running || tracing_disabled))
1892                 return 0;
1893
1894         /* Don't pollute graph traces with trace_vprintk internals */
1895         pause_graph_tracing();
1896
1897         pc = preempt_count();
1898         preempt_disable_notrace();
1899
1900         tbuffer = get_trace_buf();
1901         if (!tbuffer) {
1902                 len = 0;
1903                 goto out;
1904         }
1905
1906         len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
1907
1908         if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
1909                 goto out;
1910
1911         local_save_flags(flags);
1912         size = sizeof(*entry) + sizeof(u32) * len;
1913         buffer = tr->trace_buffer.buffer;
1914         event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
1915                                           flags, pc);
1916         if (!event)
1917                 goto out;
1918         entry = ring_buffer_event_data(event);
1919         entry->ip                       = ip;
1920         entry->fmt                      = fmt;
1921
1922         memcpy(entry->buf, tbuffer, sizeof(u32) * len);
1923         if (!filter_check_discard(call, entry, buffer, event)) {
1924                 __buffer_unlock_commit(buffer, event);
1925                 ftrace_trace_stack(buffer, flags, 6, pc);
1926         }
1927
1928 out:
1929         preempt_enable_notrace();
1930         unpause_graph_tracing();
1931
1932         return len;
1933 }
1934 EXPORT_SYMBOL_GPL(trace_vbprintk);
1935
1936 static int
1937 __trace_array_vprintk(struct ring_buffer *buffer,
1938                       unsigned long ip, const char *fmt, va_list args)
1939 {
1940         struct ftrace_event_call *call = &event_print;
1941         struct ring_buffer_event *event;
1942         int len = 0, size, pc;
1943         struct print_entry *entry;
1944         unsigned long flags;
1945         char *tbuffer;
1946
1947         if (tracing_disabled || tracing_selftest_running)
1948                 return 0;
1949
1950         /* Don't pollute graph traces with trace_vprintk internals */
1951         pause_graph_tracing();
1952
1953         pc = preempt_count();
1954         preempt_disable_notrace();
1955
1956
1957         tbuffer = get_trace_buf();
1958         if (!tbuffer) {
1959                 len = 0;
1960                 goto out;
1961         }
1962
1963         len = vsnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
1964         if (len > TRACE_BUF_SIZE)
1965                 goto out;
1966
1967         local_save_flags(flags);
1968         size = sizeof(*entry) + len + 1;
1969         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
1970                                           flags, pc);
1971         if (!event)
1972                 goto out;
1973         entry = ring_buffer_event_data(event);
1974         entry->ip = ip;
1975
1976         memcpy(&entry->buf, tbuffer, len);
1977         entry->buf[len] = '\0';
1978         if (!filter_check_discard(call, entry, buffer, event)) {
1979                 __buffer_unlock_commit(buffer, event);
1980                 ftrace_trace_stack(buffer, flags, 6, pc);
1981         }
1982  out:
1983         preempt_enable_notrace();
1984         unpause_graph_tracing();
1985
1986         return len;
1987 }
1988
1989 int trace_array_vprintk(struct trace_array *tr,
1990                         unsigned long ip, const char *fmt, va_list args)
1991 {
1992         return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
1993 }
1994
1995 int trace_array_printk(struct trace_array *tr,
1996                        unsigned long ip, const char *fmt, ...)
1997 {
1998         int ret;
1999         va_list ap;
2000
2001         if (!(trace_flags & TRACE_ITER_PRINTK))
2002                 return 0;
2003
2004         va_start(ap, fmt);
2005         ret = trace_array_vprintk(tr, ip, fmt, ap);
2006         va_end(ap);
2007         return ret;
2008 }
2009
2010 int trace_array_printk_buf(struct ring_buffer *buffer,
2011                            unsigned long ip, const char *fmt, ...)
2012 {
2013         int ret;
2014         va_list ap;
2015
2016         if (!(trace_flags & TRACE_ITER_PRINTK))
2017                 return 0;
2018
2019         va_start(ap, fmt);
2020         ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2021         va_end(ap);
2022         return ret;
2023 }
2024
2025 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2026 {
2027         return trace_array_vprintk(&global_trace, ip, fmt, args);
2028 }
2029 EXPORT_SYMBOL_GPL(trace_vprintk);
2030
2031 static void trace_iterator_increment(struct trace_iterator *iter)
2032 {
2033         struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2034
2035         iter->idx++;
2036         if (buf_iter)
2037                 ring_buffer_read(buf_iter, NULL);
2038 }
2039
2040 static struct trace_entry *
2041 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2042                 unsigned long *lost_events)
2043 {
2044         struct ring_buffer_event *event;
2045         struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2046
2047         if (buf_iter)
2048                 event = ring_buffer_iter_peek(buf_iter, ts);
2049         else
2050                 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2051                                          lost_events);
2052
2053         if (event) {
2054                 iter->ent_size = ring_buffer_event_length(event);
2055                 return ring_buffer_event_data(event);
2056         }
2057         iter->ent_size = 0;
2058         return NULL;
2059 }
2060
2061 static struct trace_entry *
2062 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2063                   unsigned long *missing_events, u64 *ent_ts)
2064 {
2065         struct ring_buffer *buffer = iter->trace_buffer->buffer;
2066         struct trace_entry *ent, *next = NULL;
2067         unsigned long lost_events = 0, next_lost = 0;
2068         int cpu_file = iter->cpu_file;
2069         u64 next_ts = 0, ts;
2070         int next_cpu = -1;
2071         int next_size = 0;
2072         int cpu;
2073
2074         /*
2075          * If we are in a per_cpu trace file, don't bother by iterating over
2076          * all cpu and peek directly.
2077          */
2078         if (cpu_file > RING_BUFFER_ALL_CPUS) {
2079                 if (ring_buffer_empty_cpu(buffer, cpu_file))
2080                         return NULL;
2081                 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2082                 if (ent_cpu)
2083                         *ent_cpu = cpu_file;
2084
2085                 return ent;
2086         }
2087
2088         for_each_tracing_cpu(cpu) {
2089
2090                 if (ring_buffer_empty_cpu(buffer, cpu))
2091                         continue;
2092
2093                 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2094
2095                 /*
2096                  * Pick the entry with the smallest timestamp:
2097                  */
2098                 if (ent && (!next || ts < next_ts)) {
2099                         next = ent;
2100                         next_cpu = cpu;
2101                         next_ts = ts;
2102                         next_lost = lost_events;
2103                         next_size = iter->ent_size;
2104                 }
2105         }
2106
2107         iter->ent_size = next_size;
2108
2109         if (ent_cpu)
2110                 *ent_cpu = next_cpu;
2111
2112         if (ent_ts)
2113                 *ent_ts = next_ts;
2114
2115         if (missing_events)
2116                 *missing_events = next_lost;
2117
2118         return next;
2119 }
2120
2121 /* Find the next real entry, without updating the iterator itself */
2122 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2123                                           int *ent_cpu, u64 *ent_ts)
2124 {
2125         return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2126 }
2127
2128 /* Find the next real entry, and increment the iterator to the next entry */
2129 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2130 {
2131         iter->ent = __find_next_entry(iter, &iter->cpu,
2132                                       &iter->lost_events, &iter->ts);
2133
2134         if (iter->ent)
2135                 trace_iterator_increment(iter);
2136
2137         return iter->ent ? iter : NULL;
2138 }
2139
2140 static void trace_consume(struct trace_iterator *iter)
2141 {
2142         ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2143                             &iter->lost_events);
2144 }
2145
2146 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2147 {
2148         struct trace_iterator *iter = m->private;
2149         int i = (int)*pos;
2150         void *ent;
2151
2152         WARN_ON_ONCE(iter->leftover);
2153
2154         (*pos)++;
2155
2156         /* can't go backwards */
2157         if (iter->idx > i)
2158                 return NULL;
2159
2160         if (iter->idx < 0)
2161                 ent = trace_find_next_entry_inc(iter);
2162         else
2163                 ent = iter;
2164
2165         while (ent && iter->idx < i)
2166                 ent = trace_find_next_entry_inc(iter);
2167
2168         iter->pos = *pos;
2169
2170         return ent;
2171 }
2172
2173 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2174 {
2175         struct ring_buffer_event *event;
2176         struct ring_buffer_iter *buf_iter;
2177         unsigned long entries = 0;
2178         u64 ts;
2179
2180         per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2181
2182         buf_iter = trace_buffer_iter(iter, cpu);
2183         if (!buf_iter)
2184                 return;
2185
2186         ring_buffer_iter_reset(buf_iter);
2187
2188         /*
2189          * We could have the case with the max latency tracers
2190          * that a reset never took place on a cpu. This is evident
2191          * by the timestamp being before the start of the buffer.
2192          */
2193         while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2194                 if (ts >= iter->trace_buffer->time_start)
2195                         break;
2196                 entries++;
2197                 ring_buffer_read(buf_iter, NULL);
2198         }
2199
2200         per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2201 }
2202
2203 /*
2204  * The current tracer is copied to avoid a global locking
2205  * all around.
2206  */
2207 static void *s_start(struct seq_file *m, loff_t *pos)
2208 {
2209         struct trace_iterator *iter = m->private;
2210         struct trace_array *tr = iter->tr;
2211         int cpu_file = iter->cpu_file;
2212         void *p = NULL;
2213         loff_t l = 0;
2214         int cpu;
2215
2216         /*
2217          * copy the tracer to avoid using a global lock all around.
2218          * iter->trace is a copy of current_trace, the pointer to the
2219          * name may be used instead of a strcmp(), as iter->trace->name
2220          * will point to the same string as current_trace->name.
2221          */
2222         mutex_lock(&trace_types_lock);
2223         if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2224                 *iter->trace = *tr->current_trace;
2225         mutex_unlock(&trace_types_lock);
2226
2227 #ifdef CONFIG_TRACER_MAX_TRACE
2228         if (iter->snapshot && iter->trace->use_max_tr)
2229                 return ERR_PTR(-EBUSY);
2230 #endif
2231
2232         if (!iter->snapshot)
2233                 atomic_inc(&trace_record_cmdline_disabled);
2234
2235         if (*pos != iter->pos) {
2236                 iter->ent = NULL;
2237                 iter->cpu = 0;
2238                 iter->idx = -1;
2239
2240                 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2241                         for_each_tracing_cpu(cpu)
2242                                 tracing_iter_reset(iter, cpu);
2243                 } else
2244                         tracing_iter_reset(iter, cpu_file);
2245
2246                 iter->leftover = 0;
2247                 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2248                         ;
2249
2250         } else {
2251                 /*
2252                  * If we overflowed the seq_file before, then we want
2253                  * to just reuse the trace_seq buffer again.
2254                  */
2255                 if (iter->leftover)
2256                         p = iter;
2257                 else {
2258                         l = *pos - 1;
2259                         p = s_next(m, p, &l);
2260                 }
2261         }
2262
2263         trace_event_read_lock();
2264         trace_access_lock(cpu_file);
2265         return p;
2266 }
2267
2268 static void s_stop(struct seq_file *m, void *p)
2269 {
2270         struct trace_iterator *iter = m->private;
2271
2272 #ifdef CONFIG_TRACER_MAX_TRACE
2273         if (iter->snapshot && iter->trace->use_max_tr)
2274                 return;
2275 #endif
2276
2277         if (!iter->snapshot)
2278                 atomic_dec(&trace_record_cmdline_disabled);
2279
2280         trace_access_unlock(iter->cpu_file);
2281         trace_event_read_unlock();
2282 }
2283
2284 static void
2285 get_total_entries(struct trace_buffer *buf,
2286                   unsigned long *total, unsigned long *entries)
2287 {
2288         unsigned long count;
2289         int cpu;
2290
2291         *total = 0;
2292         *entries = 0;
2293
2294         for_each_tracing_cpu(cpu) {
2295                 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2296                 /*
2297                  * If this buffer has skipped entries, then we hold all
2298                  * entries for the trace and we need to ignore the
2299                  * ones before the time stamp.
2300                  */
2301                 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2302                         count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2303                         /* total is the same as the entries */
2304                         *total += count;
2305                 } else
2306                         *total += count +
2307                                 ring_buffer_overrun_cpu(buf->buffer, cpu);
2308                 *entries += count;
2309         }
2310 }
2311
2312 static void print_lat_help_header(struct seq_file *m)
2313 {
2314         seq_puts(m, "#                  _------=> CPU#            \n");
2315         seq_puts(m, "#                 / _-----=> irqs-off        \n");
2316         seq_puts(m, "#                | / _----=> need-resched    \n");
2317         seq_puts(m, "#                || / _---=> hardirq/softirq \n");
2318         seq_puts(m, "#                ||| / _--=> preempt-depth   \n");
2319         seq_puts(m, "#                |||| /     delay             \n");
2320         seq_puts(m, "#  cmd     pid   ||||| time  |   caller      \n");
2321         seq_puts(m, "#     \\   /      |||||  \\    |   /           \n");
2322 }
2323
2324 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2325 {
2326         unsigned long total;
2327         unsigned long entries;
2328
2329         get_total_entries(buf, &total, &entries);
2330         seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu   #P:%d\n",
2331                    entries, total, num_online_cpus());
2332         seq_puts(m, "#\n");
2333 }
2334
2335 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2336 {
2337         print_event_info(buf, m);
2338         seq_puts(m, "#           TASK-PID   CPU#      TIMESTAMP  FUNCTION\n");
2339         seq_puts(m, "#              | |       |          |         |\n");
2340 }
2341
2342 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2343 {
2344         print_event_info(buf, m);
2345         seq_puts(m, "#                              _-----=> irqs-off\n");
2346         seq_puts(m, "#                             / _----=> need-resched\n");
2347         seq_puts(m, "#                            | / _---=> hardirq/softirq\n");
2348         seq_puts(m, "#                            || / _--=> preempt-depth\n");
2349         seq_puts(m, "#                            ||| /     delay\n");
2350         seq_puts(m, "#           TASK-PID   CPU#  ||||    TIMESTAMP  FUNCTION\n");
2351         seq_puts(m, "#              | |       |   ||||       |         |\n");
2352 }
2353
2354 void
2355 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2356 {
2357         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2358         struct trace_buffer *buf = iter->trace_buffer;
2359         struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2360         struct tracer *type = iter->trace;
2361         unsigned long entries;
2362         unsigned long total;
2363         const char *name = "preemption";
2364
2365         name = type->name;
2366
2367         get_total_entries(buf, &total, &entries);
2368
2369         seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2370                    name, UTS_RELEASE);
2371         seq_puts(m, "# -----------------------------------"
2372                  "---------------------------------\n");
2373         seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2374                    " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2375                    nsecs_to_usecs(data->saved_latency),
2376                    entries,
2377                    total,
2378                    buf->cpu,
2379 #if defined(CONFIG_PREEMPT_NONE)
2380                    "server",
2381 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2382                    "desktop",
2383 #elif defined(CONFIG_PREEMPT)
2384                    "preempt",
2385 #else
2386                    "unknown",
2387 #endif
2388                    /* These are reserved for later use */
2389                    0, 0, 0, 0);
2390 #ifdef CONFIG_SMP
2391         seq_printf(m, " #P:%d)\n", num_online_cpus());
2392 #else
2393         seq_puts(m, ")\n");
2394 #endif
2395         seq_puts(m, "#    -----------------\n");
2396         seq_printf(m, "#    | task: %.16s-%d "
2397                    "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2398                    data->comm, data->pid,
2399                    from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
2400                    data->policy, data->rt_priority);
2401         seq_puts(m, "#    -----------------\n");
2402
2403         if (data->critical_start) {
2404                 seq_puts(m, "#  => started at: ");
2405                 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
2406                 trace_print_seq(m, &iter->seq);
2407                 seq_puts(m, "\n#  => ended at:   ");
2408                 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
2409                 trace_print_seq(m, &iter->seq);
2410                 seq_puts(m, "\n#\n");
2411         }
2412
2413         seq_puts(m, "#\n");
2414 }
2415
2416 static void test_cpu_buff_start(struct trace_iterator *iter)
2417 {
2418         struct trace_seq *s = &iter->seq;
2419
2420         if (!(trace_flags & TRACE_ITER_ANNOTATE))
2421                 return;
2422
2423         if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
2424                 return;
2425
2426         if (cpumask_test_cpu(iter->cpu, iter->started))
2427                 return;
2428
2429         if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
2430                 return;
2431
2432         cpumask_set_cpu(iter->cpu, iter->started);
2433
2434         /* Don't print started cpu buffer for the first entry of the trace */
2435         if (iter->idx > 1)
2436                 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
2437                                 iter->cpu);
2438 }
2439
2440 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
2441 {
2442         struct trace_seq *s = &iter->seq;
2443         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
2444         struct trace_entry *entry;
2445         struct trace_event *event;
2446
2447         entry = iter->ent;
2448
2449         test_cpu_buff_start(iter);
2450
2451         event = ftrace_find_event(entry->type);
2452
2453         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2454                 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2455                         if (!trace_print_lat_context(iter))
2456                                 goto partial;
2457                 } else {
2458                         if (!trace_print_context(iter))
2459                                 goto partial;
2460                 }
2461         }
2462
2463         if (event)
2464                 return event->funcs->trace(iter, sym_flags, event);
2465
2466         if (!trace_seq_printf(s, "Unknown type %d\n", entry->type))
2467                 goto partial;
2468
2469         return TRACE_TYPE_HANDLED;
2470 partial:
2471         return TRACE_TYPE_PARTIAL_LINE;
2472 }
2473
2474 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
2475 {
2476         struct trace_seq *s = &iter->seq;
2477         struct trace_entry *entry;
2478         struct trace_event *event;
2479
2480         entry = iter->ent;
2481
2482         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2483                 if (!trace_seq_printf(s, "%d %d %llu ",
2484                                       entry->pid, iter->cpu, iter->ts))
2485                         goto partial;
2486         }
2487
2488         event = ftrace_find_event(entry->type);
2489         if (event)
2490                 return event->funcs->raw(iter, 0, event);
2491
2492         if (!trace_seq_printf(s, "%d ?\n", entry->type))
2493                 goto partial;
2494
2495         return TRACE_TYPE_HANDLED;
2496 partial:
2497         return TRACE_TYPE_PARTIAL_LINE;
2498 }
2499
2500 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
2501 {
2502         struct trace_seq *s = &iter->seq;
2503         unsigned char newline = '\n';
2504         struct trace_entry *entry;
2505         struct trace_event *event;
2506
2507         entry = iter->ent;
2508
2509         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2510                 SEQ_PUT_HEX_FIELD_RET(s, entry->pid);
2511                 SEQ_PUT_HEX_FIELD_RET(s, iter->cpu);
2512                 SEQ_PUT_HEX_FIELD_RET(s, iter->ts);
2513         }
2514
2515         event = ftrace_find_event(entry->type);
2516         if (event) {
2517                 enum print_line_t ret = event->funcs->hex(iter, 0, event);
2518                 if (ret != TRACE_TYPE_HANDLED)
2519                         return ret;
2520         }
2521
2522         SEQ_PUT_FIELD_RET(s, newline);
2523
2524         return TRACE_TYPE_HANDLED;
2525 }
2526
2527 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
2528 {
2529         struct trace_seq *s = &iter->seq;
2530         struct trace_entry *entry;
2531         struct trace_event *event;
2532
2533         entry = iter->ent;
2534
2535         if (trace_flags & TRACE_ITER_CONTEXT_INFO) {
2536                 SEQ_PUT_FIELD_RET(s, entry->pid);
2537                 SEQ_PUT_FIELD_RET(s, iter->cpu);
2538                 SEQ_PUT_FIELD_RET(s, iter->ts);
2539         }
2540
2541         event = ftrace_find_event(entry->type);
2542         return event ? event->funcs->binary(iter, 0, event) :
2543                 TRACE_TYPE_HANDLED;
2544 }
2545
2546 int trace_empty(struct trace_iterator *iter)
2547 {
2548         struct ring_buffer_iter *buf_iter;
2549         int cpu;
2550
2551         /* If we are looking at one CPU buffer, only check that one */
2552         if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
2553                 cpu = iter->cpu_file;
2554                 buf_iter = trace_buffer_iter(iter, cpu);
2555                 if (buf_iter) {
2556                         if (!ring_buffer_iter_empty(buf_iter))
2557                                 return 0;
2558                 } else {
2559                         if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2560                                 return 0;
2561                 }
2562                 return 1;
2563         }
2564
2565         for_each_tracing_cpu(cpu) {
2566                 buf_iter = trace_buffer_iter(iter, cpu);
2567                 if (buf_iter) {
2568                         if (!ring_buffer_iter_empty(buf_iter))
2569                                 return 0;
2570                 } else {
2571                         if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
2572                                 return 0;
2573                 }
2574         }
2575
2576         return 1;
2577 }
2578
2579 /*  Called with trace_event_read_lock() held. */
2580 enum print_line_t print_trace_line(struct trace_iterator *iter)
2581 {
2582         enum print_line_t ret;
2583
2584         if (iter->lost_events &&
2585             !trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
2586                                  iter->cpu, iter->lost_events))
2587                 return TRACE_TYPE_PARTIAL_LINE;
2588
2589         if (iter->trace && iter->trace->print_line) {
2590                 ret = iter->trace->print_line(iter);
2591                 if (ret != TRACE_TYPE_UNHANDLED)
2592                         return ret;
2593         }
2594
2595         if (iter->ent->type == TRACE_BPUTS &&
2596                         trace_flags & TRACE_ITER_PRINTK &&
2597                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2598                 return trace_print_bputs_msg_only(iter);
2599
2600         if (iter->ent->type == TRACE_BPRINT &&
2601                         trace_flags & TRACE_ITER_PRINTK &&
2602                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2603                 return trace_print_bprintk_msg_only(iter);
2604
2605         if (iter->ent->type == TRACE_PRINT &&
2606                         trace_flags & TRACE_ITER_PRINTK &&
2607                         trace_flags & TRACE_ITER_PRINTK_MSGONLY)
2608                 return trace_print_printk_msg_only(iter);
2609
2610         if (trace_flags & TRACE_ITER_BIN)
2611                 return print_bin_fmt(iter);
2612
2613         if (trace_flags & TRACE_ITER_HEX)
2614                 return print_hex_fmt(iter);
2615
2616         if (trace_flags & TRACE_ITER_RAW)
2617                 return print_raw_fmt(iter);
2618
2619         return print_trace_fmt(iter);
2620 }
2621
2622 void trace_latency_header(struct seq_file *m)
2623 {
2624         struct trace_iterator *iter = m->private;
2625
2626         /* print nothing if the buffers are empty */
2627         if (trace_empty(iter))
2628                 return;
2629
2630         if (iter->iter_flags & TRACE_FILE_LAT_FMT)
2631                 print_trace_header(m, iter);
2632
2633         if (!(trace_flags & TRACE_ITER_VERBOSE))
2634                 print_lat_help_header(m);
2635 }
2636
2637 void trace_default_header(struct seq_file *m)
2638 {
2639         struct trace_iterator *iter = m->private;
2640
2641         if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
2642                 return;
2643
2644         if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
2645                 /* print nothing if the buffers are empty */
2646                 if (trace_empty(iter))
2647                         return;
2648                 print_trace_header(m, iter);
2649                 if (!(trace_flags & TRACE_ITER_VERBOSE))
2650                         print_lat_help_header(m);
2651         } else {
2652                 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
2653                         if (trace_flags & TRACE_ITER_IRQ_INFO)
2654                                 print_func_help_header_irq(iter->trace_buffer, m);
2655                         else
2656                                 print_func_help_header(iter->trace_buffer, m);
2657                 }
2658         }
2659 }
2660
2661 static void test_ftrace_alive(struct seq_file *m)
2662 {
2663         if (!ftrace_is_dead())
2664                 return;
2665         seq_printf(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n");
2666         seq_printf(m, "#          MAY BE MISSING FUNCTION EVENTS\n");
2667 }
2668
2669 #ifdef CONFIG_TRACER_MAX_TRACE
2670 static void show_snapshot_main_help(struct seq_file *m)
2671 {
2672         seq_printf(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n");
2673         seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2674         seq_printf(m, "#                      Takes a snapshot of the main buffer.\n");
2675         seq_printf(m, "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate)\n");
2676         seq_printf(m, "#                      (Doesn't have to be '2' works with any number that\n");
2677         seq_printf(m, "#                       is not a '0' or '1')\n");
2678 }
2679
2680 static void show_snapshot_percpu_help(struct seq_file *m)
2681 {
2682         seq_printf(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
2683 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2684         seq_printf(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n");
2685         seq_printf(m, "#                      Takes a snapshot of the main buffer for this cpu.\n");
2686 #else
2687         seq_printf(m, "# echo 1 > snapshot : Not supported with this kernel.\n");
2688         seq_printf(m, "#                     Must use main snapshot file to allocate.\n");
2689 #endif
2690         seq_printf(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n");
2691         seq_printf(m, "#                      (Doesn't have to be '2' works with any number that\n");
2692         seq_printf(m, "#                       is not a '0' or '1')\n");
2693 }
2694
2695 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
2696 {
2697         if (iter->tr->allocated_snapshot)
2698                 seq_printf(m, "#\n# * Snapshot is allocated *\n#\n");
2699         else
2700                 seq_printf(m, "#\n# * Snapshot is freed *\n#\n");
2701
2702         seq_printf(m, "# Snapshot commands:\n");
2703         if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
2704                 show_snapshot_main_help(m);
2705         else
2706                 show_snapshot_percpu_help(m);
2707 }
2708 #else
2709 /* Should never be called */
2710 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
2711 #endif
2712
2713 static int s_show(struct seq_file *m, void *v)
2714 {
2715         struct trace_iterator *iter = v;
2716         int ret;
2717
2718         if (iter->ent == NULL) {
2719                 if (iter->tr) {
2720                         seq_printf(m, "# tracer: %s\n", iter->trace->name);
2721                         seq_puts(m, "#\n");
2722                         test_ftrace_alive(m);
2723                 }
2724                 if (iter->snapshot && trace_empty(iter))
2725                         print_snapshot_help(m, iter);
2726                 else if (iter->trace && iter->trace->print_header)
2727                         iter->trace->print_header(m);
2728                 else
2729                         trace_default_header(m);
2730
2731         } else if (iter->leftover) {
2732                 /*
2733                  * If we filled the seq_file buffer earlier, we
2734                  * want to just show it now.
2735                  */
2736                 ret = trace_print_seq(m, &iter->seq);
2737
2738                 /* ret should this time be zero, but you never know */
2739                 iter->leftover = ret;
2740
2741         } else {
2742                 print_trace_line(iter);
2743                 ret = trace_print_seq(m, &iter->seq);
2744                 /*
2745                  * If we overflow the seq_file buffer, then it will
2746                  * ask us for this data again at start up.
2747                  * Use that instead.
2748                  *  ret is 0 if seq_file write succeeded.
2749                  *        -1 otherwise.
2750                  */
2751                 iter->leftover = ret;
2752         }
2753
2754         return 0;
2755 }
2756
2757 static const struct seq_operations tracer_seq_ops = {
2758         .start          = s_start,
2759         .next           = s_next,
2760         .stop           = s_stop,
2761         .show           = s_show,
2762 };
2763
2764 static struct trace_iterator *
2765 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
2766 {
2767         struct trace_cpu *tc = inode->i_private;
2768         struct trace_array *tr = tc->tr;
2769         struct trace_iterator *iter;
2770         int cpu;
2771
2772         if (tracing_disabled)
2773                 return ERR_PTR(-ENODEV);
2774
2775         iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
2776         if (!iter)
2777                 return ERR_PTR(-ENOMEM);
2778
2779         iter->buffer_iter = kzalloc(sizeof(*iter->buffer_iter) * num_possible_cpus(),
2780                                     GFP_KERNEL);
2781         if (!iter->buffer_iter)
2782                 goto release;
2783
2784         /*
2785          * We make a copy of the current tracer to avoid concurrent
2786          * changes on it while we are reading.
2787          */
2788         mutex_lock(&trace_types_lock);
2789         iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
2790         if (!iter->trace)
2791                 goto fail;
2792
2793         *iter->trace = *tr->current_trace;
2794
2795         if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
2796                 goto fail;
2797
2798         iter->tr = tr;
2799
2800 #ifdef CONFIG_TRACER_MAX_TRACE
2801         /* Currently only the top directory has a snapshot */
2802         if (tr->current_trace->print_max || snapshot)
2803                 iter->trace_buffer = &tr->max_buffer;
2804         else
2805 #endif
2806                 iter->trace_buffer = &tr->trace_buffer;
2807         iter->snapshot = snapshot;
2808         iter->pos = -1;
2809         mutex_init(&iter->mutex);
2810         iter->cpu_file = tc->cpu;
2811
2812         /* Notify the tracer early; before we stop tracing. */
2813         if (iter->trace && iter->trace->open)
2814                 iter->trace->open(iter);
2815
2816         /* Annotate start of buffers if we had overruns */
2817         if (ring_buffer_overruns(iter->trace_buffer->buffer))
2818                 iter->iter_flags |= TRACE_FILE_ANNOTATE;
2819
2820         /* Output in nanoseconds only if we are using a clock in nanoseconds. */
2821         if (trace_clocks[trace_clock_id].in_ns)
2822                 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
2823
2824         /* stop the trace while dumping if we are not opening "snapshot" */
2825         if (!iter->snapshot)
2826                 tracing_stop_tr(tr);
2827
2828         if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
2829                 for_each_tracing_cpu(cpu) {
2830                         iter->buffer_iter[cpu] =
2831                                 ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
2832                 }
2833                 ring_buffer_read_prepare_sync();
2834                 for_each_tracing_cpu(cpu) {
2835                         ring_buffer_read_start(iter->buffer_iter[cpu]);
2836                         tracing_iter_reset(iter, cpu);
2837                 }
2838         } else {
2839                 cpu = iter->cpu_file;
2840                 iter->buffer_iter[cpu] =
2841                         ring_buffer_read_prepare(iter->trace_buffer->buffer, cpu);
2842                 ring_buffer_read_prepare_sync();
2843                 ring_buffer_read_start(iter->buffer_iter[cpu]);
2844                 tracing_iter_reset(iter, cpu);
2845         }
2846
2847         tr->ref++;
2848
2849         mutex_unlock(&trace_types_lock);
2850
2851         return iter;
2852
2853  fail:
2854         mutex_unlock(&trace_types_lock);
2855         kfree(iter->trace);
2856         kfree(iter->buffer_iter);
2857 release:
2858         seq_release_private(inode, file);
2859         return ERR_PTR(-ENOMEM);
2860 }
2861
2862 int tracing_open_generic(struct inode *inode, struct file *filp)
2863 {
2864         if (tracing_disabled)
2865                 return -ENODEV;
2866
2867         filp->private_data = inode->i_private;
2868         return 0;
2869 }
2870
2871 static int tracing_release(struct inode *inode, struct file *file)
2872 {
2873         struct seq_file *m = file->private_data;
2874         struct trace_iterator *iter;
2875         struct trace_array *tr;
2876         int cpu;
2877
2878         if (!(file->f_mode & FMODE_READ))
2879                 return 0;
2880
2881         iter = m->private;
2882         tr = iter->tr;
2883
2884         mutex_lock(&trace_types_lock);
2885
2886         WARN_ON(!tr->ref);
2887         tr->ref--;
2888
2889         for_each_tracing_cpu(cpu) {
2890                 if (iter->buffer_iter[cpu])
2891                         ring_buffer_read_finish(iter->buffer_iter[cpu]);
2892         }
2893
2894         if (iter->trace && iter->trace->close)
2895                 iter->trace->close(iter);
2896
2897         if (!iter->snapshot)
2898                 /* reenable tracing if it was previously enabled */
2899                 tracing_start_tr(tr);
2900         mutex_unlock(&trace_types_lock);
2901
2902         mutex_destroy(&iter->mutex);
2903         free_cpumask_var(iter->started);
2904         kfree(iter->trace);
2905         kfree(iter->buffer_iter);
2906         seq_release_private(inode, file);
2907         return 0;
2908 }
2909
2910 static int tracing_open(struct inode *inode, struct file *file)
2911 {
2912         struct trace_iterator *iter;
2913         int ret = 0;
2914
2915         /* If this file was open for write, then erase contents */
2916         if ((file->f_mode & FMODE_WRITE) &&
2917             (file->f_flags & O_TRUNC)) {
2918                 struct trace_cpu *tc = inode->i_private;
2919                 struct trace_array *tr = tc->tr;
2920
2921                 if (tc->cpu == RING_BUFFER_ALL_CPUS)
2922                         tracing_reset_online_cpus(&tr->trace_buffer);
2923                 else
2924                         tracing_reset(&tr->trace_buffer, tc->cpu);
2925         }
2926
2927         if (file->f_mode & FMODE_READ) {
2928                 iter = __tracing_open(inode, file, false);
2929                 if (IS_ERR(iter))
2930                         ret = PTR_ERR(iter);
2931                 else if (trace_flags & TRACE_ITER_LATENCY_FMT)
2932                         iter->iter_flags |= TRACE_FILE_LAT_FMT;
2933         }
2934         return ret;
2935 }
2936
2937 static void *
2938 t_next(struct seq_file *m, void *v, loff_t *pos)
2939 {
2940         struct tracer *t = v;
2941
2942         (*pos)++;
2943
2944         if (t)
2945                 t = t->next;
2946
2947         return t;
2948 }
2949
2950 static void *t_start(struct seq_file *m, loff_t *pos)
2951 {
2952         struct tracer *t;
2953         loff_t l = 0;
2954
2955         mutex_lock(&trace_types_lock);
2956         for (t = trace_types; t && l < *pos; t = t_next(m, t, &l))
2957                 ;
2958
2959         return t;
2960 }
2961
2962 static void t_stop(struct seq_file *m, void *p)
2963 {
2964         mutex_unlock(&trace_types_lock);
2965 }
2966
2967 static int t_show(struct seq_file *m, void *v)
2968 {
2969         struct tracer *t = v;
2970
2971         if (!t)
2972                 return 0;
2973
2974         seq_printf(m, "%s", t->name);
2975         if (t->next)
2976                 seq_putc(m, ' ');
2977         else
2978                 seq_putc(m, '\n');
2979
2980         return 0;
2981 }
2982
2983 static const struct seq_operations show_traces_seq_ops = {
2984         .start          = t_start,
2985         .next           = t_next,
2986         .stop           = t_stop,
2987         .show           = t_show,
2988 };
2989
2990 static int show_traces_open(struct inode *inode, struct file *file)
2991 {
2992         if (tracing_disabled)
2993                 return -ENODEV;
2994
2995         return seq_open(file, &show_traces_seq_ops);
2996 }
2997
2998 static ssize_t
2999 tracing_write_stub(struct file *filp, const char __user *ubuf,
3000                    size_t count, loff_t *ppos)
3001 {
3002         return count;
3003 }
3004
3005 static loff_t tracing_seek(struct file *file, loff_t offset, int origin)
3006 {
3007         if (file->f_mode & FMODE_READ)
3008                 return seq_lseek(file, offset, origin);
3009         else
3010                 return 0;
3011 }
3012
3013 static const struct file_operations tracing_fops = {
3014         .open           = tracing_open,
3015         .read           = seq_read,
3016         .write          = tracing_write_stub,
3017         .llseek         = tracing_seek,
3018         .release        = tracing_release,
3019 };
3020
3021 static const struct file_operations show_traces_fops = {
3022         .open           = show_traces_open,
3023         .read           = seq_read,
3024         .release        = seq_release,
3025         .llseek         = seq_lseek,
3026 };
3027
3028 /*
3029  * Only trace on a CPU if the bitmask is set:
3030  */
3031 static cpumask_var_t tracing_cpumask;
3032
3033 /*
3034  * The tracer itself will not take this lock, but still we want
3035  * to provide a consistent cpumask to user-space:
3036  */
3037 static DEFINE_MUTEX(tracing_cpumask_update_lock);
3038
3039 /*
3040  * Temporary storage for the character representation of the
3041  * CPU bitmask (and one more byte for the newline):
3042  */
3043 static char mask_str[NR_CPUS + 1];
3044
3045 static ssize_t
3046 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3047                      size_t count, loff_t *ppos)
3048 {
3049         int len;
3050
3051         mutex_lock(&tracing_cpumask_update_lock);
3052
3053         len = cpumask_scnprintf(mask_str, count, tracing_cpumask);
3054         if (count - len < 2) {
3055                 count = -EINVAL;
3056                 goto out_err;
3057         }
3058         len += sprintf(mask_str + len, "\n");
3059         count = simple_read_from_buffer(ubuf, count, ppos, mask_str, NR_CPUS+1);
3060
3061 out_err:
3062         mutex_unlock(&tracing_cpumask_update_lock);
3063
3064         return count;
3065 }
3066
3067 static ssize_t
3068 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3069                       size_t count, loff_t *ppos)
3070 {
3071         struct trace_array *tr = filp->private_data;
3072         cpumask_var_t tracing_cpumask_new;
3073         int err, cpu;
3074
3075         if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3076                 return -ENOMEM;
3077
3078         err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3079         if (err)
3080                 goto err_unlock;
3081
3082         mutex_lock(&tracing_cpumask_update_lock);
3083
3084         local_irq_disable();
3085         arch_spin_lock(&ftrace_max_lock);
3086         for_each_tracing_cpu(cpu) {
3087                 /*
3088                  * Increase/decrease the disabled counter if we are
3089                  * about to flip a bit in the cpumask:
3090                  */
3091                 if (cpumask_test_cpu(cpu, tracing_cpumask) &&
3092                                 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3093                         atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3094                         ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3095                 }
3096                 if (!cpumask_test_cpu(cpu, tracing_cpumask) &&
3097                                 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3098                         atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3099                         ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3100                 }
3101         }
3102         arch_spin_unlock(&ftrace_max_lock);
3103         local_irq_enable();
3104
3105         cpumask_copy(tracing_cpumask, tracing_cpumask_new);
3106
3107         mutex_unlock(&tracing_cpumask_update_lock);
3108         free_cpumask_var(tracing_cpumask_new);
3109
3110         return count;
3111
3112 err_unlock:
3113         free_cpumask_var(tracing_cpumask_new);
3114
3115         return err;
3116 }
3117
3118 static const struct file_operations tracing_cpumask_fops = {
3119         .open           = tracing_open_generic,
3120         .read           = tracing_cpumask_read,
3121         .write          = tracing_cpumask_write,
3122         .llseek         = generic_file_llseek,
3123 };
3124
3125 static int tracing_trace_options_show(struct seq_file *m, void *v)
3126 {
3127         struct tracer_opt *trace_opts;
3128         struct trace_array *tr = m->private;
3129         u32 tracer_flags;
3130         int i;
3131
3132         mutex_lock(&trace_types_lock);
3133         tracer_flags = tr->current_trace->flags->val;
3134         trace_opts = tr->current_trace->flags->opts;
3135
3136         for (i = 0; trace_options[i]; i++) {
3137                 if (trace_flags & (1 << i))
3138                         seq_printf(m, "%s\n", trace_options[i]);
3139                 else
3140                         seq_printf(m, "no%s\n", trace_options[i]);
3141         }
3142
3143         for (i = 0; trace_opts[i].name; i++) {
3144                 if (tracer_flags & trace_opts[i].bit)
3145                         seq_printf(m, "%s\n", trace_opts[i].name);
3146                 else
3147                         seq_printf(m, "no%s\n", trace_opts[i].name);
3148         }
3149         mutex_unlock(&trace_types_lock);
3150
3151         return 0;
3152 }
3153
3154 static int __set_tracer_option(struct tracer *trace,
3155                                struct tracer_flags *tracer_flags,
3156                                struct tracer_opt *opts, int neg)
3157 {
3158         int ret;
3159
3160         ret = trace->set_flag(tracer_flags->val, opts->bit, !neg);
3161         if (ret)
3162                 return ret;
3163
3164         if (neg)
3165                 tracer_flags->val &= ~opts->bit;
3166         else
3167                 tracer_flags->val |= opts->bit;
3168         return 0;
3169 }
3170
3171 /* Try to assign a tracer specific option */
3172 static int set_tracer_option(struct tracer *trace, char *cmp, int neg)
3173 {
3174         struct tracer_flags *tracer_flags = trace->flags;
3175         struct tracer_opt *opts = NULL;
3176         int i;
3177
3178         for (i = 0; tracer_flags->opts[i].name; i++) {
3179                 opts = &tracer_flags->opts[i];
3180
3181                 if (strcmp(cmp, opts->name) == 0)
3182                         return __set_tracer_option(trace, trace->flags,
3183                                                    opts, neg);
3184         }
3185
3186         return -EINVAL;
3187 }
3188
3189 /* Some tracers require overwrite to stay enabled */
3190 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3191 {
3192         if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3193                 return -1;
3194
3195         return 0;
3196 }
3197
3198 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3199 {
3200         /* do nothing if flag is already set */
3201         if (!!(trace_flags & mask) == !!enabled)
3202                 return 0;
3203
3204         /* Give the tracer a chance to approve the change */
3205         if (tr->current_trace->flag_changed)
3206                 if (tr->current_trace->flag_changed(tr->current_trace, mask, !!enabled))
3207                         return -EINVAL;
3208
3209         if (enabled)
3210                 trace_flags |= mask;
3211         else
3212                 trace_flags &= ~mask;
3213
3214         if (mask == TRACE_ITER_RECORD_CMD)
3215                 trace_event_enable_cmd_record(enabled);
3216
3217         if (mask == TRACE_ITER_OVERWRITE) {
3218                 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3219 #ifdef CONFIG_TRACER_MAX_TRACE
3220                 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3221 #endif
3222         }
3223
3224         if (mask == TRACE_ITER_PRINTK)
3225                 trace_printk_start_stop_comm(enabled);
3226
3227         return 0;
3228 }
3229
3230 static int trace_set_options(struct trace_array *tr, char *option)
3231 {
3232         char *cmp;
3233         int neg = 0;
3234         int ret = -ENODEV;
3235         int i;
3236
3237         cmp = strstrip(option);
3238
3239         if (strncmp(cmp, "no", 2) == 0) {
3240                 neg = 1;
3241                 cmp += 2;
3242         }
3243
3244         mutex_lock(&trace_types_lock);
3245
3246         for (i = 0; trace_options[i]; i++) {
3247                 if (strcmp(cmp, trace_options[i]) == 0) {
3248                         ret = set_tracer_flag(tr, 1 << i, !neg);
3249                         break;
3250                 }
3251         }
3252
3253         /* If no option could be set, test the specific tracer options */
3254         if (!trace_options[i])
3255                 ret = set_tracer_option(tr->current_trace, cmp, neg);
3256
3257         mutex_unlock(&trace_types_lock);
3258
3259         return ret;
3260 }
3261
3262 static ssize_t
3263 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
3264                         size_t cnt, loff_t *ppos)
3265 {
3266         struct seq_file *m = filp->private_data;
3267         struct trace_array *tr = m->private;
3268         char buf[64];
3269         int ret;
3270
3271         if (cnt >= sizeof(buf))
3272                 return -EINVAL;
3273
3274         if (copy_from_user(&buf, ubuf, cnt))
3275                 return -EFAULT;
3276
3277         buf[cnt] = 0;
3278
3279         ret = trace_set_options(tr, buf);
3280         if (ret < 0)
3281                 return ret;
3282
3283         *ppos += cnt;
3284
3285         return cnt;
3286 }
3287
3288 static int tracing_trace_options_open(struct inode *inode, struct file *file)
3289 {
3290         if (tracing_disabled)
3291                 return -ENODEV;
3292
3293         return single_open(file, tracing_trace_options_show, inode->i_private);
3294 }
3295
3296 static const struct file_operations tracing_iter_fops = {
3297         .open           = tracing_trace_options_open,
3298         .read           = seq_read,
3299         .llseek         = seq_lseek,
3300         .release        = single_release,
3301         .write          = tracing_trace_options_write,
3302 };
3303
3304 static const char readme_msg[] =
3305         "tracing mini-HOWTO:\n\n"
3306         "# echo 0 > tracing_on : quick way to disable tracing\n"
3307         "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
3308         " Important files:\n"
3309         "  trace\t\t\t- The static contents of the buffer\n"
3310         "\t\t\t  To clear the buffer write into this file: echo > trace\n"
3311         "  trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
3312         "  current_tracer\t- function and latency tracers\n"
3313         "  available_tracers\t- list of configured tracers for current_tracer\n"
3314         "  buffer_size_kb\t- view and modify size of per cpu buffer\n"
3315         "  buffer_total_size_kb  - view total size of all cpu buffers\n\n"
3316         "  trace_clock\t\t-change the clock used to order events\n"
3317         "       local:   Per cpu clock but may not be synced across CPUs\n"
3318         "      global:   Synced across CPUs but slows tracing down.\n"
3319         "     counter:   Not a clock, but just an increment\n"
3320         "      uptime:   Jiffy counter from time of boot\n"
3321         "        perf:   Same clock that perf events use\n"
3322 #ifdef CONFIG_X86_64
3323         "     x86-tsc:   TSC cycle counter\n"
3324 #endif
3325         "\n  trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
3326         "  tracing_cpumask\t- Limit which CPUs to trace\n"
3327         "  instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
3328         "\t\t\t  Remove sub-buffer with rmdir\n"
3329         "  trace_options\t\t- Set format or modify how tracing happens\n"
3330         "\t\t\t  Disable an option by adding a suffix 'no' to the option name\n"
3331 #ifdef CONFIG_DYNAMIC_FTRACE
3332         "\n  available_filter_functions - list of functions that can be filtered on\n"
3333         "  set_ftrace_filter\t- echo function name in here to only trace these functions\n"
3334         "            accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3335         "            modules: Can select a group via module\n"
3336         "             Format: :mod:<module-name>\n"
3337         "             example: echo :mod:ext3 > set_ftrace_filter\n"
3338         "            triggers: a command to perform when function is hit\n"
3339         "              Format: <function>:<trigger>[:count]\n"
3340         "             trigger: traceon, traceoff\n"
3341         "                      enable_event:<system>:<event>\n"
3342         "                      disable_event:<system>:<event>\n"
3343 #ifdef CONFIG_STACKTRACE
3344         "                      stacktrace\n"
3345 #endif
3346 #ifdef CONFIG_TRACER_SNAPSHOT
3347         "                      snapshot\n"
3348 #endif
3349         "             example: echo do_fault:traceoff > set_ftrace_filter\n"
3350         "                      echo do_trap:traceoff:3 > set_ftrace_filter\n"
3351         "             The first one will disable tracing every time do_fault is hit\n"
3352         "             The second will disable tracing at most 3 times when do_trap is hit\n"
3353         "               The first time do trap is hit and it disables tracing, the counter\n"
3354         "               will decrement to 2. If tracing is already disabled, the counter\n"
3355         "               will not decrement. It only decrements when the trigger did work\n"
3356         "             To remove trigger without count:\n"
3357         "               echo '!<function>:<trigger> > set_ftrace_filter\n"
3358         "             To remove trigger with a count:\n"
3359         "               echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
3360         "  set_ftrace_notrace\t- echo function name in here to never trace.\n"
3361         "            accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
3362         "            modules: Can select a group via module command :mod:\n"
3363         "            Does not accept triggers\n"
3364 #endif /* CONFIG_DYNAMIC_FTRACE */
3365 #ifdef CONFIG_FUNCTION_TRACER
3366         "  set_ftrace_pid\t- Write pid(s) to only function trace those pids (function)\n"
3367 #endif
3368 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3369         "  set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
3370         "  max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
3371 #endif
3372 #ifdef CONFIG_TRACER_SNAPSHOT
3373         "\n  snapshot\t\t- Like 'trace' but shows the content of the static snapshot buffer\n"
3374         "\t\t\t  Read the contents for more information\n"
3375 #endif
3376 #ifdef CONFIG_STACKTRACE
3377         "  stack_trace\t\t- Shows the max stack trace when active\n"
3378         "  stack_max_size\t- Shows current max stack size that was traced\n"
3379         "\t\t\t  Write into this file to reset the max size (trigger a new trace)\n"
3380 #ifdef CONFIG_DYNAMIC_FTRACE
3381         "  stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace traces\n"
3382 #endif
3383 #endif /* CONFIG_STACKTRACE */
3384 ;
3385
3386 static ssize_t
3387 tracing_readme_read(struct file *filp, char __user *ubuf,
3388                        size_t cnt, loff_t *ppos)
3389 {
3390         return simple_read_from_buffer(ubuf, cnt, ppos,
3391                                         readme_msg, strlen(readme_msg));
3392 }
3393
3394 static const struct file_operations tracing_readme_fops = {
3395         .open           = tracing_open_generic,
3396         .read           = tracing_readme_read,
3397         .llseek         = generic_file_llseek,
3398 };
3399
3400 static ssize_t
3401 tracing_saved_cmdlines_read(struct file *file, char __user *ubuf,
3402                                 size_t cnt, loff_t *ppos)
3403 {
3404         char *buf_comm;
3405         char *file_buf;
3406         char *buf;
3407         int len = 0;
3408         int pid;
3409         int i;
3410
3411         file_buf = kmalloc(SAVED_CMDLINES*(16+TASK_COMM_LEN), GFP_KERNEL);
3412         if (!file_buf)
3413                 return -ENOMEM;
3414
3415         buf_comm = kmalloc(TASK_COMM_LEN, GFP_KERNEL);
3416         if (!buf_comm) {
3417                 kfree(file_buf);
3418                 return -ENOMEM;
3419         }
3420
3421         buf = file_buf;
3422
3423         for (i = 0; i < SAVED_CMDLINES; i++) {
3424                 int r;
3425
3426                 pid = map_cmdline_to_pid[i];
3427                 if (pid == -1 || pid == NO_CMDLINE_MAP)
3428                         continue;
3429
3430                 trace_find_cmdline(pid, buf_comm);
3431                 r = sprintf(buf, "%d %s\n", pid, buf_comm);
3432                 buf += r;
3433                 len += r;
3434         }
3435
3436         len = simple_read_from_buffer(ubuf, cnt, ppos,
3437                                       file_buf, len);
3438
3439         kfree(file_buf);
3440         kfree(buf_comm);
3441
3442         return len;
3443 }
3444
3445 static const struct file_operations tracing_saved_cmdlines_fops = {
3446     .open       = tracing_open_generic,
3447     .read       = tracing_saved_cmdlines_read,
3448     .llseek     = generic_file_llseek,
3449 };
3450
3451 static ssize_t
3452 tracing_set_trace_read(struct file *filp, char __user *ubuf,
3453                        size_t cnt, loff_t *ppos)
3454 {
3455         struct trace_array *tr = filp->private_data;
3456         char buf[MAX_TRACER_SIZE+2];
3457         int r;
3458
3459         mutex_lock(&trace_types_lock);
3460         r = sprintf(buf, "%s\n", tr->current_trace->name);
3461         mutex_unlock(&trace_types_lock);
3462
3463         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3464 }
3465
3466 int tracer_init(struct tracer *t, struct trace_array *tr)
3467 {
3468         tracing_reset_online_cpus(&tr->trace_buffer);
3469         return t->init(tr);
3470 }
3471
3472 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
3473 {
3474         int cpu;
3475
3476         for_each_tracing_cpu(cpu)
3477                 per_cpu_ptr(buf->data, cpu)->entries = val;
3478 }
3479
3480 #ifdef CONFIG_TRACER_MAX_TRACE
3481 /* resize @tr's buffer to the size of @size_tr's entries */
3482 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
3483                                         struct trace_buffer *size_buf, int cpu_id)
3484 {
3485         int cpu, ret = 0;
3486
3487         if (cpu_id == RING_BUFFER_ALL_CPUS) {
3488                 for_each_tracing_cpu(cpu) {
3489                         ret = ring_buffer_resize(trace_buf->buffer,
3490                                  per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
3491                         if (ret < 0)
3492                                 break;
3493                         per_cpu_ptr(trace_buf->data, cpu)->entries =
3494                                 per_cpu_ptr(size_buf->data, cpu)->entries;
3495                 }
3496         } else {
3497                 ret = ring_buffer_resize(trace_buf->buffer,
3498                                  per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
3499                 if (ret == 0)
3500                         per_cpu_ptr(trace_buf->data, cpu_id)->entries =
3501                                 per_cpu_ptr(size_buf->data, cpu_id)->entries;
3502         }
3503
3504         return ret;
3505 }
3506 #endif /* CONFIG_TRACER_MAX_TRACE */
3507
3508 static int __tracing_resize_ring_buffer(struct trace_array *tr,
3509                                         unsigned long size, int cpu)
3510 {
3511         int ret;
3512
3513         /*
3514          * If kernel or user changes the size of the ring buffer
3515          * we use the size that was given, and we can forget about
3516          * expanding it later.
3517          */
3518         ring_buffer_expanded = true;
3519
3520         /* May be called before buffers are initialized */
3521         if (!tr->trace_buffer.buffer)
3522                 return 0;
3523
3524         ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
3525         if (ret < 0)
3526                 return ret;
3527
3528 #ifdef CONFIG_TRACER_MAX_TRACE
3529         if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
3530             !tr->current_trace->use_max_tr)
3531                 goto out;
3532
3533         ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
3534         if (ret < 0) {
3535                 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
3536                                                      &tr->trace_buffer, cpu);
3537                 if (r < 0) {
3538                         /*
3539                          * AARGH! We are left with different
3540                          * size max buffer!!!!
3541                          * The max buffer is our "snapshot" buffer.
3542                          * When a tracer needs a snapshot (one of the
3543                          * latency tracers), it swaps the max buffer
3544                          * with the saved snap shot. We succeeded to
3545                          * update the size of the main buffer, but failed to
3546                          * update the size of the max buffer. But when we tried
3547                          * to reset the main buffer to the original size, we
3548                          * failed there too. This is very unlikely to
3549                          * happen, but if it does, warn and kill all
3550                          * tracing.
3551                          */
3552                         WARN_ON(1);
3553                         tracing_disabled = 1;
3554                 }
3555                 return ret;
3556         }
3557
3558         if (cpu == RING_BUFFER_ALL_CPUS)
3559                 set_buffer_entries(&tr->max_buffer, size);
3560         else
3561                 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
3562
3563  out:
3564 #endif /* CONFIG_TRACER_MAX_TRACE */
3565
3566         if (cpu == RING_BUFFER_ALL_CPUS)
3567                 set_buffer_entries(&tr->trace_buffer, size);
3568         else
3569                 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
3570
3571         return ret;
3572 }
3573
3574 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
3575                                           unsigned long size, int cpu_id)
3576 {
3577         int ret = size;
3578
3579         mutex_lock(&trace_types_lock);
3580
3581         if (cpu_id != RING_BUFFER_ALL_CPUS) {
3582                 /* make sure, this cpu is enabled in the mask */
3583                 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
3584                         ret = -EINVAL;
3585                         goto out;
3586                 }
3587         }
3588
3589         ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
3590         if (ret < 0)
3591                 ret = -ENOMEM;
3592
3593 out:
3594         mutex_unlock(&trace_types_lock);
3595
3596         return ret;
3597 }
3598
3599
3600 /**
3601  * tracing_update_buffers - used by tracing facility to expand ring buffers
3602  *
3603  * To save on memory when the tracing is never used on a system with it
3604  * configured in. The ring buffers are set to a minimum size. But once
3605  * a user starts to use the tracing facility, then they need to grow
3606  * to their default size.
3607  *
3608  * This function is to be called when a tracer is about to be used.
3609  */
3610 int tracing_update_buffers(void)
3611 {
3612         int ret = 0;
3613
3614         mutex_lock(&trace_types_lock);
3615         if (!ring_buffer_expanded)
3616                 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
3617                                                 RING_BUFFER_ALL_CPUS);
3618         mutex_unlock(&trace_types_lock);
3619
3620         return ret;
3621 }
3622
3623 struct trace_option_dentry;
3624
3625 static struct trace_option_dentry *
3626 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
3627
3628 static void
3629 destroy_trace_option_files(struct trace_option_dentry *topts);
3630
3631 static int tracing_set_tracer(const char *buf)
3632 {
3633         static struct trace_option_dentry *topts;
3634         struct trace_array *tr = &global_trace;
3635         struct tracer *t;
3636 #ifdef CONFIG_TRACER_MAX_TRACE
3637         bool had_max_tr;
3638 #endif
3639         int ret = 0;
3640
3641         mutex_lock(&trace_types_lock);
3642
3643         if (!ring_buffer_expanded) {
3644                 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
3645                                                 RING_BUFFER_ALL_CPUS);
3646                 if (ret < 0)
3647                         goto out;
3648                 ret = 0;
3649         }
3650
3651         for (t = trace_types; t; t = t->next) {
3652                 if (strcmp(t->name, buf) == 0)
3653                         break;
3654         }
3655         if (!t) {
3656                 ret = -EINVAL;
3657                 goto out;
3658         }
3659         if (t == tr->current_trace)
3660                 goto out;
3661
3662         trace_branch_disable();
3663
3664         tr->current_trace->enabled = false;
3665
3666         if (tr->current_trace->reset)
3667                 tr->current_trace->reset(tr);
3668
3669         /* Current trace needs to be nop_trace before synchronize_sched */
3670         tr->current_trace = &nop_trace;
3671
3672 #ifdef CONFIG_TRACER_MAX_TRACE
3673         had_max_tr = tr->allocated_snapshot;
3674
3675         if (had_max_tr && !t->use_max_tr) {
3676                 /*
3677                  * We need to make sure that the update_max_tr sees that
3678                  * current_trace changed to nop_trace to keep it from
3679                  * swapping the buffers after we resize it.
3680                  * The update_max_tr is called from interrupts disabled
3681                  * so a synchronized_sched() is sufficient.
3682                  */
3683                 synchronize_sched();
3684                 free_snapshot(tr);
3685         }
3686 #endif
3687         destroy_trace_option_files(topts);
3688
3689         topts = create_trace_option_files(tr, t);
3690
3691 #ifdef CONFIG_TRACER_MAX_TRACE
3692         if (t->use_max_tr && !had_max_tr) {
3693                 ret = alloc_snapshot(tr);
3694                 if (ret < 0)
3695                         goto out;
3696         }
3697 #endif
3698
3699         if (t->init) {
3700                 ret = tracer_init(t, tr);
3701                 if (ret)
3702                         goto out;
3703         }
3704
3705         tr->current_trace = t;
3706         tr->current_trace->enabled = true;
3707         trace_branch_enable(tr);
3708  out:
3709         mutex_unlock(&trace_types_lock);
3710
3711         return ret;
3712 }
3713
3714 static ssize_t
3715 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
3716                         size_t cnt, loff_t *ppos)
3717 {
3718         char buf[MAX_TRACER_SIZE+1];
3719         int i;
3720         size_t ret;
3721         int err;
3722
3723         ret = cnt;
3724
3725         if (cnt > MAX_TRACER_SIZE)
3726                 cnt = MAX_TRACER_SIZE;
3727
3728         if (copy_from_user(&buf, ubuf, cnt))
3729                 return -EFAULT;
3730
3731         buf[cnt] = 0;
3732
3733         /* strip ending whitespace. */
3734         for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
3735                 buf[i] = 0;
3736
3737         err = tracing_set_tracer(buf);
3738         if (err)
3739                 return err;
3740
3741         *ppos += ret;
3742
3743         return ret;
3744 }
3745
3746 static ssize_t
3747 tracing_max_lat_read(struct file *filp, char __user *ubuf,
3748                      size_t cnt, loff_t *ppos)
3749 {
3750         unsigned long *ptr = filp->private_data;
3751         char buf[64];
3752         int r;
3753
3754         r = snprintf(buf, sizeof(buf), "%ld\n",
3755                      *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
3756         if (r > sizeof(buf))
3757                 r = sizeof(buf);
3758         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
3759 }
3760
3761 static ssize_t
3762 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
3763                       size_t cnt, loff_t *ppos)
3764 {
3765         unsigned long *ptr = filp->private_data;
3766         unsigned long val;
3767         int ret;
3768
3769         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
3770         if (ret)
3771                 return ret;
3772
3773         *ptr = val * 1000;
3774
3775         return cnt;
3776 }
3777
3778 static int tracing_open_pipe(struct inode *inode, struct file *filp)
3779 {
3780         struct trace_cpu *tc = inode->i_private;
3781         struct trace_array *tr = tc->tr;
3782         struct trace_iterator *iter;
3783         int ret = 0;
3784
3785         if (tracing_disabled)
3786                 return -ENODEV;
3787
3788         mutex_lock(&trace_types_lock);
3789
3790         /* create a buffer to store the information to pass to userspace */
3791         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
3792         if (!iter) {
3793                 ret = -ENOMEM;
3794                 goto out;
3795         }
3796
3797         /*
3798          * We make a copy of the current tracer to avoid concurrent
3799          * changes on it while we are reading.
3800          */
3801         iter->trace = kmalloc(sizeof(*iter->trace), GFP_KERNEL);
3802         if (!iter->trace) {
3803                 ret = -ENOMEM;
3804                 goto fail;
3805         }
3806         *iter->trace = *tr->current_trace;
3807
3808         if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
3809                 ret = -ENOMEM;
3810                 goto fail;
3811         }
3812
3813         /* trace pipe does not show start of buffer */
3814         cpumask_setall(iter->started);
3815
3816         if (trace_flags & TRACE_ITER_LATENCY_FMT)
3817                 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3818
3819         /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3820         if (trace_clocks[trace_clock_id].in_ns)
3821                 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3822
3823         iter->cpu_file = tc->cpu;
3824         iter->tr = tc->tr;
3825         iter->trace_buffer = &tc->tr->trace_buffer;
3826         mutex_init(&iter->mutex);
3827         filp->private_data = iter;
3828
3829         if (iter->trace->pipe_open)
3830                 iter->trace->pipe_open(iter);
3831
3832         nonseekable_open(inode, filp);
3833 out:
3834         mutex_unlock(&trace_types_lock);
3835         return ret;
3836
3837 fail:
3838         kfree(iter->trace);
3839         kfree(iter);
3840         mutex_unlock(&trace_types_lock);
3841         return ret;
3842 }
3843
3844 static int tracing_release_pipe(struct inode *inode, struct file *file)
3845 {
3846         struct trace_iterator *iter = file->private_data;
3847
3848         mutex_lock(&trace_types_lock);
3849
3850         if (iter->trace->pipe_close)
3851                 iter->trace->pipe_close(iter);
3852
3853         mutex_unlock(&trace_types_lock);
3854
3855         free_cpumask_var(iter->started);
3856         mutex_destroy(&iter->mutex);
3857         kfree(iter->trace);
3858         kfree(iter);
3859
3860         return 0;
3861 }
3862
3863 static unsigned int
3864 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
3865 {
3866         /* Iterators are static, they should be filled or empty */
3867         if (trace_buffer_iter(iter, iter->cpu_file))
3868                 return POLLIN | POLLRDNORM;
3869
3870         if (trace_flags & TRACE_ITER_BLOCK)
3871                 /*
3872                  * Always select as readable when in blocking mode
3873                  */
3874                 return POLLIN | POLLRDNORM;
3875         else
3876                 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
3877                                              filp, poll_table);
3878 }
3879
3880 static unsigned int
3881 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
3882 {
3883         struct trace_iterator *iter = filp->private_data;
3884
3885         return trace_poll(iter, filp, poll_table);
3886 }
3887
3888 /*
3889  * This is a make-shift waitqueue.
3890  * A tracer might use this callback on some rare cases:
3891  *
3892  *  1) the current tracer might hold the runqueue lock when it wakes up
3893  *     a reader, hence a deadlock (sched, function, and function graph tracers)
3894  *  2) the function tracers, trace all functions, we don't want
3895  *     the overhead of calling wake_up and friends
3896  *     (and tracing them too)
3897  *
3898  *     Anyway, this is really very primitive wakeup.
3899  */
3900 void poll_wait_pipe(struct trace_iterator *iter)
3901 {
3902         set_current_state(TASK_INTERRUPTIBLE);
3903         /* sleep for 100 msecs, and try again. */
3904         schedule_timeout(HZ / 10);
3905 }
3906
3907 /* Must be called with trace_types_lock mutex held. */
3908 static int tracing_wait_pipe(struct file *filp)
3909 {
3910         struct trace_iterator *iter = filp->private_data;
3911
3912         while (trace_empty(iter)) {
3913
3914                 if ((filp->f_flags & O_NONBLOCK)) {
3915                         return -EAGAIN;
3916                 }
3917
3918                 mutex_unlock(&iter->mutex);
3919
3920                 iter->trace->wait_pipe(iter);
3921
3922                 mutex_lock(&iter->mutex);
3923
3924                 if (signal_pending(current))
3925                         return -EINTR;
3926
3927                 /*
3928                  * We block until we read something and tracing is disabled.
3929                  * We still block if tracing is disabled, but we have never
3930                  * read anything. This allows a user to cat this file, and
3931                  * then enable tracing. But after we have read something,
3932                  * we give an EOF when tracing is again disabled.
3933                  *
3934                  * iter->pos will be 0 if we haven't read anything.
3935                  */
3936                 if (!tracing_is_enabled() && iter->pos)
3937                         break;
3938         }
3939
3940         return 1;
3941 }
3942
3943 /*
3944  * Consumer reader.
3945  */
3946 static ssize_t
3947 tracing_read_pipe(struct file *filp, char __user *ubuf,
3948                   size_t cnt, loff_t *ppos)
3949 {
3950         struct trace_iterator *iter = filp->private_data;
3951         struct trace_array *tr = iter->tr;
3952         ssize_t sret;
3953
3954         /* return any leftover data */
3955         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
3956         if (sret != -EBUSY)
3957                 return sret;
3958
3959         trace_seq_init(&iter->seq);
3960
3961         /* copy the tracer to avoid using a global lock all around */
3962         mutex_lock(&trace_types_lock);
3963         if (unlikely(iter->trace->name != tr->current_trace->name))
3964                 *iter->trace = *tr->current_trace;
3965         mutex_unlock(&trace_types_lock);
3966
3967         /*
3968          * Avoid more than one consumer on a single file descriptor
3969          * This is just a matter of traces coherency, the ring buffer itself
3970          * is protected.
3971          */
3972         mutex_lock(&iter->mutex);
3973         if (iter->trace->read) {
3974                 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
3975                 if (sret)
3976                         goto out;
3977         }
3978
3979 waitagain:
3980         sret = tracing_wait_pipe(filp);
3981         if (sret <= 0)
3982                 goto out;
3983
3984         /* stop when tracing is finished */
3985         if (trace_empty(iter)) {
3986                 sret = 0;
3987                 goto out;
3988         }
3989
3990         if (cnt >= PAGE_SIZE)
3991                 cnt = PAGE_SIZE - 1;
3992
3993         /* reset all but tr, trace, and overruns */
3994         memset(&iter->seq, 0,
3995                sizeof(struct trace_iterator) -
3996                offsetof(struct trace_iterator, seq));
3997         iter->pos = -1;
3998
3999         trace_event_read_lock();
4000         trace_access_lock(iter->cpu_file);
4001         while (trace_find_next_entry_inc(iter) != NULL) {
4002                 enum print_line_t ret;
4003                 int len = iter->seq.len;
4004
4005                 ret = print_trace_line(iter);
4006                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4007                         /* don't print partial lines */
4008                         iter->seq.len = len;
4009                         break;
4010                 }
4011                 if (ret != TRACE_TYPE_NO_CONSUME)
4012                         trace_consume(iter);
4013
4014                 if (iter->seq.len >= cnt)
4015                         break;
4016
4017                 /*
4018                  * Setting the full flag means we reached the trace_seq buffer
4019                  * size and we should leave by partial output condition above.
4020                  * One of the trace_seq_* functions is not used properly.
4021                  */
4022                 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
4023                           iter->ent->type);
4024         }
4025         trace_access_unlock(iter->cpu_file);
4026         trace_event_read_unlock();
4027
4028         /* Now copy what we have to the user */
4029         sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
4030         if (iter->seq.readpos >= iter->seq.len)
4031                 trace_seq_init(&iter->seq);
4032
4033         /*
4034          * If there was nothing to send to user, in spite of consuming trace
4035          * entries, go back to wait for more entries.
4036          */
4037         if (sret == -EBUSY)
4038                 goto waitagain;
4039
4040 out:
4041         mutex_unlock(&iter->mutex);
4042
4043         return sret;
4044 }
4045
4046 static void tracing_pipe_buf_release(struct pipe_inode_info *pipe,
4047                                      struct pipe_buffer *buf)
4048 {
4049         __free_page(buf->page);
4050 }
4051
4052 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
4053                                      unsigned int idx)
4054 {
4055         __free_page(spd->pages[idx]);
4056 }
4057
4058 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
4059         .can_merge              = 0,
4060         .map                    = generic_pipe_buf_map,
4061         .unmap                  = generic_pipe_buf_unmap,
4062         .confirm                = generic_pipe_buf_confirm,
4063         .release                = tracing_pipe_buf_release,
4064         .steal                  = generic_pipe_buf_steal,
4065         .get                    = generic_pipe_buf_get,
4066 };
4067
4068 static size_t
4069 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
4070 {
4071         size_t count;
4072         int ret;
4073
4074         /* Seq buffer is page-sized, exactly what we need. */
4075         for (;;) {
4076                 count = iter->seq.len;
4077                 ret = print_trace_line(iter);
4078                 count = iter->seq.len - count;
4079                 if (rem < count) {
4080                         rem = 0;
4081                         iter->seq.len -= count;
4082                         break;
4083                 }
4084                 if (ret == TRACE_TYPE_PARTIAL_LINE) {
4085                         iter->seq.len -= count;
4086                         break;
4087                 }
4088
4089                 if (ret != TRACE_TYPE_NO_CONSUME)
4090                         trace_consume(iter);
4091                 rem -= count;
4092                 if (!trace_find_next_entry_inc(iter))   {
4093                         rem = 0;
4094                         iter->ent = NULL;
4095                         break;
4096                 }
4097         }
4098
4099         return rem;
4100 }
4101
4102 static ssize_t tracing_splice_read_pipe(struct file *filp,
4103                                         loff_t *ppos,
4104                                         struct pipe_inode_info *pipe,
4105                                         size_t len,
4106                                         unsigned int flags)
4107 {
4108         struct page *pages_def[PIPE_DEF_BUFFERS];
4109         struct partial_page partial_def[PIPE_DEF_BUFFERS];
4110         struct trace_iterator *iter = filp->private_data;
4111         struct splice_pipe_desc spd = {
4112                 .pages          = pages_def,
4113                 .partial        = partial_def,
4114                 .nr_pages       = 0, /* This gets updated below. */
4115                 .nr_pages_max   = PIPE_DEF_BUFFERS,
4116                 .flags          = flags,
4117                 .ops            = &tracing_pipe_buf_ops,
4118                 .spd_release    = tracing_spd_release_pipe,
4119         };
4120         struct trace_array *tr = iter->tr;
4121         ssize_t ret;
4122         size_t rem;
4123         unsigned int i;
4124
4125         if (splice_grow_spd(pipe, &spd))
4126                 return -ENOMEM;
4127
4128         /* copy the tracer to avoid using a global lock all around */
4129         mutex_lock(&trace_types_lock);
4130         if (unlikely(iter->trace->name != tr->current_trace->name))
4131                 *iter->trace = *tr->current_trace;
4132         mutex_unlock(&trace_types_lock);
4133
4134         mutex_lock(&iter->mutex);
4135
4136         if (iter->trace->splice_read) {
4137                 ret = iter->trace->splice_read(iter, filp,
4138                                                ppos, pipe, len, flags);
4139                 if (ret)
4140                         goto out_err;
4141         }
4142
4143         ret = tracing_wait_pipe(filp);
4144         if (ret <= 0)
4145                 goto out_err;
4146
4147         if (!iter->ent && !trace_find_next_entry_inc(iter)) {
4148                 ret = -EFAULT;
4149                 goto out_err;
4150         }
4151
4152         trace_event_read_lock();
4153         trace_access_lock(iter->cpu_file);
4154
4155         /* Fill as many pages as possible. */
4156         for (i = 0, rem = len; i < pipe->buffers && rem; i++) {
4157                 spd.pages[i] = alloc_page(GFP_KERNEL);
4158                 if (!spd.pages[i])
4159                         break;
4160
4161                 rem = tracing_fill_pipe_page(rem, iter);
4162
4163                 /* Copy the data into the page, so we can start over. */
4164                 ret = trace_seq_to_buffer(&iter->seq,
4165                                           page_address(spd.pages[i]),
4166                                           iter->seq.len);
4167                 if (ret < 0) {
4168                         __free_page(spd.pages[i]);
4169                         break;
4170                 }
4171                 spd.partial[i].offset = 0;
4172                 spd.partial[i].len = iter->seq.len;
4173
4174                 trace_seq_init(&iter->seq);
4175         }
4176
4177         trace_access_unlock(iter->cpu_file);
4178         trace_event_read_unlock();
4179         mutex_unlock(&iter->mutex);
4180
4181         spd.nr_pages = i;
4182
4183         ret = splice_to_pipe(pipe, &spd);
4184 out:
4185         splice_shrink_spd(&spd);
4186         return ret;
4187
4188 out_err:
4189         mutex_unlock(&iter->mutex);
4190         goto out;
4191 }
4192
4193 static ssize_t
4194 tracing_entries_read(struct file *filp, char __user *ubuf,
4195                      size_t cnt, loff_t *ppos)
4196 {
4197         struct trace_cpu *tc = filp->private_data;
4198         struct trace_array *tr = tc->tr;
4199         char buf[64];
4200         int r = 0;
4201         ssize_t ret;
4202
4203         mutex_lock(&trace_types_lock);
4204
4205         if (tc->cpu == RING_BUFFER_ALL_CPUS) {
4206                 int cpu, buf_size_same;
4207                 unsigned long size;
4208
4209                 size = 0;
4210                 buf_size_same = 1;
4211                 /* check if all cpu sizes are same */
4212                 for_each_tracing_cpu(cpu) {
4213                         /* fill in the size from first enabled cpu */
4214                         if (size == 0)
4215                                 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
4216                         if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
4217                                 buf_size_same = 0;
4218                                 break;
4219                         }
4220                 }
4221
4222                 if (buf_size_same) {
4223                         if (!ring_buffer_expanded)
4224                                 r = sprintf(buf, "%lu (expanded: %lu)\n",
4225                                             size >> 10,
4226                                             trace_buf_size >> 10);
4227                         else
4228                                 r = sprintf(buf, "%lu\n", size >> 10);
4229                 } else
4230                         r = sprintf(buf, "X\n");
4231         } else
4232                 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, tc->cpu)->entries >> 10);
4233
4234         mutex_unlock(&trace_types_lock);
4235
4236         ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4237         return ret;
4238 }
4239
4240 static ssize_t
4241 tracing_entries_write(struct file *filp, const char __user *ubuf,
4242                       size_t cnt, loff_t *ppos)
4243 {
4244         struct trace_cpu *tc = filp->private_data;
4245         unsigned long val;
4246         int ret;
4247
4248         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4249         if (ret)
4250                 return ret;
4251
4252         /* must have at least 1 entry */
4253         if (!val)
4254                 return -EINVAL;
4255
4256         /* value is in KB */
4257         val <<= 10;
4258
4259         ret = tracing_resize_ring_buffer(tc->tr, val, tc->cpu);
4260         if (ret < 0)
4261                 return ret;
4262
4263         *ppos += cnt;
4264
4265         return cnt;
4266 }
4267
4268 static ssize_t
4269 tracing_total_entries_read(struct file *filp, char __user *ubuf,
4270                                 size_t cnt, loff_t *ppos)
4271 {
4272         struct trace_array *tr = filp->private_data;
4273         char buf[64];
4274         int r, cpu;
4275         unsigned long size = 0, expanded_size = 0;
4276
4277         mutex_lock(&trace_types_lock);
4278         for_each_tracing_cpu(cpu) {
4279                 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
4280                 if (!ring_buffer_expanded)
4281                         expanded_size += trace_buf_size >> 10;
4282         }
4283         if (ring_buffer_expanded)
4284                 r = sprintf(buf, "%lu\n", size);
4285         else
4286                 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
4287         mutex_unlock(&trace_types_lock);
4288
4289         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4290 }
4291
4292 static ssize_t
4293 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
4294                           size_t cnt, loff_t *ppos)
4295 {
4296         /*
4297          * There is no need to read what the user has written, this function
4298          * is just to make sure that there is no error when "echo" is used
4299          */
4300
4301         *ppos += cnt;
4302
4303         return cnt;
4304 }
4305
4306 static int
4307 tracing_free_buffer_release(struct inode *inode, struct file *filp)
4308 {
4309         struct trace_array *tr = inode->i_private;
4310
4311         /* disable tracing ? */
4312         if (trace_flags & TRACE_ITER_STOP_ON_FREE)
4313                 tracing_off();
4314         /* resize the ring buffer to 0 */
4315         tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
4316
4317         return 0;
4318 }
4319
4320 static ssize_t
4321 tracing_mark_write(struct file *filp, const char __user *ubuf,
4322                                         size_t cnt, loff_t *fpos)
4323 {
4324         unsigned long addr = (unsigned long)ubuf;
4325         struct ring_buffer_event *event;
4326         struct ring_buffer *buffer;
4327         struct print_entry *entry;
4328         unsigned long irq_flags;
4329         struct page *pages[2];
4330         void *map_page[2];
4331         int nr_pages = 1;
4332         ssize_t written;
4333         int offset;
4334         int size;
4335         int len;
4336         int ret;
4337         int i;
4338
4339         if (tracing_disabled)
4340                 return -EINVAL;
4341
4342         if (!(trace_flags & TRACE_ITER_MARKERS))
4343                 return -EINVAL;
4344
4345         if (cnt > TRACE_BUF_SIZE)
4346                 cnt = TRACE_BUF_SIZE;
4347
4348         /*
4349          * Userspace is injecting traces into the kernel trace buffer.
4350          * We want to be as non intrusive as possible.
4351          * To do so, we do not want to allocate any special buffers
4352          * or take any locks, but instead write the userspace data
4353          * straight into the ring buffer.
4354          *
4355          * First we need to pin the userspace buffer into memory,
4356          * which, most likely it is, because it just referenced it.
4357          * But there's no guarantee that it is. By using get_user_pages_fast()
4358          * and kmap_atomic/kunmap_atomic() we can get access to the
4359          * pages directly. We then write the data directly into the
4360          * ring buffer.
4361          */
4362         BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
4363
4364         /* check if we cross pages */
4365         if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
4366                 nr_pages = 2;
4367
4368         offset = addr & (PAGE_SIZE - 1);
4369         addr &= PAGE_MASK;
4370
4371         ret = get_user_pages_fast(addr, nr_pages, 0, pages);
4372         if (ret < nr_pages) {
4373                 while (--ret >= 0)
4374                         put_page(pages[ret]);
4375                 written = -EFAULT;
4376                 goto out;
4377         }
4378
4379         for (i = 0; i < nr_pages; i++)
4380                 map_page[i] = kmap_atomic(pages[i]);
4381
4382         local_save_flags(irq_flags);
4383         size = sizeof(*entry) + cnt + 2; /* possible \n added */
4384         buffer = global_trace.trace_buffer.buffer;
4385         event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
4386                                           irq_flags, preempt_count());
4387         if (!event) {
4388                 /* Ring buffer disabled, return as if not open for write */
4389                 written = -EBADF;
4390                 goto out_unlock;
4391         }
4392
4393         entry = ring_buffer_event_data(event);
4394         entry->ip = _THIS_IP_;
4395
4396         if (nr_pages == 2) {
4397                 len = PAGE_SIZE - offset;
4398                 memcpy(&entry->buf, map_page[0] + offset, len);
4399                 memcpy(&entry->buf[len], map_page[1], cnt - len);
4400         } else
4401                 memcpy(&entry->buf, map_page[0] + offset, cnt);
4402
4403         if (entry->buf[cnt - 1] != '\n') {
4404                 entry->buf[cnt] = '\n';
4405                 entry->buf[cnt + 1] = '\0';
4406         } else
4407                 entry->buf[cnt] = '\0';
4408
4409         __buffer_unlock_commit(buffer, event);
4410
4411         written = cnt;
4412
4413         *fpos += written;
4414
4415  out_unlock:
4416         for (i = 0; i < nr_pages; i++){
4417                 kunmap_atomic(map_page[i]);
4418                 put_page(pages[i]);
4419         }
4420  out:
4421         return written;
4422 }
4423
4424 static int tracing_clock_show(struct seq_file *m, void *v)
4425 {
4426         struct trace_array *tr = m->private;
4427         int i;
4428
4429         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
4430                 seq_printf(m,
4431                         "%s%s%s%s", i ? " " : "",
4432                         i == tr->clock_id ? "[" : "", trace_clocks[i].name,
4433                         i == tr->clock_id ? "]" : "");
4434         seq_putc(m, '\n');
4435
4436         return 0;
4437 }
4438
4439 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
4440                                    size_t cnt, loff_t *fpos)
4441 {
4442         struct seq_file *m = filp->private_data;
4443         struct trace_array *tr = m->private;
4444         char buf[64];
4445         const char *clockstr;
4446         int i;
4447
4448         if (cnt >= sizeof(buf))
4449                 return -EINVAL;
4450
4451         if (copy_from_user(&buf, ubuf, cnt))
4452                 return -EFAULT;
4453
4454         buf[cnt] = 0;
4455
4456         clockstr = strstrip(buf);
4457
4458         for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
4459                 if (strcmp(trace_clocks[i].name, clockstr) == 0)
4460                         break;
4461         }
4462         if (i == ARRAY_SIZE(trace_clocks))
4463                 return -EINVAL;
4464
4465         mutex_lock(&trace_types_lock);
4466
4467         tr->clock_id = i;
4468
4469         ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
4470
4471         /*
4472          * New clock may not be consistent with the previous clock.
4473          * Reset the buffer so that it doesn't have incomparable timestamps.
4474          */
4475         tracing_reset_online_cpus(&global_trace.trace_buffer);
4476
4477 #ifdef CONFIG_TRACER_MAX_TRACE
4478         if (tr->flags & TRACE_ARRAY_FL_GLOBAL && tr->max_buffer.buffer)
4479                 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
4480         tracing_reset_online_cpus(&global_trace.max_buffer);
4481 #endif
4482
4483         mutex_unlock(&trace_types_lock);
4484
4485         *fpos += cnt;
4486
4487         return cnt;
4488 }
4489
4490 static int tracing_clock_open(struct inode *inode, struct file *file)
4491 {
4492         if (tracing_disabled)
4493                 return -ENODEV;
4494
4495         return single_open(file, tracing_clock_show, inode->i_private);
4496 }
4497
4498 struct ftrace_buffer_info {
4499         struct trace_iterator   iter;
4500         void                    *spare;
4501         unsigned int            read;
4502 };
4503
4504 #ifdef CONFIG_TRACER_SNAPSHOT
4505 static int tracing_snapshot_open(struct inode *inode, struct file *file)
4506 {
4507         struct trace_cpu *tc = inode->i_private;
4508         struct trace_iterator *iter;
4509         struct seq_file *m;
4510         int ret = 0;
4511
4512         if (file->f_mode & FMODE_READ) {
4513                 iter = __tracing_open(inode, file, true);
4514                 if (IS_ERR(iter))
4515                         ret = PTR_ERR(iter);
4516         } else {
4517                 /* Writes still need the seq_file to hold the private data */
4518                 m = kzalloc(sizeof(*m), GFP_KERNEL);
4519                 if (!m)
4520                         return -ENOMEM;
4521                 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
4522                 if (!iter) {
4523                         kfree(m);
4524                         return -ENOMEM;
4525                 }
4526                 iter->tr = tc->tr;
4527                 iter->trace_buffer = &tc->tr->max_buffer;
4528                 iter->cpu_file = tc->cpu;
4529                 m->private = iter;
4530                 file->private_data = m;
4531         }
4532
4533         return ret;
4534 }
4535
4536 static ssize_t
4537 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
4538                        loff_t *ppos)
4539 {
4540         struct seq_file *m = filp->private_data;
4541         struct trace_iterator *iter = m->private;
4542         struct trace_array *tr = iter->tr;
4543         unsigned long val;
4544         int ret;
4545
4546         ret = tracing_update_buffers();
4547         if (ret < 0)
4548                 return ret;
4549
4550         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4551         if (ret)
4552                 return ret;
4553
4554         mutex_lock(&trace_types_lock);
4555
4556         if (tr->current_trace->use_max_tr) {
4557                 ret = -EBUSY;
4558                 goto out;
4559         }
4560
4561         switch (val) {
4562         case 0:
4563                 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4564                         ret = -EINVAL;
4565                         break;
4566                 }
4567                 if (tr->allocated_snapshot)
4568                         free_snapshot(tr);
4569                 break;
4570         case 1:
4571 /* Only allow per-cpu swap if the ring buffer supports it */
4572 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
4573                 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
4574                         ret = -EINVAL;
4575                         break;
4576                 }
4577 #endif
4578                 if (!tr->allocated_snapshot) {
4579                         ret = alloc_snapshot(tr);
4580                         if (ret < 0)
4581                                 break;
4582                 }
4583                 local_irq_disable();
4584                 /* Now, we're going to swap */
4585                 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4586                         update_max_tr(tr, current, smp_processor_id());
4587                 else
4588                         update_max_tr_single(tr, current, iter->cpu_file);
4589                 local_irq_enable();
4590                 break;
4591         default:
4592                 if (tr->allocated_snapshot) {
4593                         if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
4594                                 tracing_reset_online_cpus(&tr->max_buffer);
4595                         else
4596                                 tracing_reset(&tr->max_buffer, iter->cpu_file);
4597                 }
4598                 break;
4599         }
4600
4601         if (ret >= 0) {
4602                 *ppos += cnt;
4603                 ret = cnt;
4604         }
4605 out:
4606         mutex_unlock(&trace_types_lock);
4607         return ret;
4608 }
4609
4610 static int tracing_snapshot_release(struct inode *inode, struct file *file)
4611 {
4612         struct seq_file *m = file->private_data;
4613
4614         if (file->f_mode & FMODE_READ)
4615                 return tracing_release(inode, file);
4616
4617         /* If write only, the seq_file is just a stub */
4618         if (m)
4619                 kfree(m->private);
4620         kfree(m);
4621
4622         return 0;
4623 }
4624
4625 static int tracing_buffers_open(struct inode *inode, struct file *filp);
4626 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
4627                                     size_t count, loff_t *ppos);
4628 static int tracing_buffers_release(struct inode *inode, struct file *file);
4629 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4630                    struct pipe_inode_info *pipe, size_t len, unsigned int flags);
4631
4632 static int snapshot_raw_open(struct inode *inode, struct file *filp)
4633 {
4634         struct ftrace_buffer_info *info;
4635         int ret;
4636
4637         ret = tracing_buffers_open(inode, filp);
4638         if (ret < 0)
4639                 return ret;
4640
4641         info = filp->private_data;
4642
4643         if (info->iter.trace->use_max_tr) {
4644                 tracing_buffers_release(inode, filp);
4645                 return -EBUSY;
4646         }
4647
4648         info->iter.snapshot = true;
4649         info->iter.trace_buffer = &info->iter.tr->max_buffer;
4650
4651         return ret;
4652 }
4653
4654 #endif /* CONFIG_TRACER_SNAPSHOT */
4655
4656
4657 static const struct file_operations tracing_max_lat_fops = {
4658         .open           = tracing_open_generic,
4659         .read           = tracing_max_lat_read,
4660         .write          = tracing_max_lat_write,
4661         .llseek         = generic_file_llseek,
4662 };
4663
4664 static const struct file_operations set_tracer_fops = {
4665         .open           = tracing_open_generic,
4666         .read           = tracing_set_trace_read,
4667         .write          = tracing_set_trace_write,
4668         .llseek         = generic_file_llseek,
4669 };
4670
4671 static const struct file_operations tracing_pipe_fops = {
4672         .open           = tracing_open_pipe,
4673         .poll           = tracing_poll_pipe,
4674         .read           = tracing_read_pipe,
4675         .splice_read    = tracing_splice_read_pipe,
4676         .release        = tracing_release_pipe,
4677         .llseek         = no_llseek,
4678 };
4679
4680 static const struct file_operations tracing_entries_fops = {
4681         .open           = tracing_open_generic,
4682         .read           = tracing_entries_read,
4683         .write          = tracing_entries_write,
4684         .llseek         = generic_file_llseek,
4685 };
4686
4687 static const struct file_operations tracing_total_entries_fops = {
4688         .open           = tracing_open_generic,
4689         .read           = tracing_total_entries_read,
4690         .llseek         = generic_file_llseek,
4691 };
4692
4693 static const struct file_operations tracing_free_buffer_fops = {
4694         .write          = tracing_free_buffer_write,
4695         .release        = tracing_free_buffer_release,
4696 };
4697
4698 static const struct file_operations tracing_mark_fops = {
4699         .open           = tracing_open_generic,
4700         .write          = tracing_mark_write,
4701         .llseek         = generic_file_llseek,
4702 };
4703
4704 static const struct file_operations trace_clock_fops = {
4705         .open           = tracing_clock_open,
4706         .read           = seq_read,
4707         .llseek         = seq_lseek,
4708         .release        = single_release,
4709         .write          = tracing_clock_write,
4710 };
4711
4712 #ifdef CONFIG_TRACER_SNAPSHOT
4713 static const struct file_operations snapshot_fops = {
4714         .open           = tracing_snapshot_open,
4715         .read           = seq_read,
4716         .write          = tracing_snapshot_write,
4717         .llseek         = tracing_seek,
4718         .release        = tracing_snapshot_release,
4719 };
4720
4721 static const struct file_operations snapshot_raw_fops = {
4722         .open           = snapshot_raw_open,
4723         .read           = tracing_buffers_read,
4724         .release        = tracing_buffers_release,
4725         .splice_read    = tracing_buffers_splice_read,
4726         .llseek         = no_llseek,
4727 };
4728
4729 #endif /* CONFIG_TRACER_SNAPSHOT */
4730
4731 static int tracing_buffers_open(struct inode *inode, struct file *filp)
4732 {
4733         struct trace_cpu *tc = inode->i_private;
4734         struct trace_array *tr = tc->tr;
4735         struct ftrace_buffer_info *info;
4736
4737         if (tracing_disabled)
4738                 return -ENODEV;
4739
4740         info = kzalloc(sizeof(*info), GFP_KERNEL);
4741         if (!info)
4742                 return -ENOMEM;
4743
4744         mutex_lock(&trace_types_lock);
4745
4746         tr->ref++;
4747
4748         info->iter.tr           = tr;
4749         info->iter.cpu_file     = tc->cpu;
4750         info->iter.trace        = tr->current_trace;
4751         info->iter.trace_buffer = &tr->trace_buffer;
4752         info->spare             = NULL;
4753         /* Force reading ring buffer for first read */
4754         info->read              = (unsigned int)-1;
4755
4756         filp->private_data = info;
4757
4758         mutex_unlock(&trace_types_lock);
4759
4760         return nonseekable_open(inode, filp);
4761 }
4762
4763 static unsigned int
4764 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
4765 {
4766         struct ftrace_buffer_info *info = filp->private_data;
4767         struct trace_iterator *iter = &info->iter;
4768
4769         return trace_poll(iter, filp, poll_table);
4770 }
4771
4772 static ssize_t
4773 tracing_buffers_read(struct file *filp, char __user *ubuf,
4774                      size_t count, loff_t *ppos)
4775 {
4776         struct ftrace_buffer_info *info = filp->private_data;
4777         struct trace_iterator *iter = &info->iter;
4778         ssize_t ret;
4779         ssize_t size;
4780
4781         if (!count)
4782                 return 0;
4783
4784         mutex_lock(&trace_types_lock);
4785
4786 #ifdef CONFIG_TRACER_MAX_TRACE
4787         if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
4788                 size = -EBUSY;
4789                 goto out_unlock;
4790         }
4791 #endif
4792
4793         if (!info->spare)
4794                 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
4795                                                           iter->cpu_file);
4796         size = -ENOMEM;
4797         if (!info->spare)
4798                 goto out_unlock;
4799
4800         /* Do we have previous read data to read? */
4801         if (info->read < PAGE_SIZE)
4802                 goto read;
4803
4804  again:
4805         trace_access_lock(iter->cpu_file);
4806         ret = ring_buffer_read_page(iter->trace_buffer->buffer,
4807                                     &info->spare,
4808                                     count,
4809                                     iter->cpu_file, 0);
4810         trace_access_unlock(iter->cpu_file);
4811
4812         if (ret < 0) {
4813                 if (trace_empty(iter)) {
4814                         if ((filp->f_flags & O_NONBLOCK)) {
4815                                 size = -EAGAIN;
4816                                 goto out_unlock;
4817                         }
4818                         mutex_unlock(&trace_types_lock);
4819                         iter->trace->wait_pipe(iter);
4820                         mutex_lock(&trace_types_lock);
4821                         if (signal_pending(current)) {
4822                                 size = -EINTR;
4823                                 goto out_unlock;
4824                         }
4825                         goto again;
4826                 }
4827                 size = 0;
4828                 goto out_unlock;
4829         }
4830
4831         info->read = 0;
4832  read:
4833         size = PAGE_SIZE - info->read;
4834         if (size > count)
4835                 size = count;
4836
4837         ret = copy_to_user(ubuf, info->spare + info->read, size);
4838         if (ret == size) {
4839                 size = -EFAULT;
4840                 goto out_unlock;
4841         }
4842         size -= ret;
4843
4844         *ppos += size;
4845         info->read += size;
4846
4847  out_unlock:
4848         mutex_unlock(&trace_types_lock);
4849
4850         return size;
4851 }
4852
4853 static int tracing_buffers_release(struct inode *inode, struct file *file)
4854 {
4855         struct ftrace_buffer_info *info = file->private_data;
4856         struct trace_iterator *iter = &info->iter;
4857
4858         mutex_lock(&trace_types_lock);
4859
4860         WARN_ON(!iter->tr->ref);
4861         iter->tr->ref--;
4862
4863         if (info->spare)
4864                 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
4865         kfree(info);
4866
4867         mutex_unlock(&trace_types_lock);
4868
4869         return 0;
4870 }
4871
4872 struct buffer_ref {
4873         struct ring_buffer      *buffer;
4874         void                    *page;
4875         int                     ref;
4876 };
4877
4878 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
4879                                     struct pipe_buffer *buf)
4880 {
4881         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4882
4883         if (--ref->ref)
4884                 return;
4885
4886         ring_buffer_free_read_page(ref->buffer, ref->page);
4887         kfree(ref);
4888         buf->private = 0;
4889 }
4890
4891 static void buffer_pipe_buf_get(struct pipe_inode_info *pipe,
4892                                 struct pipe_buffer *buf)
4893 {
4894         struct buffer_ref *ref = (struct buffer_ref *)buf->private;
4895
4896         ref->ref++;
4897 }
4898
4899 /* Pipe buffer operations for a buffer. */
4900 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
4901         .can_merge              = 0,
4902         .map                    = generic_pipe_buf_map,
4903         .unmap                  = generic_pipe_buf_unmap,
4904         .confirm                = generic_pipe_buf_confirm,
4905         .release                = buffer_pipe_buf_release,
4906         .steal                  = generic_pipe_buf_steal,
4907         .get                    = buffer_pipe_buf_get,
4908 };
4909
4910 /*
4911  * Callback from splice_to_pipe(), if we need to release some pages
4912  * at the end of the spd in case we error'ed out in filling the pipe.
4913  */
4914 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
4915 {
4916         struct buffer_ref *ref =
4917                 (struct buffer_ref *)spd->partial[i].private;
4918
4919         if (--ref->ref)
4920                 return;
4921
4922         ring_buffer_free_read_page(ref->buffer, ref->page);
4923         kfree(ref);
4924         spd->partial[i].private = 0;
4925 }
4926
4927 static ssize_t
4928 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
4929                             struct pipe_inode_info *pipe, size_t len,
4930                             unsigned int flags)
4931 {
4932         struct ftrace_buffer_info *info = file->private_data;
4933         struct trace_iterator *iter = &info->iter;
4934         struct partial_page partial_def[PIPE_DEF_BUFFERS];
4935         struct page *pages_def[PIPE_DEF_BUFFERS];
4936         struct splice_pipe_desc spd = {
4937                 .pages          = pages_def,
4938                 .partial        = partial_def,
4939                 .nr_pages_max   = PIPE_DEF_BUFFERS,
4940                 .flags          = flags,
4941                 .ops            = &buffer_pipe_buf_ops,
4942                 .spd_release    = buffer_spd_release,
4943         };
4944         struct buffer_ref *ref;
4945         int entries, size, i;
4946         ssize_t ret;
4947
4948         mutex_lock(&trace_types_lock);
4949
4950 #ifdef CONFIG_TRACER_MAX_TRACE
4951         if (iter->snapshot && iter->tr->current_trace->use_max_tr) {
4952                 ret = -EBUSY;
4953                 goto out;
4954         }
4955 #endif
4956
4957         if (splice_grow_spd(pipe, &spd)) {
4958                 ret = -ENOMEM;
4959                 goto out;
4960         }
4961
4962         if (*ppos & (PAGE_SIZE - 1)) {
4963                 ret = -EINVAL;
4964                 goto out;
4965         }
4966
4967         if (len & (PAGE_SIZE - 1)) {
4968                 if (len < PAGE_SIZE) {
4969                         ret = -EINVAL;
4970                         goto out;
4971                 }
4972                 len &= PAGE_MASK;
4973         }
4974
4975  again:
4976         trace_access_lock(iter->cpu_file);
4977         entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
4978
4979         for (i = 0; i < pipe->buffers && len && entries; i++, len -= PAGE_SIZE) {
4980                 struct page *page;
4981                 int r;
4982
4983                 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
4984                 if (!ref)
4985                         break;
4986
4987                 ref->ref = 1;
4988                 ref->buffer = iter->trace_buffer->buffer;
4989                 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
4990                 if (!ref->page) {
4991                         kfree(ref);
4992                         break;
4993                 }
4994
4995                 r = ring_buffer_read_page(ref->buffer, &ref->page,
4996                                           len, iter->cpu_file, 1);
4997                 if (r < 0) {
4998                         ring_buffer_free_read_page(ref->buffer, ref->page);
4999                         kfree(ref);
5000                         break;
5001                 }
5002
5003                 /*
5004                  * zero out any left over data, this is going to
5005                  * user land.
5006                  */
5007                 size = ring_buffer_page_len(ref->page);
5008                 if (size < PAGE_SIZE)
5009                         memset(ref->page + size, 0, PAGE_SIZE - size);
5010
5011                 page = virt_to_page(ref->page);
5012
5013                 spd.pages[i] = page;
5014                 spd.partial[i].len = PAGE_SIZE;
5015                 spd.partial[i].offset = 0;
5016                 spd.partial[i].private = (unsigned long)ref;
5017                 spd.nr_pages++;
5018                 *ppos += PAGE_SIZE;
5019
5020                 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
5021         }
5022
5023         trace_access_unlock(iter->cpu_file);
5024         spd.nr_pages = i;
5025
5026         /* did we read anything? */
5027         if (!spd.nr_pages) {
5028                 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK)) {
5029                         ret = -EAGAIN;
5030                         goto out;
5031                 }
5032                 mutex_unlock(&trace_types_lock);
5033                 iter->trace->wait_pipe(iter);
5034                 mutex_lock(&trace_types_lock);
5035                 if (signal_pending(current)) {
5036                         ret = -EINTR;
5037                         goto out;
5038                 }
5039                 goto again;
5040         }
5041
5042         ret = splice_to_pipe(pipe, &spd);
5043         splice_shrink_spd(&spd);
5044 out:
5045         mutex_unlock(&trace_types_lock);
5046
5047         return ret;
5048 }
5049
5050 static const struct file_operations tracing_buffers_fops = {
5051         .open           = tracing_buffers_open,
5052         .read           = tracing_buffers_read,
5053         .poll           = tracing_buffers_poll,
5054         .release        = tracing_buffers_release,
5055         .splice_read    = tracing_buffers_splice_read,
5056         .llseek         = no_llseek,
5057 };
5058
5059 static ssize_t
5060 tracing_stats_read(struct file *filp, char __user *ubuf,
5061                    size_t count, loff_t *ppos)
5062 {
5063         struct trace_cpu *tc = filp->private_data;
5064         struct trace_array *tr = tc->tr;
5065         struct trace_buffer *trace_buf = &tr->trace_buffer;
5066         struct trace_seq *s;
5067         unsigned long cnt;
5068         unsigned long long t;
5069         unsigned long usec_rem;
5070         int cpu = tc->cpu;
5071
5072         s = kmalloc(sizeof(*s), GFP_KERNEL);
5073         if (!s)
5074                 return -ENOMEM;
5075
5076         trace_seq_init(s);
5077
5078         cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
5079         trace_seq_printf(s, "entries: %ld\n", cnt);
5080
5081         cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
5082         trace_seq_printf(s, "overrun: %ld\n", cnt);
5083
5084         cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
5085         trace_seq_printf(s, "commit overrun: %ld\n", cnt);
5086
5087         cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
5088         trace_seq_printf(s, "bytes: %ld\n", cnt);
5089
5090         if (trace_clocks[trace_clock_id].in_ns) {
5091                 /* local or global for trace_clock */
5092                 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5093                 usec_rem = do_div(t, USEC_PER_SEC);
5094                 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
5095                                                                 t, usec_rem);
5096
5097                 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
5098                 usec_rem = do_div(t, USEC_PER_SEC);
5099                 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
5100         } else {
5101                 /* counter or tsc mode for trace_clock */
5102                 trace_seq_printf(s, "oldest event ts: %llu\n",
5103                                 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
5104
5105                 trace_seq_printf(s, "now ts: %llu\n",
5106                                 ring_buffer_time_stamp(trace_buf->buffer, cpu));
5107         }
5108
5109         cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
5110         trace_seq_printf(s, "dropped events: %ld\n", cnt);
5111
5112         cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
5113         trace_seq_printf(s, "read events: %ld\n", cnt);
5114
5115         count = simple_read_from_buffer(ubuf, count, ppos, s->buffer, s->len);
5116
5117         kfree(s);
5118
5119         return count;
5120 }
5121
5122 static const struct file_operations tracing_stats_fops = {
5123         .open           = tracing_open_generic,
5124         .read           = tracing_stats_read,
5125         .llseek         = generic_file_llseek,
5126 };
5127
5128 #ifdef CONFIG_DYNAMIC_FTRACE
5129
5130 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
5131 {
5132         return 0;
5133 }
5134
5135 static ssize_t
5136 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
5137                   size_t cnt, loff_t *ppos)
5138 {
5139         static char ftrace_dyn_info_buffer[1024];
5140         static DEFINE_MUTEX(dyn_info_mutex);
5141         unsigned long *p = filp->private_data;
5142         char *buf = ftrace_dyn_info_buffer;
5143         int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
5144         int r;
5145
5146         mutex_lock(&dyn_info_mutex);
5147         r = sprintf(buf, "%ld ", *p);
5148
5149         r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
5150         buf[r++] = '\n';
5151
5152         r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5153
5154         mutex_unlock(&dyn_info_mutex);
5155
5156         return r;
5157 }
5158
5159 static const struct file_operations tracing_dyn_info_fops = {
5160         .open           = tracing_open_generic,
5161         .read           = tracing_read_dyn_info,
5162         .llseek         = generic_file_llseek,
5163 };
5164 #endif /* CONFIG_DYNAMIC_FTRACE */
5165
5166 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
5167 static void
5168 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5169 {
5170         tracing_snapshot();
5171 }
5172
5173 static void
5174 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
5175 {
5176         unsigned long *count = (long *)data;
5177
5178         if (!*count)
5179                 return;
5180
5181         if (*count != -1)
5182                 (*count)--;
5183
5184         tracing_snapshot();
5185 }
5186
5187 static int
5188 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
5189                       struct ftrace_probe_ops *ops, void *data)
5190 {
5191         long count = (long)data;
5192
5193         seq_printf(m, "%ps:", (void *)ip);
5194
5195         seq_printf(m, "snapshot");
5196
5197         if (count == -1)
5198                 seq_printf(m, ":unlimited\n");
5199         else
5200                 seq_printf(m, ":count=%ld\n", count);
5201
5202         return 0;
5203 }
5204
5205 static struct ftrace_probe_ops snapshot_probe_ops = {
5206         .func                   = ftrace_snapshot,
5207         .print                  = ftrace_snapshot_print,
5208 };
5209
5210 static struct ftrace_probe_ops snapshot_count_probe_ops = {
5211         .func                   = ftrace_count_snapshot,
5212         .print                  = ftrace_snapshot_print,
5213 };
5214
5215 static int
5216 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
5217                                char *glob, char *cmd, char *param, int enable)
5218 {
5219         struct ftrace_probe_ops *ops;
5220         void *count = (void *)-1;
5221         char *number;
5222         int ret;
5223
5224         /* hash funcs only work with set_ftrace_filter */
5225         if (!enable)
5226                 return -EINVAL;
5227
5228         ops = param ? &snapshot_count_probe_ops :  &snapshot_probe_ops;
5229
5230         if (glob[0] == '!') {
5231                 unregister_ftrace_function_probe_func(glob+1, ops);
5232                 return 0;
5233         }
5234
5235         if (!param)
5236                 goto out_reg;
5237
5238         number = strsep(&param, ":");
5239
5240         if (!strlen(number))
5241                 goto out_reg;
5242
5243         /*
5244          * We use the callback data field (which is a pointer)
5245          * as our counter.
5246          */
5247         ret = kstrtoul(number, 0, (unsigned long *)&count);
5248         if (ret)
5249                 return ret;
5250
5251  out_reg:
5252         ret = register_ftrace_function_probe(glob, ops, count);
5253
5254         if (ret >= 0)
5255                 alloc_snapshot(&global_trace);
5256
5257         return ret < 0 ? ret : 0;
5258 }
5259
5260 static struct ftrace_func_command ftrace_snapshot_cmd = {
5261         .name                   = "snapshot",
5262         .func                   = ftrace_trace_snapshot_callback,
5263 };
5264
5265 static int register_snapshot_cmd(void)
5266 {
5267         return register_ftrace_command(&ftrace_snapshot_cmd);
5268 }
5269 #else
5270 static inline int register_snapshot_cmd(void) { return 0; }
5271 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
5272
5273 struct dentry *tracing_init_dentry_tr(struct trace_array *tr)
5274 {
5275         if (tr->dir)
5276                 return tr->dir;
5277
5278         if (!debugfs_initialized())
5279                 return NULL;
5280
5281         if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
5282                 tr->dir = debugfs_create_dir("tracing", NULL);
5283
5284         if (!tr->dir)
5285                 pr_warn_once("Could not create debugfs directory 'tracing'\n");
5286
5287         return tr->dir;
5288 }
5289
5290 struct dentry *tracing_init_dentry(void)
5291 {
5292         return tracing_init_dentry_tr(&global_trace);
5293 }
5294
5295 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
5296 {
5297         struct dentry *d_tracer;
5298
5299         if (tr->percpu_dir)
5300                 return tr->percpu_dir;
5301
5302         d_tracer = tracing_init_dentry_tr(tr);
5303         if (!d_tracer)
5304                 return NULL;
5305
5306         tr->percpu_dir = debugfs_create_dir("per_cpu", d_tracer);
5307
5308         WARN_ONCE(!tr->percpu_dir,
5309                   "Could not create debugfs directory 'per_cpu/%d'\n", cpu);
5310
5311         return tr->percpu_dir;
5312 }
5313
5314 static void
5315 tracing_init_debugfs_percpu(struct trace_array *tr, long cpu)
5316 {
5317         struct trace_array_cpu *data = per_cpu_ptr(tr->trace_buffer.data, cpu);
5318         struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
5319         struct dentry *d_cpu;
5320         char cpu_dir[30]; /* 30 characters should be more than enough */
5321
5322         if (!d_percpu)
5323                 return;
5324
5325         snprintf(cpu_dir, 30, "cpu%ld", cpu);
5326         d_cpu = debugfs_create_dir(cpu_dir, d_percpu);
5327         if (!d_cpu) {
5328                 pr_warning("Could not create debugfs '%s' entry\n", cpu_dir);
5329                 return;
5330         }
5331
5332         /* per cpu trace_pipe */
5333         trace_create_file("trace_pipe", 0444, d_cpu,
5334                         (void *)&data->trace_cpu, &tracing_pipe_fops);
5335
5336         /* per cpu trace */
5337         trace_create_file("trace", 0644, d_cpu,
5338                         (void *)&data->trace_cpu, &tracing_fops);
5339
5340         trace_create_file("trace_pipe_raw", 0444, d_cpu,
5341                         (void *)&data->trace_cpu, &tracing_buffers_fops);
5342
5343         trace_create_file("stats", 0444, d_cpu,
5344                         (void *)&data->trace_cpu, &tracing_stats_fops);
5345
5346         trace_create_file("buffer_size_kb", 0444, d_cpu,
5347                         (void *)&data->trace_cpu, &tracing_entries_fops);
5348
5349 #ifdef CONFIG_TRACER_SNAPSHOT
5350         trace_create_file("snapshot", 0644, d_cpu,
5351                           (void *)&data->trace_cpu, &snapshot_fops);
5352
5353         trace_create_file("snapshot_raw", 0444, d_cpu,
5354                         (void *)&data->trace_cpu, &snapshot_raw_fops);
5355 #endif
5356 }
5357
5358 #ifdef CONFIG_FTRACE_SELFTEST
5359 /* Let selftest have access to static functions in this file */
5360 #include "trace_selftest.c"
5361 #endif
5362
5363 struct trace_option_dentry {
5364         struct tracer_opt               *opt;
5365         struct tracer_flags             *flags;
5366         struct trace_array              *tr;
5367         struct dentry                   *entry;
5368 };
5369
5370 static ssize_t
5371 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
5372                         loff_t *ppos)
5373 {
5374         struct trace_option_dentry *topt = filp->private_data;
5375         char *buf;
5376
5377         if (topt->flags->val & topt->opt->bit)
5378                 buf = "1\n";
5379         else
5380                 buf = "0\n";
5381
5382         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5383 }
5384
5385 static ssize_t
5386 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
5387                          loff_t *ppos)
5388 {
5389         struct trace_option_dentry *topt = filp->private_data;
5390         unsigned long val;
5391         int ret;
5392
5393         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5394         if (ret)
5395                 return ret;
5396
5397         if (val != 0 && val != 1)
5398                 return -EINVAL;
5399
5400         if (!!(topt->flags->val & topt->opt->bit) != val) {
5401                 mutex_lock(&trace_types_lock);
5402                 ret = __set_tracer_option(topt->tr->current_trace, topt->flags,
5403                                           topt->opt, !val);
5404                 mutex_unlock(&trace_types_lock);
5405                 if (ret)
5406                         return ret;
5407         }
5408
5409         *ppos += cnt;
5410
5411         return cnt;
5412 }
5413
5414
5415 static const struct file_operations trace_options_fops = {
5416         .open = tracing_open_generic,
5417         .read = trace_options_read,
5418         .write = trace_options_write,
5419         .llseek = generic_file_llseek,
5420 };
5421
5422 static ssize_t
5423 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
5424                         loff_t *ppos)
5425 {
5426         long index = (long)filp->private_data;
5427         char *buf;
5428
5429         if (trace_flags & (1 << index))
5430                 buf = "1\n";
5431         else
5432                 buf = "0\n";
5433
5434         return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
5435 }
5436
5437 static ssize_t
5438 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
5439                          loff_t *ppos)
5440 {
5441         struct trace_array *tr = &global_trace;
5442         long index = (long)filp->private_data;
5443         unsigned long val;
5444         int ret;
5445
5446         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5447         if (ret)
5448                 return ret;
5449
5450         if (val != 0 && val != 1)
5451                 return -EINVAL;
5452
5453         mutex_lock(&trace_types_lock);
5454         ret = set_tracer_flag(tr, 1 << index, val);
5455         mutex_unlock(&trace_types_lock);
5456
5457         if (ret < 0)
5458                 return ret;
5459
5460         *ppos += cnt;
5461
5462         return cnt;
5463 }
5464
5465 static const struct file_operations trace_options_core_fops = {
5466         .open = tracing_open_generic,
5467         .read = trace_options_core_read,
5468         .write = trace_options_core_write,
5469         .llseek = generic_file_llseek,
5470 };
5471
5472 struct dentry *trace_create_file(const char *name,
5473                                  umode_t mode,
5474                                  struct dentry *parent,
5475                                  void *data,
5476                                  const struct file_operations *fops)
5477 {
5478         struct dentry *ret;
5479
5480         ret = debugfs_create_file(name, mode, parent, data, fops);
5481         if (!ret)
5482                 pr_warning("Could not create debugfs '%s' entry\n", name);
5483
5484         return ret;
5485 }
5486
5487
5488 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
5489 {
5490         struct dentry *d_tracer;
5491
5492         if (tr->options)
5493                 return tr->options;
5494
5495         d_tracer = tracing_init_dentry_tr(tr);
5496         if (!d_tracer)
5497                 return NULL;
5498
5499         tr->options = debugfs_create_dir("options", d_tracer);
5500         if (!tr->options) {
5501                 pr_warning("Could not create debugfs directory 'options'\n");
5502                 return NULL;
5503         }
5504
5505         return tr->options;
5506 }
5507
5508 static void
5509 create_trace_option_file(struct trace_array *tr,
5510                          struct trace_option_dentry *topt,
5511                          struct tracer_flags *flags,
5512                          struct tracer_opt *opt)
5513 {
5514         struct dentry *t_options;
5515
5516         t_options = trace_options_init_dentry(tr);
5517         if (!t_options)
5518                 return;
5519
5520         topt->flags = flags;
5521         topt->opt = opt;
5522         topt->tr = tr;
5523
5524         topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
5525                                     &trace_options_fops);
5526
5527 }
5528
5529 static struct trace_option_dentry *
5530 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
5531 {
5532         struct trace_option_dentry *topts;
5533         struct tracer_flags *flags;
5534         struct tracer_opt *opts;
5535         int cnt;
5536
5537         if (!tracer)
5538                 return NULL;
5539
5540         flags = tracer->flags;
5541
5542         if (!flags || !flags->opts)
5543                 return NULL;
5544
5545         opts = flags->opts;
5546
5547         for (cnt = 0; opts[cnt].name; cnt++)
5548                 ;
5549
5550         topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
5551         if (!topts)
5552                 return NULL;
5553
5554         for (cnt = 0; opts[cnt].name; cnt++)
5555                 create_trace_option_file(tr, &topts[cnt], flags,
5556                                          &opts[cnt]);
5557
5558         return topts;
5559 }
5560
5561 static void
5562 destroy_trace_option_files(struct trace_option_dentry *topts)
5563 {
5564         int cnt;
5565
5566         if (!topts)
5567                 return;
5568
5569         for (cnt = 0; topts[cnt].opt; cnt++) {
5570                 if (topts[cnt].entry)
5571                         debugfs_remove(topts[cnt].entry);
5572         }
5573
5574         kfree(topts);
5575 }
5576
5577 static struct dentry *
5578 create_trace_option_core_file(struct trace_array *tr,
5579                               const char *option, long index)
5580 {
5581         struct dentry *t_options;
5582
5583         t_options = trace_options_init_dentry(tr);
5584         if (!t_options)
5585                 return NULL;
5586
5587         return trace_create_file(option, 0644, t_options, (void *)index,
5588                                     &trace_options_core_fops);
5589 }
5590
5591 static __init void create_trace_options_dir(struct trace_array *tr)
5592 {
5593         struct dentry *t_options;
5594         int i;
5595
5596         t_options = trace_options_init_dentry(tr);
5597         if (!t_options)
5598                 return;
5599
5600         for (i = 0; trace_options[i]; i++)
5601                 create_trace_option_core_file(tr, trace_options[i], i);
5602 }
5603
5604 static ssize_t
5605 rb_simple_read(struct file *filp, char __user *ubuf,
5606                size_t cnt, loff_t *ppos)
5607 {
5608         struct trace_array *tr = filp->private_data;
5609         struct ring_buffer *buffer = tr->trace_buffer.buffer;
5610         char buf[64];
5611         int r;
5612
5613         if (buffer)
5614                 r = ring_buffer_record_is_on(buffer);
5615         else
5616                 r = 0;
5617
5618         r = sprintf(buf, "%d\n", r);
5619
5620         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5621 }
5622
5623 static ssize_t
5624 rb_simple_write(struct file *filp, const char __user *ubuf,
5625                 size_t cnt, loff_t *ppos)
5626 {
5627         struct trace_array *tr = filp->private_data;
5628         struct ring_buffer *buffer = tr->trace_buffer.buffer;
5629         unsigned long val;
5630         int ret;
5631
5632         ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5633         if (ret)
5634                 return ret;
5635
5636         if (buffer) {
5637                 mutex_lock(&trace_types_lock);
5638                 if (val) {
5639                         ring_buffer_record_on(buffer);
5640                         if (tr->current_trace->start)
5641                                 tr->current_trace->start(tr);
5642                 } else {
5643                         ring_buffer_record_off(buffer);
5644                         if (tr->current_trace->stop)
5645                                 tr->current_trace->stop(tr);
5646                 }
5647                 mutex_unlock(&trace_types_lock);
5648         }
5649
5650         (*ppos)++;
5651
5652         return cnt;
5653 }
5654
5655 static const struct file_operations rb_simple_fops = {
5656         .open           = tracing_open_generic,
5657         .read           = rb_simple_read,
5658         .write          = rb_simple_write,
5659         .llseek         = default_llseek,
5660 };
5661
5662 struct dentry *trace_instance_dir;
5663
5664 static void
5665 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer);
5666
5667 static void init_trace_buffers(struct trace_array *tr, struct trace_buffer *buf)
5668 {
5669         int cpu;
5670
5671         for_each_tracing_cpu(cpu) {
5672                 memset(per_cpu_ptr(buf->data, cpu), 0, sizeof(struct trace_array_cpu));
5673                 per_cpu_ptr(buf->data, cpu)->trace_cpu.cpu = cpu;
5674                 per_cpu_ptr(buf->data, cpu)->trace_cpu.tr = tr;
5675         }
5676 }
5677
5678 static int
5679 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
5680 {
5681         enum ring_buffer_flags rb_flags;
5682
5683         rb_flags = trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
5684
5685         buf->buffer = ring_buffer_alloc(size, rb_flags);
5686         if (!buf->buffer)
5687                 return -ENOMEM;
5688
5689         buf->data = alloc_percpu(struct trace_array_cpu);
5690         if (!buf->data) {
5691                 ring_buffer_free(buf->buffer);
5692                 return -ENOMEM;
5693         }
5694
5695         init_trace_buffers(tr, buf);
5696
5697         /* Allocate the first page for all buffers */
5698         set_buffer_entries(&tr->trace_buffer,
5699                            ring_buffer_size(tr->trace_buffer.buffer, 0));
5700
5701         return 0;
5702 }
5703
5704 static int allocate_trace_buffers(struct trace_array *tr, int size)
5705 {
5706         int ret;
5707
5708         ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
5709         if (ret)
5710                 return ret;
5711
5712 #ifdef CONFIG_TRACER_MAX_TRACE
5713         ret = allocate_trace_buffer(tr, &tr->max_buffer,
5714                                     allocate_snapshot ? size : 1);
5715         if (WARN_ON(ret)) {
5716                 ring_buffer_free(tr->trace_buffer.buffer);
5717                 free_percpu(tr->trace_buffer.data);
5718                 return -ENOMEM;
5719         }
5720         tr->allocated_snapshot = allocate_snapshot;
5721
5722         /*
5723          * Only the top level trace array gets its snapshot allocated
5724          * from the kernel command line.
5725          */
5726         allocate_snapshot = false;
5727 #endif
5728         return 0;
5729 }
5730
5731 static int new_instance_create(const char *name)
5732 {
5733         struct trace_array *tr;
5734         int ret;
5735
5736         mutex_lock(&trace_types_lock);
5737
5738         ret = -EEXIST;
5739         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5740                 if (tr->name && strcmp(tr->name, name) == 0)
5741                         goto out_unlock;
5742         }
5743
5744         ret = -ENOMEM;
5745         tr = kzalloc(sizeof(*tr), GFP_KERNEL);
5746         if (!tr)
5747                 goto out_unlock;
5748
5749         tr->name = kstrdup(name, GFP_KERNEL);
5750         if (!tr->name)
5751                 goto out_free_tr;
5752
5753         raw_spin_lock_init(&tr->start_lock);
5754
5755         tr->current_trace = &nop_trace;
5756
5757         INIT_LIST_HEAD(&tr->systems);
5758         INIT_LIST_HEAD(&tr->events);
5759
5760         if (allocate_trace_buffers(tr, trace_buf_size) < 0)
5761                 goto out_free_tr;
5762
5763         /* Holder for file callbacks */
5764         tr->trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
5765         tr->trace_cpu.tr = tr;
5766
5767         tr->dir = debugfs_create_dir(name, trace_instance_dir);
5768         if (!tr->dir)
5769                 goto out_free_tr;
5770
5771         ret = event_trace_add_tracer(tr->dir, tr);
5772         if (ret)
5773                 goto out_free_tr;
5774
5775         init_tracer_debugfs(tr, tr->dir);
5776
5777         list_add(&tr->list, &ftrace_trace_arrays);
5778
5779         mutex_unlock(&trace_types_lock);
5780
5781         return 0;
5782
5783  out_free_tr:
5784         if (tr->trace_buffer.buffer)
5785                 ring_buffer_free(tr->trace_buffer.buffer);
5786         kfree(tr->name);
5787         kfree(tr);
5788
5789  out_unlock:
5790         mutex_unlock(&trace_types_lock);
5791
5792         return ret;
5793
5794 }
5795
5796 static int instance_delete(const char *name)
5797 {
5798         struct trace_array *tr;
5799         int found = 0;
5800         int ret;
5801
5802         mutex_lock(&trace_types_lock);
5803
5804         ret = -ENODEV;
5805         list_for_each_entry(tr, &ftrace_trace_arrays, list) {
5806                 if (tr->name && strcmp(tr->name, name) == 0) {
5807                         found = 1;
5808                         break;
5809                 }
5810         }
5811         if (!found)
5812                 goto out_unlock;
5813
5814         ret = -EBUSY;
5815         if (tr->ref)
5816                 goto out_unlock;
5817
5818         list_del(&tr->list);
5819
5820         event_trace_del_tracer(tr);
5821         debugfs_remove_recursive(tr->dir);
5822         free_percpu(tr->trace_buffer.data);
5823         ring_buffer_free(tr->trace_buffer.buffer);
5824
5825         kfree(tr->name);
5826         kfree(tr);
5827
5828         ret = 0;
5829
5830  out_unlock:
5831         mutex_unlock(&trace_types_lock);
5832
5833         return ret;
5834 }
5835
5836 static int instance_mkdir (struct inode *inode, struct dentry *dentry, umode_t mode)
5837 {
5838         struct dentry *parent;
5839         int ret;
5840
5841         /* Paranoid: Make sure the parent is the "instances" directory */
5842         parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
5843         if (WARN_ON_ONCE(parent != trace_instance_dir))
5844                 return -ENOENT;
5845
5846         /*
5847          * The inode mutex is locked, but debugfs_create_dir() will also
5848          * take the mutex. As the instances directory can not be destroyed
5849          * or changed in any other way, it is safe to unlock it, and
5850          * let the dentry try. If two users try to make the same dir at
5851          * the same time, then the new_instance_create() will determine the
5852          * winner.
5853          */
5854         mutex_unlock(&inode->i_mutex);
5855
5856         ret = new_instance_create(dentry->d_iname);
5857
5858         mutex_lock(&inode->i_mutex);
5859
5860         return ret;
5861 }
5862
5863 static int instance_rmdir(struct inode *inode, struct dentry *dentry)
5864 {
5865         struct dentry *parent;
5866         int ret;
5867
5868         /* Paranoid: Make sure the parent is the "instances" directory */
5869         parent = hlist_entry(inode->i_dentry.first, struct dentry, d_alias);
5870         if (WARN_ON_ONCE(parent != trace_instance_dir))
5871                 return -ENOENT;
5872
5873         /* The caller did a dget() on dentry */
5874         mutex_unlock(&dentry->d_inode->i_mutex);
5875
5876         /*
5877          * The inode mutex is locked, but debugfs_create_dir() will also
5878          * take the mutex. As the instances directory can not be destroyed
5879          * or changed in any other way, it is safe to unlock it, and
5880          * let the dentry try. If two users try to make the same dir at
5881          * the same time, then the instance_delete() will determine the
5882          * winner.
5883          */
5884         mutex_unlock(&inode->i_mutex);
5885
5886         ret = instance_delete(dentry->d_iname);
5887
5888         mutex_lock_nested(&inode->i_mutex, I_MUTEX_PARENT);
5889         mutex_lock(&dentry->d_inode->i_mutex);
5890
5891         return ret;
5892 }
5893
5894 static const struct inode_operations instance_dir_inode_operations = {
5895         .lookup         = simple_lookup,
5896         .mkdir          = instance_mkdir,
5897         .rmdir          = instance_rmdir,
5898 };
5899
5900 static __init void create_trace_instances(struct dentry *d_tracer)
5901 {
5902         trace_instance_dir = debugfs_create_dir("instances", d_tracer);
5903         if (WARN_ON(!trace_instance_dir))
5904                 return;
5905
5906         /* Hijack the dir inode operations, to allow mkdir */
5907         trace_instance_dir->d_inode->i_op = &instance_dir_inode_operations;
5908 }
5909
5910 static void
5911 init_tracer_debugfs(struct trace_array *tr, struct dentry *d_tracer)
5912 {
5913         int cpu;
5914
5915         trace_create_file("trace_options", 0644, d_tracer,
5916                           tr, &tracing_iter_fops);
5917
5918         trace_create_file("trace", 0644, d_tracer,
5919                         (void *)&tr->trace_cpu, &tracing_fops);
5920
5921         trace_create_file("trace_pipe", 0444, d_tracer,
5922                         (void *)&tr->trace_cpu, &tracing_pipe_fops);
5923
5924         trace_create_file("buffer_size_kb", 0644, d_tracer,
5925                         (void *)&tr->trace_cpu, &tracing_entries_fops);
5926
5927         trace_create_file("buffer_total_size_kb", 0444, d_tracer,
5928                           tr, &tracing_total_entries_fops);
5929
5930         trace_create_file("free_buffer", 0644, d_tracer,
5931                           tr, &tracing_free_buffer_fops);
5932
5933         trace_create_file("trace_marker", 0220, d_tracer,
5934                           tr, &tracing_mark_fops);
5935
5936         trace_create_file("trace_clock", 0644, d_tracer, tr,
5937                           &trace_clock_fops);
5938
5939         trace_create_file("tracing_on", 0644, d_tracer,
5940                             tr, &rb_simple_fops);
5941
5942 #ifdef CONFIG_TRACER_SNAPSHOT
5943         trace_create_file("snapshot", 0644, d_tracer,
5944                           (void *)&tr->trace_cpu, &snapshot_fops);
5945 #endif
5946
5947         for_each_tracing_cpu(cpu)
5948                 tracing_init_debugfs_percpu(tr, cpu);
5949
5950 }
5951
5952 static __init int tracer_init_debugfs(void)
5953 {
5954         struct dentry *d_tracer;
5955
5956         trace_access_lock_init();
5957
5958         d_tracer = tracing_init_dentry();
5959         if (!d_tracer)
5960                 return 0;
5961
5962         init_tracer_debugfs(&global_trace, d_tracer);
5963
5964         trace_create_file("tracing_cpumask", 0644, d_tracer,
5965                         &global_trace, &tracing_cpumask_fops);
5966
5967         trace_create_file("available_tracers", 0444, d_tracer,
5968                         &global_trace, &show_traces_fops);
5969
5970         trace_create_file("current_tracer", 0644, d_tracer,
5971                         &global_trace, &set_tracer_fops);
5972
5973 #ifdef CONFIG_TRACER_MAX_TRACE
5974         trace_create_file("tracing_max_latency", 0644, d_tracer,
5975                         &tracing_max_latency, &tracing_max_lat_fops);
5976 #endif
5977
5978         trace_create_file("tracing_thresh", 0644, d_tracer,
5979                         &tracing_thresh, &tracing_max_lat_fops);
5980
5981         trace_create_file("README", 0444, d_tracer,
5982                         NULL, &tracing_readme_fops);
5983
5984         trace_create_file("saved_cmdlines", 0444, d_tracer,
5985                         NULL, &tracing_saved_cmdlines_fops);
5986
5987 #ifdef CONFIG_DYNAMIC_FTRACE
5988         trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
5989                         &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
5990 #endif
5991
5992         create_trace_instances(d_tracer);
5993
5994         create_trace_options_dir(&global_trace);
5995
5996         return 0;
5997 }
5998
5999 static int trace_panic_handler(struct notifier_block *this,
6000                                unsigned long event, void *unused)
6001 {
6002         if (ftrace_dump_on_oops)
6003                 ftrace_dump(ftrace_dump_on_oops);
6004         return NOTIFY_OK;
6005 }
6006
6007 static struct notifier_block trace_panic_notifier = {
6008         .notifier_call  = trace_panic_handler,
6009         .next           = NULL,
6010         .priority       = 150   /* priority: INT_MAX >= x >= 0 */
6011 };
6012
6013 static int trace_die_handler(struct notifier_block *self,
6014                              unsigned long val,
6015                              void *data)
6016 {
6017         switch (val) {
6018         case DIE_OOPS:
6019                 if (ftrace_dump_on_oops)
6020                         ftrace_dump(ftrace_dump_on_oops);
6021                 break;
6022         default:
6023                 break;
6024         }
6025         return NOTIFY_OK;
6026 }
6027
6028 static struct notifier_block trace_die_notifier = {
6029         .notifier_call = trace_die_handler,
6030         .priority = 200
6031 };
6032
6033 /*
6034  * printk is set to max of 1024, we really don't need it that big.
6035  * Nothing should be printing 1000 characters anyway.
6036  */
6037 #define TRACE_MAX_PRINT         1000
6038
6039 /*
6040  * Define here KERN_TRACE so that we have one place to modify
6041  * it if we decide to change what log level the ftrace dump
6042  * should be at.
6043  */
6044 #define KERN_TRACE              KERN_EMERG
6045
6046 void
6047 trace_printk_seq(struct trace_seq *s)
6048 {
6049         /* Probably should print a warning here. */
6050         if (s->len >= TRACE_MAX_PRINT)
6051                 s->len = TRACE_MAX_PRINT;
6052
6053         /* should be zero ended, but we are paranoid. */
6054         s->buffer[s->len] = 0;
6055
6056         printk(KERN_TRACE "%s", s->buffer);
6057
6058         trace_seq_init(s);
6059 }
6060
6061 void trace_init_global_iter(struct trace_iterator *iter)
6062 {
6063         iter->tr = &global_trace;
6064         iter->trace = iter->tr->current_trace;
6065         iter->cpu_file = RING_BUFFER_ALL_CPUS;
6066         iter->trace_buffer = &global_trace.trace_buffer;
6067 }
6068
6069 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
6070 {
6071         /* use static because iter can be a bit big for the stack */
6072         static struct trace_iterator iter;
6073         static atomic_t dump_running;
6074         unsigned int old_userobj;
6075         unsigned long flags;
6076         int cnt = 0, cpu;
6077
6078         /* Only allow one dump user at a time. */
6079         if (atomic_inc_return(&dump_running) != 1) {
6080                 atomic_dec(&dump_running);
6081                 return;
6082         }
6083
6084         /*
6085          * Always turn off tracing when we dump.
6086          * We don't need to show trace output of what happens
6087          * between multiple crashes.
6088          *
6089          * If the user does a sysrq-z, then they can re-enable
6090          * tracing with echo 1 > tracing_on.
6091          */
6092         tracing_off();
6093
6094         local_irq_save(flags);
6095
6096         /* Simulate the iterator */
6097         trace_init_global_iter(&iter);
6098
6099         for_each_tracing_cpu(cpu) {
6100                 atomic_inc(&per_cpu_ptr(iter.tr->trace_buffer.data, cpu)->disabled);
6101         }
6102
6103         old_userobj = trace_flags & TRACE_ITER_SYM_USEROBJ;
6104
6105         /* don't look at user memory in panic mode */
6106         trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
6107
6108         switch (oops_dump_mode) {
6109         case DUMP_ALL:
6110                 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6111                 break;
6112         case DUMP_ORIG:
6113                 iter.cpu_file = raw_smp_processor_id();
6114                 break;
6115         case DUMP_NONE:
6116                 goto out_enable;
6117         default:
6118                 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
6119                 iter.cpu_file = RING_BUFFER_ALL_CPUS;
6120         }
6121
6122         printk(KERN_TRACE "Dumping ftrace buffer:\n");
6123
6124         /* Did function tracer already get disabled? */
6125         if (ftrace_is_dead()) {
6126                 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
6127                 printk("#          MAY BE MISSING FUNCTION EVENTS\n");
6128         }
6129
6130         /*
6131          * We need to stop all tracing on all CPUS to read the
6132          * the next buffer. This is a bit expensive, but is
6133          * not done often. We fill all what we can read,
6134          * and then release the locks again.
6135          */
6136
6137         while (!trace_empty(&iter)) {
6138
6139                 if (!cnt)
6140                         printk(KERN_TRACE "---------------------------------\n");
6141
6142                 cnt++;
6143
6144                 /* reset all but tr, trace, and overruns */
6145                 memset(&iter.seq, 0,
6146                        sizeof(struct trace_iterator) -
6147                        offsetof(struct trace_iterator, seq));
6148                 iter.iter_flags |= TRACE_FILE_LAT_FMT;
6149                 iter.pos = -1;
6150
6151                 if (trace_find_next_entry_inc(&iter) != NULL) {
6152                         int ret;
6153
6154                         ret = print_trace_line(&iter);
6155                         if (ret != TRACE_TYPE_NO_CONSUME)
6156                                 trace_consume(&iter);
6157                 }
6158                 touch_nmi_watchdog();
6159
6160                 trace_printk_seq(&iter.seq);
6161         }
6162
6163         if (!cnt)
6164                 printk(KERN_TRACE "   (ftrace buffer empty)\n");
6165         else
6166                 printk(KERN_TRACE "---------------------------------\n");
6167
6168  out_enable:
6169         trace_flags |= old_userobj;
6170
6171         for_each_tracing_cpu(cpu) {
6172                 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
6173         }
6174         atomic_dec(&dump_running);
6175         local_irq_restore(flags);
6176 }
6177 EXPORT_SYMBOL_GPL(ftrace_dump);
6178
6179 __init static int tracer_alloc_buffers(void)
6180 {
6181         int ring_buf_size;
6182         int ret = -ENOMEM;
6183
6184
6185         if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
6186                 goto out;
6187
6188         if (!alloc_cpumask_var(&tracing_cpumask, GFP_KERNEL))
6189                 goto out_free_buffer_mask;
6190
6191         /* Only allocate trace_printk buffers if a trace_printk exists */
6192         if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
6193                 /* Must be called before global_trace.buffer is allocated */
6194                 trace_printk_init_buffers();
6195
6196         /* To save memory, keep the ring buffer size to its minimum */
6197         if (ring_buffer_expanded)
6198                 ring_buf_size = trace_buf_size;
6199         else
6200                 ring_buf_size = 1;
6201
6202         cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
6203         cpumask_copy(tracing_cpumask, cpu_all_mask);
6204
6205         raw_spin_lock_init(&global_trace.start_lock);
6206
6207         /* TODO: make the number of buffers hot pluggable with CPUS */
6208         if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
6209                 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
6210                 WARN_ON(1);
6211                 goto out_free_cpumask;
6212         }
6213
6214         if (global_trace.buffer_disabled)
6215                 tracing_off();
6216
6217         trace_init_cmdlines();
6218
6219         /*
6220          * register_tracer() might reference current_trace, so it
6221          * needs to be set before we register anything. This is
6222          * just a bootstrap of current_trace anyway.
6223          */
6224         global_trace.current_trace = &nop_trace;
6225
6226         register_tracer(&nop_trace);
6227
6228         /* All seems OK, enable tracing */
6229         tracing_disabled = 0;
6230
6231         atomic_notifier_chain_register(&panic_notifier_list,
6232                                        &trace_panic_notifier);
6233
6234         register_die_notifier(&trace_die_notifier);
6235
6236         global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
6237
6238         /* Holder for file callbacks */
6239         global_trace.trace_cpu.cpu = RING_BUFFER_ALL_CPUS;
6240         global_trace.trace_cpu.tr = &global_trace;
6241
6242         INIT_LIST_HEAD(&global_trace.systems);
6243         INIT_LIST_HEAD(&global_trace.events);
6244         list_add(&global_trace.list, &ftrace_trace_arrays);
6245
6246         while (trace_boot_options) {
6247                 char *option;
6248
6249                 option = strsep(&trace_boot_options, ",");
6250                 trace_set_options(&global_trace, option);
6251         }
6252
6253         register_snapshot_cmd();
6254
6255         return 0;
6256
6257 out_free_cpumask:
6258         free_percpu(global_trace.trace_buffer.data);
6259 #ifdef CONFIG_TRACER_MAX_TRACE
6260         free_percpu(global_trace.max_buffer.data);
6261 #endif
6262         free_cpumask_var(tracing_cpumask);
6263 out_free_buffer_mask:
6264         free_cpumask_var(tracing_buffer_mask);
6265 out:
6266         return ret;
6267 }
6268
6269 __init static int clear_boot_tracer(void)
6270 {
6271         /*
6272          * The default tracer at boot buffer is an init section.
6273          * This function is called in lateinit. If we did not
6274          * find the boot tracer, then clear it out, to prevent
6275          * later registration from accessing the buffer that is
6276          * about to be freed.
6277          */
6278         if (!default_bootup_tracer)
6279                 return 0;
6280
6281         printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
6282                default_bootup_tracer);
6283         default_bootup_tracer = NULL;
6284
6285         return 0;
6286 }
6287
6288 early_initcall(tracer_alloc_buffers);
6289 fs_initcall(tracer_init_debugfs);
6290 late_initcall(clear_boot_tracer);