5 #include "util/evlist.h"
6 #include "util/cache.h"
7 #include "util/evsel.h"
8 #include "util/symbol.h"
9 #include "util/thread.h"
10 #include "util/header.h"
11 #include "util/session.h"
12 #include "util/tool.h"
14 #include "util/parse-options.h"
15 #include "util/trace-event.h"
17 #include "util/debug.h"
19 #include <sys/prctl.h>
20 #include <sys/resource.h>
22 #include <semaphore.h>
26 static const char *input_name;
28 static char default_sort_order[] = "avg, max, switch, runtime";
29 static const char *sort_order = default_sort_order;
31 static int profile_cpu = -1;
33 #define PR_SET_NAME 15 /* Set process name */
36 static u64 run_measurement_overhead;
37 static u64 sleep_measurement_overhead;
44 static unsigned long nr_tasks;
53 unsigned long nr_events;
54 unsigned long curr_event;
55 struct sched_atom **atoms;
66 enum sched_event_type {
70 SCHED_EVENT_MIGRATION,
74 enum sched_event_type type;
80 struct task_desc *wakee;
83 static struct task_desc *pid_to_task[MAX_PID];
85 static struct task_desc **tasks;
87 static pthread_mutex_t start_work_mutex = PTHREAD_MUTEX_INITIALIZER;
88 static u64 start_time;
90 static pthread_mutex_t work_done_wait_mutex = PTHREAD_MUTEX_INITIALIZER;
92 static unsigned long nr_run_events;
93 static unsigned long nr_sleep_events;
94 static unsigned long nr_wakeup_events;
96 static unsigned long nr_sleep_corrections;
97 static unsigned long nr_run_events_optimized;
99 static unsigned long targetless_wakeups;
100 static unsigned long multitarget_wakeups;
102 static u64 cpu_usage;
103 static u64 runavg_cpu_usage;
104 static u64 parent_cpu_usage;
105 static u64 runavg_parent_cpu_usage;
107 static unsigned long nr_runs;
108 static u64 sum_runtime;
109 static u64 sum_fluct;
112 static unsigned int replay_repeat = 10;
113 static unsigned long nr_timestamps;
114 static unsigned long nr_unordered_timestamps;
115 static unsigned long nr_state_machine_bugs;
116 static unsigned long nr_context_switch_bugs;
117 static unsigned long nr_events;
118 static unsigned long nr_lost_chunks;
119 static unsigned long nr_lost_events;
121 #define TASK_STATE_TO_CHAR_STR "RSDTtZX"
131 struct list_head list;
132 enum thread_state state;
140 struct list_head work_list;
141 struct thread *thread;
150 typedef int (*sort_fn_t)(struct work_atoms *, struct work_atoms *);
152 static struct rb_root atom_root, sorted_atom_root;
154 static u64 all_runtime;
155 static u64 all_count;
158 static u64 get_nsecs(void)
162 clock_gettime(CLOCK_MONOTONIC, &ts);
164 return ts.tv_sec * 1000000000ULL + ts.tv_nsec;
167 static void burn_nsecs(u64 nsecs)
169 u64 T0 = get_nsecs(), T1;
173 } while (T1 + run_measurement_overhead < T0 + nsecs);
176 static void sleep_nsecs(u64 nsecs)
180 ts.tv_nsec = nsecs % 999999999;
181 ts.tv_sec = nsecs / 999999999;
183 nanosleep(&ts, NULL);
186 static void calibrate_run_measurement_overhead(void)
188 u64 T0, T1, delta, min_delta = 1000000000ULL;
191 for (i = 0; i < 10; i++) {
196 min_delta = min(min_delta, delta);
198 run_measurement_overhead = min_delta;
200 printf("run measurement overhead: %" PRIu64 " nsecs\n", min_delta);
203 static void calibrate_sleep_measurement_overhead(void)
205 u64 T0, T1, delta, min_delta = 1000000000ULL;
208 for (i = 0; i < 10; i++) {
213 min_delta = min(min_delta, delta);
216 sleep_measurement_overhead = min_delta;
218 printf("sleep measurement overhead: %" PRIu64 " nsecs\n", min_delta);
221 static struct sched_atom *
222 get_new_event(struct task_desc *task, u64 timestamp)
224 struct sched_atom *event = zalloc(sizeof(*event));
225 unsigned long idx = task->nr_events;
228 event->timestamp = timestamp;
232 size = sizeof(struct sched_atom *) * task->nr_events;
233 task->atoms = realloc(task->atoms, size);
234 BUG_ON(!task->atoms);
236 task->atoms[idx] = event;
241 static struct sched_atom *last_event(struct task_desc *task)
243 if (!task->nr_events)
246 return task->atoms[task->nr_events - 1];
250 add_sched_event_run(struct task_desc *task, u64 timestamp, u64 duration)
252 struct sched_atom *event, *curr_event = last_event(task);
255 * optimize an existing RUN event by merging this one
258 if (curr_event && curr_event->type == SCHED_EVENT_RUN) {
259 nr_run_events_optimized++;
260 curr_event->duration += duration;
264 event = get_new_event(task, timestamp);
266 event->type = SCHED_EVENT_RUN;
267 event->duration = duration;
273 add_sched_event_wakeup(struct task_desc *task, u64 timestamp,
274 struct task_desc *wakee)
276 struct sched_atom *event, *wakee_event;
278 event = get_new_event(task, timestamp);
279 event->type = SCHED_EVENT_WAKEUP;
280 event->wakee = wakee;
282 wakee_event = last_event(wakee);
283 if (!wakee_event || wakee_event->type != SCHED_EVENT_SLEEP) {
284 targetless_wakeups++;
287 if (wakee_event->wait_sem) {
288 multitarget_wakeups++;
292 wakee_event->wait_sem = zalloc(sizeof(*wakee_event->wait_sem));
293 sem_init(wakee_event->wait_sem, 0, 0);
294 wakee_event->specific_wait = 1;
295 event->wait_sem = wakee_event->wait_sem;
301 add_sched_event_sleep(struct task_desc *task, u64 timestamp,
302 u64 task_state __used)
304 struct sched_atom *event = get_new_event(task, timestamp);
306 event->type = SCHED_EVENT_SLEEP;
311 static struct task_desc *register_pid(unsigned long pid, const char *comm)
313 struct task_desc *task;
315 BUG_ON(pid >= MAX_PID);
317 task = pid_to_task[pid];
322 task = zalloc(sizeof(*task));
325 strcpy(task->comm, comm);
327 * every task starts in sleeping state - this gets ignored
328 * if there's no wakeup pointing to this sleep state:
330 add_sched_event_sleep(task, 0, 0);
332 pid_to_task[pid] = task;
334 tasks = realloc(tasks, nr_tasks*sizeof(struct task_task *));
336 tasks[task->nr] = task;
339 printf("registered task #%ld, PID %ld (%s)\n", nr_tasks, pid, comm);
345 static void print_task_traces(void)
347 struct task_desc *task;
350 for (i = 0; i < nr_tasks; i++) {
352 printf("task %6ld (%20s:%10ld), nr_events: %ld\n",
353 task->nr, task->comm, task->pid, task->nr_events);
357 static void add_cross_task_wakeups(void)
359 struct task_desc *task1, *task2;
362 for (i = 0; i < nr_tasks; i++) {
368 add_sched_event_wakeup(task1, 0, task2);
373 process_sched_event(struct task_desc *this_task __used, struct sched_atom *atom)
377 switch (atom->type) {
378 case SCHED_EVENT_RUN:
379 burn_nsecs(atom->duration);
381 case SCHED_EVENT_SLEEP:
383 ret = sem_wait(atom->wait_sem);
386 case SCHED_EVENT_WAKEUP:
388 ret = sem_post(atom->wait_sem);
391 case SCHED_EVENT_MIGRATION:
398 static u64 get_cpu_usage_nsec_parent(void)
404 err = getrusage(RUSAGE_SELF, &ru);
407 sum = ru.ru_utime.tv_sec*1e9 + ru.ru_utime.tv_usec*1e3;
408 sum += ru.ru_stime.tv_sec*1e9 + ru.ru_stime.tv_usec*1e3;
413 static int self_open_counters(void)
415 struct perf_event_attr attr;
418 memset(&attr, 0, sizeof(attr));
420 attr.type = PERF_TYPE_SOFTWARE;
421 attr.config = PERF_COUNT_SW_TASK_CLOCK;
423 fd = sys_perf_event_open(&attr, 0, -1, -1, 0);
426 die("Error: sys_perf_event_open() syscall returned"
427 "with %d (%s)\n", fd, strerror(errno));
431 static u64 get_cpu_usage_nsec_self(int fd)
436 ret = read(fd, &runtime, sizeof(runtime));
437 BUG_ON(ret != sizeof(runtime));
442 static void *thread_func(void *ctx)
444 struct task_desc *this_task = ctx;
445 u64 cpu_usage_0, cpu_usage_1;
446 unsigned long i, ret;
450 sprintf(comm2, ":%s", this_task->comm);
451 prctl(PR_SET_NAME, comm2);
452 fd = self_open_counters();
455 ret = sem_post(&this_task->ready_for_work);
457 ret = pthread_mutex_lock(&start_work_mutex);
459 ret = pthread_mutex_unlock(&start_work_mutex);
462 cpu_usage_0 = get_cpu_usage_nsec_self(fd);
464 for (i = 0; i < this_task->nr_events; i++) {
465 this_task->curr_event = i;
466 process_sched_event(this_task, this_task->atoms[i]);
469 cpu_usage_1 = get_cpu_usage_nsec_self(fd);
470 this_task->cpu_usage = cpu_usage_1 - cpu_usage_0;
471 ret = sem_post(&this_task->work_done_sem);
474 ret = pthread_mutex_lock(&work_done_wait_mutex);
476 ret = pthread_mutex_unlock(&work_done_wait_mutex);
482 static void create_tasks(void)
484 struct task_desc *task;
489 err = pthread_attr_init(&attr);
491 err = pthread_attr_setstacksize(&attr,
492 (size_t) max(16 * 1024, PTHREAD_STACK_MIN));
494 err = pthread_mutex_lock(&start_work_mutex);
496 err = pthread_mutex_lock(&work_done_wait_mutex);
498 for (i = 0; i < nr_tasks; i++) {
500 sem_init(&task->sleep_sem, 0, 0);
501 sem_init(&task->ready_for_work, 0, 0);
502 sem_init(&task->work_done_sem, 0, 0);
503 task->curr_event = 0;
504 err = pthread_create(&task->thread, &attr, thread_func, task);
509 static void wait_for_tasks(void)
511 u64 cpu_usage_0, cpu_usage_1;
512 struct task_desc *task;
513 unsigned long i, ret;
515 start_time = get_nsecs();
517 pthread_mutex_unlock(&work_done_wait_mutex);
519 for (i = 0; i < nr_tasks; i++) {
521 ret = sem_wait(&task->ready_for_work);
523 sem_init(&task->ready_for_work, 0, 0);
525 ret = pthread_mutex_lock(&work_done_wait_mutex);
528 cpu_usage_0 = get_cpu_usage_nsec_parent();
530 pthread_mutex_unlock(&start_work_mutex);
532 for (i = 0; i < nr_tasks; i++) {
534 ret = sem_wait(&task->work_done_sem);
536 sem_init(&task->work_done_sem, 0, 0);
537 cpu_usage += task->cpu_usage;
541 cpu_usage_1 = get_cpu_usage_nsec_parent();
542 if (!runavg_cpu_usage)
543 runavg_cpu_usage = cpu_usage;
544 runavg_cpu_usage = (runavg_cpu_usage*9 + cpu_usage)/10;
546 parent_cpu_usage = cpu_usage_1 - cpu_usage_0;
547 if (!runavg_parent_cpu_usage)
548 runavg_parent_cpu_usage = parent_cpu_usage;
549 runavg_parent_cpu_usage = (runavg_parent_cpu_usage*9 +
550 parent_cpu_usage)/10;
552 ret = pthread_mutex_lock(&start_work_mutex);
555 for (i = 0; i < nr_tasks; i++) {
557 sem_init(&task->sleep_sem, 0, 0);
558 task->curr_event = 0;
562 static void run_one_test(void)
564 u64 T0, T1, delta, avg_delta, fluct;
571 sum_runtime += delta;
574 avg_delta = sum_runtime / nr_runs;
575 if (delta < avg_delta)
576 fluct = avg_delta - delta;
578 fluct = delta - avg_delta;
582 run_avg = (run_avg*9 + delta)/10;
584 printf("#%-3ld: %0.3f, ",
585 nr_runs, (double)delta/1000000.0);
587 printf("ravg: %0.2f, ",
588 (double)run_avg/1e6);
590 printf("cpu: %0.2f / %0.2f",
591 (double)cpu_usage/1e6, (double)runavg_cpu_usage/1e6);
595 * rusage statistics done by the parent, these are less
596 * accurate than the sum_exec_runtime based statistics:
598 printf(" [%0.2f / %0.2f]",
599 (double)parent_cpu_usage/1e6,
600 (double)runavg_parent_cpu_usage/1e6);
605 if (nr_sleep_corrections)
606 printf(" (%ld sleep corrections)\n", nr_sleep_corrections);
607 nr_sleep_corrections = 0;
610 static void test_calibrations(void)
618 printf("the run test took %" PRIu64 " nsecs\n", T1 - T0);
624 printf("the sleep test took %" PRIu64 " nsecs\n", T1 - T0);
627 #define FILL_FIELD(ptr, field, event, data) \
628 ptr.field = (typeof(ptr.field)) raw_field_value(event, #field, data)
630 #define FILL_ARRAY(ptr, array, event, data) \
632 void *__array = raw_field_ptr(event, #array, data); \
633 memcpy(ptr.array, __array, sizeof(ptr.array)); \
636 #define FILL_COMMON_FIELDS(ptr, event, data) \
638 FILL_FIELD(ptr, common_type, event, data); \
639 FILL_FIELD(ptr, common_flags, event, data); \
640 FILL_FIELD(ptr, common_preempt_count, event, data); \
641 FILL_FIELD(ptr, common_pid, event, data); \
642 FILL_FIELD(ptr, common_tgid, event, data); \
647 struct trace_switch_event {
652 u8 common_preempt_count;
665 struct trace_runtime_event {
670 u8 common_preempt_count;
680 struct trace_wakeup_event {
685 u8 common_preempt_count;
697 struct trace_fork_event {
702 u8 common_preempt_count;
706 char parent_comm[16];
712 struct trace_migrate_task_event {
717 u8 common_preempt_count;
728 struct trace_sched_handler {
729 void (*switch_event)(struct trace_switch_event *,
731 struct event_format *,
732 struct perf_sample *sample);
734 void (*runtime_event)(struct trace_runtime_event *,
736 struct perf_sample *sample);
738 void (*wakeup_event)(struct trace_wakeup_event *,
740 struct event_format *,
741 struct perf_sample *sample);
743 void (*fork_event)(struct trace_fork_event *,
744 struct event_format *event);
746 void (*migrate_task_event)(struct trace_migrate_task_event *,
747 struct machine *machine,
748 struct perf_sample *sample);
753 replay_wakeup_event(struct trace_wakeup_event *wakeup_event,
754 struct machine *machine __used,
755 struct event_format *event, struct perf_sample *sample)
757 struct task_desc *waker, *wakee;
760 printf("sched_wakeup event %p\n", event);
762 printf(" ... pid %d woke up %s/%d\n",
763 wakeup_event->common_pid,
768 waker = register_pid(wakeup_event->common_pid, "<unknown>");
769 wakee = register_pid(wakeup_event->pid, wakeup_event->comm);
771 add_sched_event_wakeup(waker, sample->time, wakee);
774 static u64 cpu_last_switched[MAX_CPUS];
777 replay_switch_event(struct trace_switch_event *switch_event,
778 struct machine *machine __used,
779 struct event_format *event,
780 struct perf_sample *sample)
782 struct task_desc *prev, __used *next;
783 u64 timestamp0, timestamp = sample->time;
784 int cpu = sample->cpu;
788 printf("sched_switch event %p\n", event);
790 if (cpu >= MAX_CPUS || cpu < 0)
793 timestamp0 = cpu_last_switched[cpu];
795 delta = timestamp - timestamp0;
800 die("hm, delta: %" PRIu64 " < 0 ?\n", delta);
803 printf(" ... switch from %s/%d to %s/%d [ran %" PRIu64 " nsecs]\n",
804 switch_event->prev_comm, switch_event->prev_pid,
805 switch_event->next_comm, switch_event->next_pid,
809 prev = register_pid(switch_event->prev_pid, switch_event->prev_comm);
810 next = register_pid(switch_event->next_pid, switch_event->next_comm);
812 cpu_last_switched[cpu] = timestamp;
814 add_sched_event_run(prev, timestamp, delta);
815 add_sched_event_sleep(prev, timestamp, switch_event->prev_state);
820 replay_fork_event(struct trace_fork_event *fork_event,
821 struct event_format *event)
824 printf("sched_fork event %p\n", event);
825 printf("... parent: %s/%d\n", fork_event->parent_comm, fork_event->parent_pid);
826 printf("... child: %s/%d\n", fork_event->child_comm, fork_event->child_pid);
828 register_pid(fork_event->parent_pid, fork_event->parent_comm);
829 register_pid(fork_event->child_pid, fork_event->child_comm);
832 static struct trace_sched_handler replay_ops = {
833 .wakeup_event = replay_wakeup_event,
834 .switch_event = replay_switch_event,
835 .fork_event = replay_fork_event,
838 struct sort_dimension {
841 struct list_head list;
844 static LIST_HEAD(cmp_pid);
847 thread_lat_cmp(struct list_head *list, struct work_atoms *l, struct work_atoms *r)
849 struct sort_dimension *sort;
852 BUG_ON(list_empty(list));
854 list_for_each_entry(sort, list, list) {
855 ret = sort->cmp(l, r);
863 static struct work_atoms *
864 thread_atoms_search(struct rb_root *root, struct thread *thread,
865 struct list_head *sort_list)
867 struct rb_node *node = root->rb_node;
868 struct work_atoms key = { .thread = thread };
871 struct work_atoms *atoms;
874 atoms = container_of(node, struct work_atoms, node);
876 cmp = thread_lat_cmp(sort_list, &key, atoms);
878 node = node->rb_left;
880 node = node->rb_right;
882 BUG_ON(thread != atoms->thread);
890 __thread_latency_insert(struct rb_root *root, struct work_atoms *data,
891 struct list_head *sort_list)
893 struct rb_node **new = &(root->rb_node), *parent = NULL;
896 struct work_atoms *this;
899 this = container_of(*new, struct work_atoms, node);
902 cmp = thread_lat_cmp(sort_list, data, this);
905 new = &((*new)->rb_left);
907 new = &((*new)->rb_right);
910 rb_link_node(&data->node, parent, new);
911 rb_insert_color(&data->node, root);
914 static void thread_atoms_insert(struct thread *thread)
916 struct work_atoms *atoms = zalloc(sizeof(*atoms));
920 atoms->thread = thread;
921 INIT_LIST_HEAD(&atoms->work_list);
922 __thread_latency_insert(&atom_root, atoms, &cmp_pid);
926 latency_fork_event(struct trace_fork_event *fork_event __used,
927 struct event_format *event __used)
929 /* should insert the newcomer */
933 static char sched_out_state(struct trace_switch_event *switch_event)
935 const char *str = TASK_STATE_TO_CHAR_STR;
937 return str[switch_event->prev_state];
941 add_sched_out_event(struct work_atoms *atoms,
945 struct work_atom *atom = zalloc(sizeof(*atom));
949 atom->sched_out_time = timestamp;
951 if (run_state == 'R') {
952 atom->state = THREAD_WAIT_CPU;
953 atom->wake_up_time = atom->sched_out_time;
956 list_add_tail(&atom->list, &atoms->work_list);
960 add_runtime_event(struct work_atoms *atoms, u64 delta, u64 timestamp __used)
962 struct work_atom *atom;
964 BUG_ON(list_empty(&atoms->work_list));
966 atom = list_entry(atoms->work_list.prev, struct work_atom, list);
968 atom->runtime += delta;
969 atoms->total_runtime += delta;
973 add_sched_in_event(struct work_atoms *atoms, u64 timestamp)
975 struct work_atom *atom;
978 if (list_empty(&atoms->work_list))
981 atom = list_entry(atoms->work_list.prev, struct work_atom, list);
983 if (atom->state != THREAD_WAIT_CPU)
986 if (timestamp < atom->wake_up_time) {
987 atom->state = THREAD_IGNORE;
991 atom->state = THREAD_SCHED_IN;
992 atom->sched_in_time = timestamp;
994 delta = atom->sched_in_time - atom->wake_up_time;
995 atoms->total_lat += delta;
996 if (delta > atoms->max_lat) {
997 atoms->max_lat = delta;
998 atoms->max_lat_at = timestamp;
1004 latency_switch_event(struct trace_switch_event *switch_event,
1005 struct machine *machine,
1006 struct event_format *event __used,
1007 struct perf_sample *sample)
1009 struct work_atoms *out_events, *in_events;
1010 struct thread *sched_out, *sched_in;
1011 u64 timestamp0, timestamp = sample->time;
1012 int cpu = sample->cpu;
1015 BUG_ON(cpu >= MAX_CPUS || cpu < 0);
1017 timestamp0 = cpu_last_switched[cpu];
1018 cpu_last_switched[cpu] = timestamp;
1020 delta = timestamp - timestamp0;
1025 die("hm, delta: %" PRIu64 " < 0 ?\n", delta);
1028 sched_out = machine__findnew_thread(machine, switch_event->prev_pid);
1029 sched_in = machine__findnew_thread(machine, switch_event->next_pid);
1031 out_events = thread_atoms_search(&atom_root, sched_out, &cmp_pid);
1033 thread_atoms_insert(sched_out);
1034 out_events = thread_atoms_search(&atom_root, sched_out, &cmp_pid);
1036 die("out-event: Internal tree error");
1038 add_sched_out_event(out_events, sched_out_state(switch_event), timestamp);
1040 in_events = thread_atoms_search(&atom_root, sched_in, &cmp_pid);
1042 thread_atoms_insert(sched_in);
1043 in_events = thread_atoms_search(&atom_root, sched_in, &cmp_pid);
1045 die("in-event: Internal tree error");
1047 * Take came in we have not heard about yet,
1048 * add in an initial atom in runnable state:
1050 add_sched_out_event(in_events, 'R', timestamp);
1052 add_sched_in_event(in_events, timestamp);
1056 latency_runtime_event(struct trace_runtime_event *runtime_event,
1057 struct machine *machine, struct perf_sample *sample)
1059 struct thread *thread = machine__findnew_thread(machine, runtime_event->pid);
1060 struct work_atoms *atoms = thread_atoms_search(&atom_root, thread, &cmp_pid);
1061 u64 timestamp = sample->time;
1062 int cpu = sample->cpu;
1064 BUG_ON(cpu >= MAX_CPUS || cpu < 0);
1066 thread_atoms_insert(thread);
1067 atoms = thread_atoms_search(&atom_root, thread, &cmp_pid);
1069 die("in-event: Internal tree error");
1070 add_sched_out_event(atoms, 'R', timestamp);
1073 add_runtime_event(atoms, runtime_event->runtime, timestamp);
1077 latency_wakeup_event(struct trace_wakeup_event *wakeup_event,
1078 struct machine *machine, struct event_format *event __used,
1079 struct perf_sample *sample)
1081 struct work_atoms *atoms;
1082 struct work_atom *atom;
1083 struct thread *wakee;
1084 u64 timestamp = sample->time;
1086 /* Note for later, it may be interesting to observe the failing cases */
1087 if (!wakeup_event->success)
1090 wakee = machine__findnew_thread(machine, wakeup_event->pid);
1091 atoms = thread_atoms_search(&atom_root, wakee, &cmp_pid);
1093 thread_atoms_insert(wakee);
1094 atoms = thread_atoms_search(&atom_root, wakee, &cmp_pid);
1096 die("wakeup-event: Internal tree error");
1097 add_sched_out_event(atoms, 'S', timestamp);
1100 BUG_ON(list_empty(&atoms->work_list));
1102 atom = list_entry(atoms->work_list.prev, struct work_atom, list);
1105 * You WILL be missing events if you've recorded only
1106 * one CPU, or are only looking at only one, so don't
1107 * make useless noise.
1109 if (profile_cpu == -1 && atom->state != THREAD_SLEEPING)
1110 nr_state_machine_bugs++;
1113 if (atom->sched_out_time > timestamp) {
1114 nr_unordered_timestamps++;
1118 atom->state = THREAD_WAIT_CPU;
1119 atom->wake_up_time = timestamp;
1123 latency_migrate_task_event(struct trace_migrate_task_event *migrate_task_event,
1124 struct machine *machine, struct perf_sample *sample)
1126 u64 timestamp = sample->time;
1127 struct work_atoms *atoms;
1128 struct work_atom *atom;
1129 struct thread *migrant;
1132 * Only need to worry about migration when profiling one CPU.
1134 if (profile_cpu == -1)
1137 migrant = machine__findnew_thread(machine, migrate_task_event->pid);
1138 atoms = thread_atoms_search(&atom_root, migrant, &cmp_pid);
1140 thread_atoms_insert(migrant);
1141 register_pid(migrant->pid, migrant->comm);
1142 atoms = thread_atoms_search(&atom_root, migrant, &cmp_pid);
1144 die("migration-event: Internal tree error");
1145 add_sched_out_event(atoms, 'R', timestamp);
1148 BUG_ON(list_empty(&atoms->work_list));
1150 atom = list_entry(atoms->work_list.prev, struct work_atom, list);
1151 atom->sched_in_time = atom->sched_out_time = atom->wake_up_time = timestamp;
1155 if (atom->sched_out_time > timestamp)
1156 nr_unordered_timestamps++;
1159 static struct trace_sched_handler lat_ops = {
1160 .wakeup_event = latency_wakeup_event,
1161 .switch_event = latency_switch_event,
1162 .runtime_event = latency_runtime_event,
1163 .fork_event = latency_fork_event,
1164 .migrate_task_event = latency_migrate_task_event,
1167 static void output_lat_thread(struct work_atoms *work_list)
1173 if (!work_list->nb_atoms)
1176 * Ignore idle threads:
1178 if (!strcmp(work_list->thread->comm, "swapper"))
1181 all_runtime += work_list->total_runtime;
1182 all_count += work_list->nb_atoms;
1184 ret = printf(" %s:%d ", work_list->thread->comm, work_list->thread->pid);
1186 for (i = 0; i < 24 - ret; i++)
1189 avg = work_list->total_lat / work_list->nb_atoms;
1191 printf("|%11.3f ms |%9" PRIu64 " | avg:%9.3f ms | max:%9.3f ms | max at: %9.6f s\n",
1192 (double)work_list->total_runtime / 1e6,
1193 work_list->nb_atoms, (double)avg / 1e6,
1194 (double)work_list->max_lat / 1e6,
1195 (double)work_list->max_lat_at / 1e9);
1198 static int pid_cmp(struct work_atoms *l, struct work_atoms *r)
1200 if (l->thread->pid < r->thread->pid)
1202 if (l->thread->pid > r->thread->pid)
1208 static struct sort_dimension pid_sort_dimension = {
1213 static int avg_cmp(struct work_atoms *l, struct work_atoms *r)
1223 avgl = l->total_lat / l->nb_atoms;
1224 avgr = r->total_lat / r->nb_atoms;
1234 static struct sort_dimension avg_sort_dimension = {
1239 static int max_cmp(struct work_atoms *l, struct work_atoms *r)
1241 if (l->max_lat < r->max_lat)
1243 if (l->max_lat > r->max_lat)
1249 static struct sort_dimension max_sort_dimension = {
1254 static int switch_cmp(struct work_atoms *l, struct work_atoms *r)
1256 if (l->nb_atoms < r->nb_atoms)
1258 if (l->nb_atoms > r->nb_atoms)
1264 static struct sort_dimension switch_sort_dimension = {
1269 static int runtime_cmp(struct work_atoms *l, struct work_atoms *r)
1271 if (l->total_runtime < r->total_runtime)
1273 if (l->total_runtime > r->total_runtime)
1279 static struct sort_dimension runtime_sort_dimension = {
1284 static struct sort_dimension *available_sorts[] = {
1285 &pid_sort_dimension,
1286 &avg_sort_dimension,
1287 &max_sort_dimension,
1288 &switch_sort_dimension,
1289 &runtime_sort_dimension,
1292 #define NB_AVAILABLE_SORTS (int)(sizeof(available_sorts) / sizeof(struct sort_dimension *))
1294 static LIST_HEAD(sort_list);
1296 static int sort_dimension__add(const char *tok, struct list_head *list)
1300 for (i = 0; i < NB_AVAILABLE_SORTS; i++) {
1301 if (!strcmp(available_sorts[i]->name, tok)) {
1302 list_add_tail(&available_sorts[i]->list, list);
1311 static void setup_sorting(void);
1313 static void sort_lat(void)
1315 struct rb_node *node;
1318 struct work_atoms *data;
1319 node = rb_first(&atom_root);
1323 rb_erase(node, &atom_root);
1324 data = rb_entry(node, struct work_atoms, node);
1325 __thread_latency_insert(&sorted_atom_root, data, &sort_list);
1329 static struct trace_sched_handler *trace_handler;
1332 process_sched_wakeup_event(struct perf_tool *tool __used,
1333 struct event_format *event,
1334 struct perf_sample *sample,
1335 struct machine *machine,
1336 struct thread *thread __used)
1338 void *data = sample->raw_data;
1339 struct trace_wakeup_event wakeup_event;
1341 FILL_COMMON_FIELDS(wakeup_event, event, data);
1343 FILL_ARRAY(wakeup_event, comm, event, data);
1344 FILL_FIELD(wakeup_event, pid, event, data);
1345 FILL_FIELD(wakeup_event, prio, event, data);
1346 FILL_FIELD(wakeup_event, success, event, data);
1347 FILL_FIELD(wakeup_event, cpu, event, data);
1349 if (trace_handler->wakeup_event)
1350 trace_handler->wakeup_event(&wakeup_event, machine, event, sample);
1354 * Track the current task - that way we can know whether there's any
1355 * weird events, such as a task being switched away that is not current.
1359 static u32 curr_pid[MAX_CPUS] = { [0 ... MAX_CPUS-1] = -1 };
1361 static struct thread *curr_thread[MAX_CPUS];
1363 static char next_shortname1 = 'A';
1364 static char next_shortname2 = '0';
1367 map_switch_event(struct trace_switch_event *switch_event,
1368 struct machine *machine,
1369 struct event_format *event __used,
1370 struct perf_sample *sample)
1372 struct thread *sched_out __used, *sched_in;
1374 u64 timestamp0, timestamp = sample->time;
1376 int cpu, this_cpu = sample->cpu;
1378 BUG_ON(this_cpu >= MAX_CPUS || this_cpu < 0);
1380 if (this_cpu > max_cpu)
1383 timestamp0 = cpu_last_switched[this_cpu];
1384 cpu_last_switched[this_cpu] = timestamp;
1386 delta = timestamp - timestamp0;
1391 die("hm, delta: %" PRIu64 " < 0 ?\n", delta);
1394 sched_out = machine__findnew_thread(machine, switch_event->prev_pid);
1395 sched_in = machine__findnew_thread(machine, switch_event->next_pid);
1397 curr_thread[this_cpu] = sched_in;
1402 if (!sched_in->shortname[0]) {
1403 sched_in->shortname[0] = next_shortname1;
1404 sched_in->shortname[1] = next_shortname2;
1406 if (next_shortname1 < 'Z') {
1409 next_shortname1='A';
1410 if (next_shortname2 < '9') {
1413 next_shortname2='0';
1419 for (cpu = 0; cpu <= max_cpu; cpu++) {
1420 if (cpu != this_cpu)
1425 if (curr_thread[cpu]) {
1426 if (curr_thread[cpu]->pid)
1427 printf("%2s ", curr_thread[cpu]->shortname);
1434 printf(" %12.6f secs ", (double)timestamp/1e9);
1435 if (new_shortname) {
1436 printf("%s => %s:%d\n",
1437 sched_in->shortname, sched_in->comm, sched_in->pid);
1444 process_sched_switch_event(struct perf_tool *tool __used,
1445 struct event_format *event,
1446 struct perf_sample *sample,
1447 struct machine *machine,
1448 struct thread *thread __used)
1450 int this_cpu = sample->cpu;
1451 void *data = sample->raw_data;
1452 struct trace_switch_event switch_event;
1454 FILL_COMMON_FIELDS(switch_event, event, data);
1456 FILL_ARRAY(switch_event, prev_comm, event, data);
1457 FILL_FIELD(switch_event, prev_pid, event, data);
1458 FILL_FIELD(switch_event, prev_prio, event, data);
1459 FILL_FIELD(switch_event, prev_state, event, data);
1460 FILL_ARRAY(switch_event, next_comm, event, data);
1461 FILL_FIELD(switch_event, next_pid, event, data);
1462 FILL_FIELD(switch_event, next_prio, event, data);
1464 if (curr_pid[this_cpu] != (u32)-1) {
1466 * Are we trying to switch away a PID that is
1469 if (curr_pid[this_cpu] != switch_event.prev_pid)
1470 nr_context_switch_bugs++;
1472 if (trace_handler->switch_event)
1473 trace_handler->switch_event(&switch_event, machine, event, sample);
1475 curr_pid[this_cpu] = switch_event.next_pid;
1479 process_sched_runtime_event(struct perf_tool *tool __used,
1480 struct event_format *event,
1481 struct perf_sample *sample,
1482 struct machine *machine,
1483 struct thread *thread __used)
1485 void *data = sample->raw_data;
1486 struct trace_runtime_event runtime_event;
1488 FILL_ARRAY(runtime_event, comm, event, data);
1489 FILL_FIELD(runtime_event, pid, event, data);
1490 FILL_FIELD(runtime_event, runtime, event, data);
1491 FILL_FIELD(runtime_event, vruntime, event, data);
1493 if (trace_handler->runtime_event)
1494 trace_handler->runtime_event(&runtime_event, machine, sample);
1498 process_sched_fork_event(struct perf_tool *tool __used,
1499 struct event_format *event,
1500 struct perf_sample *sample,
1501 struct machine *machine __used,
1502 struct thread *thread __used)
1504 void *data = sample->raw_data;
1505 struct trace_fork_event fork_event;
1507 FILL_COMMON_FIELDS(fork_event, event, data);
1509 FILL_ARRAY(fork_event, parent_comm, event, data);
1510 FILL_FIELD(fork_event, parent_pid, event, data);
1511 FILL_ARRAY(fork_event, child_comm, event, data);
1512 FILL_FIELD(fork_event, child_pid, event, data);
1514 if (trace_handler->fork_event)
1515 trace_handler->fork_event(&fork_event, event);
1519 process_sched_exit_event(struct perf_tool *tool __used,
1520 struct event_format *event,
1521 struct perf_sample *sample __used,
1522 struct machine *machine __used,
1523 struct thread *thread __used)
1526 printf("sched_exit event %p\n", event);
1530 process_sched_migrate_task_event(struct perf_tool *tool __used,
1531 struct event_format *event,
1532 struct perf_sample *sample,
1533 struct machine *machine,
1534 struct thread *thread __used)
1536 void *data = sample->raw_data;
1537 struct trace_migrate_task_event migrate_task_event;
1539 FILL_COMMON_FIELDS(migrate_task_event, event, data);
1541 FILL_ARRAY(migrate_task_event, comm, event, data);
1542 FILL_FIELD(migrate_task_event, pid, event, data);
1543 FILL_FIELD(migrate_task_event, prio, event, data);
1544 FILL_FIELD(migrate_task_event, cpu, event, data);
1546 if (trace_handler->migrate_task_event)
1547 trace_handler->migrate_task_event(&migrate_task_event, machine, sample);
1550 typedef void (*tracepoint_handler)(struct perf_tool *tool, struct event_format *event,
1551 struct perf_sample *sample,
1552 struct machine *machine,
1553 struct thread *thread);
1555 static int perf_sched__process_tracepoint_sample(struct perf_tool *tool __used,
1556 union perf_event *event __used,
1557 struct perf_sample *sample,
1558 struct perf_evsel *evsel,
1559 struct machine *machine)
1561 struct thread *thread = machine__findnew_thread(machine, sample->pid);
1563 if (thread == NULL) {
1564 pr_debug("problem processing %s event, skipping it.\n",
1565 perf_evsel__name(evsel));
1569 evsel->hists.stats.total_period += sample->period;
1570 hists__inc_nr_events(&evsel->hists, PERF_RECORD_SAMPLE);
1572 if (evsel->handler.func != NULL) {
1573 tracepoint_handler f = evsel->handler.func;
1574 f(tool, evsel->tp_format, sample, machine, thread);
1580 static struct perf_tool perf_sched = {
1581 .sample = perf_sched__process_tracepoint_sample,
1582 .comm = perf_event__process_comm,
1583 .lost = perf_event__process_lost,
1584 .fork = perf_event__process_task,
1585 .ordered_samples = true,
1588 static void read_events(bool destroy, struct perf_session **psession)
1591 const struct perf_evsel_str_handler handlers[] = {
1592 { "sched:sched_switch", process_sched_switch_event, },
1593 { "sched:sched_stat_runtime", process_sched_runtime_event, },
1594 { "sched:sched_wakeup", process_sched_wakeup_event, },
1595 { "sched:sched_wakeup_new", process_sched_wakeup_event, },
1596 { "sched:sched_process_fork", process_sched_fork_event, },
1597 { "sched:sched_process_exit", process_sched_exit_event, },
1598 { "sched:sched_migrate_task", process_sched_migrate_task_event, },
1600 struct perf_session *session;
1602 session = perf_session__new(input_name, O_RDONLY, 0, false, &perf_sched);
1603 if (session == NULL)
1606 err = perf_session__set_tracepoints_handlers(session, handlers);
1609 if (perf_session__has_traces(session, "record -R")) {
1610 err = perf_session__process_events(session, &perf_sched);
1612 die("Failed to process events, error %d", err);
1614 nr_events = session->hists.stats.nr_events[0];
1615 nr_lost_events = session->hists.stats.total_lost;
1616 nr_lost_chunks = session->hists.stats.nr_events[PERF_RECORD_LOST];
1620 perf_session__delete(session);
1623 *psession = session;
1626 static void print_bad_events(void)
1628 if (nr_unordered_timestamps && nr_timestamps) {
1629 printf(" INFO: %.3f%% unordered timestamps (%ld out of %ld)\n",
1630 (double)nr_unordered_timestamps/(double)nr_timestamps*100.0,
1631 nr_unordered_timestamps, nr_timestamps);
1633 if (nr_lost_events && nr_events) {
1634 printf(" INFO: %.3f%% lost events (%ld out of %ld, in %ld chunks)\n",
1635 (double)nr_lost_events/(double)nr_events*100.0,
1636 nr_lost_events, nr_events, nr_lost_chunks);
1638 if (nr_state_machine_bugs && nr_timestamps) {
1639 printf(" INFO: %.3f%% state machine bugs (%ld out of %ld)",
1640 (double)nr_state_machine_bugs/(double)nr_timestamps*100.0,
1641 nr_state_machine_bugs, nr_timestamps);
1643 printf(" (due to lost events?)");
1646 if (nr_context_switch_bugs && nr_timestamps) {
1647 printf(" INFO: %.3f%% context switch bugs (%ld out of %ld)",
1648 (double)nr_context_switch_bugs/(double)nr_timestamps*100.0,
1649 nr_context_switch_bugs, nr_timestamps);
1651 printf(" (due to lost events?)");
1656 static void __cmd_lat(void)
1658 struct rb_node *next;
1659 struct perf_session *session;
1662 read_events(false, &session);
1665 printf("\n ---------------------------------------------------------------------------------------------------------------\n");
1666 printf(" Task | Runtime ms | Switches | Average delay ms | Maximum delay ms | Maximum delay at |\n");
1667 printf(" ---------------------------------------------------------------------------------------------------------------\n");
1669 next = rb_first(&sorted_atom_root);
1672 struct work_atoms *work_list;
1674 work_list = rb_entry(next, struct work_atoms, node);
1675 output_lat_thread(work_list);
1676 next = rb_next(next);
1679 printf(" -----------------------------------------------------------------------------------------\n");
1680 printf(" TOTAL: |%11.3f ms |%9" PRIu64 " |\n",
1681 (double)all_runtime/1e6, all_count);
1683 printf(" ---------------------------------------------------\n");
1688 perf_session__delete(session);
1691 static struct trace_sched_handler map_ops = {
1692 .wakeup_event = NULL,
1693 .switch_event = map_switch_event,
1694 .runtime_event = NULL,
1698 static void __cmd_map(void)
1700 max_cpu = sysconf(_SC_NPROCESSORS_CONF);
1703 read_events(true, NULL);
1707 static void __cmd_replay(void)
1711 calibrate_run_measurement_overhead();
1712 calibrate_sleep_measurement_overhead();
1714 test_calibrations();
1716 read_events(true, NULL);
1718 printf("nr_run_events: %ld\n", nr_run_events);
1719 printf("nr_sleep_events: %ld\n", nr_sleep_events);
1720 printf("nr_wakeup_events: %ld\n", nr_wakeup_events);
1722 if (targetless_wakeups)
1723 printf("target-less wakeups: %ld\n", targetless_wakeups);
1724 if (multitarget_wakeups)
1725 printf("multi-target wakeups: %ld\n", multitarget_wakeups);
1726 if (nr_run_events_optimized)
1727 printf("run atoms optimized: %ld\n",
1728 nr_run_events_optimized);
1730 print_task_traces();
1731 add_cross_task_wakeups();
1734 printf("------------------------------------------------------------\n");
1735 for (i = 0; i < replay_repeat; i++)
1740 static const char * const sched_usage[] = {
1741 "perf sched [<options>] {record|latency|map|replay|script}",
1745 static const struct option sched_options[] = {
1746 OPT_STRING('i', "input", &input_name, "file",
1748 OPT_INCR('v', "verbose", &verbose,
1749 "be more verbose (show symbol address, etc)"),
1750 OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
1751 "dump raw trace in ASCII"),
1755 static const char * const latency_usage[] = {
1756 "perf sched latency [<options>]",
1760 static const struct option latency_options[] = {
1761 OPT_STRING('s', "sort", &sort_order, "key[,key2...]",
1762 "sort by key(s): runtime, switch, avg, max"),
1763 OPT_INCR('v', "verbose", &verbose,
1764 "be more verbose (show symbol address, etc)"),
1765 OPT_INTEGER('C', "CPU", &profile_cpu,
1766 "CPU to profile on"),
1767 OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
1768 "dump raw trace in ASCII"),
1772 static const char * const replay_usage[] = {
1773 "perf sched replay [<options>]",
1777 static const struct option replay_options[] = {
1778 OPT_UINTEGER('r', "repeat", &replay_repeat,
1779 "repeat the workload replay N times (-1: infinite)"),
1780 OPT_INCR('v', "verbose", &verbose,
1781 "be more verbose (show symbol address, etc)"),
1782 OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
1783 "dump raw trace in ASCII"),
1787 static void setup_sorting(void)
1789 char *tmp, *tok, *str = strdup(sort_order);
1791 for (tok = strtok_r(str, ", ", &tmp);
1792 tok; tok = strtok_r(NULL, ", ", &tmp)) {
1793 if (sort_dimension__add(tok, &sort_list) < 0) {
1794 error("Unknown --sort key: `%s'", tok);
1795 usage_with_options(latency_usage, latency_options);
1801 sort_dimension__add("pid", &cmp_pid);
1804 static const char *record_args[] = {
1811 "-e", "sched:sched_switch",
1812 "-e", "sched:sched_stat_wait",
1813 "-e", "sched:sched_stat_sleep",
1814 "-e", "sched:sched_stat_iowait",
1815 "-e", "sched:sched_stat_runtime",
1816 "-e", "sched:sched_process_exit",
1817 "-e", "sched:sched_process_fork",
1818 "-e", "sched:sched_wakeup",
1819 "-e", "sched:sched_migrate_task",
1822 static int __cmd_record(int argc, const char **argv)
1824 unsigned int rec_argc, i, j;
1825 const char **rec_argv;
1827 rec_argc = ARRAY_SIZE(record_args) + argc - 1;
1828 rec_argv = calloc(rec_argc + 1, sizeof(char *));
1830 if (rec_argv == NULL)
1833 for (i = 0; i < ARRAY_SIZE(record_args); i++)
1834 rec_argv[i] = strdup(record_args[i]);
1836 for (j = 1; j < (unsigned int)argc; j++, i++)
1837 rec_argv[i] = argv[j];
1839 BUG_ON(i != rec_argc);
1841 return cmd_record(i, rec_argv, NULL);
1844 int cmd_sched(int argc, const char **argv, const char *prefix __used)
1846 argc = parse_options(argc, argv, sched_options, sched_usage,
1847 PARSE_OPT_STOP_AT_NON_OPTION);
1849 usage_with_options(sched_usage, sched_options);
1852 * Aliased to 'perf script' for now:
1854 if (!strcmp(argv[0], "script"))
1855 return cmd_script(argc, argv, prefix);
1858 if (!strncmp(argv[0], "rec", 3)) {
1859 return __cmd_record(argc, argv);
1860 } else if (!strncmp(argv[0], "lat", 3)) {
1861 trace_handler = &lat_ops;
1863 argc = parse_options(argc, argv, latency_options, latency_usage, 0);
1865 usage_with_options(latency_usage, latency_options);
1869 } else if (!strcmp(argv[0], "map")) {
1870 trace_handler = &map_ops;
1873 } else if (!strncmp(argv[0], "rep", 3)) {
1874 trace_handler = &replay_ops;
1876 argc = parse_options(argc, argv, replay_options, replay_usage, 0);
1878 usage_with_options(replay_usage, replay_options);
1882 usage_with_options(sched_usage, sched_options);