2 * Copyright (C) ARM Limited 2010-2014. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
10 #include <trace/events/sched.h>
13 #define TASK_MAP_ENTRIES 1024 /* must be power of 2 */
14 #define TASK_MAX_COLLISIONS 2
17 STATE_WAIT_ON_OTHER = 0,
23 static DEFINE_PER_CPU(uint64_t *, taskname_keys);
24 static DEFINE_PER_CPU(int, collecting);
26 // this array is never read as the cpu wait charts are derived counters
27 // the files are needed, nonetheless, to show that these counters are available
28 static ulong cpu_wait_enabled[CPU_WAIT_TOTAL];
29 static ulong sched_cpu_key[CPU_WAIT_TOTAL];
31 static int sched_trace_create_files(struct super_block *sb, struct dentry *root)
35 // CPU Wait - Contention
36 dir = gatorfs_mkdir(sb, root, "Linux_cpu_wait_contention");
40 gatorfs_create_ulong(sb, dir, "enabled", &cpu_wait_enabled[STATE_CONTENTION]);
41 gatorfs_create_ro_ulong(sb, dir, "key", &sched_cpu_key[STATE_CONTENTION]);
44 dir = gatorfs_mkdir(sb, root, "Linux_cpu_wait_io");
48 gatorfs_create_ulong(sb, dir, "enabled", &cpu_wait_enabled[STATE_WAIT_ON_IO]);
49 gatorfs_create_ro_ulong(sb, dir, "key", &sched_cpu_key[STATE_WAIT_ON_IO]);
54 static void emit_pid_name(struct task_struct *task)
57 char taskcomm[TASK_COMM_LEN + 3];
58 unsigned long x, cpu = get_physical_cpu();
59 uint64_t *keys = &(per_cpu(taskname_keys, cpu)[(task->pid & 0xFF) * TASK_MAX_COLLISIONS]);
62 value = gator_chksum_crc32(task->comm);
63 value = (value << 32) | (uint32_t)task->pid;
65 // determine if the thread name was emitted already
66 for (x = 0; x < TASK_MAX_COLLISIONS; x++) {
67 if (keys[x] == value) {
74 // shift values, new value always in front
75 uint64_t oldv, newv = value;
76 for (x = 0; x < TASK_MAX_COLLISIONS; x++) {
82 // emit pid names, cannot use get_task_comm, as it's not exported on all kernel versions
83 if (strlcpy(taskcomm, task->comm, TASK_COMM_LEN) == TASK_COMM_LEN - 1) {
84 // append ellipses if task->comm has length of TASK_COMM_LEN - 1
85 strcat(taskcomm, "...");
88 marshal_thread_name(task->pid, taskcomm);
92 static void collect_counters(u64 time, struct task_struct *task)
94 int *buffer, len, cpu = get_physical_cpu();
96 struct gator_interface *gi;
98 if (marshal_event_header(time)) {
99 list_for_each_entry(gi, &gator_events, list) {
101 len = gi->read(&buffer);
102 marshal_event(len, buffer);
103 } else if (gi->read64) {
104 len = gi->read64(&buffer64);
105 marshal_event64(len, buffer64);
107 if (gi->read_proc && task != NULL) {
108 len = gi->read_proc(&buffer64, task);
109 marshal_event64(len, buffer64);
112 // Only check after writing all counters so that time and corresponding counters appear in the same frame
113 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
115 // Commit buffers on timeout
116 if (gator_live_rate > 0 && time >= per_cpu(gator_buffer_commit_time, cpu)) {
117 static const int buftypes[] = { NAME_BUF, COUNTER_BUF, BLOCK_COUNTER_BUF, SCHED_TRACE_BUF, ACTIVITY_BUF };
120 for (i = 0; i < ARRAY_SIZE(buftypes); ++i) {
121 gator_commit_buffer(cpu, buftypes[i], time);
124 // spinlocks are noops on uniprocessor machines and mutexes do not work in sched_switch context in
125 // RT-Preempt full, so disable proactive flushing of the annotate frame on uniprocessor machines.
127 // Try to preemptively flush the annotate buffer to reduce the chance of the buffer being full
128 if (on_primary_core() && spin_trylock(&annotate_lock)) {
129 gator_commit_buffer(0, ANNOTATE_BUF, time);
130 spin_unlock(&annotate_lock);
137 // special case used during a suspend of the system
138 static void trace_sched_insert_idle(void)
140 marshal_sched_trace_switch(0, 0);
143 static void gator_trace_emit_link(struct task_struct *p)
146 int cpu = get_physical_cpu();
148 cookie = get_exec_cookie(cpu, p);
151 marshal_link(cookie, p->tgid, p->pid);
154 GATOR_DEFINE_PROBE(sched_process_fork, TP_PROTO(struct task_struct *parent, struct task_struct *child))
156 gator_trace_emit_link(child);
159 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0)
160 GATOR_DEFINE_PROBE(sched_process_exec, TP_PROTO(struct task_struct *p, pid_t old_pid, struct linux_binprm *bprm))
162 gator_trace_emit_link(p);
166 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35)
167 GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct rq *rq, struct task_struct *prev, struct task_struct *next))
169 GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_struct *next))
173 int cpu = get_physical_cpu();
175 per_cpu(in_scheduler_context, cpu) = true;
177 // do as much work as possible before disabling interrupts
178 if (prev->state == TASK_RUNNING) {
179 state = STATE_CONTENTION;
180 } else if (prev->in_iowait) {
181 state = STATE_WAIT_ON_IO;
183 state = STATE_WAIT_ON_OTHER;
186 per_cpu(collecting, cpu) = 1;
187 collect_counters(gator_get_time(), prev);
188 per_cpu(collecting, cpu) = 0;
190 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)
191 gator_trace_emit_link(next);
193 marshal_sched_trace_switch(next->pid, state);
195 per_cpu(in_scheduler_context, cpu) = false;
198 GATOR_DEFINE_PROBE(sched_process_free, TP_PROTO(struct task_struct *p))
200 marshal_sched_trace_exit(p->tgid, p->pid);
203 static void do_nothing(void *info)
205 // Intentionally do nothing
209 static int register_scheduler_tracepoints(void)
211 // register tracepoints
212 if (GATOR_REGISTER_TRACE(sched_process_fork))
213 goto fail_sched_process_fork;
214 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0)
215 if (GATOR_REGISTER_TRACE(sched_process_exec))
216 goto fail_sched_process_exec;
218 if (GATOR_REGISTER_TRACE(sched_switch))
219 goto fail_sched_switch;
220 if (GATOR_REGISTER_TRACE(sched_process_free))
221 goto fail_sched_process_free;
222 pr_debug("gator: registered tracepoints\n");
224 // Now that the scheduler tracepoint is registered, force a context switch
225 // on all cpus to capture what is currently running.
226 on_each_cpu(do_nothing, NULL, 0);
230 // unregister tracepoints on error
231 fail_sched_process_free:
232 GATOR_UNREGISTER_TRACE(sched_switch);
234 GATOR_UNREGISTER_TRACE(sched_process_fork);
235 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0)
236 fail_sched_process_exec:
237 GATOR_UNREGISTER_TRACE(sched_process_exec);
239 fail_sched_process_fork:
240 pr_err("gator: tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n");
245 static void unregister_scheduler_tracepoints(void)
247 GATOR_UNREGISTER_TRACE(sched_process_fork);
248 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0)
249 GATOR_UNREGISTER_TRACE(sched_process_exec);
251 GATOR_UNREGISTER_TRACE(sched_switch);
252 GATOR_UNREGISTER_TRACE(sched_process_free);
253 pr_debug("gator: unregistered tracepoints\n");
256 static void gator_trace_sched_stop(void)
260 unregister_scheduler_tracepoints();
262 for_each_present_cpu(cpu) {
263 kfree(per_cpu(taskname_keys, cpu));
267 static int gator_trace_sched_start(void)
272 for_each_present_cpu(cpu) {
273 size = TASK_MAP_ENTRIES * TASK_MAX_COLLISIONS * sizeof(uint64_t);
274 per_cpu(taskname_keys, cpu) = (uint64_t *)kmalloc(size, GFP_KERNEL);
275 if (!per_cpu(taskname_keys, cpu))
277 memset(per_cpu(taskname_keys, cpu), 0, size);
280 ret = register_scheduler_tracepoints();
285 static void gator_trace_sched_offline(void)
287 trace_sched_insert_idle();
290 static void gator_trace_sched_init(void)
293 for (i = 0; i < CPU_WAIT_TOTAL; i++) {
294 cpu_wait_enabled[i] = 0;
295 sched_cpu_key[i] = gator_events_get_key();