drivers/gator/gator_trace_sched.c

   1 /**
   2  * Copyright (C) ARM Limited 2010-2014. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  */
   9
  10 #include <trace/events/sched.h>
  11 #include "gator.h"
  12
  13 #define TASK_MAP_ENTRIES                1024    /* must be power of 2 */
  14 #define TASK_MAX_COLLISIONS             2
  15
  16 enum {
  17         STATE_WAIT_ON_OTHER = 0,
  18         STATE_CONTENTION,
  19         STATE_WAIT_ON_IO,
  20         CPU_WAIT_TOTAL
  21 };
  22
  23 static DEFINE_PER_CPU(uint64_t *, taskname_keys);
  24 static DEFINE_PER_CPU(int, collecting);
  25
  26 // this array is never read as the cpu wait charts are derived counters
  27 // the files are needed, nonetheless, to show that these counters are available
  28 static ulong cpu_wait_enabled[CPU_WAIT_TOTAL];
  29 static ulong sched_cpu_key[CPU_WAIT_TOTAL];
  30
  31 static int sched_trace_create_files(struct super_block *sb, struct dentry *root)
  32 {
  33         struct dentry *dir;
  34
  35         // CPU Wait - Contention
  36         dir = gatorfs_mkdir(sb, root, "Linux_cpu_wait_contention");
  37         if (!dir) {
  38                 return -1;
  39         }
  40         gatorfs_create_ulong(sb, dir, "enabled", &cpu_wait_enabled[STATE_CONTENTION]);
  41         gatorfs_create_ro_ulong(sb, dir, "key", &sched_cpu_key[STATE_CONTENTION]);
  42
  43         // CPU Wait - I/O
  44         dir = gatorfs_mkdir(sb, root, "Linux_cpu_wait_io");
  45         if (!dir) {
  46                 return -1;
  47         }
  48         gatorfs_create_ulong(sb, dir, "enabled", &cpu_wait_enabled[STATE_WAIT_ON_IO]);
  49         gatorfs_create_ro_ulong(sb, dir, "key", &sched_cpu_key[STATE_WAIT_ON_IO]);
  50
  51         return 0;
  52 }
  53
  54 static void emit_pid_name(struct task_struct *task)
  55 {
  56         bool found = false;
  57         char taskcomm[TASK_COMM_LEN + 3];
  58         unsigned long x, cpu = get_physical_cpu();
  59         uint64_t *keys = &(per_cpu(taskname_keys, cpu)[(task->pid & 0xFF) * TASK_MAX_COLLISIONS]);
  60         uint64_t value;
  61
  62         value = gator_chksum_crc32(task->comm);
  63         value = (value << 32) | (uint32_t)task->pid;
  64
  65         // determine if the thread name was emitted already
  66         for (x = 0; x < TASK_MAX_COLLISIONS; x++) {
  67                 if (keys[x] == value) {
  68                         found = true;
  69                         break;
  70                 }
  71         }
  72
  73         if (!found) {
  74                 // shift values, new value always in front
  75                 uint64_t oldv, newv = value;
  76                 for (x = 0; x < TASK_MAX_COLLISIONS; x++) {
  77                         oldv = keys[x];
  78                         keys[x] = newv;
  79                         newv = oldv;
  80                 }
  81
  82                 // emit pid names, cannot use get_task_comm, as it's not exported on all kernel versions
  83                 if (strlcpy(taskcomm, task->comm, TASK_COMM_LEN) == TASK_COMM_LEN - 1) {
  84                         // append ellipses if task->comm has length of TASK_COMM_LEN - 1
  85                         strcat(taskcomm, "...");
  86                 }
  87
  88                 marshal_thread_name(task->pid, taskcomm);
  89         }
  90 }
  91
  92 static void collect_counters(u64 time, struct task_struct *task)
  93 {
  94         int *buffer, len, cpu = get_physical_cpu();
  95         long long *buffer64;
  96         struct gator_interface *gi;
  97
  98         if (marshal_event_header(time)) {
  99                 list_for_each_entry(gi, &gator_events, list) {
 100                         if (gi->read) {
 101                                 len = gi->read(&buffer);
 102                                 marshal_event(len, buffer);
 103                         } else if (gi->read64) {
 104                                 len = gi->read64(&buffer64);
 105                                 marshal_event64(len, buffer64);
 106                         }
 107                         if (gi->read_proc && task != NULL) {
 108                                 len = gi->read_proc(&buffer64, task);
 109                                 marshal_event64(len, buffer64);
 110                         }
 111                 }
 112                 // Only check after writing all counters so that time and corresponding counters appear in the same frame
 113                 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
 114
 115                 // Commit buffers on timeout
 116                 if (gator_live_rate > 0 && time >= per_cpu(gator_buffer_commit_time, cpu)) {
 117                         static const int buftypes[] = { NAME_BUF, COUNTER_BUF, BLOCK_COUNTER_BUF, SCHED_TRACE_BUF, ACTIVITY_BUF };
 118                         int i;
 119
 120                         for (i = 0; i < ARRAY_SIZE(buftypes); ++i) {
 121                                 gator_commit_buffer(cpu, buftypes[i], time);
 122                         }
 123
 124                         // spinlocks are noops on uniprocessor machines and mutexes do not work in sched_switch context in
 125                         // RT-Preempt full, so disable proactive flushing of the annotate frame on uniprocessor machines.
 126 #ifdef CONFIG_SMP
 127                         // Try to preemptively flush the annotate buffer to reduce the chance of the buffer being full
 128                         if (on_primary_core() && spin_trylock(&annotate_lock)) {
 129                                 gator_commit_buffer(0, ANNOTATE_BUF, time);
 130                                 spin_unlock(&annotate_lock);
 131                         }
 132 #endif
 133                 }
 134         }
 135 }
 136
 137 // special case used during a suspend of the system
 138 static void trace_sched_insert_idle(void)
 139 {
 140         marshal_sched_trace_switch(0, 0);
 141 }
 142
 143 static void gator_trace_emit_link(struct task_struct *p)
 144 {
 145         int cookie;
 146         int cpu = get_physical_cpu();
 147
 148         cookie = get_exec_cookie(cpu, p);
 149         emit_pid_name(p);
 150
 151         marshal_link(cookie, p->tgid, p->pid);
 152 }
 153
 154 GATOR_DEFINE_PROBE(sched_process_fork, TP_PROTO(struct task_struct *parent, struct task_struct *child))
 155 {
 156         gator_trace_emit_link(child);
 157 }
 158
 159 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0)
 160 GATOR_DEFINE_PROBE(sched_process_exec, TP_PROTO(struct task_struct *p, pid_t old_pid, struct linux_binprm *bprm))
 161 {
 162         gator_trace_emit_link(p);
 163 }
 164 #endif
 165
 166 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 35)
 167 GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct rq *rq, struct task_struct *prev, struct task_struct *next))
 168 #else
 169 GATOR_DEFINE_PROBE(sched_switch, TP_PROTO(struct task_struct *prev, struct task_struct *next))
 170 #endif
 171 {
 172         int state;
 173         int cpu = get_physical_cpu();
 174
 175         per_cpu(in_scheduler_context, cpu) = true;
 176
 177         // do as much work as possible before disabling interrupts
 178         if (prev->state == TASK_RUNNING) {
 179                 state = STATE_CONTENTION;
 180         } else if (prev->in_iowait) {
 181                 state = STATE_WAIT_ON_IO;
 182         } else {
 183                 state = STATE_WAIT_ON_OTHER;
 184         }
 185
 186         per_cpu(collecting, cpu) = 1;
 187         collect_counters(gator_get_time(), prev);
 188         per_cpu(collecting, cpu) = 0;
 189
 190 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)
 191         gator_trace_emit_link(next);
 192 #endif
 193         marshal_sched_trace_switch(next->pid, state);
 194
 195         per_cpu(in_scheduler_context, cpu) = false;
 196 }
 197
 198 GATOR_DEFINE_PROBE(sched_process_free, TP_PROTO(struct task_struct *p))
 199 {
 200         marshal_sched_trace_exit(p->tgid, p->pid);
 201 }
 202
 203 static void do_nothing(void *info)
 204 {
 205         // Intentionally do nothing
 206         (void)info;
 207 }
 208
 209 static int register_scheduler_tracepoints(void)
 210 {
 211         // register tracepoints
 212         if (GATOR_REGISTER_TRACE(sched_process_fork))
 213                 goto fail_sched_process_fork;
 214 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0)
 215         if (GATOR_REGISTER_TRACE(sched_process_exec))
 216                 goto fail_sched_process_exec;
 217 #endif
 218         if (GATOR_REGISTER_TRACE(sched_switch))
 219                 goto fail_sched_switch;
 220         if (GATOR_REGISTER_TRACE(sched_process_free))
 221                 goto fail_sched_process_free;
 222         pr_debug("gator: registered tracepoints\n");
 223
 224         // Now that the scheduler tracepoint is registered, force a context switch
 225         // on all cpus to capture what is currently running.
 226         on_each_cpu(do_nothing, NULL, 0);
 227
 228         return 0;
 229
 230         // unregister tracepoints on error
 231 fail_sched_process_free:
 232         GATOR_UNREGISTER_TRACE(sched_switch);
 233 fail_sched_switch:
 234         GATOR_UNREGISTER_TRACE(sched_process_fork);
 235 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0)
 236 fail_sched_process_exec:
 237         GATOR_UNREGISTER_TRACE(sched_process_exec);
 238 #endif
 239 fail_sched_process_fork:
 240         pr_err("gator: tracepoints failed to activate, please verify that tracepoints are enabled in the linux kernel\n");
 241
 242         return -1;
 243 }
 244
 245 static void unregister_scheduler_tracepoints(void)
 246 {
 247         GATOR_UNREGISTER_TRACE(sched_process_fork);
 248 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 4, 0)
 249         GATOR_UNREGISTER_TRACE(sched_process_exec);
 250 #endif
 251         GATOR_UNREGISTER_TRACE(sched_switch);
 252         GATOR_UNREGISTER_TRACE(sched_process_free);
 253         pr_debug("gator: unregistered tracepoints\n");
 254 }
 255
 256 static void gator_trace_sched_stop(void)
 257 {
 258         int cpu;
 259
 260         unregister_scheduler_tracepoints();
 261
 262         for_each_present_cpu(cpu) {
 263                 kfree(per_cpu(taskname_keys, cpu));
 264         }
 265 }
 266
 267 static int gator_trace_sched_start(void)
 268 {
 269         int cpu, size;
 270         int ret;
 271
 272         for_each_present_cpu(cpu) {
 273                 size = TASK_MAP_ENTRIES * TASK_MAX_COLLISIONS * sizeof(uint64_t);
 274                 per_cpu(taskname_keys, cpu) = (uint64_t *)kmalloc(size, GFP_KERNEL);
 275                 if (!per_cpu(taskname_keys, cpu))
 276                         return -1;
 277                 memset(per_cpu(taskname_keys, cpu), 0, size);
 278         }
 279
 280         ret = register_scheduler_tracepoints();
 281
 282         return ret;
 283 }
 284
 285 static void gator_trace_sched_offline(void)
 286 {
 287         trace_sched_insert_idle();
 288 }
 289
 290 static void gator_trace_sched_init(void)
 291 {
 292         int i;
 293         for (i = 0; i < CPU_WAIT_TOTAL; i++) {
 294                 cpu_wait_enabled[i] = 0;
 295                 sched_cpu_key[i] = gator_events_get_key();
 296         }
 297 }