drivers/gator/gator_main.c

   1 /**
   2  * Copyright (C) ARM Limited 2010-2014. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  */
   9
  10 /* This version must match the gator daemon version */
  11 #define PROTOCOL_VERSION 20
  12 static unsigned long gator_protocol_version = PROTOCOL_VERSION;
  13
  14 #include <linux/slab.h>
  15 #include <linux/cpu.h>
  16 #include <linux/sched.h>
  17 #include <linux/irq.h>
  18 #include <linux/vmalloc.h>
  19 #include <linux/hardirq.h>
  20 #include <linux/highmem.h>
  21 #include <linux/pagemap.h>
  22 #include <linux/suspend.h>
  23 #include <linux/module.h>
  24 #include <linux/perf_event.h>
  25 #include <linux/utsname.h>
  26 #include <linux/kthread.h>
  27 #include <asm/stacktrace.h>
  28 #include <linux/uaccess.h>
  29
  30 #include "gator.h"
  31
  32 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 32)
  33 #error kernels prior to 2.6.32 are not supported
  34 #endif
  35
  36 #if defined(MODULE) && !defined(CONFIG_MODULES)
  37 #error Cannot build a module against a kernel that does not support modules. To resolve, either rebuild the kernel to support modules or build gator as part of the kernel.
  38 #endif
  39
  40 #if !defined(CONFIG_GENERIC_TRACER) && !defined(CONFIG_TRACING)
  41 #error gator requires the kernel to have CONFIG_GENERIC_TRACER or CONFIG_TRACING defined
  42 #endif
  43
  44 #ifndef CONFIG_PROFILING
  45 #error gator requires the kernel to have CONFIG_PROFILING defined
  46 #endif
  47
  48 #ifndef CONFIG_HIGH_RES_TIMERS
  49 #error gator requires the kernel to have CONFIG_HIGH_RES_TIMERS defined to support PC sampling
  50 #endif
  51
  52 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 12, 0) && defined(__arm__) && defined(CONFIG_SMP) && !defined(CONFIG_LOCAL_TIMERS)
  53 #error gator requires the kernel to have CONFIG_LOCAL_TIMERS defined on SMP systems
  54 #endif
  55
  56 #if (GATOR_PERF_SUPPORT) && (!(GATOR_PERF_PMU_SUPPORT))
  57 #ifndef CONFIG_PERF_EVENTS
  58 #error gator requires the kernel to have CONFIG_PERF_EVENTS defined to support pmu hardware counters
  59 #elif !defined CONFIG_HW_PERF_EVENTS
  60 #error gator requires the kernel to have CONFIG_HW_PERF_EVENTS defined to support pmu hardware counters
  61 #endif
  62 #endif
  63
  64 /******************************************************************************
  65  * DEFINES
  66  ******************************************************************************/
  67 #define SUMMARY_BUFFER_SIZE       (1*1024)
  68 #define BACKTRACE_BUFFER_SIZE     (128*1024)
  69 #define NAME_BUFFER_SIZE          (64*1024)
  70 #define COUNTER_BUFFER_SIZE       (64*1024)     /* counters have the core as part of the data and the core value in the frame header may be discarded */
  71 #define BLOCK_COUNTER_BUFFER_SIZE (128*1024)
  72 #define ANNOTATE_BUFFER_SIZE      (128*1024)    /* annotate counters have the core as part of the data and the core value in the frame header may be discarded */
  73 #define SCHED_TRACE_BUFFER_SIZE   (128*1024)
  74 #define IDLE_BUFFER_SIZE          (32*1024)     /* idle counters have the core as part of the data and the core value in the frame header may be discarded */
  75 #define ACTIVITY_BUFFER_SIZE      (128*1024)
  76
  77 #define NO_COOKIE      0U
  78 #define UNRESOLVED_COOKIE ~0U
  79
  80 #define FRAME_SUMMARY       1
  81 #define FRAME_BACKTRACE     2
  82 #define FRAME_NAME          3
  83 #define FRAME_COUNTER       4
  84 #define FRAME_BLOCK_COUNTER 5
  85 #define FRAME_ANNOTATE      6
  86 #define FRAME_SCHED_TRACE   7
  87 #define FRAME_IDLE          9
  88 #define FRAME_ACTIVITY     13
  89
  90 #define MESSAGE_END_BACKTRACE 1
  91
  92 /* Name Frame Messages */
  93 #define MESSAGE_COOKIE      1
  94 #define MESSAGE_THREAD_NAME 2
  95 #define MESSAGE_LINK        4
  96
  97 /* Scheduler Trace Frame Messages */
  98 #define MESSAGE_SCHED_SWITCH 1
  99 #define MESSAGE_SCHED_EXIT   2
 100
 101 /* Idle Frame Messages */
 102 #define MESSAGE_IDLE_ENTER 1
 103 #define MESSAGE_IDLE_EXIT  2
 104
 105 /* Summary Frame Messages */
 106 #define MESSAGE_SUMMARY   1
 107 #define MESSAGE_CORE_NAME 3
 108
 109 /* Activity Frame Messages */
 110 #define MESSAGE_SWITCH 2
 111 #define MESSAGE_EXIT   3
 112
 113 #define MAXSIZE_PACK32     5
 114 #define MAXSIZE_PACK64    10
 115
 116 #define FRAME_HEADER_SIZE 3
 117
 118 #if defined(__arm__)
 119 #define PC_REG regs->ARM_pc
 120 #elif defined(__aarch64__)
 121 #define PC_REG regs->pc
 122 #else
 123 #define PC_REG regs->ip
 124 #endif
 125
 126 enum {
 127         SUMMARY_BUF,
 128         BACKTRACE_BUF,
 129         NAME_BUF,
 130         COUNTER_BUF,
 131         BLOCK_COUNTER_BUF,
 132         ANNOTATE_BUF,
 133         SCHED_TRACE_BUF,
 134         IDLE_BUF,
 135         ACTIVITY_BUF,
 136         NUM_GATOR_BUFS
 137 };
 138
 139 /******************************************************************************
 140  * Globals
 141  ******************************************************************************/
 142 static unsigned long gator_cpu_cores;
 143 /* Size of the largest buffer. Effectively constant, set in gator_op_create_files */
 144 static unsigned long userspace_buffer_size;
 145 static unsigned long gator_backtrace_depth;
 146 /* How often to commit the buffers for live in nanoseconds */
 147 static u64 gator_live_rate;
 148
 149 static unsigned long gator_started;
 150 static u64 gator_monotonic_started;
 151 static u64 gator_sync_time;
 152 static u64 gator_hibernate_time;
 153 static unsigned long gator_buffer_opened;
 154 static unsigned long gator_timer_count;
 155 static unsigned long gator_response_type;
 156 static DEFINE_MUTEX(start_mutex);
 157 static DEFINE_MUTEX(gator_buffer_mutex);
 158
 159 bool event_based_sampling;
 160
 161 static DECLARE_WAIT_QUEUE_HEAD(gator_buffer_wait);
 162 static DECLARE_WAIT_QUEUE_HEAD(gator_annotate_wait);
 163 static struct timer_list gator_buffer_wake_up_timer;
 164 static bool gator_buffer_wake_run;
 165 /* Initialize semaphore unlocked to initialize memory values */
 166 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 36)
 167 static DECLARE_MUTEX(gator_buffer_wake_sem);
 168 #else
 169 static DEFINE_SEMAPHORE(gator_buffer_wake_sem);
 170 #endif
 171 static struct task_struct *gator_buffer_wake_thread;
 172 static LIST_HEAD(gator_events);
 173
 174 static DEFINE_PER_CPU(u64, last_timestamp);
 175
 176 static bool printed_monotonic_warning;
 177
 178 static u32 gator_cpuids[NR_CPUS];
 179 static bool sent_core_name[NR_CPUS];
 180
 181 static DEFINE_PER_CPU(bool, in_scheduler_context);
 182
 183 /******************************************************************************
 184  * Prototypes
 185  ******************************************************************************/
 186 static u64 gator_get_time(void);
 187 static void gator_emit_perf_time(u64 time);
 188 static void gator_op_create_files(struct super_block *sb, struct dentry *root);
 189
 190 /* gator_buffer is protected by being per_cpu and by having IRQs
 191  * disabled when writing to it. Most marshal_* calls take care of this
 192  * except for marshal_cookie*, marshal_backtrace* and marshal_frame
 193  * where the caller is responsible for doing so. No synchronization is
 194  * needed with the backtrace buffer as it is per cpu and is only used
 195  * from the hrtimer. The annotate_lock must be held when using the
 196  * annotation buffer as it is not per cpu. collect_counters which is
 197  * the sole writer to the block counter frame is additionally
 198  * protected by the per cpu collecting flag.
 199  */
 200
 201 /* Size of the buffer, must be a power of 2. Effectively constant, set in gator_op_setup. */
 202 static uint32_t gator_buffer_size[NUM_GATOR_BUFS];
 203 /* gator_buffer_size - 1, bitwise and with pos to get offset into the array. Effectively constant, set in gator_op_setup. */
 204 static uint32_t gator_buffer_mask[NUM_GATOR_BUFS];
 205 /* Read position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are read by userspace in userspace_buffer_read */
 206 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_read);
 207 /* Write position in the buffer. Initialized to zero in gator_op_setup and incremented after bytes are written to the buffer */
 208 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_write);
 209 /* Commit position in the buffer. Initialized to zero in gator_op_setup and incremented after a frame is ready to be read by userspace */
 210 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], gator_buffer_commit);
 211 /* If set to false, decreases the number of bytes returned by
 212  * buffer_bytes_available. Set in buffer_check_space if no space is
 213  * remaining. Initialized to true in gator_op_setup. This means that
 214  * if we run out of space, continue to report that no space is
 215  * available until bytes are read by userspace
 216  */
 217 static DEFINE_PER_CPU(int[NUM_GATOR_BUFS], buffer_space_available);
 218 /* The buffer. Allocated in gator_op_setup */
 219 static DEFINE_PER_CPU(char *[NUM_GATOR_BUFS], gator_buffer);
 220 /* The time after which the buffer should be committed for live display */
 221 static DEFINE_PER_CPU(u64, gator_buffer_commit_time);
 222
 223 /* List of all gator events - new events must be added to this list */
 224 #define GATOR_EVENTS_LIST \
 225         GATOR_EVENT(gator_events_armv6_init) \
 226         GATOR_EVENT(gator_events_armv7_init) \
 227         GATOR_EVENT(gator_events_block_init) \
 228         GATOR_EVENT(gator_events_ccn504_init) \
 229         GATOR_EVENT(gator_events_irq_init) \
 230         GATOR_EVENT(gator_events_l2c310_init) \
 231         GATOR_EVENT(gator_events_mali_init) \
 232         GATOR_EVENT(gator_events_mali_midgard_hw_init) \
 233         GATOR_EVENT(gator_events_mali_midgard_init) \
 234         GATOR_EVENT(gator_events_meminfo_init) \
 235         GATOR_EVENT(gator_events_mmapped_init) \
 236         GATOR_EVENT(gator_events_net_init) \
 237         GATOR_EVENT(gator_events_perf_pmu_init) \
 238         GATOR_EVENT(gator_events_sched_init) \
 239         GATOR_EVENT(gator_events_scorpion_init) \
 240
 241 #define GATOR_EVENT(EVENT_INIT) __weak int EVENT_INIT(void);
 242 GATOR_EVENTS_LIST
 243 #undef GATOR_EVENT
 244
 245 static int (*gator_events_list[])(void) = {
 246 #define GATOR_EVENT(EVENT_INIT) EVENT_INIT,
 247 GATOR_EVENTS_LIST
 248 #undef GATOR_EVENT
 249 };
 250
 251 /******************************************************************************
 252  * Application Includes
 253  ******************************************************************************/
 254 #include "gator_fs.c"
 255 #include "gator_buffer_write.c"
 256 #include "gator_buffer.c"
 257 #include "gator_marshaling.c"
 258 #include "gator_hrtimer_gator.c"
 259 #include "gator_cookies.c"
 260 #include "gator_annotate.c"
 261 #include "gator_trace_sched.c"
 262 #include "gator_trace_power.c"
 263 #include "gator_trace_gpu.c"
 264 #include "gator_backtrace.c"
 265
 266 /******************************************************************************
 267  * Misc
 268  ******************************************************************************/
 269
 270 static const struct gator_cpu gator_cpus[] = {
 271         {
 272                 .cpuid = ARM1136,
 273                 .core_name = "ARM1136",
 274                 .pmnc_name = "ARM_ARM11",
 275                 .dt_name = "arm,arm1136",
 276                 .pmnc_counters = 3,
 277         },
 278         {
 279                 .cpuid = ARM1156,
 280                 .core_name = "ARM1156",
 281                 .pmnc_name = "ARM_ARM11",
 282                 .dt_name = "arm,arm1156",
 283                 .pmnc_counters = 3,
 284         },
 285         {
 286                 .cpuid = ARM1176,
 287                 .core_name = "ARM1176",
 288                 .pmnc_name = "ARM_ARM11",
 289                 .dt_name = "arm,arm1176",
 290                 .pmnc_counters = 3,
 291         },
 292         {
 293                 .cpuid = ARM11MPCORE,
 294                 .core_name = "ARM11MPCore",
 295                 .pmnc_name = "ARM_ARM11MPCore",
 296                 .dt_name = "arm,arm11mpcore",
 297                 .pmnc_counters = 3,
 298         },
 299         {
 300                 .cpuid = CORTEX_A5,
 301                 .core_name = "Cortex-A5",
 302                 .pmnc_name = "ARMv7_Cortex_A5",
 303                 .dt_name = "arm,cortex-a5",
 304                 .pmnc_counters = 2,
 305         },
 306         {
 307                 .cpuid = CORTEX_A7,
 308                 .core_name = "Cortex-A7",
 309                 .pmnc_name = "ARMv7_Cortex_A7",
 310                 .dt_name = "arm,cortex-a7",
 311                 .pmnc_counters = 4,
 312         },
 313         {
 314                 .cpuid = CORTEX_A8,
 315                 .core_name = "Cortex-A8",
 316                 .pmnc_name = "ARMv7_Cortex_A8",
 317                 .dt_name = "arm,cortex-a8",
 318                 .pmnc_counters = 4,
 319         },
 320         {
 321                 .cpuid = CORTEX_A9,
 322                 .core_name = "Cortex-A9",
 323                 .pmnc_name = "ARMv7_Cortex_A9",
 324                 .dt_name = "arm,cortex-a9",
 325                 .pmnc_counters = 6,
 326         },
 327         {
 328                 .cpuid = CORTEX_A15,
 329                 .core_name = "Cortex-A15",
 330                 .pmnc_name = "ARMv7_Cortex_A15",
 331                 .dt_name = "arm,cortex-a15",
 332                 .pmnc_counters = 6,
 333         },
 334         {
 335                 .cpuid = CORTEX_A17,
 336                 .core_name = "Cortex-A17",
 337                 .pmnc_name = "ARMv7_Cortex_A17",
 338                 .dt_name = "arm,cortex-a17",
 339                 .pmnc_counters = 6,
 340         },
 341         {
 342                 .cpuid = SCORPION,
 343                 .core_name = "Scorpion",
 344                 .pmnc_name = "Scorpion",
 345                 .pmnc_counters = 4,
 346         },
 347         {
 348                 .cpuid = SCORPIONMP,
 349                 .core_name = "ScorpionMP",
 350                 .pmnc_name = "ScorpionMP",
 351                 .pmnc_counters = 4,
 352         },
 353         {
 354                 .cpuid = KRAITSIM,
 355                 .core_name = "KraitSIM",
 356                 .pmnc_name = "Krait",
 357                 .pmnc_counters = 4,
 358         },
 359         {
 360                 .cpuid = KRAIT,
 361                 .core_name = "Krait",
 362                 .pmnc_name = "Krait",
 363                 .pmnc_counters = 4,
 364         },
 365         {
 366                 .cpuid = KRAIT_S4_PRO,
 367                 .core_name = "Krait S4 Pro",
 368                 .pmnc_name = "Krait",
 369                 .pmnc_counters = 4,
 370         },
 371         {
 372                 .cpuid = CORTEX_A53,
 373                 .core_name = "Cortex-A53",
 374                 .pmnc_name = "ARM_Cortex-A53",
 375                 .dt_name = "arm,cortex-a53",
 376                 .pmnc_counters = 6,
 377         },
 378         {
 379                 .cpuid = CORTEX_A57,
 380                 .core_name = "Cortex-A57",
 381                 .pmnc_name = "ARM_Cortex-A57",
 382                 .dt_name = "arm,cortex-a57",
 383                 .pmnc_counters = 6,
 384         },
 385         {
 386                 .cpuid = AARCH64,
 387                 .core_name = "AArch64",
 388                 .pmnc_name = "ARM_AArch64",
 389                 .pmnc_counters = 6,
 390         },
 391         {
 392                 .cpuid = OTHER,
 393                 .core_name = "Other",
 394                 .pmnc_name = "Other",
 395                 .pmnc_counters = 6,
 396         },
 397         {}
 398 };
 399
 400 const struct gator_cpu *gator_find_cpu_by_cpuid(const u32 cpuid)
 401 {
 402         int i;
 403
 404         for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
 405                 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
 406
 407                 if (gator_cpu->cpuid == cpuid)
 408                         return gator_cpu;
 409         }
 410
 411         return NULL;
 412 }
 413
 414 static const char OLD_PMU_PREFIX[] = "ARMv7 Cortex-";
 415 static const char NEW_PMU_PREFIX[] = "ARMv7_Cortex_";
 416
 417 const struct gator_cpu *gator_find_cpu_by_pmu_name(const char *const name)
 418 {
 419         int i;
 420
 421         for (i = 0; gator_cpus[i].cpuid != 0; ++i) {
 422                 const struct gator_cpu *const gator_cpu = &gator_cpus[i];
 423
 424                 if (gator_cpu->pmnc_name != NULL &&
 425                     /* Do the names match exactly? */
 426                     (strcasecmp(gator_cpu->pmnc_name, name) == 0 ||
 427                      /* Do these names match but have the old vs new prefix? */
 428                      ((strncasecmp(name, OLD_PMU_PREFIX, sizeof(OLD_PMU_PREFIX) - 1) == 0 &&
 429                        strncasecmp(gator_cpu->pmnc_name, NEW_PMU_PREFIX, sizeof(NEW_PMU_PREFIX) - 1) == 0 &&
 430                        strcasecmp(name + sizeof(OLD_PMU_PREFIX) - 1, gator_cpu->pmnc_name + sizeof(NEW_PMU_PREFIX) - 1) == 0))))
 431                         return gator_cpu;
 432         }
 433
 434         return NULL;
 435 }
 436
 437 u32 gator_cpuid(void)
 438 {
 439 #if defined(__arm__) || defined(__aarch64__)
 440         u32 val;
 441 #if !defined(__aarch64__)
 442         asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r" (val));
 443 #else
 444         asm volatile("mrs %0, midr_el1" : "=r" (val));
 445 #endif
 446         return (val >> 4) & 0xfff;
 447 #else
 448         return OTHER;
 449 #endif
 450 }
 451
 452 static void gator_buffer_wake_up(unsigned long data)
 453 {
 454         wake_up(&gator_buffer_wait);
 455 }
 456
 457 static int gator_buffer_wake_func(void *data)
 458 {
 459         for (;;) {
 460                 if (down_killable(&gator_buffer_wake_sem))
 461                         break;
 462
 463                 /* Eat up any pending events */
 464                 while (!down_trylock(&gator_buffer_wake_sem))
 465                         ;
 466
 467                 if (!gator_buffer_wake_run)
 468                         break;
 469
 470                 gator_buffer_wake_up(0);
 471         }
 472
 473         return 0;
 474 }
 475
 476 /******************************************************************************
 477  * Commit interface
 478  ******************************************************************************/
 479 static bool buffer_commit_ready(int *cpu, int *buftype)
 480 {
 481         int cpu_x, x;
 482
 483         for_each_present_cpu(cpu_x) {
 484                 for (x = 0; x < NUM_GATOR_BUFS; x++)
 485                         if (per_cpu(gator_buffer_commit, cpu_x)[x] != per_cpu(gator_buffer_read, cpu_x)[x]) {
 486                                 *cpu = cpu_x;
 487                                 *buftype = x;
 488                                 return true;
 489                         }
 490         }
 491         *cpu = -1;
 492         *buftype = -1;
 493         return false;
 494 }
 495
 496 /******************************************************************************
 497  * hrtimer interrupt processing
 498  ******************************************************************************/
 499 static void gator_timer_interrupt(void)
 500 {
 501         struct pt_regs *const regs = get_irq_regs();
 502
 503         gator_backtrace_handler(regs);
 504 }
 505
 506 void gator_backtrace_handler(struct pt_regs *const regs)
 507 {
 508         u64 time = gator_get_time();
 509         int cpu = get_physical_cpu();
 510
 511         /* Output backtrace */
 512         gator_add_sample(cpu, regs, time);
 513
 514         /* Collect counters */
 515         if (!per_cpu(collecting, cpu))
 516                 collect_counters(time, current, false);
 517
 518         /* No buffer flushing occurs during sched switch for RT-Preempt full. The block counter frame will be flushed by collect_counters, but the sched buffer needs to be explicitly flushed */
 519 #ifdef CONFIG_PREEMPT_RT_FULL
 520         buffer_check(cpu, SCHED_TRACE_BUF, time);
 521 #endif
 522 }
 523
 524 static int gator_running;
 525
 526 /* This function runs in interrupt context and on the appropriate core */
 527 static void gator_timer_offline(void *migrate)
 528 {
 529         struct gator_interface *gi;
 530         int i, len, cpu = get_physical_cpu();
 531         int *buffer;
 532         u64 time;
 533
 534         gator_trace_sched_offline();
 535         gator_trace_power_offline();
 536
 537         if (!migrate)
 538                 gator_hrtimer_offline();
 539
 540         /* Offline any events and output counters */
 541         time = gator_get_time();
 542         if (marshal_event_header(time)) {
 543                 list_for_each_entry(gi, &gator_events, list) {
 544                         if (gi->offline) {
 545                                 len = gi->offline(&buffer, migrate);
 546                                 marshal_event(len, buffer);
 547                         }
 548                 }
 549                 /* Only check after writing all counters so that time and corresponding counters appear in the same frame */
 550                 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
 551         }
 552
 553         /* Flush all buffers on this core */
 554         for (i = 0; i < NUM_GATOR_BUFS; i++)
 555                 gator_commit_buffer(cpu, i, time);
 556 }
 557
 558 /* This function runs in interrupt context and may be running on a core other than core 'cpu' */
 559 static void gator_timer_offline_dispatch(int cpu, bool migrate)
 560 {
 561         struct gator_interface *gi;
 562
 563         list_for_each_entry(gi, &gator_events, list) {
 564                 if (gi->offline_dispatch)
 565                         gi->offline_dispatch(cpu, migrate);
 566         }
 567 }
 568
 569 static void gator_timer_stop(void)
 570 {
 571         int cpu;
 572
 573         if (gator_running) {
 574                 on_each_cpu(gator_timer_offline, NULL, 1);
 575                 for_each_online_cpu(cpu) {
 576                         gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
 577                 }
 578
 579                 gator_running = 0;
 580                 gator_hrtimer_shutdown();
 581         }
 582 }
 583
 584 static void gator_send_core_name(const int cpu, const u32 cpuid)
 585 {
 586 #if defined(__arm__) || defined(__aarch64__)
 587         if (!sent_core_name[cpu] || (cpuid != gator_cpuids[cpu])) {
 588                 const struct gator_cpu *const gator_cpu = gator_find_cpu_by_cpuid(cpuid);
 589                 const char *core_name = NULL;
 590                 char core_name_buf[32];
 591
 592                 /* Save off this cpuid */
 593                 gator_cpuids[cpu] = cpuid;
 594                 if (gator_cpu != NULL) {
 595                         core_name = gator_cpu->core_name;
 596                 } else {
 597                         if (cpuid == -1)
 598                                 snprintf(core_name_buf, sizeof(core_name_buf), "Unknown");
 599                         else
 600                                 snprintf(core_name_buf, sizeof(core_name_buf), "Unknown (0x%.3x)", cpuid);
 601                         core_name = core_name_buf;
 602                 }
 603
 604                 marshal_core_name(cpu, cpuid, core_name);
 605                 sent_core_name[cpu] = true;
 606         }
 607 #endif
 608 }
 609
 610 static void gator_read_cpuid(void *arg)
 611 {
 612         gator_cpuids[get_physical_cpu()] = gator_cpuid();
 613 }
 614
 615 /* This function runs in interrupt context and on the appropriate core */
 616 static void gator_timer_online(void *migrate)
 617 {
 618         struct gator_interface *gi;
 619         int len, cpu = get_physical_cpu();
 620         int *buffer;
 621         u64 time;
 622
 623         /* Send what is currently running on this core */
 624         marshal_sched_trace_switch(current->pid, 0);
 625
 626         gator_trace_power_online();
 627
 628         /* online any events and output counters */
 629         time = gator_get_time();
 630         if (marshal_event_header(time)) {
 631                 list_for_each_entry(gi, &gator_events, list) {
 632                         if (gi->online) {
 633                                 len = gi->online(&buffer, migrate);
 634                                 marshal_event(len, buffer);
 635                         }
 636                 }
 637                 /* Only check after writing all counters so that time and corresponding counters appear in the same frame */
 638                 buffer_check(cpu, BLOCK_COUNTER_BUF, time);
 639         }
 640
 641         if (!migrate)
 642                 gator_hrtimer_online();
 643
 644         gator_send_core_name(cpu, gator_cpuid());
 645 }
 646
 647 /* This function runs in interrupt context and may be running on a core other than core 'cpu' */
 648 static void gator_timer_online_dispatch(int cpu, bool migrate)
 649 {
 650         struct gator_interface *gi;
 651
 652         list_for_each_entry(gi, &gator_events, list) {
 653                 if (gi->online_dispatch)
 654                         gi->online_dispatch(cpu, migrate);
 655         }
 656 }
 657
 658 #include "gator_iks.c"
 659
 660 static int gator_timer_start(unsigned long sample_rate)
 661 {
 662         int cpu;
 663
 664         if (gator_running) {
 665                 pr_notice("gator: already running\n");
 666                 return 0;
 667         }
 668
 669         gator_running = 1;
 670
 671         /* event based sampling trumps hr timer based sampling */
 672         if (event_based_sampling)
 673                 sample_rate = 0;
 674
 675         if (gator_hrtimer_init(sample_rate, gator_timer_interrupt) == -1)
 676                 return -1;
 677
 678         /* Send off the previously saved cpuids */
 679         for_each_present_cpu(cpu) {
 680                 preempt_disable();
 681                 gator_send_core_name(cpu, gator_cpuids[cpu]);
 682                 preempt_enable();
 683         }
 684
 685         gator_send_iks_core_names();
 686         for_each_online_cpu(cpu) {
 687                 gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
 688         }
 689         on_each_cpu(gator_timer_online, NULL, 1);
 690
 691         return 0;
 692 }
 693
 694 static u64 gator_get_time(void)
 695 {
 696         struct timespec ts;
 697         u64 timestamp;
 698         u64 prev_timestamp;
 699         u64 delta;
 700         int cpu = smp_processor_id();
 701
 702         /* Match clock_gettime(CLOCK_MONOTONIC_RAW, &ts) from userspace */
 703         getrawmonotonic(&ts);
 704         timestamp = timespec_to_ns(&ts);
 705
 706         /* getrawmonotonic is not monotonic on all systems. Detect and
 707          * attempt to correct these cases. up to 0.5ms delta has been seen
 708          * on some systems, which can skew Streamline data when viewing at
 709          * high resolution. This doesn't work well with interrupts, but that
 710          * it's OK - the real concern is to catch big jumps in time
 711          */
 712         prev_timestamp = per_cpu(last_timestamp, cpu);
 713         if (prev_timestamp <= timestamp) {
 714                 per_cpu(last_timestamp, cpu) = timestamp;
 715         } else {
 716                 delta = prev_timestamp - timestamp;
 717                 /* Log the error once */
 718                 if (!printed_monotonic_warning && delta > 500000) {
 719                         pr_err("%s: getrawmonotonic is not monotonic  cpu: %i  delta: %lli\nSkew in Streamline data may be present at the fine zoom levels\n", __func__, cpu, delta);
 720                         printed_monotonic_warning = true;
 721                 }
 722                 timestamp = prev_timestamp;
 723         }
 724
 725         return timestamp - gator_monotonic_started;
 726 }
 727
 728 static void gator_emit_perf_time(u64 time)
 729 {
 730 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 10, 0)
 731         if (time >= gator_sync_time) {
 732                 int cpu = get_physical_cpu();
 733
 734                 marshal_event_single64(0, -1, local_clock());
 735                 gator_sync_time += NSEC_PER_SEC;
 736                 gator_commit_buffer(cpu, COUNTER_BUF, time);
 737         }
 738 #endif
 739 }
 740
 741 /******************************************************************************
 742  * cpu hotplug and pm notifiers
 743  ******************************************************************************/
 744 static int __cpuinit gator_hotcpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
 745 {
 746         int cpu = lcpu_to_pcpu((long)hcpu);
 747
 748         switch (action) {
 749         case CPU_DOWN_PREPARE:
 750         case CPU_DOWN_PREPARE_FROZEN:
 751                 smp_call_function_single(cpu, gator_timer_offline, NULL, 1);
 752                 gator_timer_offline_dispatch(cpu, false);
 753                 break;
 754         case CPU_ONLINE:
 755         case CPU_ONLINE_FROZEN:
 756                 gator_timer_online_dispatch(cpu, false);
 757                 smp_call_function_single(cpu, gator_timer_online, NULL, 1);
 758                 break;
 759         }
 760
 761         return NOTIFY_OK;
 762 }
 763
 764 static struct notifier_block __refdata gator_hotcpu_notifier = {
 765         .notifier_call = gator_hotcpu_notify,
 766 };
 767
 768 /* n.b. calling "on_each_cpu" only runs on those that are online.
 769  * Registered linux events are not disabled, so their counters will
 770  * continue to collect
 771  */
 772 static int gator_pm_notify(struct notifier_block *nb, unsigned long event, void *dummy)
 773 {
 774         int cpu;
 775         struct timespec ts;
 776
 777         switch (event) {
 778         case PM_HIBERNATION_PREPARE:
 779         case PM_SUSPEND_PREPARE:
 780                 unregister_hotcpu_notifier(&gator_hotcpu_notifier);
 781                 unregister_scheduler_tracepoints();
 782                 on_each_cpu(gator_timer_offline, NULL, 1);
 783                 for_each_online_cpu(cpu) {
 784                         gator_timer_offline_dispatch(lcpu_to_pcpu(cpu), false);
 785                 }
 786
 787                 /* Record the wallclock hibernate time */
 788                 getnstimeofday(&ts);
 789                 gator_hibernate_time = timespec_to_ns(&ts) - gator_get_time();
 790                 break;
 791         case PM_POST_HIBERNATION:
 792         case PM_POST_SUSPEND:
 793                 /* Adjust gator_monotonic_started for the time spent sleeping, as gator_get_time does not account for it */
 794                 if (gator_hibernate_time > 0) {
 795                         getnstimeofday(&ts);
 796                         gator_monotonic_started += gator_hibernate_time + gator_get_time() - timespec_to_ns(&ts);
 797                         gator_hibernate_time = 0;
 798                 }
 799
 800                 for_each_online_cpu(cpu) {
 801                         gator_timer_online_dispatch(lcpu_to_pcpu(cpu), false);
 802                 }
 803                 on_each_cpu(gator_timer_online, NULL, 1);
 804                 register_scheduler_tracepoints();
 805                 register_hotcpu_notifier(&gator_hotcpu_notifier);
 806                 break;
 807         }
 808
 809         return NOTIFY_OK;
 810 }
 811
 812 static struct notifier_block gator_pm_notifier = {
 813         .notifier_call = gator_pm_notify,
 814 };
 815
 816 static int gator_notifier_start(void)
 817 {
 818         int retval;
 819
 820         retval = register_hotcpu_notifier(&gator_hotcpu_notifier);
 821         if (retval == 0)
 822                 retval = register_pm_notifier(&gator_pm_notifier);
 823         return retval;
 824 }
 825
 826 static void gator_notifier_stop(void)
 827 {
 828         unregister_pm_notifier(&gator_pm_notifier);
 829         unregister_hotcpu_notifier(&gator_hotcpu_notifier);
 830 }
 831
 832 /******************************************************************************
 833  * Main
 834  ******************************************************************************/
 835 static void gator_summary(void)
 836 {
 837         u64 timestamp, uptime;
 838         struct timespec ts;
 839         char uname_buf[512];
 840
 841         snprintf(uname_buf, sizeof(uname_buf), "%s %s %s %s %s GNU/Linux", utsname()->sysname, utsname()->nodename, utsname()->release, utsname()->version, utsname()->machine);
 842
 843         getnstimeofday(&ts);
 844         timestamp = timespec_to_ns(&ts);
 845
 846         /* Similar to reading /proc/uptime from fs/proc/uptime.c, calculate uptime */
 847 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 11, 0)
 848         {
 849                 void (*m2b)(struct timespec *ts);
 850
 851                 do_posix_clock_monotonic_gettime(&ts);
 852                 /* monotonic_to_bootbased is not defined for some versions of Android */
 853                 m2b = symbol_get(monotonic_to_bootbased);
 854                 if (m2b)
 855                         m2b(&ts);
 856         }
 857 #else
 858         get_monotonic_boottime(&ts);
 859 #endif
 860         uptime = timespec_to_ns(&ts);
 861
 862         /* Disable preemption as gator_get_time calls smp_processor_id to verify time is monotonic */
 863         preempt_disable();
 864         /* Set monotonic_started to zero as gator_get_time is uptime minus monotonic_started */
 865         gator_monotonic_started = 0;
 866         gator_monotonic_started = gator_get_time();
 867
 868         marshal_summary(timestamp, uptime, gator_monotonic_started, uname_buf);
 869         gator_sync_time = 0;
 870         gator_emit_perf_time(gator_monotonic_started);
 871         preempt_enable();
 872 }
 873
 874 int gator_events_install(struct gator_interface *interface)
 875 {
 876         list_add_tail(&interface->list, &gator_events);
 877
 878         return 0;
 879 }
 880
 881 int gator_events_get_key(void)
 882 {
 883         /* key 0 is reserved as a timestamp. key 1 is reserved as the marker
 884          * for thread specific counters. key 2 is reserved as the marker for
 885          * core. Odd keys are assigned by the driver, even keys by the
 886          * daemon.
 887          */
 888         static int key = 3;
 889         const int ret = key;
 890
 891         key += 2;
 892         return ret;
 893 }
 894
 895 static int gator_init(void)
 896 {
 897         int i;
 898
 899         calc_first_cluster_size();
 900
 901         /* events sources */
 902         for (i = 0; i < ARRAY_SIZE(gator_events_list); i++)
 903                 if (gator_events_list[i])
 904                         gator_events_list[i]();
 905
 906         gator_trace_sched_init();
 907         gator_trace_power_init();
 908
 909         return 0;
 910 }
 911
 912 static void gator_exit(void)
 913 {
 914         struct gator_interface *gi;
 915
 916         list_for_each_entry(gi, &gator_events, list)
 917                 if (gi->shutdown)
 918                         gi->shutdown();
 919 }
 920
 921 static int gator_start(void)
 922 {
 923         unsigned long cpu, i;
 924         struct gator_interface *gi;
 925
 926         gator_buffer_wake_run = true;
 927         gator_buffer_wake_thread = kthread_run(gator_buffer_wake_func, NULL, "gator_bwake");
 928         if (IS_ERR(gator_buffer_wake_thread))
 929                 goto bwake_failure;
 930
 931         if (gator_migrate_start())
 932                 goto migrate_failure;
 933
 934         /* Initialize the buffer with the frame type and core */
 935         for_each_present_cpu(cpu) {
 936                 for (i = 0; i < NUM_GATOR_BUFS; i++)
 937                         marshal_frame(cpu, i);
 938                 per_cpu(last_timestamp, cpu) = 0;
 939         }
 940         printed_monotonic_warning = false;
 941
 942         /* Capture the start time */
 943         gator_summary();
 944
 945         /* start all events */
 946         list_for_each_entry(gi, &gator_events, list) {
 947                 if (gi->start && gi->start() != 0) {
 948                         struct list_head *ptr = gi->list.prev;
 949
 950                         while (ptr != &gator_events) {
 951                                 gi = list_entry(ptr, struct gator_interface, list);
 952
 953                                 if (gi->stop)
 954                                         gi->stop();
 955
 956                                 ptr = ptr->prev;
 957                         }
 958                         goto events_failure;
 959                 }
 960         }
 961
 962         /* cookies shall be initialized before trace_sched_start() and gator_timer_start() */
 963         if (cookies_initialize())
 964                 goto cookies_failure;
 965         if (gator_annotate_start())
 966                 goto annotate_failure;
 967         if (gator_trace_sched_start())
 968                 goto sched_failure;
 969         if (gator_trace_power_start())
 970                 goto power_failure;
 971         if (gator_trace_gpu_start())
 972                 goto gpu_failure;
 973         if (gator_timer_start(gator_timer_count))
 974                 goto timer_failure;
 975         if (gator_notifier_start())
 976                 goto notifier_failure;
 977
 978         return 0;
 979
 980 notifier_failure:
 981         gator_timer_stop();
 982 timer_failure:
 983         gator_trace_gpu_stop();
 984 gpu_failure:
 985         gator_trace_power_stop();
 986 power_failure:
 987         gator_trace_sched_stop();
 988 sched_failure:
 989         gator_annotate_stop();
 990 annotate_failure:
 991         cookies_release();
 992 cookies_failure:
 993         /* stop all events */
 994         list_for_each_entry(gi, &gator_events, list)
 995                 if (gi->stop)
 996                         gi->stop();
 997 events_failure:
 998         gator_migrate_stop();
 999 migrate_failure:
1000         gator_buffer_wake_run = false;
1001         up(&gator_buffer_wake_sem);
1002         gator_buffer_wake_thread = NULL;
1003 bwake_failure:
1004
1005         return -1;
1006 }
1007
1008 static void gator_stop(void)
1009 {
1010         struct gator_interface *gi;
1011
1012         gator_annotate_stop();
1013         gator_trace_sched_stop();
1014         gator_trace_power_stop();
1015         gator_trace_gpu_stop();
1016
1017         /* stop all interrupt callback reads before tearing down other interfaces */
1018         gator_notifier_stop();  /* should be called before gator_timer_stop to avoid re-enabling the hrtimer after it has been offlined */
1019         gator_timer_stop();
1020
1021         /* stop all events */
1022         list_for_each_entry(gi, &gator_events, list)
1023                 if (gi->stop)
1024                         gi->stop();
1025
1026         gator_migrate_stop();
1027
1028         gator_buffer_wake_run = false;
1029         up(&gator_buffer_wake_sem);
1030         gator_buffer_wake_thread = NULL;
1031 }
1032
1033 /******************************************************************************
1034  * Filesystem
1035  ******************************************************************************/
1036 /* fopen("buffer") */
1037 static int gator_op_setup(void)
1038 {
1039         int err = 0;
1040         int cpu, i;
1041
1042         mutex_lock(&start_mutex);
1043
1044         gator_buffer_size[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE;
1045         gator_buffer_mask[SUMMARY_BUF] = SUMMARY_BUFFER_SIZE - 1;
1046
1047         gator_buffer_size[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE;
1048         gator_buffer_mask[BACKTRACE_BUF] = BACKTRACE_BUFFER_SIZE - 1;
1049
1050         gator_buffer_size[NAME_BUF] = NAME_BUFFER_SIZE;
1051         gator_buffer_mask[NAME_BUF] = NAME_BUFFER_SIZE - 1;
1052
1053         gator_buffer_size[COUNTER_BUF] = COUNTER_BUFFER_SIZE;
1054         gator_buffer_mask[COUNTER_BUF] = COUNTER_BUFFER_SIZE - 1;
1055
1056         gator_buffer_size[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE;
1057         gator_buffer_mask[BLOCK_COUNTER_BUF] = BLOCK_COUNTER_BUFFER_SIZE - 1;
1058
1059         gator_buffer_size[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE;
1060         gator_buffer_mask[ANNOTATE_BUF] = ANNOTATE_BUFFER_SIZE - 1;
1061
1062         gator_buffer_size[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE;
1063         gator_buffer_mask[SCHED_TRACE_BUF] = SCHED_TRACE_BUFFER_SIZE - 1;
1064
1065         gator_buffer_size[IDLE_BUF] = IDLE_BUFFER_SIZE;
1066         gator_buffer_mask[IDLE_BUF] = IDLE_BUFFER_SIZE - 1;
1067
1068         gator_buffer_size[ACTIVITY_BUF] = ACTIVITY_BUFFER_SIZE;
1069         gator_buffer_mask[ACTIVITY_BUF] = ACTIVITY_BUFFER_SIZE - 1;
1070
1071         /* Initialize percpu per buffer variables */
1072         for (i = 0; i < NUM_GATOR_BUFS; i++) {
1073                 /* Verify buffers are a power of 2 */
1074                 if (gator_buffer_size[i] & (gator_buffer_size[i] - 1)) {
1075                         err = -ENOEXEC;
1076                         goto setup_error;
1077                 }
1078
1079                 for_each_present_cpu(cpu) {
1080                         per_cpu(gator_buffer_read, cpu)[i] = 0;
1081                         per_cpu(gator_buffer_write, cpu)[i] = 0;
1082                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
1083                         per_cpu(buffer_space_available, cpu)[i] = true;
1084                         per_cpu(gator_buffer_commit_time, cpu) = gator_live_rate;
1085
1086                         /* Annotation is a special case that only uses a single buffer */
1087                         if (cpu > 0 && i == ANNOTATE_BUF) {
1088                                 per_cpu(gator_buffer, cpu)[i] = NULL;
1089                                 continue;
1090                         }
1091
1092                         per_cpu(gator_buffer, cpu)[i] = vmalloc(gator_buffer_size[i]);
1093                         if (!per_cpu(gator_buffer, cpu)[i]) {
1094                                 err = -ENOMEM;
1095                                 goto setup_error;
1096                         }
1097                 }
1098         }
1099
1100 setup_error:
1101         mutex_unlock(&start_mutex);
1102         return err;
1103 }
1104
1105 /* Actually start profiling (echo 1>/dev/gator/enable) */
1106 static int gator_op_start(void)
1107 {
1108         int err = 0;
1109
1110         mutex_lock(&start_mutex);
1111
1112         if (gator_started || gator_start())
1113                 err = -EINVAL;
1114         else
1115                 gator_started = 1;
1116
1117         mutex_unlock(&start_mutex);
1118
1119         return err;
1120 }
1121
1122 /* echo 0>/dev/gator/enable */
1123 static void gator_op_stop(void)
1124 {
1125         mutex_lock(&start_mutex);
1126
1127         if (gator_started) {
1128                 gator_stop();
1129
1130                 mutex_lock(&gator_buffer_mutex);
1131
1132                 gator_started = 0;
1133                 gator_monotonic_started = 0;
1134                 cookies_release();
1135                 wake_up(&gator_buffer_wait);
1136
1137                 mutex_unlock(&gator_buffer_mutex);
1138         }
1139
1140         mutex_unlock(&start_mutex);
1141 }
1142
1143 static void gator_shutdown(void)
1144 {
1145         int cpu, i;
1146
1147         mutex_lock(&start_mutex);
1148
1149         for_each_present_cpu(cpu) {
1150                 mutex_lock(&gator_buffer_mutex);
1151                 for (i = 0; i < NUM_GATOR_BUFS; i++) {
1152                         vfree(per_cpu(gator_buffer, cpu)[i]);
1153                         per_cpu(gator_buffer, cpu)[i] = NULL;
1154                         per_cpu(gator_buffer_read, cpu)[i] = 0;
1155                         per_cpu(gator_buffer_write, cpu)[i] = 0;
1156                         per_cpu(gator_buffer_commit, cpu)[i] = 0;
1157                         per_cpu(buffer_space_available, cpu)[i] = true;
1158                         per_cpu(gator_buffer_commit_time, cpu) = 0;
1159                 }
1160                 mutex_unlock(&gator_buffer_mutex);
1161         }
1162
1163         memset(&sent_core_name, 0, sizeof(sent_core_name));
1164
1165         mutex_unlock(&start_mutex);
1166 }
1167
1168 static int gator_set_backtrace(unsigned long val)
1169 {
1170         int err = 0;
1171
1172         mutex_lock(&start_mutex);
1173
1174         if (gator_started)
1175                 err = -EBUSY;
1176         else
1177                 gator_backtrace_depth = val;
1178
1179         mutex_unlock(&start_mutex);
1180
1181         return err;
1182 }
1183
1184 static ssize_t enable_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1185 {
1186         return gatorfs_ulong_to_user(gator_started, buf, count, offset);
1187 }
1188
1189 static ssize_t enable_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1190 {
1191         unsigned long val;
1192         int retval;
1193
1194         if (*offset)
1195                 return -EINVAL;
1196
1197         retval = gatorfs_ulong_from_user(&val, buf, count);
1198         if (retval)
1199                 return retval;
1200
1201         if (val)
1202                 retval = gator_op_start();
1203         else
1204                 gator_op_stop();
1205
1206         if (retval)
1207                 return retval;
1208         return count;
1209 }
1210
1211 static const struct file_operations enable_fops = {
1212         .read = enable_read,
1213         .write = enable_write,
1214 };
1215
1216 static int userspace_buffer_open(struct inode *inode, struct file *file)
1217 {
1218         int err = -EPERM;
1219
1220         if (!capable(CAP_SYS_ADMIN))
1221                 return -EPERM;
1222
1223         if (test_and_set_bit_lock(0, &gator_buffer_opened))
1224                 return -EBUSY;
1225
1226         err = gator_op_setup();
1227         if (err)
1228                 goto fail;
1229
1230         /* NB: the actual start happens from userspace
1231          * echo 1 >/dev/gator/enable
1232          */
1233
1234         return 0;
1235
1236 fail:
1237         __clear_bit_unlock(0, &gator_buffer_opened);
1238         return err;
1239 }
1240
1241 static int userspace_buffer_release(struct inode *inode, struct file *file)
1242 {
1243         gator_op_stop();
1244         gator_shutdown();
1245         __clear_bit_unlock(0, &gator_buffer_opened);
1246         return 0;
1247 }
1248
1249 static ssize_t userspace_buffer_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1250 {
1251         int commit, length1, length2, read;
1252         char *buffer1;
1253         char *buffer2;
1254         int cpu, buftype;
1255         int written = 0;
1256
1257         /* ensure there is enough space for a whole frame */
1258         if (count < userspace_buffer_size || *offset)
1259                 return -EINVAL;
1260
1261         /* sleep until the condition is true or a signal is received the
1262          * condition is checked each time gator_buffer_wait is woken up
1263          */
1264         wait_event_interruptible(gator_buffer_wait, buffer_commit_ready(&cpu, &buftype) || !gator_started);
1265
1266         if (signal_pending(current))
1267                 return -EINTR;
1268
1269         if (buftype == -1 || cpu == -1)
1270                 return 0;
1271
1272         mutex_lock(&gator_buffer_mutex);
1273
1274         do {
1275                 read = per_cpu(gator_buffer_read, cpu)[buftype];
1276                 commit = per_cpu(gator_buffer_commit, cpu)[buftype];
1277
1278                 /* May happen if the buffer is freed during pending reads. */
1279                 if (!per_cpu(gator_buffer, cpu)[buftype])
1280                         break;
1281
1282                 /* determine the size of two halves */
1283                 length1 = commit - read;
1284                 length2 = 0;
1285                 buffer1 = &(per_cpu(gator_buffer, cpu)[buftype][read]);
1286                 buffer2 = &(per_cpu(gator_buffer, cpu)[buftype][0]);
1287                 if (length1 < 0) {
1288                         length1 = gator_buffer_size[buftype] - read;
1289                         length2 = commit;
1290                 }
1291
1292                 if (length1 + length2 > count - written)
1293                         break;
1294
1295                 /* start, middle or end */
1296                 if (length1 > 0 && copy_to_user(&buf[written], buffer1, length1))
1297                         break;
1298
1299                 /* possible wrap around */
1300                 if (length2 > 0 && copy_to_user(&buf[written + length1], buffer2, length2))
1301                         break;
1302
1303                 per_cpu(gator_buffer_read, cpu)[buftype] = commit;
1304                 written += length1 + length2;
1305
1306                 /* Wake up annotate_write if more space is available */
1307                 if (buftype == ANNOTATE_BUF)
1308                         wake_up(&gator_annotate_wait);
1309         } while (buffer_commit_ready(&cpu, &buftype));
1310
1311         mutex_unlock(&gator_buffer_mutex);
1312
1313         /* kick just in case we've lost an SMP event */
1314         wake_up(&gator_buffer_wait);
1315
1316         return written > 0 ? written : -EFAULT;
1317 }
1318
1319 static const struct file_operations gator_event_buffer_fops = {
1320         .open = userspace_buffer_open,
1321         .release = userspace_buffer_release,
1322         .read = userspace_buffer_read,
1323 };
1324
1325 static ssize_t depth_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
1326 {
1327         return gatorfs_ulong_to_user(gator_backtrace_depth, buf, count, offset);
1328 }
1329
1330 static ssize_t depth_write(struct file *file, char const __user *buf, size_t count, loff_t *offset)
1331 {
1332         unsigned long val;
1333         int retval;
1334
1335         if (*offset)
1336                 return -EINVAL;
1337
1338         retval = gatorfs_ulong_from_user(&val, buf, count);
1339         if (retval)
1340                 return retval;
1341
1342         retval = gator_set_backtrace(val);
1343
1344         if (retval)
1345                 return retval;
1346         return count;
1347 }
1348
1349 static const struct file_operations depth_fops = {
1350         .read = depth_read,
1351         .write = depth_write
1352 };
1353
1354 static void gator_op_create_files(struct super_block *sb, struct dentry *root)
1355 {
1356         struct dentry *dir;
1357         struct gator_interface *gi;
1358         int cpu;
1359
1360         /* reinitialize default values */
1361         gator_cpu_cores = 0;
1362         for_each_present_cpu(cpu) {
1363                 gator_cpu_cores++;
1364         }
1365         userspace_buffer_size = BACKTRACE_BUFFER_SIZE;
1366         gator_response_type = 1;
1367         gator_live_rate = 0;
1368
1369         gatorfs_create_file(sb, root, "enable", &enable_fops);
1370         gatorfs_create_file(sb, root, "buffer", &gator_event_buffer_fops);
1371         gatorfs_create_file(sb, root, "backtrace_depth", &depth_fops);
1372         gatorfs_create_ro_ulong(sb, root, "cpu_cores", &gator_cpu_cores);
1373         gatorfs_create_ro_ulong(sb, root, "buffer_size", &userspace_buffer_size);
1374         gatorfs_create_ulong(sb, root, "tick", &gator_timer_count);
1375         gatorfs_create_ulong(sb, root, "response_type", &gator_response_type);
1376         gatorfs_create_ro_ulong(sb, root, "version", &gator_protocol_version);
1377         gatorfs_create_ro_u64(sb, root, "started", &gator_monotonic_started);
1378         gatorfs_create_u64(sb, root, "live_rate", &gator_live_rate);
1379
1380         /* Annotate interface */
1381         gator_annotate_create_files(sb, root);
1382
1383         /* Linux Events */
1384         dir = gatorfs_mkdir(sb, root, "events");
1385         list_for_each_entry(gi, &gator_events, list)
1386                 if (gi->create_files)
1387                         gi->create_files(sb, dir);
1388
1389         /* Sched Events */
1390         sched_trace_create_files(sb, dir);
1391
1392         /* Power interface */
1393         gator_trace_power_create_files(sb, dir);
1394 }
1395
1396 /******************************************************************************
1397  * Module
1398  ******************************************************************************/
1399
1400 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 15, 0)
1401
1402 #define GATOR_TRACEPOINTS \
1403         GATOR_HANDLE_TRACEPOINT(block_rq_complete); \
1404         GATOR_HANDLE_TRACEPOINT(cpu_frequency); \
1405         GATOR_HANDLE_TRACEPOINT(cpu_idle); \
1406         GATOR_HANDLE_TRACEPOINT(cpu_migrate_begin); \
1407         GATOR_HANDLE_TRACEPOINT(cpu_migrate_current); \
1408         GATOR_HANDLE_TRACEPOINT(cpu_migrate_finish); \
1409         GATOR_HANDLE_TRACEPOINT(irq_handler_exit); \
1410         GATOR_HANDLE_TRACEPOINT(mali_hw_counter); \
1411         GATOR_HANDLE_TRACEPOINT(mali_job_slots_event); \
1412         GATOR_HANDLE_TRACEPOINT(mali_mmu_as_in_use); \
1413         GATOR_HANDLE_TRACEPOINT(mali_mmu_as_released); \
1414         GATOR_HANDLE_TRACEPOINT(mali_page_fault_insert_pages); \
1415         GATOR_HANDLE_TRACEPOINT(mali_pm_status); \
1416         GATOR_HANDLE_TRACEPOINT(mali_sw_counter); \
1417         GATOR_HANDLE_TRACEPOINT(mali_sw_counters); \
1418         GATOR_HANDLE_TRACEPOINT(mali_timeline_event); \
1419         GATOR_HANDLE_TRACEPOINT(mali_total_alloc_pages_change); \
1420         GATOR_HANDLE_TRACEPOINT(mm_page_alloc); \
1421         GATOR_HANDLE_TRACEPOINT(mm_page_free); \
1422         GATOR_HANDLE_TRACEPOINT(mm_page_free_batched); \
1423         GATOR_HANDLE_TRACEPOINT(sched_process_exec); \
1424         GATOR_HANDLE_TRACEPOINT(sched_process_fork); \
1425         GATOR_HANDLE_TRACEPOINT(sched_process_free); \
1426         GATOR_HANDLE_TRACEPOINT(sched_switch); \
1427         GATOR_HANDLE_TRACEPOINT(softirq_exit); \
1428         GATOR_HANDLE_TRACEPOINT(task_rename); \
1429
1430 #define GATOR_HANDLE_TRACEPOINT(probe_name) \
1431         struct tracepoint *gator_tracepoint_##probe_name
1432 GATOR_TRACEPOINTS;
1433 #undef GATOR_HANDLE_TRACEPOINT
1434
1435 static void gator_save_tracepoint(struct tracepoint *tp, void *priv)
1436 {
1437 #define GATOR_HANDLE_TRACEPOINT(probe_name) \
1438         do { \
1439                 if (strcmp(tp->name, #probe_name) == 0) { \
1440                         gator_tracepoint_##probe_name = tp; \
1441                         return; \
1442                 } \
1443         } while (0)
1444 GATOR_TRACEPOINTS;
1445 #undef GATOR_HANDLE_TRACEPOINT
1446 }
1447
1448 #else
1449
1450 #define for_each_kernel_tracepoint(fct, priv)
1451
1452 #endif
1453
1454 static int __init gator_module_init(void)
1455 {
1456         for_each_kernel_tracepoint(gator_save_tracepoint, NULL);
1457
1458         if (gatorfs_register())
1459                 return -1;
1460
1461         if (gator_init()) {
1462                 gatorfs_unregister();
1463                 return -1;
1464         }
1465
1466         setup_timer(&gator_buffer_wake_up_timer, gator_buffer_wake_up, 0);
1467
1468         /* Initialize the list of cpuids */
1469         memset(gator_cpuids, -1, sizeof(gator_cpuids));
1470         on_each_cpu(gator_read_cpuid, NULL, 1);
1471
1472         return 0;
1473 }
1474
1475 static void __exit gator_module_exit(void)
1476 {
1477         del_timer_sync(&gator_buffer_wake_up_timer);
1478         tracepoint_synchronize_unregister();
1479         gator_exit();
1480         gatorfs_unregister();
1481 }
1482
1483 module_init(gator_module_init);
1484 module_exit(gator_module_exit);
1485
1486 MODULE_LICENSE("GPL");
1487 MODULE_AUTHOR("ARM Ltd");
1488 MODULE_DESCRIPTION("Gator system profiler");
1489 #define STRIFY2(ARG) #ARG
1490 #define STRIFY(ARG) STRIFY2(ARG)
1491 MODULE_VERSION(STRIFY(PROTOCOL_VERSION));