drivers/gator/gator_events_meminfo.c

   1 /**
   2  * Copyright (C) ARM Limited 2010-2014. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  */
   9
  10 #include "gator.h"
  11
  12 #include <linux/hardirq.h>
  13 #include <linux/kthread.h>
  14 #include <linux/sched.h>
  15 #include <linux/semaphore.h>
  16 #include <linux/workqueue.h>
  17 #include <trace/events/kmem.h>
  18
  19 #define USE_THREAD defined(CONFIG_PREEMPT_RT_FULL)
  20
  21 enum {
  22         MEMINFO_MEMFREE,
  23         MEMINFO_MEMUSED,
  24         MEMINFO_BUFFERRAM,
  25         MEMINFO_TOTAL,
  26 };
  27
  28 enum {
  29         PROC_SIZE,
  30         PROC_SHARE,
  31         PROC_TEXT,
  32         PROC_DATA,
  33         PROC_COUNT,
  34 };
  35
  36 static const char * const meminfo_names[] = {
  37         "Linux_meminfo_memfree",
  38         "Linux_meminfo_memused",
  39         "Linux_meminfo_bufferram",
  40 };
  41
  42 static const char * const proc_names[] = {
  43         "Linux_proc_statm_size",
  44         "Linux_proc_statm_share",
  45         "Linux_proc_statm_text",
  46         "Linux_proc_statm_data",
  47 };
  48
  49 static bool meminfo_global_enabled;
  50 static ulong meminfo_enabled[MEMINFO_TOTAL];
  51 static ulong meminfo_keys[MEMINFO_TOTAL];
  52 static long long meminfo_buffer[2 * (MEMINFO_TOTAL + 2)];
  53 static int meminfo_length;
  54 static bool new_data_avail;
  55
  56 static bool proc_global_enabled;
  57 static ulong proc_enabled[PROC_COUNT];
  58 static ulong proc_keys[PROC_COUNT];
  59 static DEFINE_PER_CPU(long long, proc_buffer[2 * (PROC_COUNT + 3)]);
  60
  61 #if USE_THREAD
  62
  63 static int gator_meminfo_func(void *data);
  64 static bool gator_meminfo_run;
  65 /* Initialize semaphore unlocked to initialize memory values */
  66 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 36)
  67 static DECLARE_MUTEX(gator_meminfo_sem);
  68 #else
  69 static DEFINE_SEMAPHORE(gator_meminfo_sem);
  70 #endif
  71
  72 static void notify(void)
  73 {
  74         up(&gator_meminfo_sem);
  75 }
  76
  77 #else
  78
  79 static unsigned int mem_event;
  80 static void wq_sched_handler(struct work_struct *wsptr);
  81 DECLARE_WORK(work, wq_sched_handler);
  82 static struct timer_list meminfo_wake_up_timer;
  83 static void meminfo_wake_up_handler(unsigned long unused_data);
  84
  85 static void notify(void)
  86 {
  87         mem_event++;
  88 }
  89
  90 #endif
  91
  92 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 3, 0)
  93 GATOR_DEFINE_PROBE(mm_page_free_direct, TP_PROTO(struct page *page, unsigned int order))
  94 #else
  95 GATOR_DEFINE_PROBE(mm_page_free, TP_PROTO(struct page *page, unsigned int order))
  96 #endif
  97 {
  98         notify();
  99 }
 100
 101 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 3, 0)
 102 GATOR_DEFINE_PROBE(mm_pagevec_free, TP_PROTO(struct page *page, int cold))
 103 #else
 104 GATOR_DEFINE_PROBE(mm_page_free_batched, TP_PROTO(struct page *page, int cold))
 105 #endif
 106 {
 107         notify();
 108 }
 109
 110 GATOR_DEFINE_PROBE(mm_page_alloc, TP_PROTO(struct page *page, unsigned int order, gfp_t gfp_flags, int migratetype))
 111 {
 112         notify();
 113 }
 114
 115 static int gator_events_meminfo_create_files(struct super_block *sb, struct dentry *root)
 116 {
 117         struct dentry *dir;
 118         int i;
 119
 120         for (i = 0; i < MEMINFO_TOTAL; i++) {
 121                 dir = gatorfs_mkdir(sb, root, meminfo_names[i]);
 122                 if (!dir)
 123                         return -1;
 124                 gatorfs_create_ulong(sb, dir, "enabled", &meminfo_enabled[i]);
 125                 gatorfs_create_ro_ulong(sb, dir, "key", &meminfo_keys[i]);
 126         }
 127
 128         for (i = 0; i < PROC_COUNT; ++i) {
 129                 dir = gatorfs_mkdir(sb, root, proc_names[i]);
 130                 if (!dir)
 131                         return -1;
 132                 gatorfs_create_ulong(sb, dir, "enabled", &proc_enabled[i]);
 133                 gatorfs_create_ro_ulong(sb, dir, "key", &proc_keys[i]);
 134         }
 135
 136         return 0;
 137 }
 138
 139 static int gator_events_meminfo_start(void)
 140 {
 141         int i;
 142
 143         new_data_avail = false;
 144         meminfo_global_enabled = 0;
 145         for (i = 0; i < MEMINFO_TOTAL; i++) {
 146                 if (meminfo_enabled[i]) {
 147                         meminfo_global_enabled = 1;
 148                         break;
 149                 }
 150         }
 151
 152         proc_global_enabled = 0;
 153         for (i = 0; i < PROC_COUNT; ++i) {
 154                 if (proc_enabled[i]) {
 155                         proc_global_enabled = 1;
 156                         break;
 157                 }
 158         }
 159         if (meminfo_enabled[MEMINFO_MEMUSED])
 160                 proc_global_enabled = 1;
 161
 162         if (meminfo_global_enabled == 0)
 163                 return 0;
 164
 165 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 3, 0)
 166         if (GATOR_REGISTER_TRACE(mm_page_free_direct))
 167 #else
 168         if (GATOR_REGISTER_TRACE(mm_page_free))
 169 #endif
 170                 goto mm_page_free_exit;
 171 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 3, 0)
 172         if (GATOR_REGISTER_TRACE(mm_pagevec_free))
 173 #else
 174         if (GATOR_REGISTER_TRACE(mm_page_free_batched))
 175 #endif
 176                 goto mm_page_free_batched_exit;
 177         if (GATOR_REGISTER_TRACE(mm_page_alloc))
 178                 goto mm_page_alloc_exit;
 179
 180 #if USE_THREAD
 181         /* Start worker thread */
 182         gator_meminfo_run = true;
 183         /* Since the mutex starts unlocked, memory values will be initialized */
 184         if (IS_ERR(kthread_run(gator_meminfo_func, NULL, "gator_meminfo")))
 185                 goto kthread_run_exit;
 186 #else
 187         setup_timer(&meminfo_wake_up_timer, meminfo_wake_up_handler, 0);
 188 #endif
 189
 190         return 0;
 191
 192 #if USE_THREAD
 193 kthread_run_exit:
 194         GATOR_UNREGISTER_TRACE(mm_page_alloc);
 195 #endif
 196 mm_page_alloc_exit:
 197 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 3, 0)
 198         GATOR_UNREGISTER_TRACE(mm_pagevec_free);
 199 #else
 200         GATOR_UNREGISTER_TRACE(mm_page_free_batched);
 201 #endif
 202 mm_page_free_batched_exit:
 203 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 3, 0)
 204         GATOR_UNREGISTER_TRACE(mm_page_free_direct);
 205 #else
 206         GATOR_UNREGISTER_TRACE(mm_page_free);
 207 #endif
 208 mm_page_free_exit:
 209         return -1;
 210 }
 211
 212 static void gator_events_meminfo_stop(void)
 213 {
 214         if (meminfo_global_enabled) {
 215 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 3, 0)
 216                 GATOR_UNREGISTER_TRACE(mm_page_free_direct);
 217                 GATOR_UNREGISTER_TRACE(mm_pagevec_free);
 218 #else
 219                 GATOR_UNREGISTER_TRACE(mm_page_free);
 220                 GATOR_UNREGISTER_TRACE(mm_page_free_batched);
 221 #endif
 222                 GATOR_UNREGISTER_TRACE(mm_page_alloc);
 223
 224 #if USE_THREAD
 225                 /* Stop worker thread */
 226                 gator_meminfo_run = false;
 227                 up(&gator_meminfo_sem);
 228 #else
 229                 del_timer_sync(&meminfo_wake_up_timer);
 230 #endif
 231         }
 232 }
 233
 234 static void do_read(void)
 235 {
 236         struct sysinfo info;
 237         int i, len;
 238         unsigned long long value;
 239
 240         meminfo_length = len = 0;
 241
 242         si_meminfo(&info);
 243         for (i = 0; i < MEMINFO_TOTAL; i++) {
 244                 if (meminfo_enabled[i]) {
 245                         switch (i) {
 246                         case MEMINFO_MEMFREE:
 247                                 value = info.freeram * PAGE_SIZE;
 248                                 break;
 249                         case MEMINFO_MEMUSED:
 250                                 /* pid -1 means system wide */
 251                                 meminfo_buffer[len++] = 1;
 252                                 meminfo_buffer[len++] = -1;
 253                                 /* Emit value */
 254                                 meminfo_buffer[len++] = meminfo_keys[MEMINFO_MEMUSED];
 255                                 meminfo_buffer[len++] = (info.totalram - info.freeram) * PAGE_SIZE;
 256                                 /* Clear pid */
 257                                 meminfo_buffer[len++] = 1;
 258                                 meminfo_buffer[len++] = 0;
 259                                 continue;
 260                         case MEMINFO_BUFFERRAM:
 261                                 value = info.bufferram * PAGE_SIZE;
 262                                 break;
 263                         default:
 264                                 value = 0;
 265                                 break;
 266                         }
 267                         meminfo_buffer[len++] = meminfo_keys[i];
 268                         meminfo_buffer[len++] = value;
 269                 }
 270         }
 271
 272         meminfo_length = len;
 273         new_data_avail = true;
 274 }
 275
 276 #if USE_THREAD
 277
 278 static int gator_meminfo_func(void *data)
 279 {
 280         for (;;) {
 281                 if (down_killable(&gator_meminfo_sem))
 282                         break;
 283
 284                 /* Eat up any pending events */
 285                 while (!down_trylock(&gator_meminfo_sem))
 286                         ;
 287
 288                 if (!gator_meminfo_run)
 289                         break;
 290
 291                 do_read();
 292         }
 293
 294         return 0;
 295 }
 296
 297 #else
 298
 299 /* Must be run in process context as the kernel function si_meminfo() can sleep */
 300 static void wq_sched_handler(struct work_struct *wsptr)
 301 {
 302         do_read();
 303 }
 304
 305 static void meminfo_wake_up_handler(unsigned long unused_data)
 306 {
 307         /* had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater */
 308         schedule_work(&work);
 309 }
 310
 311 #endif
 312
 313 static int gator_events_meminfo_read(long long **buffer)
 314 {
 315 #if !USE_THREAD
 316         static unsigned int last_mem_event;
 317 #endif
 318
 319         if (!on_primary_core() || !meminfo_global_enabled)
 320                 return 0;
 321
 322 #if !USE_THREAD
 323         if (last_mem_event != mem_event) {
 324                 last_mem_event = mem_event;
 325                 mod_timer(&meminfo_wake_up_timer, jiffies + 1);
 326         }
 327 #endif
 328
 329         if (!new_data_avail)
 330                 return 0;
 331
 332         new_data_avail = false;
 333
 334         if (buffer)
 335                 *buffer = meminfo_buffer;
 336
 337         return meminfo_length;
 338 }
 339
 340 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 34) && LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)
 341
 342 static inline unsigned long gator_get_mm_counter(struct mm_struct *mm, int member)
 343 {
 344 #ifdef SPLIT_RSS_COUNTING
 345         long val = atomic_long_read(&mm->rss_stat.count[member]);
 346
 347         if (val < 0)
 348                 val = 0;
 349         return (unsigned long)val;
 350 #else
 351 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0)
 352         return mm->rss_stat.count[member];
 353 #else
 354         return atomic_long_read(&mm->rss_stat.count[member]);
 355 #endif
 356 #endif
 357 }
 358
 359 #define get_mm_counter(mm, member) gator_get_mm_counter(mm, member)
 360
 361 #endif
 362
 363 static int gator_events_meminfo_read_proc(long long **buffer, struct task_struct *task)
 364 {
 365         struct mm_struct *mm;
 366         u64 share = 0;
 367         int i;
 368         long long value;
 369         int len = 0;
 370         int cpu = get_physical_cpu();
 371         long long *buf = per_cpu(proc_buffer, cpu);
 372
 373         if (!proc_global_enabled)
 374                 return 0;
 375
 376         /* Collect the memory stats of the process instead of the thread */
 377         if (task->group_leader != NULL)
 378                 task = task->group_leader;
 379
 380         /* get_task_mm/mmput is not needed in this context because the task and it's mm are required as part of the sched_switch */
 381         mm = task->mm;
 382         if (mm == NULL)
 383                 return 0;
 384
 385         /* Derived from task_statm in fs/proc/task_mmu.c */
 386         if (meminfo_enabled[MEMINFO_MEMUSED] || proc_enabled[PROC_SHARE]) {
 387                 share = get_mm_counter(mm,
 388 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 32) && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 34)
 389                                                            file_rss
 390 #else
 391                                                            MM_FILEPAGES
 392 #endif
 393                                                            );
 394         }
 395
 396         /* key of 1 indicates a pid */
 397         buf[len++] = 1;
 398         buf[len++] = task->pid;
 399
 400         for (i = 0; i < PROC_COUNT; ++i) {
 401                 if (proc_enabled[i]) {
 402                         switch (i) {
 403                         case PROC_SIZE:
 404                                 value = mm->total_vm;
 405                                 break;
 406                         case PROC_SHARE:
 407                                 value = share;
 408                                 break;
 409                         case PROC_TEXT:
 410                                 value = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) >> PAGE_SHIFT;
 411                                 break;
 412                         case PROC_DATA:
 413                                 value = mm->total_vm - mm->shared_vm;
 414                                 break;
 415                         }
 416
 417                         buf[len++] = proc_keys[i];
 418                         buf[len++] = value * PAGE_SIZE;
 419                 }
 420         }
 421
 422         if (meminfo_enabled[MEMINFO_MEMUSED]) {
 423                 value = share + get_mm_counter(mm,
 424 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 32) && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 34)
 425                                                                            anon_rss
 426 #else
 427                                                                            MM_ANONPAGES
 428 #endif
 429                                                                            );
 430                 /* Send resident for this pid */
 431                 buf[len++] = meminfo_keys[MEMINFO_MEMUSED];
 432                 buf[len++] = value * PAGE_SIZE;
 433         }
 434
 435         /* Clear pid */
 436         buf[len++] = 1;
 437         buf[len++] = 0;
 438
 439         if (buffer)
 440                 *buffer = buf;
 441
 442         return len;
 443 }
 444
 445 static struct gator_interface gator_events_meminfo_interface = {
 446         .create_files = gator_events_meminfo_create_files,
 447         .start = gator_events_meminfo_start,
 448         .stop = gator_events_meminfo_stop,
 449         .read64 = gator_events_meminfo_read,
 450         .read_proc = gator_events_meminfo_read_proc,
 451 };
 452
 453 int gator_events_meminfo_init(void)
 454 {
 455         int i;
 456
 457         meminfo_global_enabled = 0;
 458         for (i = 0; i < MEMINFO_TOTAL; i++) {
 459                 meminfo_enabled[i] = 0;
 460                 meminfo_keys[i] = gator_events_get_key();
 461         }
 462
 463         proc_global_enabled = 0;
 464         for (i = 0; i < PROC_COUNT; ++i) {
 465                 proc_enabled[i] = 0;
 466                 proc_keys[i] = gator_events_get_key();
 467         }
 468
 469         return gator_events_install(&gator_events_meminfo_interface);
 470 }