2 * Copyright (C) ARM Limited 2010-2014. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
12 #include <linux/hardirq.h>
13 #include <linux/kthread.h>
14 #include <linux/sched.h>
15 #include <linux/semaphore.h>
16 #include <linux/workqueue.h>
17 #include <trace/events/kmem.h>
34 static const char * const meminfo_names[] = {
35 "Linux_meminfo_memfree",
36 "Linux_meminfo_memused",
37 "Linux_meminfo_bufferram",
40 static const char * const proc_names[] = {
41 "Linux_proc_statm_size",
42 "Linux_proc_statm_share",
43 "Linux_proc_statm_text",
44 "Linux_proc_statm_data",
47 static bool meminfo_global_enabled;
48 static ulong meminfo_enabled[MEMINFO_TOTAL];
49 static ulong meminfo_keys[MEMINFO_TOTAL];
50 static long long meminfo_buffer[2 * (MEMINFO_TOTAL + 2)];
51 static int meminfo_length = 0;
52 static bool new_data_avail;
54 static bool proc_global_enabled;
55 static ulong proc_enabled[PROC_COUNT];
56 static ulong proc_keys[PROC_COUNT];
57 static DEFINE_PER_CPU(long long, proc_buffer[2 * (PROC_COUNT + 3)]);
59 static int gator_meminfo_func(void *data);
60 static bool gator_meminfo_run;
61 // Initialize semaphore unlocked to initialize memory values
62 #if LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 36)
63 static DECLARE_MUTEX(gator_meminfo_sem);
65 static DEFINE_SEMAPHORE(gator_meminfo_sem);
68 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 3, 0)
69 GATOR_DEFINE_PROBE(mm_page_free_direct, TP_PROTO(struct page *page, unsigned int order))
71 GATOR_DEFINE_PROBE(mm_page_free, TP_PROTO(struct page *page, unsigned int order))
74 up(&gator_meminfo_sem);
77 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 3, 0)
78 GATOR_DEFINE_PROBE(mm_pagevec_free, TP_PROTO(struct page *page, int cold))
80 GATOR_DEFINE_PROBE(mm_page_free_batched, TP_PROTO(struct page *page, int cold))
83 up(&gator_meminfo_sem);
86 GATOR_DEFINE_PROBE(mm_page_alloc, TP_PROTO(struct page *page, unsigned int order, gfp_t gfp_flags, int migratetype))
88 up(&gator_meminfo_sem);
91 static int gator_events_meminfo_create_files(struct super_block *sb, struct dentry *root)
96 for (i = 0; i < MEMINFO_TOTAL; i++) {
97 dir = gatorfs_mkdir(sb, root, meminfo_names[i]);
101 gatorfs_create_ulong(sb, dir, "enabled", &meminfo_enabled[i]);
102 gatorfs_create_ro_ulong(sb, dir, "key", &meminfo_keys[i]);
105 for (i = 0; i < PROC_COUNT; ++i) {
106 dir = gatorfs_mkdir(sb, root, proc_names[i]);
110 gatorfs_create_ulong(sb, dir, "enabled", &proc_enabled[i]);
111 gatorfs_create_ro_ulong(sb, dir, "key", &proc_keys[i]);
117 static int gator_events_meminfo_start(void)
121 new_data_avail = false;
122 meminfo_global_enabled = 0;
123 for (i = 0; i < MEMINFO_TOTAL; i++) {
124 if (meminfo_enabled[i]) {
125 meminfo_global_enabled = 1;
130 proc_global_enabled = 0;
131 for (i = 0; i < PROC_COUNT; ++i) {
132 if (proc_enabled[i]) {
133 proc_global_enabled = 1;
137 if (meminfo_enabled[MEMINFO_MEMUSED]) {
138 proc_global_enabled = 1;
141 if (meminfo_global_enabled == 0)
144 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 3, 0)
145 if (GATOR_REGISTER_TRACE(mm_page_free_direct))
147 if (GATOR_REGISTER_TRACE(mm_page_free))
149 goto mm_page_free_exit;
150 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 3, 0)
151 if (GATOR_REGISTER_TRACE(mm_pagevec_free))
153 if (GATOR_REGISTER_TRACE(mm_page_free_batched))
155 goto mm_page_free_batched_exit;
156 if (GATOR_REGISTER_TRACE(mm_page_alloc))
157 goto mm_page_alloc_exit;
159 // Start worker thread
160 gator_meminfo_run = true;
161 // Since the mutex starts unlocked, memory values will be initialized
162 if (IS_ERR(kthread_run(gator_meminfo_func, NULL, "gator_meminfo")))
163 goto kthread_run_exit;
168 GATOR_UNREGISTER_TRACE(mm_page_alloc);
170 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 3, 0)
171 GATOR_UNREGISTER_TRACE(mm_pagevec_free);
173 GATOR_UNREGISTER_TRACE(mm_page_free_batched);
175 mm_page_free_batched_exit:
176 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 3, 0)
177 GATOR_UNREGISTER_TRACE(mm_page_free_direct);
179 GATOR_UNREGISTER_TRACE(mm_page_free);
185 static void gator_events_meminfo_stop(void)
187 if (meminfo_global_enabled) {
188 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 3, 0)
189 GATOR_UNREGISTER_TRACE(mm_page_free_direct);
190 GATOR_UNREGISTER_TRACE(mm_pagevec_free);
192 GATOR_UNREGISTER_TRACE(mm_page_free);
193 GATOR_UNREGISTER_TRACE(mm_page_free_batched);
195 GATOR_UNREGISTER_TRACE(mm_page_alloc);
197 // Stop worker thread
198 gator_meminfo_run = false;
199 up(&gator_meminfo_sem);
203 // Must be run in process context as the kernel function si_meminfo() can sleep
204 static int gator_meminfo_func(void *data)
208 unsigned long long value;
211 if (down_killable(&gator_meminfo_sem)) {
215 // Eat up any pending events
216 while (!down_trylock(&gator_meminfo_sem));
218 if (!gator_meminfo_run) {
222 meminfo_length = len = 0;
225 for (i = 0; i < MEMINFO_TOTAL; i++) {
226 if (meminfo_enabled[i]) {
228 case MEMINFO_MEMFREE:
229 value = info.freeram * PAGE_SIZE;
231 case MEMINFO_MEMUSED:
232 // pid -1 means system wide
233 meminfo_buffer[len++] = 1;
234 meminfo_buffer[len++] = -1;
236 meminfo_buffer[len++] = meminfo_keys[MEMINFO_MEMUSED];
237 meminfo_buffer[len++] = (info.totalram - info.freeram) * PAGE_SIZE;
239 meminfo_buffer[len++] = 1;
240 meminfo_buffer[len++] = 0;
242 case MEMINFO_BUFFERRAM:
243 value = info.bufferram * PAGE_SIZE;
249 meminfo_buffer[len++] = meminfo_keys[i];
250 meminfo_buffer[len++] = value;
254 meminfo_length = len;
255 new_data_avail = true;
261 static int gator_events_meminfo_read(long long **buffer)
263 if (!on_primary_core() || !meminfo_global_enabled)
269 new_data_avail = false;
272 *buffer = meminfo_buffer;
274 return meminfo_length;
277 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 34) && LINUX_VERSION_CODE < KERNEL_VERSION(3, 4, 0)
279 static inline unsigned long gator_get_mm_counter(struct mm_struct *mm, int member)
281 #ifdef SPLIT_RSS_COUNTING
282 long val = atomic_long_read(&mm->rss_stat.count[member]);
285 return (unsigned long)val;
287 #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 0, 0)
288 return mm->rss_stat.count[member];
290 return atomic_long_read(&mm->rss_stat.count[member]);
295 #define get_mm_counter(mm, member) gator_get_mm_counter(mm, member)
299 static int gator_events_meminfo_read_proc(long long **buffer, struct task_struct *task)
301 struct mm_struct *mm;
306 int cpu = get_physical_cpu();
307 long long *buf = per_cpu(proc_buffer, cpu);
309 if (!proc_global_enabled) {
313 // Collect the memory stats of the process instead of the thread
314 if (task->group_leader != NULL) {
315 task = task->group_leader;
318 // get_task_mm/mmput is not needed in this context because the task and it's mm are required as part of the sched_switch
324 // Derived from task_statm in fs/proc/task_mmu.c
325 if (meminfo_enabled[MEMINFO_MEMUSED] || proc_enabled[PROC_SHARE]) {
326 share = get_mm_counter(mm,
327 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 32) && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 34)
335 // key of 1 indicates a pid
337 buf[len++] = task->pid;
339 for (i = 0; i < PROC_COUNT; ++i) {
340 if (proc_enabled[i]) {
343 value = mm->total_vm;
349 value = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) >> PAGE_SHIFT;
352 value = mm->total_vm - mm->shared_vm;
356 buf[len++] = proc_keys[i];
357 buf[len++] = value * PAGE_SIZE;
361 if (meminfo_enabled[MEMINFO_MEMUSED]) {
362 value = share + get_mm_counter(mm,
363 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 32) && LINUX_VERSION_CODE < KERNEL_VERSION(2, 6, 34)
369 // Send resident for this pid
370 buf[len++] = meminfo_keys[MEMINFO_MEMUSED];
371 buf[len++] = value * PAGE_SIZE;
384 static struct gator_interface gator_events_meminfo_interface = {
385 .create_files = gator_events_meminfo_create_files,
386 .start = gator_events_meminfo_start,
387 .stop = gator_events_meminfo_stop,
388 .read64 = gator_events_meminfo_read,
389 .read_proc = gator_events_meminfo_read_proc,
392 int gator_events_meminfo_init(void)
396 meminfo_global_enabled = 0;
397 for (i = 0; i < MEMINFO_TOTAL; i++) {
398 meminfo_enabled[i] = 0;
399 meminfo_keys[i] = gator_events_get_key();
402 proc_global_enabled = 0;
403 for (i = 0; i < PROC_COUNT; ++i) {
405 proc_keys[i] = gator_events_get_key();
408 return gator_events_install(&gator_events_meminfo_interface);