2 * Copyright (C) ARM Limited 2010-2014. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
10 #define COOKIEMAP_ENTRIES 1024 /* must be power of 2 */
11 #define TRANSLATE_BUFFER_SIZE 512 // must be a power of 2 - 512/4 = 128 entries
12 #define TRANSLATE_TEXT_SIZE 256
13 #define MAX_COLLISIONS 2
15 static uint32_t *gator_crc32_table;
16 static unsigned int translate_buffer_mask;
19 struct task_struct *task;
23 static DEFINE_PER_CPU(char *, translate_text);
24 static DEFINE_PER_CPU(uint32_t, cookie_next_key);
25 static DEFINE_PER_CPU(uint64_t *, cookie_keys);
26 static DEFINE_PER_CPU(uint32_t *, cookie_values);
27 static DEFINE_PER_CPU(int, translate_buffer_read);
28 static DEFINE_PER_CPU(int, translate_buffer_write);
29 static DEFINE_PER_CPU(struct cookie_args *, translate_buffer);
31 static uint32_t get_cookie(int cpu, struct task_struct *task, const char *text, bool from_wq);
32 static void wq_cookie_handler(struct work_struct *unused);
33 static DECLARE_WORK(cookie_work, wq_cookie_handler);
34 static struct timer_list app_process_wake_up_timer;
35 static void app_process_wake_up_handler(unsigned long unused_data);
37 static uint32_t cookiemap_code(uint64_t value64)
39 uint32_t value = (uint32_t)((value64 >> 32) + value64);
40 uint32_t cookiecode = (value >> 24) & 0xff;
41 cookiecode = cookiecode * 31 + ((value >> 16) & 0xff);
42 cookiecode = cookiecode * 31 + ((value >> 8) & 0xff);
43 cookiecode = cookiecode * 31 + ((value >> 0) & 0xff);
44 cookiecode &= (COOKIEMAP_ENTRIES - 1);
45 return cookiecode * MAX_COLLISIONS;
48 static uint32_t gator_chksum_crc32(const char *data)
50 register unsigned long crc;
51 const unsigned char *block = data;
52 int i, length = strlen(data);
55 for (i = 0; i < length; i++) {
56 crc = ((crc >> 8) & 0x00FFFFFF) ^ gator_crc32_table[(crc ^ *block++) & 0xFF];
59 return (crc ^ 0xFFFFFFFF);
64 * Pre: [0][1][v][3]..[n-1]
65 * Post: [v][0][1][3]..[n-1]
67 static uint32_t cookiemap_exists(uint64_t key)
69 unsigned long x, flags, retval = 0;
70 int cpu = get_physical_cpu();
71 uint32_t cookiecode = cookiemap_code(key);
72 uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
73 uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
75 // Can be called from interrupt handler or from work queue
76 local_irq_save(flags);
77 for (x = 0; x < MAX_COLLISIONS; x++) {
79 uint32_t value = values[x];
81 keys[x] = keys[x - 1];
82 values[x] = values[x - 1];
90 local_irq_restore(flags);
97 * Pre: [0][1][2][3]..[n-1]
98 * Post: [v][0][1][2]..[n-2]
100 static void cookiemap_add(uint64_t key, uint32_t value)
102 int cpu = get_physical_cpu();
103 int cookiecode = cookiemap_code(key);
104 uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
105 uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
108 for (x = MAX_COLLISIONS - 1; x > 0; x--) {
109 keys[x] = keys[x - 1];
110 values[x] = values[x - 1];
116 #ifndef CONFIG_PREEMPT_RT_FULL
117 static void translate_buffer_write_args(int cpu, struct task_struct *task, const char *text)
122 struct cookie_args *args;
124 local_irq_save(flags);
126 write = per_cpu(translate_buffer_write, cpu);
127 next_write = (write + 1) & translate_buffer_mask;
129 // At least one entry must always remain available as when read == write, the queue is empty not full
130 if (next_write != per_cpu(translate_buffer_read, cpu)) {
131 args = &per_cpu(translate_buffer, cpu)[write];
134 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)
135 get_task_struct(task);
137 per_cpu(translate_buffer_write, cpu) = next_write;
140 local_irq_restore(flags);
144 static void translate_buffer_read_args(int cpu, struct cookie_args *args)
149 local_irq_save(flags);
151 read = per_cpu(translate_buffer_read, cpu);
152 *args = per_cpu(translate_buffer, cpu)[read];
153 per_cpu(translate_buffer_read, cpu) = (read + 1) & translate_buffer_mask;
155 local_irq_restore(flags);
158 static void wq_cookie_handler(struct work_struct *unused)
160 struct cookie_args args;
161 int cpu = get_physical_cpu(), cookie;
163 mutex_lock(&start_mutex);
165 if (gator_started != 0) {
166 while (per_cpu(translate_buffer_read, cpu) != per_cpu(translate_buffer_write, cpu)) {
167 translate_buffer_read_args(cpu, &args);
168 cookie = get_cookie(cpu, args.task, args.text, true);
169 marshal_link(cookie, args.task->tgid, args.task->pid);
170 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)
171 put_task_struct(args.task);
176 mutex_unlock(&start_mutex);
179 static void app_process_wake_up_handler(unsigned long unused_data)
181 // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
182 schedule_work(&cookie_work);
185 // Retrieve full name from proc/pid/cmdline for java processes on Android
186 static int translate_app_process(const char **text, int cpu, struct task_struct *task, bool from_wq)
191 struct mm_struct *mm;
192 struct page *page = NULL;
193 struct vm_area_struct *page_vma;
194 int bytes, offset, retval = 0;
195 char *buf = per_cpu(translate_text, cpu);
197 #ifndef CONFIG_PREEMPT_RT_FULL
198 // Push work into a work queue if in atomic context as the kernel functions below might sleep
199 // Rely on the in_interrupt variable rather than in_irq() or in_interrupt() kernel functions, as the value of these functions seems
200 // inconsistent during a context switch between android/linux versions
202 // Check if already in buffer
203 int pos = per_cpu(translate_buffer_read, cpu);
204 while (pos != per_cpu(translate_buffer_write, cpu)) {
205 if (per_cpu(translate_buffer, cpu)[pos].task == task)
207 pos = (pos + 1) & translate_buffer_mask;
210 translate_buffer_write_args(cpu, task, *text);
212 // Not safe to call in RT-Preempt full in schedule switch context
213 mod_timer(&app_process_wake_up_timer, jiffies + 1);
218 mm = get_task_mm(task);
223 addr = mm->arg_start;
224 len = mm->arg_end - mm->arg_start;
226 if (len > TRANSLATE_TEXT_SIZE)
227 len = TRANSLATE_TEXT_SIZE;
229 down_read(&mm->mmap_sem);
231 if (get_user_pages(task, mm, addr, 1, 0, 1, &page, &page_vma) <= 0)
235 offset = addr & (PAGE_SIZE - 1);
237 if (bytes > PAGE_SIZE - offset)
238 bytes = PAGE_SIZE - offset;
240 copy_from_user_page(page_vma, page, addr, buf, maddr + offset, bytes);
242 kunmap(page); // release page allocated by get_user_pages()
243 page_cache_release(page);
249 *text = per_cpu(translate_text, cpu);
253 // On app_process startup, /proc/pid/cmdline is initially "zygote" then "<pre-initialized>" but changes after an initial startup period
254 if (strcmp(*text, "zygote") == 0 || strcmp(*text, "<pre-initialized>") == 0)
258 up_read(&mm->mmap_sem);
265 static uint32_t get_cookie(int cpu, struct task_struct *task, const char *text, bool from_wq)
267 unsigned long flags, cookie;
270 key = gator_chksum_crc32(text);
271 key = (key << 32) | (uint32_t)task->tgid;
273 cookie = cookiemap_exists(key);
278 if (strcmp(text, "app_process") == 0) {
279 if (!translate_app_process(&text, cpu, task, from_wq))
280 return UNRESOLVED_COOKIE;
283 // Can be called from interrupt handler or from work queue or from scheduler trace
284 local_irq_save(flags);
286 cookie = UNRESOLVED_COOKIE;
287 if (marshal_cookie_header(text)) {
288 cookie = per_cpu(cookie_next_key, cpu) += nr_cpu_ids;
289 cookiemap_add(key, cookie);
290 marshal_cookie(cookie, text);
293 local_irq_restore(flags);
298 static int get_exec_cookie(int cpu, struct task_struct *task)
300 struct mm_struct *mm = task->mm;
303 // kernel threads have no address space
307 if (task && task->mm && task->mm->exe_file) {
308 text = task->mm->exe_file->f_path.dentry->d_name.name;
309 return get_cookie(cpu, task, text, false);
312 return UNRESOLVED_COOKIE;
315 static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsigned long addr, off_t *offset)
317 unsigned long cookie = NO_COOKIE;
318 struct mm_struct *mm = task->mm;
319 struct vm_area_struct *vma;
325 for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
326 if (addr < vma->vm_start || addr >= vma->vm_end)
330 text = vma->vm_file->f_path.dentry->d_name.name;
331 cookie = get_cookie(cpu, task, text, false);
332 *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start;
334 /* must be an anonymous map */
342 cookie = UNRESOLVED_COOKIE;
347 static int cookies_initialize(void)
350 int i, j, cpu, size, err = 0;
352 translate_buffer_mask = TRANSLATE_BUFFER_SIZE / sizeof(per_cpu(translate_buffer, 0)[0]) - 1;
354 for_each_present_cpu(cpu) {
355 per_cpu(cookie_next_key, cpu) = nr_cpu_ids + cpu;
357 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint64_t);
358 per_cpu(cookie_keys, cpu) = (uint64_t *)kmalloc(size, GFP_KERNEL);
359 if (!per_cpu(cookie_keys, cpu)) {
361 goto cookie_setup_error;
363 memset(per_cpu(cookie_keys, cpu), 0, size);
365 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t);
366 per_cpu(cookie_values, cpu) = (uint32_t *)kmalloc(size, GFP_KERNEL);
367 if (!per_cpu(cookie_values, cpu)) {
369 goto cookie_setup_error;
371 memset(per_cpu(cookie_values, cpu), 0, size);
373 per_cpu(translate_buffer, cpu) = (struct cookie_args *)kmalloc(TRANSLATE_BUFFER_SIZE, GFP_KERNEL);
374 if (!per_cpu(translate_buffer, cpu)) {
376 goto cookie_setup_error;
379 per_cpu(translate_buffer_write, cpu) = 0;
380 per_cpu(translate_buffer_read, cpu) = 0;
382 per_cpu(translate_text, cpu) = (char *)kmalloc(TRANSLATE_TEXT_SIZE, GFP_KERNEL);
383 if (!per_cpu(translate_text, cpu)) {
385 goto cookie_setup_error;
391 gator_crc32_table = (uint32_t *)kmalloc(256 * sizeof(uint32_t), GFP_KERNEL);
392 if (!gator_crc32_table) {
394 goto cookie_setup_error;
396 for (i = 0; i < 256; i++) {
398 for (j = 8; j > 0; j--) {
400 crc = (crc >> 1) ^ poly;
405 gator_crc32_table[i] = crc;
408 setup_timer(&app_process_wake_up_timer, app_process_wake_up_handler, 0);
414 static void cookies_release(void)
418 for_each_present_cpu(cpu) {
419 kfree(per_cpu(cookie_keys, cpu));
420 per_cpu(cookie_keys, cpu) = NULL;
422 kfree(per_cpu(cookie_values, cpu));
423 per_cpu(cookie_values, cpu) = NULL;
425 kfree(per_cpu(translate_buffer, cpu));
426 per_cpu(translate_buffer, cpu) = NULL;
427 per_cpu(translate_buffer_read, cpu) = 0;
428 per_cpu(translate_buffer_write, cpu) = 0;
430 kfree(per_cpu(translate_text, cpu));
431 per_cpu(translate_text, cpu) = NULL;
434 del_timer_sync(&app_process_wake_up_timer);
435 kfree(gator_crc32_table);
436 gator_crc32_table = NULL;