2 * Copyright (C) ARM Limited 2010-2014. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
10 /* must be power of 2 */
11 #define COOKIEMAP_ENTRIES 1024
12 /* must be a power of 2 - 512/4 = 128 entries */
13 #define TRANSLATE_BUFFER_SIZE 512
14 #define TRANSLATE_TEXT_SIZE 256
15 #define MAX_COLLISIONS 2
17 static uint32_t *gator_crc32_table;
18 static unsigned int translate_buffer_mask;
21 struct task_struct *task;
25 static DEFINE_PER_CPU(char *, translate_text);
26 static DEFINE_PER_CPU(uint32_t, cookie_next_key);
27 static DEFINE_PER_CPU(uint64_t *, cookie_keys);
28 static DEFINE_PER_CPU(uint32_t *, cookie_values);
29 static DEFINE_PER_CPU(int, translate_buffer_read);
30 static DEFINE_PER_CPU(int, translate_buffer_write);
31 static DEFINE_PER_CPU(struct cookie_args *, translate_buffer);
33 static uint32_t get_cookie(int cpu, struct task_struct *task, const char *text, bool from_wq);
34 static void wq_cookie_handler(struct work_struct *unused);
35 static DECLARE_WORK(cookie_work, wq_cookie_handler);
36 static struct timer_list app_process_wake_up_timer;
37 static void app_process_wake_up_handler(unsigned long unused_data);
39 static uint32_t cookiemap_code(uint64_t value64)
41 uint32_t value = (uint32_t)((value64 >> 32) + value64);
42 uint32_t cookiecode = (value >> 24) & 0xff;
44 cookiecode = cookiecode * 31 + ((value >> 16) & 0xff);
45 cookiecode = cookiecode * 31 + ((value >> 8) & 0xff);
46 cookiecode = cookiecode * 31 + ((value >> 0) & 0xff);
47 cookiecode &= (COOKIEMAP_ENTRIES - 1);
48 return cookiecode * MAX_COLLISIONS;
51 static uint32_t gator_chksum_crc32(const char *data)
53 register unsigned long crc;
54 const unsigned char *block = data;
55 int i, length = strlen(data);
58 for (i = 0; i < length; i++)
59 crc = ((crc >> 8) & 0x00FFFFFF) ^ gator_crc32_table[(crc ^ *block++) & 0xFF];
61 return (crc ^ 0xFFFFFFFF);
66 * Pre: [0][1][v][3]..[n-1]
67 * Post: [v][0][1][3]..[n-1]
69 static uint32_t cookiemap_exists(uint64_t key)
71 unsigned long x, flags, retval = 0;
72 int cpu = get_physical_cpu();
73 uint32_t cookiecode = cookiemap_code(key);
74 uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
75 uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
77 /* Can be called from interrupt handler or from work queue */
78 local_irq_save(flags);
79 for (x = 0; x < MAX_COLLISIONS; x++) {
81 uint32_t value = values[x];
84 keys[x] = keys[x - 1];
85 values[x] = values[x - 1];
93 local_irq_restore(flags);
100 * Pre: [0][1][2][3]..[n-1]
101 * Post: [v][0][1][2]..[n-2]
103 static void cookiemap_add(uint64_t key, uint32_t value)
105 int cpu = get_physical_cpu();
106 int cookiecode = cookiemap_code(key);
107 uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
108 uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
111 for (x = MAX_COLLISIONS - 1; x > 0; x--) {
112 keys[x] = keys[x - 1];
113 values[x] = values[x - 1];
119 #ifndef CONFIG_PREEMPT_RT_FULL
120 static void translate_buffer_write_args(int cpu, struct task_struct *task, const char *text)
125 struct cookie_args *args;
127 local_irq_save(flags);
129 write = per_cpu(translate_buffer_write, cpu);
130 next_write = (write + 1) & translate_buffer_mask;
132 /* At least one entry must always remain available as when read == write, the queue is empty not full */
133 if (next_write != per_cpu(translate_buffer_read, cpu)) {
134 args = &per_cpu(translate_buffer, cpu)[write];
137 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)
138 get_task_struct(task);
140 per_cpu(translate_buffer_write, cpu) = next_write;
143 local_irq_restore(flags);
147 static void translate_buffer_read_args(int cpu, struct cookie_args *args)
152 local_irq_save(flags);
154 read = per_cpu(translate_buffer_read, cpu);
155 *args = per_cpu(translate_buffer, cpu)[read];
156 per_cpu(translate_buffer_read, cpu) = (read + 1) & translate_buffer_mask;
158 local_irq_restore(flags);
161 static void wq_cookie_handler(struct work_struct *unused)
163 struct cookie_args args;
164 int cpu = get_physical_cpu(), cookie;
166 mutex_lock(&start_mutex);
168 if (gator_started != 0) {
169 while (per_cpu(translate_buffer_read, cpu) != per_cpu(translate_buffer_write, cpu)) {
170 translate_buffer_read_args(cpu, &args);
171 cookie = get_cookie(cpu, args.task, args.text, true);
172 marshal_link(cookie, args.task->tgid, args.task->pid);
173 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)
174 put_task_struct(args.task);
179 mutex_unlock(&start_mutex);
182 static void app_process_wake_up_handler(unsigned long unused_data)
184 /* had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater */
185 schedule_work(&cookie_work);
188 /* Retrieve full name from proc/pid/cmdline for java processes on Android */
189 static int translate_app_process(const char **text, int cpu, struct task_struct *task, bool from_wq)
194 struct mm_struct *mm;
195 struct page *page = NULL;
196 struct vm_area_struct *page_vma;
197 int bytes, offset, retval = 0;
198 char *buf = per_cpu(translate_text, cpu);
200 #ifndef CONFIG_PREEMPT_RT_FULL
201 /* Push work into a work queue if in atomic context as the kernel
202 * functions below might sleep. Rely on the in_interrupt variable
203 * rather than in_irq() or in_interrupt() kernel functions, as the
204 * value of these functions seems inconsistent during a context
205 * switch between android/linux versions
208 /* Check if already in buffer */
209 int pos = per_cpu(translate_buffer_read, cpu);
211 while (pos != per_cpu(translate_buffer_write, cpu)) {
212 if (per_cpu(translate_buffer, cpu)[pos].task == task)
214 pos = (pos + 1) & translate_buffer_mask;
217 translate_buffer_write_args(cpu, task, *text);
219 /* Not safe to call in RT-Preempt full in schedule switch context */
220 mod_timer(&app_process_wake_up_timer, jiffies + 1);
225 mm = get_task_mm(task);
230 addr = mm->arg_start;
231 len = mm->arg_end - mm->arg_start;
233 if (len > TRANSLATE_TEXT_SIZE)
234 len = TRANSLATE_TEXT_SIZE;
236 down_read(&mm->mmap_sem);
238 if (get_user_pages(task, mm, addr, 1, 0, 1, &page, &page_vma) <= 0)
242 offset = addr & (PAGE_SIZE - 1);
244 if (bytes > PAGE_SIZE - offset)
245 bytes = PAGE_SIZE - offset;
247 copy_from_user_page(page_vma, page, addr, buf, maddr + offset, bytes);
249 /* release page allocated by get_user_pages() */
251 page_cache_release(page);
257 *text = per_cpu(translate_text, cpu);
261 /* On app_process startup, /proc/pid/cmdline is initially "zygote" then "<pre-initialized>" but changes after an initial startup period */
262 if (strcmp(*text, "zygote") == 0 || strcmp(*text, "<pre-initialized>") == 0)
266 up_read(&mm->mmap_sem);
273 static const char APP_PROCESS[] = "app_process";
275 static uint32_t get_cookie(int cpu, struct task_struct *task, const char *text, bool from_wq)
277 unsigned long flags, cookie;
280 key = gator_chksum_crc32(text);
281 key = (key << 32) | (uint32_t)task->tgid;
283 cookie = cookiemap_exists(key);
287 /* On 64-bit android app_process can be app_process32 or app_process64 */
288 if (strncmp(text, APP_PROCESS, sizeof(APP_PROCESS) - 1) == 0) {
289 if (!translate_app_process(&text, cpu, task, from_wq))
290 return UNRESOLVED_COOKIE;
293 /* Can be called from interrupt handler or from work queue or from scheduler trace */
294 local_irq_save(flags);
296 cookie = UNRESOLVED_COOKIE;
297 if (marshal_cookie_header(text)) {
298 cookie = per_cpu(cookie_next_key, cpu) += nr_cpu_ids;
299 cookiemap_add(key, cookie);
300 marshal_cookie(cookie, text);
303 local_irq_restore(flags);
308 static int get_exec_cookie(int cpu, struct task_struct *task)
310 struct mm_struct *mm = task->mm;
313 /* kernel threads have no address space */
317 if (task && task->mm && task->mm->exe_file) {
318 text = task->mm->exe_file->f_path.dentry->d_name.name;
319 return get_cookie(cpu, task, text, false);
322 return UNRESOLVED_COOKIE;
325 static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsigned long addr, off_t *offset)
327 unsigned long cookie = NO_COOKIE;
328 struct mm_struct *mm = task->mm;
329 struct vm_area_struct *vma;
335 for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
336 if (addr < vma->vm_start || addr >= vma->vm_end)
340 text = vma->vm_file->f_path.dentry->d_name.name;
341 cookie = get_cookie(cpu, task, text, false);
342 *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start;
344 /* must be an anonymous map */
352 cookie = UNRESOLVED_COOKIE;
357 static int cookies_initialize(void)
360 int i, j, cpu, size, err = 0;
362 translate_buffer_mask = TRANSLATE_BUFFER_SIZE / sizeof(per_cpu(translate_buffer, 0)[0]) - 1;
364 for_each_present_cpu(cpu) {
365 per_cpu(cookie_next_key, cpu) = nr_cpu_ids + cpu;
367 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint64_t);
368 per_cpu(cookie_keys, cpu) = kmalloc(size, GFP_KERNEL);
369 if (!per_cpu(cookie_keys, cpu)) {
371 goto cookie_setup_error;
373 memset(per_cpu(cookie_keys, cpu), 0, size);
375 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t);
376 per_cpu(cookie_values, cpu) = kmalloc(size, GFP_KERNEL);
377 if (!per_cpu(cookie_values, cpu)) {
379 goto cookie_setup_error;
381 memset(per_cpu(cookie_values, cpu), 0, size);
383 per_cpu(translate_buffer, cpu) = kmalloc(TRANSLATE_BUFFER_SIZE, GFP_KERNEL);
384 if (!per_cpu(translate_buffer, cpu)) {
386 goto cookie_setup_error;
389 per_cpu(translate_buffer_write, cpu) = 0;
390 per_cpu(translate_buffer_read, cpu) = 0;
392 per_cpu(translate_text, cpu) = kmalloc(TRANSLATE_TEXT_SIZE, GFP_KERNEL);
393 if (!per_cpu(translate_text, cpu)) {
395 goto cookie_setup_error;
399 /* build CRC32 table */
401 gator_crc32_table = kmalloc(256 * sizeof(*gator_crc32_table), GFP_KERNEL);
402 if (!gator_crc32_table) {
404 goto cookie_setup_error;
406 for (i = 0; i < 256; i++) {
408 for (j = 8; j > 0; j--) {
410 crc = (crc >> 1) ^ poly;
414 gator_crc32_table[i] = crc;
417 setup_timer(&app_process_wake_up_timer, app_process_wake_up_handler, 0);
423 static void cookies_release(void)
427 for_each_present_cpu(cpu) {
428 kfree(per_cpu(cookie_keys, cpu));
429 per_cpu(cookie_keys, cpu) = NULL;
431 kfree(per_cpu(cookie_values, cpu));
432 per_cpu(cookie_values, cpu) = NULL;
434 kfree(per_cpu(translate_buffer, cpu));
435 per_cpu(translate_buffer, cpu) = NULL;
436 per_cpu(translate_buffer_read, cpu) = 0;
437 per_cpu(translate_buffer_write, cpu) = 0;
439 kfree(per_cpu(translate_text, cpu));
440 per_cpu(translate_text, cpu) = NULL;
443 del_timer_sync(&app_process_wake_up_timer);
444 kfree(gator_crc32_table);
445 gator_crc32_table = NULL;