drivers/gator/gator_cookies.c

   1 /**
   2  * Copyright (C) ARM Limited 2010-2015. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  */
   9
  10 /* must be power of 2 */
  11 #define COOKIEMAP_ENTRIES       1024
  12 /* must be a power of 2 - 512/4 = 128 entries */
  13 #define TRANSLATE_BUFFER_SIZE 512
  14 #define TRANSLATE_TEXT_SIZE             256
  15 #define MAX_COLLISIONS          2
  16
  17 static uint32_t *gator_crc32_table;
  18 static unsigned int translate_buffer_mask;
  19
  20 struct cookie_args {
  21         struct task_struct *task;
  22         const char *text;
  23 };
  24
  25 static DEFINE_PER_CPU(char *, translate_text);
  26 static DEFINE_PER_CPU(uint32_t, cookie_next_key);
  27 static DEFINE_PER_CPU(uint64_t *, cookie_keys);
  28 static DEFINE_PER_CPU(uint32_t *, cookie_values);
  29 static DEFINE_PER_CPU(int, translate_buffer_read);
  30 static DEFINE_PER_CPU(int, translate_buffer_write);
  31 static DEFINE_PER_CPU(struct cookie_args *, translate_buffer);
  32
  33 static uint32_t get_cookie(int cpu, struct task_struct *task, const char *text, bool from_wq);
  34 static void wq_cookie_handler(struct work_struct *unused);
  35 static DECLARE_WORK(cookie_work, wq_cookie_handler);
  36 static struct timer_list app_process_wake_up_timer;
  37 static void app_process_wake_up_handler(unsigned long unused_data);
  38
  39 static uint32_t cookiemap_code(uint64_t value64)
  40 {
  41         uint32_t value = (uint32_t)((value64 >> 32) + value64);
  42         uint32_t cookiecode = (value >> 24) & 0xff;
  43
  44         cookiecode = cookiecode * 31 + ((value >> 16) & 0xff);
  45         cookiecode = cookiecode * 31 + ((value >> 8) & 0xff);
  46         cookiecode = cookiecode * 31 + ((value >> 0) & 0xff);
  47         cookiecode &= (COOKIEMAP_ENTRIES - 1);
  48         return cookiecode * MAX_COLLISIONS;
  49 }
  50
  51 static uint32_t gator_chksum_crc32(const char *data)
  52 {
  53         register unsigned long crc;
  54         const unsigned char *block = data;
  55         int i, length = strlen(data);
  56
  57         crc = 0xFFFFFFFF;
  58         for (i = 0; i < length; i++)
  59                 crc = ((crc >> 8) & 0x00FFFFFF) ^ gator_crc32_table[(crc ^ *block++) & 0xFF];
  60
  61         return (crc ^ 0xFFFFFFFF);
  62 }
  63
  64 /*
  65  * Exists
  66  *  Pre:  [0][1][v][3]..[n-1]
  67  *  Post: [v][0][1][3]..[n-1]
  68  */
  69 static uint32_t cookiemap_exists(uint64_t key)
  70 {
  71         unsigned long x, flags, retval = 0;
  72         int cpu = get_physical_cpu();
  73         uint32_t cookiecode = cookiemap_code(key);
  74         uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
  75         uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
  76
  77         /* Can be called from interrupt handler or from work queue */
  78         local_irq_save(flags);
  79         for (x = 0; x < MAX_COLLISIONS; x++) {
  80                 if (keys[x] == key) {
  81                         uint32_t value = values[x];
  82
  83                         for (; x > 0; x--) {
  84                                 keys[x] = keys[x - 1];
  85                                 values[x] = values[x - 1];
  86                         }
  87                         keys[0] = key;
  88                         values[0] = value;
  89                         retval = value;
  90                         break;
  91                 }
  92         }
  93         local_irq_restore(flags);
  94
  95         return retval;
  96 }
  97
  98 /*
  99  * Add
 100  *  Pre:  [0][1][2][3]..[n-1]
 101  *  Post: [v][0][1][2]..[n-2]
 102  */
 103 static void cookiemap_add(uint64_t key, uint32_t value)
 104 {
 105         int cpu = get_physical_cpu();
 106         int cookiecode = cookiemap_code(key);
 107         uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
 108         uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
 109         int x;
 110
 111         for (x = MAX_COLLISIONS - 1; x > 0; x--) {
 112                 keys[x] = keys[x - 1];
 113                 values[x] = values[x - 1];
 114         }
 115         keys[0] = key;
 116         values[0] = value;
 117 }
 118
 119 #ifndef CONFIG_PREEMPT_RT_FULL
 120 static void translate_buffer_write_args(int cpu, struct task_struct *task, const char *text)
 121 {
 122         unsigned long flags;
 123         int write;
 124         int next_write;
 125         struct cookie_args *args;
 126
 127         local_irq_save(flags);
 128
 129         write = per_cpu(translate_buffer_write, cpu);
 130         next_write = (write + 1) & translate_buffer_mask;
 131
 132         /* At least one entry must always remain available as when read == write, the queue is empty not full */
 133         if (next_write != per_cpu(translate_buffer_read, cpu)) {
 134                 args = &per_cpu(translate_buffer, cpu)[write];
 135                 args->task = task;
 136                 args->text = text;
 137 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)
 138                 get_task_struct(task);
 139 #endif
 140                 per_cpu(translate_buffer_write, cpu) = next_write;
 141         }
 142
 143         local_irq_restore(flags);
 144 }
 145 #endif
 146
 147 static void translate_buffer_read_args(int cpu, struct cookie_args *args)
 148 {
 149         unsigned long flags;
 150         int read;
 151
 152         local_irq_save(flags);
 153
 154         read = per_cpu(translate_buffer_read, cpu);
 155         *args = per_cpu(translate_buffer, cpu)[read];
 156         per_cpu(translate_buffer_read, cpu) = (read + 1) & translate_buffer_mask;
 157
 158         local_irq_restore(flags);
 159 }
 160
 161 static void wq_cookie_handler(struct work_struct *unused)
 162 {
 163         struct cookie_args args;
 164         int cpu = get_physical_cpu(), cookie;
 165
 166         mutex_lock(&start_mutex);
 167
 168         if (gator_started != 0) {
 169                 while (per_cpu(translate_buffer_read, cpu) != per_cpu(translate_buffer_write, cpu)) {
 170                         translate_buffer_read_args(cpu, &args);
 171                         cookie = get_cookie(cpu, args.task, args.text, true);
 172                         marshal_link(cookie, args.task->tgid, args.task->pid);
 173 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)
 174                         put_task_struct(args.task);
 175 #endif
 176                 }
 177         }
 178
 179         mutex_unlock(&start_mutex);
 180 }
 181
 182 static void app_process_wake_up_handler(unsigned long unused_data)
 183 {
 184         /* had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater */
 185         schedule_work(&cookie_work);
 186 }
 187
 188 /* Retrieve full name from proc/pid/cmdline for java processes on Android */
 189 static int translate_app_process(const char **text, int cpu, struct task_struct *task, bool from_wq)
 190 {
 191         void *maddr;
 192         unsigned int len;
 193         unsigned long addr;
 194         struct mm_struct *mm;
 195         struct page *page = NULL;
 196         struct vm_area_struct *page_vma;
 197         int bytes, offset, retval = 0;
 198         char *buf = per_cpu(translate_text, cpu);
 199
 200 #ifndef CONFIG_PREEMPT_RT_FULL
 201         /* Push work into a work queue if in atomic context as the kernel
 202          * functions below might sleep. Rely on the in_interrupt variable
 203          * rather than in_irq() or in_interrupt() kernel functions, as the
 204          * value of these functions seems inconsistent during a context
 205          * switch between android/linux versions
 206          */
 207         if (!from_wq) {
 208                 /* Check if already in buffer */
 209                 int pos = per_cpu(translate_buffer_read, cpu);
 210
 211                 while (pos != per_cpu(translate_buffer_write, cpu)) {
 212                         if (per_cpu(translate_buffer, cpu)[pos].task == task)
 213                                 goto out;
 214                         pos = (pos + 1) & translate_buffer_mask;
 215                 }
 216
 217                 translate_buffer_write_args(cpu, task, *text);
 218
 219                 /* Not safe to call in RT-Preempt full in schedule switch context */
 220                 mod_timer(&app_process_wake_up_timer, jiffies + 1);
 221                 goto out;
 222         }
 223 #endif
 224
 225         mm = get_task_mm(task);
 226         if (!mm)
 227                 goto out;
 228         if (!mm->arg_end)
 229                 goto outmm;
 230         addr = mm->arg_start;
 231         len = mm->arg_end - mm->arg_start;
 232
 233         if (len > TRANSLATE_TEXT_SIZE)
 234                 len = TRANSLATE_TEXT_SIZE;
 235
 236         down_read(&mm->mmap_sem);
 237         while (len) {
 238                 if (get_user_pages(task, mm, addr, 1, 0, 1, &page, &page_vma) <= 0)
 239                         goto outsem;
 240
 241                 maddr = kmap(page);
 242                 offset = addr & (PAGE_SIZE - 1);
 243                 bytes = len;
 244                 if (bytes > PAGE_SIZE - offset)
 245                         bytes = PAGE_SIZE - offset;
 246
 247                 copy_from_user_page(page_vma, page, addr, buf, maddr + offset, bytes);
 248
 249                 /* release page allocated by get_user_pages() */
 250                 kunmap(page);
 251                 page_cache_release(page);
 252
 253                 len -= bytes;
 254                 buf += bytes;
 255                 addr += bytes;
 256
 257                 *text = per_cpu(translate_text, cpu);
 258                 retval = 1;
 259         }
 260
 261         /* On app_process startup, /proc/pid/cmdline is initially "zygote" then "<pre-initialized>" but changes after an initial startup period */
 262         if (strcmp(*text, "zygote") == 0 || strcmp(*text, "<pre-initialized>") == 0)
 263                 retval = 0;
 264
 265 outsem:
 266         up_read(&mm->mmap_sem);
 267 outmm:
 268         mmput(mm);
 269 out:
 270         return retval;
 271 }
 272
 273 static const char APP_PROCESS[] = "app_process";
 274
 275 static uint32_t get_cookie(int cpu, struct task_struct *task, const char *text, bool from_wq)
 276 {
 277         unsigned long flags, cookie;
 278         uint64_t key;
 279
 280         key = gator_chksum_crc32(text);
 281         key = (key << 32) | (uint32_t)task->tgid;
 282
 283         cookie = cookiemap_exists(key);
 284         if (cookie)
 285                 return cookie;
 286
 287         /* On 64-bit android app_process can be app_process32 or app_process64 */
 288         if (strncmp(text, APP_PROCESS, sizeof(APP_PROCESS) - 1) == 0) {
 289                 if (!translate_app_process(&text, cpu, task, from_wq))
 290                         return UNRESOLVED_COOKIE;
 291         }
 292
 293         /* Can be called from interrupt handler or from work queue or from scheduler trace */
 294         local_irq_save(flags);
 295
 296         cookie = UNRESOLVED_COOKIE;
 297         if (marshal_cookie_header(text)) {
 298                 cookie = per_cpu(cookie_next_key, cpu) += nr_cpu_ids;
 299                 cookiemap_add(key, cookie);
 300                 marshal_cookie(cookie, text);
 301         }
 302
 303         local_irq_restore(flags);
 304
 305         return cookie;
 306 }
 307
 308 static int get_exec_cookie(int cpu, struct task_struct *task)
 309 {
 310         struct mm_struct *mm = task->mm;
 311         const char *text;
 312
 313         /* kernel threads have no address space */
 314         if (!mm)
 315                 return NO_COOKIE;
 316
 317         if (task && task->mm && task->mm->exe_file) {
 318                 text = task->mm->exe_file->f_path.dentry->d_name.name;
 319                 return get_cookie(cpu, task, text, false);
 320         }
 321
 322         return UNRESOLVED_COOKIE;
 323 }
 324
 325 static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsigned long addr, off_t *offset)
 326 {
 327         unsigned long cookie = NO_COOKIE;
 328         struct mm_struct *mm = task->mm;
 329         struct vm_area_struct *vma;
 330         const char *text;
 331
 332         if (!mm)
 333                 return cookie;
 334
 335         for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
 336                 if (addr < vma->vm_start || addr >= vma->vm_end)
 337                         continue;
 338
 339                 if (vma->vm_file) {
 340                         text = vma->vm_file->f_path.dentry->d_name.name;
 341                         cookie = get_cookie(cpu, task, text, false);
 342                         *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start;
 343                 } else {
 344                         /* must be an anonymous map */
 345                         *offset = addr;
 346                 }
 347
 348                 break;
 349         }
 350
 351         if (!vma)
 352                 cookie = UNRESOLVED_COOKIE;
 353
 354         return cookie;
 355 }
 356
 357 static int cookies_initialize(void)
 358 {
 359         uint32_t crc, poly;
 360         int i, j, cpu, size, err = 0;
 361
 362         translate_buffer_mask = TRANSLATE_BUFFER_SIZE / sizeof(per_cpu(translate_buffer, 0)[0]) - 1;
 363
 364         for_each_present_cpu(cpu) {
 365                 per_cpu(cookie_next_key, cpu) = nr_cpu_ids + cpu;
 366
 367                 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint64_t);
 368                 per_cpu(cookie_keys, cpu) = kmalloc(size, GFP_KERNEL);
 369                 if (!per_cpu(cookie_keys, cpu)) {
 370                         err = -ENOMEM;
 371                         goto cookie_setup_error;
 372                 }
 373                 memset(per_cpu(cookie_keys, cpu), 0, size);
 374
 375                 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t);
 376                 per_cpu(cookie_values, cpu) = kmalloc(size, GFP_KERNEL);
 377                 if (!per_cpu(cookie_values, cpu)) {
 378                         err = -ENOMEM;
 379                         goto cookie_setup_error;
 380                 }
 381                 memset(per_cpu(cookie_values, cpu), 0, size);
 382
 383                 per_cpu(translate_buffer, cpu) = kmalloc(TRANSLATE_BUFFER_SIZE, GFP_KERNEL);
 384                 if (!per_cpu(translate_buffer, cpu)) {
 385                         err = -ENOMEM;
 386                         goto cookie_setup_error;
 387                 }
 388
 389                 per_cpu(translate_buffer_write, cpu) = 0;
 390                 per_cpu(translate_buffer_read, cpu) = 0;
 391
 392                 per_cpu(translate_text, cpu) = kmalloc(TRANSLATE_TEXT_SIZE, GFP_KERNEL);
 393                 if (!per_cpu(translate_text, cpu)) {
 394                         err = -ENOMEM;
 395                         goto cookie_setup_error;
 396                 }
 397         }
 398
 399         /* build CRC32 table */
 400         poly = 0x04c11db7;
 401         gator_crc32_table = kmalloc(256 * sizeof(*gator_crc32_table), GFP_KERNEL);
 402         if (!gator_crc32_table) {
 403                 err = -ENOMEM;
 404                 goto cookie_setup_error;
 405         }
 406         for (i = 0; i < 256; i++) {
 407                 crc = i;
 408                 for (j = 8; j > 0; j--) {
 409                         if (crc & 1)
 410                                 crc = (crc >> 1) ^ poly;
 411                         else
 412                                 crc >>= 1;
 413                 }
 414                 gator_crc32_table[i] = crc;
 415         }
 416
 417         setup_timer(&app_process_wake_up_timer, app_process_wake_up_handler, 0);
 418
 419 cookie_setup_error:
 420         return err;
 421 }
 422
 423 static void cookies_release(void)
 424 {
 425         int cpu;
 426
 427         for_each_present_cpu(cpu) {
 428                 kfree(per_cpu(cookie_keys, cpu));
 429                 per_cpu(cookie_keys, cpu) = NULL;
 430
 431                 kfree(per_cpu(cookie_values, cpu));
 432                 per_cpu(cookie_values, cpu) = NULL;
 433
 434                 kfree(per_cpu(translate_buffer, cpu));
 435                 per_cpu(translate_buffer, cpu) = NULL;
 436                 per_cpu(translate_buffer_read, cpu) = 0;
 437                 per_cpu(translate_buffer_write, cpu) = 0;
 438
 439                 kfree(per_cpu(translate_text, cpu));
 440                 per_cpu(translate_text, cpu) = NULL;
 441         }
 442
 443         del_timer_sync(&app_process_wake_up_timer);
 444         kfree(gator_crc32_table);
 445         gator_crc32_table = NULL;
 446 }