drivers/gator/gator_cookies.c

   1 /**
   2  * Copyright (C) ARM Limited 2010-2014. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  */
   9
  10 #define COOKIEMAP_ENTRIES       1024    /* must be power of 2 */
  11 #define TRANSLATE_BUFFER_SIZE 512  // must be a power of 2 - 512/4 = 128 entries
  12 #define TRANSLATE_TEXT_SIZE             256
  13 #define MAX_COLLISIONS          2
  14
  15 static uint32_t *gator_crc32_table;
  16 static unsigned int translate_buffer_mask;
  17
  18 struct cookie_args {
  19         struct task_struct *task;
  20         const char *text;
  21 };
  22
  23 static DEFINE_PER_CPU(char *, translate_text);
  24 static DEFINE_PER_CPU(uint32_t, cookie_next_key);
  25 static DEFINE_PER_CPU(uint64_t *, cookie_keys);
  26 static DEFINE_PER_CPU(uint32_t *, cookie_values);
  27 static DEFINE_PER_CPU(int, translate_buffer_read);
  28 static DEFINE_PER_CPU(int, translate_buffer_write);
  29 static DEFINE_PER_CPU(struct cookie_args *, translate_buffer);
  30
  31 static uint32_t get_cookie(int cpu, struct task_struct *task, const char *text, bool from_wq);
  32 static void wq_cookie_handler(struct work_struct *unused);
  33 static DECLARE_WORK(cookie_work, wq_cookie_handler);
  34 static struct timer_list app_process_wake_up_timer;
  35 static void app_process_wake_up_handler(unsigned long unused_data);
  36
  37 static uint32_t cookiemap_code(uint64_t value64)
  38 {
  39         uint32_t value = (uint32_t)((value64 >> 32) + value64);
  40         uint32_t cookiecode = (value >> 24) & 0xff;
  41         cookiecode = cookiecode * 31 + ((value >> 16) & 0xff);
  42         cookiecode = cookiecode * 31 + ((value >> 8) & 0xff);
  43         cookiecode = cookiecode * 31 + ((value >> 0) & 0xff);
  44         cookiecode &= (COOKIEMAP_ENTRIES - 1);
  45         return cookiecode * MAX_COLLISIONS;
  46 }
  47
  48 static uint32_t gator_chksum_crc32(const char *data)
  49 {
  50         register unsigned long crc;
  51         const unsigned char *block = data;
  52         int i, length = strlen(data);
  53
  54         crc = 0xFFFFFFFF;
  55         for (i = 0; i < length; i++) {
  56                 crc = ((crc >> 8) & 0x00FFFFFF) ^ gator_crc32_table[(crc ^ *block++) & 0xFF];
  57         }
  58
  59         return (crc ^ 0xFFFFFFFF);
  60 }
  61
  62 /*
  63  * Exists
  64  *  Pre:  [0][1][v][3]..[n-1]
  65  *  Post: [v][0][1][3]..[n-1]
  66  */
  67 static uint32_t cookiemap_exists(uint64_t key)
  68 {
  69         unsigned long x, flags, retval = 0;
  70         int cpu = get_physical_cpu();
  71         uint32_t cookiecode = cookiemap_code(key);
  72         uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
  73         uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
  74
  75         // Can be called from interrupt handler or from work queue
  76         local_irq_save(flags);
  77         for (x = 0; x < MAX_COLLISIONS; x++) {
  78                 if (keys[x] == key) {
  79                         uint32_t value = values[x];
  80                         for (; x > 0; x--) {
  81                                 keys[x] = keys[x - 1];
  82                                 values[x] = values[x - 1];
  83                         }
  84                         keys[0] = key;
  85                         values[0] = value;
  86                         retval = value;
  87                         break;
  88                 }
  89         }
  90         local_irq_restore(flags);
  91
  92         return retval;
  93 }
  94
  95 /*
  96  * Add
  97  *  Pre:  [0][1][2][3]..[n-1]
  98  *  Post: [v][0][1][2]..[n-2]
  99  */
 100 static void cookiemap_add(uint64_t key, uint32_t value)
 101 {
 102         int cpu = get_physical_cpu();
 103         int cookiecode = cookiemap_code(key);
 104         uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
 105         uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
 106         int x;
 107
 108         for (x = MAX_COLLISIONS - 1; x > 0; x--) {
 109                 keys[x] = keys[x - 1];
 110                 values[x] = values[x - 1];
 111         }
 112         keys[0] = key;
 113         values[0] = value;
 114 }
 115
 116 #ifndef CONFIG_PREEMPT_RT_FULL
 117 static void translate_buffer_write_args(int cpu, struct task_struct *task, const char *text)
 118 {
 119         unsigned long flags;
 120         int write;
 121         int next_write;
 122         struct cookie_args *args;
 123
 124         local_irq_save(flags);
 125
 126         write = per_cpu(translate_buffer_write, cpu);
 127         next_write = (write + 1) & translate_buffer_mask;
 128
 129         // At least one entry must always remain available as when read == write, the queue is empty not full
 130         if (next_write != per_cpu(translate_buffer_read, cpu)) {
 131                 args = &per_cpu(translate_buffer, cpu)[write];
 132                 args->task = task;
 133                 args->text = text;
 134 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)
 135                 get_task_struct(task);
 136 #endif
 137                 per_cpu(translate_buffer_write, cpu) = next_write;
 138         }
 139
 140         local_irq_restore(flags);
 141 }
 142 #endif
 143
 144 static void translate_buffer_read_args(int cpu, struct cookie_args *args)
 145 {
 146         unsigned long flags;
 147         int read;
 148
 149         local_irq_save(flags);
 150
 151         read = per_cpu(translate_buffer_read, cpu);
 152         *args = per_cpu(translate_buffer, cpu)[read];
 153         per_cpu(translate_buffer_read, cpu) = (read + 1) & translate_buffer_mask;
 154
 155         local_irq_restore(flags);
 156 }
 157
 158 static void wq_cookie_handler(struct work_struct *unused)
 159 {
 160         struct cookie_args args;
 161         int cpu = get_physical_cpu(), cookie;
 162
 163         mutex_lock(&start_mutex);
 164
 165         if (gator_started != 0) {
 166                 while (per_cpu(translate_buffer_read, cpu) != per_cpu(translate_buffer_write, cpu)) {
 167                         translate_buffer_read_args(cpu, &args);
 168                         cookie = get_cookie(cpu, args.task, args.text, true);
 169                         marshal_link(cookie, args.task->tgid, args.task->pid);
 170 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2, 6, 39)
 171                         put_task_struct(args.task);
 172 #endif
 173                 }
 174         }
 175
 176         mutex_unlock(&start_mutex);
 177 }
 178
 179 static void app_process_wake_up_handler(unsigned long unused_data)
 180 {
 181         // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
 182         schedule_work(&cookie_work);
 183 }
 184
 185 // Retrieve full name from proc/pid/cmdline for java processes on Android
 186 static int translate_app_process(const char **text, int cpu, struct task_struct *task, bool from_wq)
 187 {
 188         void *maddr;
 189         unsigned int len;
 190         unsigned long addr;
 191         struct mm_struct *mm;
 192         struct page *page = NULL;
 193         struct vm_area_struct *page_vma;
 194         int bytes, offset, retval = 0;
 195         char *buf = per_cpu(translate_text, cpu);
 196
 197 #ifndef CONFIG_PREEMPT_RT_FULL
 198         // Push work into a work queue if in atomic context as the kernel functions below might sleep
 199         // Rely on the in_interrupt variable rather than in_irq() or in_interrupt() kernel functions, as the value of these functions seems
 200         //   inconsistent during a context switch between android/linux versions
 201         if (!from_wq) {
 202                 // Check if already in buffer
 203                 int pos = per_cpu(translate_buffer_read, cpu);
 204                 while (pos != per_cpu(translate_buffer_write, cpu)) {
 205                         if (per_cpu(translate_buffer, cpu)[pos].task == task)
 206                                 goto out;
 207                         pos = (pos + 1) & translate_buffer_mask;
 208                 }
 209
 210                 translate_buffer_write_args(cpu, task, *text);
 211
 212                 // Not safe to call in RT-Preempt full in schedule switch context
 213                 mod_timer(&app_process_wake_up_timer, jiffies + 1);
 214                 goto out;
 215         }
 216 #endif
 217
 218         mm = get_task_mm(task);
 219         if (!mm)
 220                 goto out;
 221         if (!mm->arg_end)
 222                 goto outmm;
 223         addr = mm->arg_start;
 224         len = mm->arg_end - mm->arg_start;
 225
 226         if (len > TRANSLATE_TEXT_SIZE)
 227                 len = TRANSLATE_TEXT_SIZE;
 228
 229         down_read(&mm->mmap_sem);
 230         while (len) {
 231                 if (get_user_pages(task, mm, addr, 1, 0, 1, &page, &page_vma) <= 0)
 232                         goto outsem;
 233
 234                 maddr = kmap(page);
 235                 offset = addr & (PAGE_SIZE - 1);
 236                 bytes = len;
 237                 if (bytes > PAGE_SIZE - offset)
 238                         bytes = PAGE_SIZE - offset;
 239
 240                 copy_from_user_page(page_vma, page, addr, buf, maddr + offset, bytes);
 241
 242                 kunmap(page);   // release page allocated by get_user_pages()
 243                 page_cache_release(page);
 244
 245                 len -= bytes;
 246                 buf += bytes;
 247                 addr += bytes;
 248
 249                 *text = per_cpu(translate_text, cpu);
 250                 retval = 1;
 251         }
 252
 253         // On app_process startup, /proc/pid/cmdline is initially "zygote" then "<pre-initialized>" but changes after an initial startup period
 254         if (strcmp(*text, "zygote") == 0 || strcmp(*text, "<pre-initialized>") == 0)
 255                 retval = 0;
 256
 257 outsem:
 258         up_read(&mm->mmap_sem);
 259 outmm:
 260         mmput(mm);
 261 out:
 262         return retval;
 263 }
 264
 265 static uint32_t get_cookie(int cpu, struct task_struct *task, const char *text, bool from_wq)
 266 {
 267         unsigned long flags, cookie;
 268         uint64_t key;
 269
 270         key = gator_chksum_crc32(text);
 271         key = (key << 32) | (uint32_t)task->tgid;
 272
 273         cookie = cookiemap_exists(key);
 274         if (cookie) {
 275                 return cookie;
 276         }
 277
 278         if (strcmp(text, "app_process") == 0) {
 279                 if (!translate_app_process(&text, cpu, task, from_wq))
 280                         return UNRESOLVED_COOKIE;
 281         }
 282
 283         // Can be called from interrupt handler or from work queue or from scheduler trace
 284         local_irq_save(flags);
 285
 286         cookie = UNRESOLVED_COOKIE;
 287         if (marshal_cookie_header(text)) {
 288                 cookie = per_cpu(cookie_next_key, cpu) += nr_cpu_ids;
 289                 cookiemap_add(key, cookie);
 290                 marshal_cookie(cookie, text);
 291         }
 292
 293         local_irq_restore(flags);
 294
 295         return cookie;
 296 }
 297
 298 static int get_exec_cookie(int cpu, struct task_struct *task)
 299 {
 300         struct mm_struct *mm = task->mm;
 301         const char *text;
 302
 303         // kernel threads have no address space
 304         if (!mm)
 305                 return NO_COOKIE;
 306
 307         if (task && task->mm && task->mm->exe_file) {
 308                 text = task->mm->exe_file->f_path.dentry->d_name.name;
 309                 return get_cookie(cpu, task, text, false);
 310         }
 311
 312         return UNRESOLVED_COOKIE;
 313 }
 314
 315 static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsigned long addr, off_t *offset)
 316 {
 317         unsigned long cookie = NO_COOKIE;
 318         struct mm_struct *mm = task->mm;
 319         struct vm_area_struct *vma;
 320         const char *text;
 321
 322         if (!mm)
 323                 return cookie;
 324
 325         for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
 326                 if (addr < vma->vm_start || addr >= vma->vm_end)
 327                         continue;
 328
 329                 if (vma->vm_file) {
 330                         text = vma->vm_file->f_path.dentry->d_name.name;
 331                         cookie = get_cookie(cpu, task, text, false);
 332                         *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start;
 333                 } else {
 334                         /* must be an anonymous map */
 335                         *offset = addr;
 336                 }
 337
 338                 break;
 339         }
 340
 341         if (!vma)
 342                 cookie = UNRESOLVED_COOKIE;
 343
 344         return cookie;
 345 }
 346
 347 static int cookies_initialize(void)
 348 {
 349         uint32_t crc, poly;
 350         int i, j, cpu, size, err = 0;
 351
 352         translate_buffer_mask = TRANSLATE_BUFFER_SIZE / sizeof(per_cpu(translate_buffer, 0)[0]) - 1;
 353
 354         for_each_present_cpu(cpu) {
 355                 per_cpu(cookie_next_key, cpu) = nr_cpu_ids + cpu;
 356
 357                 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint64_t);
 358                 per_cpu(cookie_keys, cpu) = (uint64_t *)kmalloc(size, GFP_KERNEL);
 359                 if (!per_cpu(cookie_keys, cpu)) {
 360                         err = -ENOMEM;
 361                         goto cookie_setup_error;
 362                 }
 363                 memset(per_cpu(cookie_keys, cpu), 0, size);
 364
 365                 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t);
 366                 per_cpu(cookie_values, cpu) = (uint32_t *)kmalloc(size, GFP_KERNEL);
 367                 if (!per_cpu(cookie_values, cpu)) {
 368                         err = -ENOMEM;
 369                         goto cookie_setup_error;
 370                 }
 371                 memset(per_cpu(cookie_values, cpu), 0, size);
 372
 373                 per_cpu(translate_buffer, cpu) = (struct cookie_args *)kmalloc(TRANSLATE_BUFFER_SIZE, GFP_KERNEL);
 374                 if (!per_cpu(translate_buffer, cpu)) {
 375                         err = -ENOMEM;
 376                         goto cookie_setup_error;
 377                 }
 378
 379                 per_cpu(translate_buffer_write, cpu) = 0;
 380                 per_cpu(translate_buffer_read, cpu) = 0;
 381
 382                 per_cpu(translate_text, cpu) = (char *)kmalloc(TRANSLATE_TEXT_SIZE, GFP_KERNEL);
 383                 if (!per_cpu(translate_text, cpu)) {
 384                         err = -ENOMEM;
 385                         goto cookie_setup_error;
 386                 }
 387         }
 388
 389         // build CRC32 table
 390         poly = 0x04c11db7;
 391         gator_crc32_table = (uint32_t *)kmalloc(256 * sizeof(uint32_t), GFP_KERNEL);
 392         if (!gator_crc32_table) {
 393                 err = -ENOMEM;
 394                 goto cookie_setup_error;
 395         }
 396         for (i = 0; i < 256; i++) {
 397                 crc = i;
 398                 for (j = 8; j > 0; j--) {
 399                         if (crc & 1) {
 400                                 crc = (crc >> 1) ^ poly;
 401                         } else {
 402                                 crc >>= 1;
 403                         }
 404                 }
 405                 gator_crc32_table[i] = crc;
 406         }
 407
 408         setup_timer(&app_process_wake_up_timer, app_process_wake_up_handler, 0);
 409
 410 cookie_setup_error:
 411         return err;
 412 }
 413
 414 static void cookies_release(void)
 415 {
 416         int cpu;
 417
 418         for_each_present_cpu(cpu) {
 419                 kfree(per_cpu(cookie_keys, cpu));
 420                 per_cpu(cookie_keys, cpu) = NULL;
 421
 422                 kfree(per_cpu(cookie_values, cpu));
 423                 per_cpu(cookie_values, cpu) = NULL;
 424
 425                 kfree(per_cpu(translate_buffer, cpu));
 426                 per_cpu(translate_buffer, cpu) = NULL;
 427                 per_cpu(translate_buffer_read, cpu) = 0;
 428                 per_cpu(translate_buffer_write, cpu) = 0;
 429
 430                 kfree(per_cpu(translate_text, cpu));
 431                 per_cpu(translate_text, cpu) = NULL;
 432         }
 433
 434         del_timer_sync(&app_process_wake_up_timer);
 435         kfree(gator_crc32_table);
 436         gator_crc32_table = NULL;
 437 }