drivers/gator/gator_cookies.c

   1 /**
   2  * Copyright (C) ARM Limited 2010-2013. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  */
   9
  10 #define COOKIEMAP_ENTRIES       1024    /* must be power of 2 */
  11 #define TRANSLATE_BUFFER_SIZE 512  // must be a power of 2 - 512/4 = 128 entries
  12 #define TRANSLATE_TEXT_SIZE             256
  13 #define MAX_COLLISIONS          2
  14
  15 static uint32_t *gator_crc32_table;
  16 static unsigned int translate_buffer_mask;
  17
  18 static DEFINE_PER_CPU(char *, translate_text);
  19 static DEFINE_PER_CPU(uint32_t, cookie_next_key);
  20 static DEFINE_PER_CPU(uint64_t *, cookie_keys);
  21 static DEFINE_PER_CPU(uint32_t *, cookie_values);
  22 static DEFINE_PER_CPU(int, translate_buffer_read);
  23 static DEFINE_PER_CPU(int, translate_buffer_write);
  24 static DEFINE_PER_CPU(void **, translate_buffer);
  25
  26 static uint32_t get_cookie(int cpu, struct task_struct *task, const char *text, bool from_wq);
  27 static void wq_cookie_handler(struct work_struct *unused);
  28 DECLARE_WORK(cookie_work, wq_cookie_handler);
  29 static struct timer_list app_process_wake_up_timer;
  30 static void app_process_wake_up_handler(unsigned long unused_data);
  31
  32 static uint32_t cookiemap_code(uint64_t value64)
  33 {
  34         uint32_t value = (uint32_t)((value64 >> 32) + value64);
  35         uint32_t cookiecode = (value >> 24) & 0xff;
  36         cookiecode = cookiecode * 31 + ((value >> 16) & 0xff);
  37         cookiecode = cookiecode * 31 + ((value >> 8) & 0xff);
  38         cookiecode = cookiecode * 31 + ((value >> 0) & 0xff);
  39         cookiecode &= (COOKIEMAP_ENTRIES - 1);
  40         return cookiecode * MAX_COLLISIONS;
  41 }
  42
  43 static uint32_t gator_chksum_crc32(const char *data)
  44 {
  45         register unsigned long crc;
  46         const unsigned char *block = data;
  47         int i, length = strlen(data);
  48
  49         crc = 0xFFFFFFFF;
  50         for (i = 0; i < length; i++) {
  51                 crc = ((crc >> 8) & 0x00FFFFFF) ^ gator_crc32_table[(crc ^ *block++) & 0xFF];
  52         }
  53
  54         return (crc ^ 0xFFFFFFFF);
  55 }
  56
  57 /*
  58  * Exists
  59  *  Pre:  [0][1][v][3]..[n-1]
  60  *  Post: [v][0][1][3]..[n-1]
  61  */
  62 static uint32_t cookiemap_exists(uint64_t key)
  63 {
  64         unsigned long x, flags, retval = 0;
  65         int cpu = get_physical_cpu();
  66         uint32_t cookiecode = cookiemap_code(key);
  67         uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
  68         uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
  69
  70         // Can be called from interrupt handler or from work queue
  71         local_irq_save(flags);
  72         for (x = 0; x < MAX_COLLISIONS; x++) {
  73                 if (keys[x] == key) {
  74                         uint32_t value = values[x];
  75                         for (; x > 0; x--) {
  76                                 keys[x] = keys[x - 1];
  77                                 values[x] = values[x - 1];
  78                         }
  79                         keys[0] = key;
  80                         values[0] = value;
  81                         retval = value;
  82                         break;
  83                 }
  84         }
  85         local_irq_restore(flags);
  86
  87         return retval;
  88 }
  89
  90 /*
  91  * Add
  92  *  Pre:  [0][1][2][3]..[n-1]
  93  *  Post: [v][0][1][2]..[n-2]
  94  */
  95 static void cookiemap_add(uint64_t key, uint32_t value)
  96 {
  97         int cpu = get_physical_cpu();
  98         int cookiecode = cookiemap_code(key);
  99         uint64_t *keys = &(per_cpu(cookie_keys, cpu)[cookiecode]);
 100         uint32_t *values = &(per_cpu(cookie_values, cpu)[cookiecode]);
 101         int x;
 102
 103         for (x = MAX_COLLISIONS - 1; x > 0; x--) {
 104                 keys[x] = keys[x - 1];
 105                 values[x] = values[x - 1];
 106         }
 107         keys[0] = key;
 108         values[0] = value;
 109 }
 110
 111 #ifndef CONFIG_PREEMPT_RT_FULL
 112 static void translate_buffer_write_ptr(int cpu, void *x)
 113 {
 114         per_cpu(translate_buffer, cpu)[per_cpu(translate_buffer_write, cpu)++] = x;
 115         per_cpu(translate_buffer_write, cpu) &= translate_buffer_mask;
 116 }
 117 #endif
 118
 119 static void *translate_buffer_read_ptr(int cpu)
 120 {
 121         void *value = per_cpu(translate_buffer, cpu)[per_cpu(translate_buffer_read, cpu)++];
 122         per_cpu(translate_buffer_read, cpu) &= translate_buffer_mask;
 123         return value;
 124 }
 125
 126 static void wq_cookie_handler(struct work_struct *unused)
 127 {
 128         struct task_struct *task;
 129         char *text;
 130         int cpu = get_physical_cpu(), cookie;
 131         unsigned int commit;
 132
 133         mutex_lock(&start_mutex);
 134
 135         if (gator_started != 0) {
 136                 commit = per_cpu(translate_buffer_write, cpu);
 137                 while (per_cpu(translate_buffer_read, cpu) != commit) {
 138                         task = (struct task_struct *)translate_buffer_read_ptr(cpu);
 139                         text = (char *)translate_buffer_read_ptr(cpu);
 140                         cookie = get_cookie(cpu, task, text, true);
 141                         marshal_link(cookie, task->tgid, task->pid);
 142                 }
 143         }
 144
 145         mutex_unlock(&start_mutex);
 146 }
 147
 148 static void app_process_wake_up_handler(unsigned long unused_data)
 149 {
 150         // had to delay scheduling work as attempting to schedule work during the context switch is illegal in kernel versions 3.5 and greater
 151         schedule_work(&cookie_work);
 152 }
 153
 154 // Retrieve full name from proc/pid/cmdline for java processes on Android
 155 static int translate_app_process(const char **text, int cpu, struct task_struct *task, bool from_wq)
 156 {
 157         void *maddr;
 158         unsigned int len;
 159         unsigned long addr;
 160         struct mm_struct *mm;
 161         struct page *page = NULL;
 162         struct vm_area_struct *page_vma;
 163         int bytes, offset, retval = 0;
 164         char *buf = per_cpu(translate_text, cpu);
 165
 166 #ifndef CONFIG_PREEMPT_RT_FULL
 167         // Push work into a work queue if in atomic context as the kernel functions below might sleep
 168         // Rely on the in_interrupt variable rather than in_irq() or in_interrupt() kernel functions, as the value of these functions seems
 169         //   inconsistent during a context switch between android/linux versions
 170         if (!from_wq) {
 171                 // Check if already in buffer
 172                 int ptr = per_cpu(translate_buffer_read, cpu);
 173                 while (ptr != per_cpu(translate_buffer_write, cpu)) {
 174                         if (per_cpu(translate_buffer, cpu)[ptr] == (void *)task)
 175                                 goto out;
 176                         ptr = (ptr + 2) & translate_buffer_mask;
 177                 }
 178
 179                 translate_buffer_write_ptr(cpu, (void *)task);
 180                 translate_buffer_write_ptr(cpu, (void *)*text);
 181
 182                 // Not safe to call in RT-Preempt full in schedule switch context
 183                 mod_timer(&app_process_wake_up_timer, jiffies + 1);
 184                 goto out;
 185         }
 186 #endif
 187
 188         mm = get_task_mm(task);
 189         if (!mm)
 190                 goto out;
 191         if (!mm->arg_end)
 192                 goto outmm;
 193         addr = mm->arg_start;
 194         len = mm->arg_end - mm->arg_start;
 195
 196         if (len > TRANSLATE_TEXT_SIZE)
 197                 len = TRANSLATE_TEXT_SIZE;
 198
 199         down_read(&mm->mmap_sem);
 200         while (len) {
 201                 if (get_user_pages(task, mm, addr, 1, 0, 1, &page, &page_vma) <= 0)
 202                         goto outsem;
 203
 204                 maddr = kmap(page);
 205                 offset = addr & (PAGE_SIZE - 1);
 206                 bytes = len;
 207                 if (bytes > PAGE_SIZE - offset)
 208                         bytes = PAGE_SIZE - offset;
 209
 210                 copy_from_user_page(page_vma, page, addr, buf, maddr + offset, bytes);
 211
 212                 kunmap(page);   // release page allocated by get_user_pages()
 213                 page_cache_release(page);
 214
 215                 len -= bytes;
 216                 buf += bytes;
 217                 addr += bytes;
 218
 219                 *text = per_cpu(translate_text, cpu);
 220                 retval = 1;
 221         }
 222
 223         // On app_process startup, /proc/pid/cmdline is initially "zygote" then "<pre-initialized>" but changes after an initial startup period
 224         if (strcmp(*text, "zygote") == 0 || strcmp(*text, "<pre-initialized>") == 0)
 225                 retval = 0;
 226
 227 outsem:
 228         up_read(&mm->mmap_sem);
 229 outmm:
 230         mmput(mm);
 231 out:
 232         return retval;
 233 }
 234
 235 static uint32_t get_cookie(int cpu, struct task_struct *task, const char *text, bool from_wq)
 236 {
 237         unsigned long flags, cookie;
 238         uint64_t key;
 239
 240         key = gator_chksum_crc32(text);
 241         key = (key << 32) | (uint32_t)task->tgid;
 242
 243         cookie = cookiemap_exists(key);
 244         if (cookie) {
 245                 return cookie;
 246         }
 247
 248         if (strcmp(text, "app_process") == 0) {
 249                 if (!translate_app_process(&text, cpu, task, from_wq))
 250                         return UNRESOLVED_COOKIE;
 251         }
 252
 253         // Can be called from interrupt handler or from work queue or from scheduler trace
 254         local_irq_save(flags);
 255
 256         cookie = UNRESOLVED_COOKIE;
 257         if (marshal_cookie_header(text)) {
 258                 cookie = per_cpu(cookie_next_key, cpu) += nr_cpu_ids;
 259                 cookiemap_add(key, cookie);
 260                 marshal_cookie(cookie, text);
 261         }
 262
 263         local_irq_restore(flags);
 264
 265         return cookie;
 266 }
 267
 268 static int get_exec_cookie(int cpu, struct task_struct *task)
 269 {
 270         struct mm_struct *mm = task->mm;
 271         const char *text;
 272
 273         // kernel threads have no address space
 274         if (!mm)
 275                 return NO_COOKIE;
 276
 277         if (task && task->mm && task->mm->exe_file) {
 278                 text = task->mm->exe_file->f_path.dentry->d_name.name;
 279                 return get_cookie(cpu, task, text, false);
 280         }
 281
 282         return UNRESOLVED_COOKIE;
 283 }
 284
 285 static unsigned long get_address_cookie(int cpu, struct task_struct *task, unsigned long addr, off_t *offset)
 286 {
 287         unsigned long cookie = NO_COOKIE;
 288         struct mm_struct *mm = task->mm;
 289         struct vm_area_struct *vma;
 290         const char *text;
 291
 292         if (!mm)
 293                 return cookie;
 294
 295         for (vma = find_vma(mm, addr); vma; vma = vma->vm_next) {
 296                 if (addr < vma->vm_start || addr >= vma->vm_end)
 297                         continue;
 298
 299                 if (vma->vm_file) {
 300                         text = vma->vm_file->f_path.dentry->d_name.name;
 301                         cookie = get_cookie(cpu, task, text, false);
 302                         *offset = (vma->vm_pgoff << PAGE_SHIFT) + addr - vma->vm_start;
 303                 } else {
 304                         /* must be an anonymous map */
 305                         *offset = addr;
 306                 }
 307
 308                 break;
 309         }
 310
 311         if (!vma)
 312                 cookie = UNRESOLVED_COOKIE;
 313
 314         return cookie;
 315 }
 316
 317 static int cookies_initialize(void)
 318 {
 319         uint32_t crc, poly;
 320         int i, j, cpu, size, err = 0;
 321
 322         translate_buffer_mask = TRANSLATE_BUFFER_SIZE / sizeof(per_cpu(translate_buffer, 0)[0]) - 1;
 323
 324         for_each_present_cpu(cpu) {
 325                 per_cpu(cookie_next_key, cpu) = nr_cpu_ids + cpu;
 326
 327                 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint64_t);
 328                 per_cpu(cookie_keys, cpu) = (uint64_t *)kmalloc(size, GFP_KERNEL);
 329                 if (!per_cpu(cookie_keys, cpu)) {
 330                         err = -ENOMEM;
 331                         goto cookie_setup_error;
 332                 }
 333                 memset(per_cpu(cookie_keys, cpu), 0, size);
 334
 335                 size = COOKIEMAP_ENTRIES * MAX_COLLISIONS * sizeof(uint32_t);
 336                 per_cpu(cookie_values, cpu) = (uint32_t *)kmalloc(size, GFP_KERNEL);
 337                 if (!per_cpu(cookie_values, cpu)) {
 338                         err = -ENOMEM;
 339                         goto cookie_setup_error;
 340                 }
 341                 memset(per_cpu(cookie_values, cpu), 0, size);
 342
 343                 per_cpu(translate_buffer, cpu) = (void **)kmalloc(TRANSLATE_BUFFER_SIZE, GFP_KERNEL);
 344                 if (!per_cpu(translate_buffer, cpu)) {
 345                         err = -ENOMEM;
 346                         goto cookie_setup_error;
 347                 }
 348
 349                 per_cpu(translate_buffer_write, cpu) = 0;
 350                 per_cpu(translate_buffer_read, cpu) = 0;
 351
 352                 per_cpu(translate_text, cpu) = (char *)kmalloc(TRANSLATE_TEXT_SIZE, GFP_KERNEL);
 353                 if (!per_cpu(translate_text, cpu)) {
 354                         err = -ENOMEM;
 355                         goto cookie_setup_error;
 356                 }
 357         }
 358
 359         // build CRC32 table
 360         poly = 0x04c11db7;
 361         gator_crc32_table = (uint32_t *)kmalloc(256 * sizeof(uint32_t), GFP_KERNEL);
 362         if (!gator_crc32_table) {
 363                 err = -ENOMEM;
 364                 goto cookie_setup_error;
 365         }
 366         for (i = 0; i < 256; i++) {
 367                 crc = i;
 368                 for (j = 8; j > 0; j--) {
 369                         if (crc & 1) {
 370                                 crc = (crc >> 1) ^ poly;
 371                         } else {
 372                                 crc >>= 1;
 373                         }
 374                 }
 375                 gator_crc32_table[i] = crc;
 376         }
 377
 378         setup_timer(&app_process_wake_up_timer, app_process_wake_up_handler, 0);
 379
 380 cookie_setup_error:
 381         return err;
 382 }
 383
 384 static void cookies_release(void)
 385 {
 386         int cpu;
 387
 388         for_each_present_cpu(cpu) {
 389                 kfree(per_cpu(cookie_keys, cpu));
 390                 per_cpu(cookie_keys, cpu) = NULL;
 391
 392                 kfree(per_cpu(cookie_values, cpu));
 393                 per_cpu(cookie_values, cpu) = NULL;
 394
 395                 kfree(per_cpu(translate_buffer, cpu));
 396                 per_cpu(translate_buffer, cpu) = NULL;
 397                 per_cpu(translate_buffer_read, cpu) = 0;
 398                 per_cpu(translate_buffer_write, cpu) = 0;
 399
 400                 kfree(per_cpu(translate_text, cpu));
 401                 per_cpu(translate_text, cpu) = NULL;
 402         }
 403
 404         del_timer_sync(&app_process_wake_up_timer);
 405         kfree(gator_crc32_table);
 406         gator_crc32_table = NULL;
 407 }