drivers/gpu/arm/mali400/mali/linux/mali_osk_profiling.c

   1 /*
   2  * Copyright (C) 2012-2016 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10 #include <linux/hrtimer.h>
  11 #include <linux/module.h>
  12 #include <linux/file.h>
  13 #include <linux/poll.h>
  14 #include <linux/anon_inodes.h>
  15 #include <linux/sched.h>
  16
  17 #include <mali_profiling_gator_api.h>
  18 #include "mali_kernel_common.h"
  19 #include "mali_osk.h"
  20 #include "mali_ukk.h"
  21 #include "mali_uk_types.h"
  22 #include "mali_osk_profiling.h"
  23 #include "mali_linux_trace.h"
  24 #include "mali_gp.h"
  25 #include "mali_pp.h"
  26 #include "mali_l2_cache.h"
  27 #include "mali_user_settings_db.h"
  28 #include "mali_executor.h"
  29 #include "mali_memory_manager.h"
  30
  31 #define MALI_PROFILING_STREAM_DATA_DEFAULT_SIZE 100
  32 #define MALI_PROFILING_STREAM_HOLD_TIME 1000000         /*1 ms */
  33
  34 #define MALI_PROFILING_STREAM_BUFFER_SIZE       (1 << 12)
  35 #define MALI_PROFILING_STREAM_BUFFER_NUM        100
  36
  37 /**
  38  * Define the mali profiling stream struct.
  39  */
  40 typedef struct mali_profiling_stream {
  41         u8 data[MALI_PROFILING_STREAM_BUFFER_SIZE];
  42         u32 used_size;
  43         struct list_head list;
  44 } mali_profiling_stream;
  45
  46 typedef struct mali_profiling_stream_list {
  47         spinlock_t spin_lock;
  48         struct list_head free_list;
  49         struct list_head queue_list;
  50 } mali_profiling_stream_list;
  51
  52 static const char mali_name[] = "4xx";
  53 static const char utgard_setup_version[] = "ANNOTATE_SETUP 1\n";
  54
  55 static u32 profiling_sample_rate = 0;
  56 static u32 first_sw_counter_index = 0;
  57
  58 static mali_bool l2_cache_counter_if_enabled = MALI_FALSE;
  59 static u32 num_counters_enabled = 0;
  60 static u32 mem_counters_enabled = 0;
  61
  62 static _mali_osk_atomic_t stream_fd_if_used;
  63
  64 static wait_queue_head_t stream_fd_wait_queue;
  65 static mali_profiling_counter *global_mali_profiling_counters = NULL;
  66 static u32 num_global_mali_profiling_counters = 0;
  67
  68 static mali_profiling_stream_list *global_mali_stream_list = NULL;
  69 static mali_profiling_stream *mali_counter_stream = NULL;
  70 static mali_profiling_stream *mali_core_activity_stream = NULL;
  71 static u64 mali_core_activity_stream_dequeue_time = 0;
  72 static spinlock_t mali_activity_lock;
  73 static u32 mali_activity_cores_num =  0;
  74 static struct hrtimer profiling_sampling_timer;
  75
  76 const char *_mali_mem_counter_descriptions[] = _MALI_MEM_COUTNER_DESCRIPTIONS;
  77 const char *_mali_special_counter_descriptions[] = _MALI_SPCIAL_COUNTER_DESCRIPTIONS;
  78
  79 static u32 current_profiling_pid = 0;
  80
  81 static void _mali_profiling_stream_list_destory(mali_profiling_stream_list *profiling_stream_list)
  82 {
  83         mali_profiling_stream *profiling_stream, *tmp_profiling_stream;
  84         MALI_DEBUG_ASSERT_POINTER(profiling_stream_list);
  85
  86         list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &profiling_stream_list->free_list, list) {
  87                 list_del(&profiling_stream->list);
  88                 kfree(profiling_stream);
  89         }
  90
  91         list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &profiling_stream_list->queue_list, list) {
  92                 list_del(&profiling_stream->list);
  93                 kfree(profiling_stream);
  94         }
  95
  96         kfree(profiling_stream_list);
  97 }
  98
  99 static void _mali_profiling_global_stream_list_free(void)
 100 {
 101         mali_profiling_stream *profiling_stream, *tmp_profiling_stream;
 102         unsigned long irq_flags;
 103
 104         MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list);
 105         spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags);
 106         list_for_each_entry_safe(profiling_stream, tmp_profiling_stream, &global_mali_stream_list->queue_list, list) {
 107                 profiling_stream->used_size = 0;
 108                 list_move(&profiling_stream->list, &global_mali_stream_list->free_list);
 109         }
 110         spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags);
 111 }
 112
 113 static _mali_osk_errcode_t _mali_profiling_global_stream_list_dequeue(struct list_head *stream_list, mali_profiling_stream **new_mali_profiling_stream)
 114 {
 115         unsigned long irq_flags;
 116         _mali_osk_errcode_t ret = _MALI_OSK_ERR_OK;
 117         MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list);
 118         MALI_DEBUG_ASSERT_POINTER(stream_list);
 119
 120         spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags);
 121
 122         if (!list_empty(stream_list)) {
 123                 *new_mali_profiling_stream = list_entry(stream_list->next, mali_profiling_stream, list);
 124                 list_del_init(&(*new_mali_profiling_stream)->list);
 125         } else {
 126                 ret = _MALI_OSK_ERR_NOMEM;
 127         }
 128
 129         spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags);
 130
 131         return ret;
 132 }
 133
 134 static void _mali_profiling_global_stream_list_queue(struct list_head *stream_list, mali_profiling_stream *current_mali_profiling_stream)
 135 {
 136         unsigned long irq_flags;
 137         MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list);
 138         MALI_DEBUG_ASSERT_POINTER(stream_list);
 139
 140         spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags);
 141         list_add_tail(&current_mali_profiling_stream->list, stream_list);
 142         spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags);
 143 }
 144
 145 static mali_bool _mali_profiling_global_stream_queue_list_if_empty(void)
 146 {
 147         MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list);
 148         return list_empty(&global_mali_stream_list->queue_list);
 149 }
 150
 151 static u32 _mali_profiling_global_stream_queue_list_next_size(void)
 152 {
 153         unsigned long irq_flags;
 154         u32 size = 0;
 155         MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list);
 156
 157         spin_lock_irqsave(&global_mali_stream_list->spin_lock, irq_flags);
 158         if (!list_empty(&global_mali_stream_list->queue_list)) {
 159                 mali_profiling_stream *next_mali_profiling_stream =
 160                         list_entry(global_mali_stream_list->queue_list.next, mali_profiling_stream, list);
 161                 size = next_mali_profiling_stream->used_size;
 162         }
 163         spin_unlock_irqrestore(&global_mali_stream_list->spin_lock, irq_flags);
 164         return size;
 165 }
 166
 167 /* The mali profiling stream file operations functions. */
 168 static ssize_t _mali_profiling_stream_read(
 169         struct file *filp,
 170         char __user *buffer,
 171         size_t      size,
 172         loff_t      *f_pos);
 173
 174 static unsigned int  _mali_profiling_stream_poll(struct file *filp, poll_table *wait);
 175
 176 static int  _mali_profiling_stream_release(struct inode *inode, struct file *filp);
 177
 178 /* The timeline stream file operations structure. */
 179 static const struct file_operations mali_profiling_stream_fops = {
 180         .release = _mali_profiling_stream_release,
 181         .read    = _mali_profiling_stream_read,
 182         .poll    = _mali_profiling_stream_poll,
 183 };
 184
 185 static ssize_t _mali_profiling_stream_read(
 186         struct file *filp,
 187         char __user *buffer,
 188         size_t      size,
 189         loff_t      *f_pos)
 190 {
 191         u32 copy_len = 0;
 192         mali_profiling_stream *current_mali_profiling_stream;
 193         u32 used_size;
 194         MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list);
 195
 196         while (!_mali_profiling_global_stream_queue_list_if_empty()) {
 197                 used_size = _mali_profiling_global_stream_queue_list_next_size();
 198                 if (used_size <= ((u32)size - copy_len)) {
 199                         current_mali_profiling_stream = NULL;
 200                         _mali_profiling_global_stream_list_dequeue(&global_mali_stream_list->queue_list,
 201                                         &current_mali_profiling_stream);
 202                         MALI_DEBUG_ASSERT_POINTER(current_mali_profiling_stream);
 203                         if (copy_to_user(&buffer[copy_len], current_mali_profiling_stream->data, current_mali_profiling_stream->used_size)) {
 204                                 current_mali_profiling_stream->used_size = 0;
 205                                 _mali_profiling_global_stream_list_queue(&global_mali_stream_list->free_list, current_mali_profiling_stream);
 206                                 return -EFAULT;
 207                         }
 208                         copy_len += current_mali_profiling_stream->used_size;
 209                         current_mali_profiling_stream->used_size = 0;
 210                         _mali_profiling_global_stream_list_queue(&global_mali_stream_list->free_list, current_mali_profiling_stream);
 211                 } else {
 212                         break;
 213                 }
 214         }
 215         return (ssize_t)copy_len;
 216 }
 217
 218 static unsigned int  _mali_profiling_stream_poll(struct file *filp, poll_table *wait)
 219 {
 220         poll_wait(filp, &stream_fd_wait_queue, wait);
 221         if (!_mali_profiling_global_stream_queue_list_if_empty())
 222                 return POLLIN;
 223         return 0;
 224 }
 225
 226 static int  _mali_profiling_stream_release(struct inode *inode, struct file *filp)
 227 {
 228         _mali_osk_atomic_init(&stream_fd_if_used, 0);
 229         return 0;
 230 }
 231
 232 /* The funs for control packet and stream data.*/
 233 static void _mali_profiling_set_packet_size(unsigned char *const buf, const u32 size)
 234 {
 235         u32 i;
 236
 237         for (i = 0; i < sizeof(size); ++i)
 238                 buf[i] = (size >> 8 * i) & 0xFF;
 239 }
 240
 241 static u32 _mali_profiling_get_packet_size(unsigned char *const buf)
 242 {
 243         u32 i;
 244         u32 size = 0;
 245         for (i = 0; i < sizeof(size); ++i)
 246                 size |= (u32)buf[i] << 8 * i;
 247         return size;
 248 }
 249
 250 static u32 _mali_profiling_read_packet_int(unsigned char *const buf, u32 *const pos, u32 const packet_size)
 251 {
 252         u64 int_value = 0;
 253         u8 shift = 0;
 254         u8 byte_value = ~0;
 255
 256         while ((byte_value & 0x80) != 0) {
 257                 if ((*pos) >= packet_size)
 258                         return -1;
 259                 byte_value = buf[*pos];
 260                 *pos += 1;
 261                 int_value |= (u32)(byte_value & 0x7f) << shift;
 262                 shift += 7;
 263         }
 264
 265         if (shift < 8 * sizeof(int_value) && (byte_value & 0x40) != 0) {
 266                 int_value |= -(1 << shift);
 267         }
 268
 269         return int_value;
 270 }
 271
 272 static u32 _mali_profiling_pack_int(u8 *const buf, u32 const buf_size, u32 const pos, s32 value)
 273 {
 274         u32 add_bytes = 0;
 275         int more = 1;
 276         while (more) {
 277                 /* low order 7 bits of val */
 278                 char byte_value = value & 0x7f;
 279                 value >>= 7;
 280
 281                 if ((value == 0 && (byte_value & 0x40) == 0) || (value == -1 && (byte_value & 0x40) != 0)) {
 282                         more = 0;
 283                 } else {
 284                         byte_value |= 0x80;
 285                 }
 286
 287                 if ((pos + add_bytes) >= buf_size)
 288                         return 0;
 289                 buf[pos + add_bytes] = byte_value;
 290                 add_bytes++;
 291         }
 292
 293         return add_bytes;
 294 }
 295
 296 static int _mali_profiling_pack_long(uint8_t *const buf, u32 const buf_size, u32 const pos, s64 val)
 297 {
 298         int add_bytes = 0;
 299         int more = 1;
 300         while (more) {
 301                 /* low order 7 bits of x */
 302                 char byte_value = val & 0x7f;
 303                 val >>= 7;
 304
 305                 if ((val == 0 && (byte_value & 0x40) == 0) || (val == -1 && (byte_value & 0x40) != 0)) {
 306                         more = 0;
 307                 } else {
 308                         byte_value |= 0x80;
 309                 }
 310
 311                 MALI_DEBUG_ASSERT((pos + add_bytes) < buf_size);
 312                 buf[pos + add_bytes] = byte_value;
 313                 add_bytes++;
 314         }
 315
 316         return add_bytes;
 317 }
 318
 319 static void _mali_profiling_stream_add_counter(mali_profiling_stream *profiling_stream, s64 current_time, u32 key, u32 counter_value)
 320 {
 321         u32 add_size = STREAM_HEADER_SIZE;
 322         MALI_DEBUG_ASSERT_POINTER(profiling_stream);
 323         MALI_DEBUG_ASSERT((profiling_stream->used_size) < MALI_PROFILING_STREAM_BUFFER_SIZE);
 324
 325         profiling_stream->data[profiling_stream->used_size] = STREAM_HEADER_COUNTER_VALUE;
 326
 327         add_size += _mali_profiling_pack_long(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE,
 328                                               profiling_stream->used_size + add_size, current_time);
 329         add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE,
 330                                              profiling_stream->used_size + add_size, (s32)0);
 331         add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE,
 332                                              profiling_stream->used_size + add_size, (s32)key);
 333         add_size += _mali_profiling_pack_int(profiling_stream->data, MALI_PROFILING_STREAM_BUFFER_SIZE,
 334                                              profiling_stream->used_size + add_size, (s32)counter_value);
 335
 336         _mali_profiling_set_packet_size(profiling_stream->data + profiling_stream->used_size + 1,
 337                                         add_size - STREAM_HEADER_SIZE);
 338
 339         profiling_stream->used_size += add_size;
 340 }
 341
 342 /* The callback function for sampling timer.*/
 343 static enum hrtimer_restart  _mali_profiling_sampling_counters(struct hrtimer *timer)
 344 {
 345         u32 counter_index;
 346         s64 current_time;
 347         MALI_DEBUG_ASSERT_POINTER(global_mali_profiling_counters);
 348         MALI_DEBUG_ASSERT_POINTER(global_mali_stream_list);
 349
 350         MALI_DEBUG_ASSERT(NULL == mali_counter_stream);
 351         if (_MALI_OSK_ERR_OK == _mali_profiling_global_stream_list_dequeue(
 352                     &global_mali_stream_list->free_list, &mali_counter_stream)) {
 353
 354                 MALI_DEBUG_ASSERT_POINTER(mali_counter_stream);
 355                 MALI_DEBUG_ASSERT(0 == mali_counter_stream->used_size);
 356
 357                 /* Capture l2 cache counter values if enabled */
 358                 if (MALI_TRUE == l2_cache_counter_if_enabled) {
 359                         int i, j = 0;
 360                         _mali_profiling_l2_counter_values l2_counters_values;
 361                         _mali_profiling_get_l2_counters(&l2_counters_values);
 362
 363                         for (i  = COUNTER_L2_0_C0; i <= COUNTER_L2_2_C1; i++) {
 364                                 if (0 == (j % 2))
 365                                         _mali_osk_profiling_record_global_counters(i, l2_counters_values.cores[j / 2].value0);
 366                                 else
 367                                         _mali_osk_profiling_record_global_counters(i, l2_counters_values.cores[j / 2].value1);
 368                                 j++;
 369                         }
 370                 }
 371
 372                 current_time = (s64)_mali_osk_boot_time_get_ns();
 373
 374                 /* Add all enabled counter values into stream */
 375                 for (counter_index = 0; counter_index < num_global_mali_profiling_counters; counter_index++) {
 376                         /* No need to sample these couners here. */
 377                         if (global_mali_profiling_counters[counter_index].enabled) {
 378                                 if ((global_mali_profiling_counters[counter_index].counter_id >= FIRST_MEM_COUNTER &&
 379                                      global_mali_profiling_counters[counter_index].counter_id <= LAST_MEM_COUNTER)
 380                                     || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_VP_ACTIVITY)
 381                                     || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_FP_ACTIVITY)
 382                                     || (global_mali_profiling_counters[counter_index].counter_id == COUNTER_FILMSTRIP)) {
 383
 384                                         continue;
 385                                 }
 386
 387                                 if (global_mali_profiling_counters[counter_index].counter_id >= COUNTER_L2_0_C0 &&
 388                                     global_mali_profiling_counters[counter_index].counter_id <= COUNTER_L2_2_C1) {
 389
 390                                         u32 prev_val = global_mali_profiling_counters[counter_index].prev_counter_value;
 391
 392                                         _mali_profiling_stream_add_counter(mali_counter_stream, current_time, global_mali_profiling_counters[counter_index].key,
 393                                                                            global_mali_profiling_counters[counter_index].current_counter_value - prev_val);
 394
 395                                         prev_val = global_mali_profiling_counters[counter_index].current_counter_value;
 396
 397                                         global_mali_profiling_counters[counter_index].prev_counter_value = prev_val;
 398                                 } else {
 399
 400                                         if (global_mali_profiling_counters[counter_index].counter_id == COUNTER_TOTAL_ALLOC_PAGES) {
 401                                                 u32 total_alloc_mem = _mali_ukk_report_memory_usage();
 402                                                 global_mali_profiling_counters[counter_index].current_counter_value = total_alloc_mem / _MALI_OSK_MALI_PAGE_SIZE;
 403                                         }
 404                                         _mali_profiling_stream_add_counter(mali_counter_stream, current_time, global_mali_profiling_counters[counter_index].key,
 405                                                                            global_mali_profiling_counters[counter_index].current_counter_value);
 406                                         if (global_mali_profiling_counters[counter_index].counter_id < FIRST_SPECIAL_COUNTER)
 407                                                 global_mali_profiling_counters[counter_index].current_counter_value = 0;
 408                                 }
 409                         }
 410                 }
 411                 _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_counter_stream);
 412                 mali_counter_stream = NULL;
 413         } else {
 414                 MALI_DEBUG_PRINT(1, ("Not enough mali profiling stream buffer!\n"));
 415         }
 416
 417         wake_up_interruptible(&stream_fd_wait_queue);
 418
 419         /*Enable the sampling timer again*/
 420         if (0 != num_counters_enabled && 0 != profiling_sample_rate) {
 421                 hrtimer_forward_now(&profiling_sampling_timer, ns_to_ktime(profiling_sample_rate));
 422                 return HRTIMER_RESTART;
 423         }
 424         return HRTIMER_NORESTART;
 425 }
 426
 427 static void _mali_profiling_sampling_core_activity_switch(int counter_id, int core, u32 activity, u32 pid)
 428 {
 429         unsigned long irq_flags;
 430
 431         spin_lock_irqsave(&mali_activity_lock, irq_flags);
 432         if (activity == 0)
 433                 mali_activity_cores_num--;
 434         else
 435                 mali_activity_cores_num++;
 436         spin_unlock_irqrestore(&mali_activity_lock, irq_flags);
 437
 438         if (NULL != global_mali_profiling_counters) {
 439                 int i ;
 440                 for (i = 0; i < num_global_mali_profiling_counters; i++) {
 441                         if (counter_id == global_mali_profiling_counters[i].counter_id && global_mali_profiling_counters[i].enabled) {
 442                                 u64 current_time = _mali_osk_boot_time_get_ns();
 443                                 u32 add_size = STREAM_HEADER_SIZE;
 444
 445                                 if (NULL != mali_core_activity_stream) {
 446                                         if ((mali_core_activity_stream_dequeue_time +  MALI_PROFILING_STREAM_HOLD_TIME < current_time) ||
 447                                             (MALI_PROFILING_STREAM_DATA_DEFAULT_SIZE > MALI_PROFILING_STREAM_BUFFER_SIZE
 448                                              - mali_core_activity_stream->used_size)) {
 449                                                 _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_core_activity_stream);
 450                                                 mali_core_activity_stream = NULL;
 451                                                 wake_up_interruptible(&stream_fd_wait_queue);
 452                                         }
 453                                 }
 454
 455                                 if (NULL == mali_core_activity_stream) {
 456                                         if (_MALI_OSK_ERR_OK == _mali_profiling_global_stream_list_dequeue(
 457                                                     &global_mali_stream_list->free_list, &mali_core_activity_stream)) {
 458                                                 mali_core_activity_stream_dequeue_time = current_time;
 459                                         } else {
 460                                                 MALI_DEBUG_PRINT(1, ("Not enough mali profiling stream buffer!\n"));
 461                                                 wake_up_interruptible(&stream_fd_wait_queue);
 462                                                 break;
 463                                         }
 464
 465                                 }
 466
 467                                 mali_core_activity_stream->data[mali_core_activity_stream->used_size] = STREAM_HEADER_CORE_ACTIVITY;
 468
 469                                 add_size += _mali_profiling_pack_long(mali_core_activity_stream->data,
 470                                                                       MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, (s64)current_time);
 471                                 add_size += _mali_profiling_pack_int(mali_core_activity_stream->data,
 472                                                                      MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, core);
 473                                 add_size += _mali_profiling_pack_int(mali_core_activity_stream->data,
 474                                                                      MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, (s32)global_mali_profiling_counters[i].key);
 475                                 add_size += _mali_profiling_pack_int(mali_core_activity_stream->data,
 476                                                                      MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, activity);
 477                                 add_size += _mali_profiling_pack_int(mali_core_activity_stream->data,
 478                                                                      MALI_PROFILING_STREAM_BUFFER_SIZE, mali_core_activity_stream->used_size + add_size, pid);
 479
 480                                 _mali_profiling_set_packet_size(mali_core_activity_stream->data + mali_core_activity_stream->used_size + 1,
 481                                                                 add_size - STREAM_HEADER_SIZE);
 482
 483                                 mali_core_activity_stream->used_size += add_size;
 484
 485                                 if (0 == mali_activity_cores_num) {
 486                                         _mali_profiling_global_stream_list_queue(&global_mali_stream_list->queue_list, mali_core_activity_stream);
 487                                         mali_core_activity_stream = NULL;
 488                                         wake_up_interruptible(&stream_fd_wait_queue);
 489                                 }
 490
 491                                 break;
 492                         }
 493                 }
 494         }
 495 }
 496
 497 static mali_bool _mali_profiling_global_counters_init(void)
 498 {
 499         int core_id, counter_index, counter_number, counter_id;
 500         u32 num_l2_cache_cores;
 501         u32 num_pp_cores;
 502         u32 num_gp_cores = 1;
 503
 504         MALI_DEBUG_ASSERT(NULL == global_mali_profiling_counters);
 505         num_pp_cores = mali_pp_get_glob_num_pp_cores();
 506         num_l2_cache_cores =    mali_l2_cache_core_get_glob_num_l2_cores();
 507
 508         num_global_mali_profiling_counters = 3 * (num_gp_cores + num_pp_cores) + 2 * num_l2_cache_cores
 509                                              + MALI_PROFILING_SW_COUNTERS_NUM
 510                                              + MALI_PROFILING_SPECIAL_COUNTERS_NUM
 511                                              + MALI_PROFILING_MEM_COUNTERS_NUM;
 512         global_mali_profiling_counters = _mali_osk_calloc(num_global_mali_profiling_counters, sizeof(mali_profiling_counter));
 513
 514         if (NULL == global_mali_profiling_counters)
 515                 return MALI_FALSE;
 516
 517         counter_index = 0;
 518         /*Vertex processor counters */
 519         for (core_id = 0; core_id < num_gp_cores; core_id ++) {
 520                 global_mali_profiling_counters[counter_index].counter_id = ACTIVITY_VP_0 + core_id;
 521                 _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name,
 522                                    sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_VP_%d_active", mali_name, core_id);
 523
 524                 for (counter_number = 0; counter_number < 2; counter_number++) {
 525                         counter_index++;
 526                         global_mali_profiling_counters[counter_index].counter_id = COUNTER_VP_0_C0 + (2 * core_id) + counter_number;
 527                         _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name,
 528                                            sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_VP_%d_cnt%d", mali_name, core_id, counter_number);
 529                 }
 530         }
 531
 532         /* Fragment processors' counters */
 533         for (core_id = 0; core_id < num_pp_cores; core_id++) {
 534                 counter_index++;
 535                 global_mali_profiling_counters[counter_index].counter_id = ACTIVITY_FP_0 + core_id;
 536                 _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name,
 537                                    sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_FP_%d_active", mali_name, core_id);
 538
 539                 for (counter_number = 0; counter_number < 2; counter_number++) {
 540                         counter_index++;
 541                         global_mali_profiling_counters[counter_index].counter_id = COUNTER_FP_0_C0 + (2 * core_id) + counter_number;
 542                         _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name,
 543                                            sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_FP_%d_cnt%d", mali_name, core_id, counter_number);
 544                 }
 545         }
 546
 547         /* L2 Cache counters */
 548         for (core_id = 0; core_id < num_l2_cache_cores; core_id++) {
 549                 for (counter_number = 0; counter_number < 2; counter_number++) {
 550                         counter_index++;
 551                         global_mali_profiling_counters[counter_index].counter_id = COUNTER_L2_0_C0 + (2 * core_id) + counter_number;
 552                         _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name,
 553                                            sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_L2_%d_cnt%d", mali_name, core_id, counter_number);
 554                 }
 555         }
 556
 557         /* Now set up the software counter entries */
 558         for (counter_id = FIRST_SW_COUNTER; counter_id <= LAST_SW_COUNTER; counter_id++) {
 559                 counter_index++;
 560
 561                 if (0 == first_sw_counter_index)
 562                         first_sw_counter_index = counter_index;
 563
 564                 global_mali_profiling_counters[counter_index].counter_id = counter_id;
 565                 _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name,
 566                                    sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_SW_%d", mali_name, counter_id - FIRST_SW_COUNTER);
 567         }
 568
 569         /* Now set up the special counter entries */
 570         for (counter_id = FIRST_SPECIAL_COUNTER; counter_id <= LAST_SPECIAL_COUNTER; counter_id++) {
 571
 572                 counter_index++;
 573                 _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name,
 574                                    sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_%s",
 575                                    mali_name, _mali_special_counter_descriptions[counter_id - FIRST_SPECIAL_COUNTER]);
 576
 577                 global_mali_profiling_counters[counter_index].counter_id = counter_id;
 578         }
 579
 580         /* Now set up the mem counter entries*/
 581         for (counter_id = FIRST_MEM_COUNTER; counter_id <= LAST_MEM_COUNTER; counter_id++) {
 582
 583                 counter_index++;
 584                 _mali_osk_snprintf(global_mali_profiling_counters[counter_index].counter_name,
 585                                    sizeof(global_mali_profiling_counters[counter_index].counter_name), "ARM_Mali-%s_%s",
 586                                    mali_name, _mali_mem_counter_descriptions[counter_id - FIRST_MEM_COUNTER]);
 587
 588                 global_mali_profiling_counters[counter_index].counter_id = counter_id;
 589         }
 590
 591         MALI_DEBUG_ASSERT((counter_index + 1) == num_global_mali_profiling_counters);
 592
 593         return MALI_TRUE;
 594 }
 595
 596 void _mali_profiling_notification_mem_counter(struct mali_session_data *session, u32 counter_id, u32 key, int enable)
 597 {
 598
 599         MALI_DEBUG_ASSERT_POINTER(session);
 600
 601         if (NULL != session) {
 602                 _mali_osk_notification_t *notification;
 603                 _mali_osk_notification_queue_t *queue;
 604
 605                 queue = session->ioctl_queue;
 606                 MALI_DEBUG_ASSERT(NULL != queue);
 607
 608                 notification = _mali_osk_notification_create(_MALI_NOTIFICATION_ANNOTATE_PROFILING_MEM_COUNTER,
 609                                 sizeof(_mali_uk_annotate_profiling_mem_counter_s));
 610
 611                 if (NULL != notification) {
 612                         _mali_uk_annotate_profiling_mem_counter_s *data = notification->result_buffer;
 613                         data->counter_id = counter_id;
 614                         data->key = key;
 615                         data->enable = enable;
 616
 617                         _mali_osk_notification_queue_send(queue, notification);
 618                 } else {
 619                         MALI_PRINT_ERROR(("Failed to create notification object!\n"));
 620                 }
 621         } else {
 622                 MALI_PRINT_ERROR(("Failed to find the right session!\n"));
 623         }
 624 }
 625
 626 void _mali_profiling_notification_enable(struct mali_session_data *session, u32 sampling_rate, int enable)
 627 {
 628         MALI_DEBUG_ASSERT_POINTER(session);
 629
 630         if (NULL != session) {
 631                 _mali_osk_notification_t *notification;
 632                 _mali_osk_notification_queue_t *queue;
 633
 634                 queue = session->ioctl_queue;
 635                 MALI_DEBUG_ASSERT(NULL != queue);
 636
 637                 notification = _mali_osk_notification_create(_MALI_NOTIFICATION_ANNOTATE_PROFILING_ENABLE,
 638                                 sizeof(_mali_uk_annotate_profiling_enable_s));
 639
 640                 if (NULL != notification) {
 641                         _mali_uk_annotate_profiling_enable_s *data = notification->result_buffer;
 642                         data->sampling_rate = sampling_rate;
 643                         data->enable = enable;
 644
 645                         _mali_osk_notification_queue_send(queue, notification);
 646                 } else {
 647                         MALI_PRINT_ERROR(("Failed to create notification object!\n"));
 648                 }
 649         } else {
 650                 MALI_PRINT_ERROR(("Failed to find the right session!\n"));
 651         }
 652 }
 653
 654
 655 _mali_osk_errcode_t _mali_osk_profiling_init(mali_bool auto_start)
 656 {
 657         int i;
 658         mali_profiling_stream *new_mali_profiling_stream = NULL;
 659         mali_profiling_stream_list *new_mali_profiling_stream_list = NULL;
 660         if (MALI_TRUE == auto_start) {
 661                 mali_set_user_setting(_MALI_UK_USER_SETTING_SW_EVENTS_ENABLE, MALI_TRUE);
 662         }
 663
 664         /*Init the global_mali_stream_list*/
 665         MALI_DEBUG_ASSERT(NULL == global_mali_stream_list);
 666         new_mali_profiling_stream_list = (mali_profiling_stream_list *)kmalloc(sizeof(mali_profiling_stream_list), GFP_KERNEL);
 667
 668         if (NULL == new_mali_profiling_stream_list) {
 669                 return _MALI_OSK_ERR_NOMEM;
 670         }
 671
 672         spin_lock_init(&new_mali_profiling_stream_list->spin_lock);
 673         INIT_LIST_HEAD(&new_mali_profiling_stream_list->free_list);
 674         INIT_LIST_HEAD(&new_mali_profiling_stream_list->queue_list);
 675
 676         spin_lock_init(&mali_activity_lock);
 677         mali_activity_cores_num =  0;
 678
 679         for (i = 0; i < MALI_PROFILING_STREAM_BUFFER_NUM; i++) {
 680                 new_mali_profiling_stream = (mali_profiling_stream *)kmalloc(sizeof(mali_profiling_stream), GFP_KERNEL);
 681                 if (NULL == new_mali_profiling_stream) {
 682                         _mali_profiling_stream_list_destory(new_mali_profiling_stream_list);
 683                         return _MALI_OSK_ERR_NOMEM;
 684                 }
 685
 686                 INIT_LIST_HEAD(&new_mali_profiling_stream->list);
 687                 new_mali_profiling_stream->used_size = 0;
 688                 list_add_tail(&new_mali_profiling_stream->list, &new_mali_profiling_stream_list->free_list);
 689
 690         }
 691
 692         _mali_osk_atomic_init(&stream_fd_if_used, 0);
 693         init_waitqueue_head(&stream_fd_wait_queue);
 694
 695         hrtimer_init(&profiling_sampling_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 696
 697         profiling_sampling_timer.function = _mali_profiling_sampling_counters;
 698
 699         global_mali_stream_list = new_mali_profiling_stream_list;
 700
 701         return _MALI_OSK_ERR_OK;
 702 }
 703
 704 void _mali_osk_profiling_term(void)
 705 {
 706         if (0 != profiling_sample_rate) {
 707                 hrtimer_cancel(&profiling_sampling_timer);
 708                 profiling_sample_rate = 0;
 709         }
 710         _mali_osk_atomic_term(&stream_fd_if_used);
 711
 712         if (NULL != global_mali_profiling_counters) {
 713                 _mali_osk_free(global_mali_profiling_counters);
 714                 global_mali_profiling_counters = NULL;
 715                 num_global_mali_profiling_counters = 0;
 716         }
 717
 718         if (NULL != global_mali_stream_list) {
 719                 _mali_profiling_stream_list_destory(global_mali_stream_list);
 720                 global_mali_stream_list = NULL;
 721         }
 722
 723 }
 724
 725 void _mali_osk_profiling_stop_sampling(u32 pid)
 726 {
 727         if (pid == current_profiling_pid) {
 728
 729                 int i;
 730                 /* Reset all counter states when closing connection.*/
 731                 for (i = 0; i < num_global_mali_profiling_counters; ++i) {
 732                         _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id, MALI_HW_CORE_NO_COUNTER);
 733                         global_mali_profiling_counters[i].enabled = 0;
 734                         global_mali_profiling_counters[i].prev_counter_value = 0;
 735                         global_mali_profiling_counters[i].current_counter_value = 0;
 736                 }
 737                 l2_cache_counter_if_enabled = MALI_FALSE;
 738                 num_counters_enabled = 0;
 739                 mem_counters_enabled = 0;
 740                 _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 0);
 741                 _mali_profiling_control(SW_COUNTER_ENABLE, 0);
 742                 /* Delete sampling timer when closing connection. */
 743                 if (0 != profiling_sample_rate) {
 744                         hrtimer_cancel(&profiling_sampling_timer);
 745                         profiling_sample_rate = 0;
 746                 }
 747                 current_profiling_pid = 0;
 748         }
 749 }
 750
 751 void    _mali_osk_profiling_add_event(u32 event_id, u32 data0, u32 data1, u32 data2, u32 data3, u32 data4)
 752 {
 753         /*Record the freq & volt to global_mali_profiling_counters here. */
 754         if (0 != profiling_sample_rate) {
 755                 u32 channel;
 756                 u32 state;
 757                 channel = (event_id >> 16) & 0xFF;
 758                 state = ((event_id >> 24) & 0xF) << 24;
 759
 760                 switch (state) {
 761                 case MALI_PROFILING_EVENT_TYPE_SINGLE:
 762                         if ((MALI_PROFILING_EVENT_CHANNEL_GPU >> 16) == channel) {
 763                                 u32 reason = (event_id & 0xFFFF);
 764                                 if (MALI_PROFILING_EVENT_REASON_SINGLE_GPU_FREQ_VOLT_CHANGE == reason) {
 765                                         _mali_osk_profiling_record_global_counters(COUNTER_FREQUENCY, data0);
 766                                         _mali_osk_profiling_record_global_counters(COUNTER_VOLTAGE, data1);
 767                                 }
 768                         }
 769                         break;
 770                 case MALI_PROFILING_EVENT_TYPE_START:
 771                         if ((MALI_PROFILING_EVENT_CHANNEL_GP0 >> 16) == channel) {
 772                                 _mali_profiling_sampling_core_activity_switch(COUNTER_VP_ACTIVITY, 0, 1, data1);
 773                         } else if (channel >= (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16) &&
 774                                    (MALI_PROFILING_EVENT_CHANNEL_PP7 >> 16) >= channel) {
 775                                 u32 core_id = channel - (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16);
 776                                 _mali_profiling_sampling_core_activity_switch(COUNTER_FP_ACTIVITY, core_id, 1, data1);
 777                         }
 778                         break;
 779                 case MALI_PROFILING_EVENT_TYPE_STOP:
 780                         if ((MALI_PROFILING_EVENT_CHANNEL_GP0 >> 16) == channel) {
 781                                 _mali_profiling_sampling_core_activity_switch(COUNTER_VP_ACTIVITY, 0, 0, 0);
 782                         } else if (channel >= (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16) &&
 783                                    (MALI_PROFILING_EVENT_CHANNEL_PP7 >> 16) >= channel) {
 784                                 u32 core_id = channel - (MALI_PROFILING_EVENT_CHANNEL_PP0 >> 16);
 785                                 _mali_profiling_sampling_core_activity_switch(COUNTER_FP_ACTIVITY, core_id, 0, 0);
 786                         }
 787                         break;
 788                 default:
 789                         break;
 790                 }
 791         }
 792         trace_mali_timeline_event(event_id, data0, data1, data2, data3, data4);
 793 }
 794
 795 void _mali_osk_profiling_report_sw_counters(u32 *counters)
 796 {
 797         trace_mali_sw_counters(_mali_osk_get_pid(), _mali_osk_get_tid(), NULL, counters);
 798 }
 799
 800 void _mali_osk_profiling_record_global_counters(int counter_id, u32 value)
 801 {
 802         if (NULL != global_mali_profiling_counters) {
 803                 int i ;
 804                 for (i = 0; i < num_global_mali_profiling_counters; i++) {
 805                         if (counter_id == global_mali_profiling_counters[i].counter_id && global_mali_profiling_counters[i].enabled) {
 806                                 global_mali_profiling_counters[i].current_counter_value = value;
 807                                 break;
 808                         }
 809                 }
 810         }
 811 }
 812
 813 _mali_osk_errcode_t _mali_ukk_profiling_add_event(_mali_uk_profiling_add_event_s *args)
 814 {
 815         /* Always add process and thread identificator in the first two data elements for events from user space */
 816         _mali_osk_profiling_add_event(args->event_id, _mali_osk_get_pid(), _mali_osk_get_tid(), args->data[2], args->data[3], args->data[4]);
 817
 818         return _MALI_OSK_ERR_OK;
 819 }
 820
 821 _mali_osk_errcode_t _mali_ukk_sw_counters_report(_mali_uk_sw_counters_report_s *args)
 822 {
 823         u32 *counters = (u32 *)(uintptr_t)args->counters;
 824
 825         _mali_osk_profiling_report_sw_counters(counters);
 826
 827         if (NULL != global_mali_profiling_counters) {
 828                 int i;
 829                 for (i = 0; i < MALI_PROFILING_SW_COUNTERS_NUM; i ++) {
 830                         if (global_mali_profiling_counters[first_sw_counter_index + i].enabled) {
 831                                 global_mali_profiling_counters[first_sw_counter_index + i].current_counter_value = *(counters + i);
 832                         }
 833                 }
 834         }
 835
 836         return _MALI_OSK_ERR_OK;
 837 }
 838
 839 _mali_osk_errcode_t _mali_ukk_profiling_stream_fd_get(_mali_uk_profiling_stream_fd_get_s *args)
 840 {
 841         struct  mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx;
 842         MALI_DEBUG_ASSERT_POINTER(session);
 843
 844         if (1 == _mali_osk_atomic_inc_return(&stream_fd_if_used)) {
 845
 846                 s32 fd = anon_inode_getfd("[mali_profiling_stream]", &mali_profiling_stream_fops,
 847                                           session,
 848                                           O_RDONLY | O_CLOEXEC);
 849
 850                 args->stream_fd = fd;
 851                 if (0 > fd) {
 852                         _mali_osk_atomic_dec(&stream_fd_if_used);
 853                         return _MALI_OSK_ERR_FAULT;
 854                 }
 855                 args->stream_fd = fd;
 856         } else {
 857                 _mali_osk_atomic_dec(&stream_fd_if_used);
 858                 args->stream_fd = -1;
 859                 return _MALI_OSK_ERR_BUSY;
 860         }
 861
 862         return _MALI_OSK_ERR_OK;
 863 }
 864
 865 _mali_osk_errcode_t _mali_ukk_profiling_control_set(_mali_uk_profiling_control_set_s *args)
 866 {
 867         u32 control_packet_size;
 868         u32 output_buffer_size;
 869
 870         struct  mali_session_data *session = (struct mali_session_data *)(uintptr_t)args->ctx;
 871         MALI_DEBUG_ASSERT_POINTER(session);
 872
 873         if (NULL == global_mali_profiling_counters && MALI_FALSE == _mali_profiling_global_counters_init()) {
 874                 MALI_PRINT_ERROR(("Failed to create global_mali_profiling_counters.\n"));
 875                 return _MALI_OSK_ERR_FAULT;
 876         }
 877
 878         control_packet_size = args->control_packet_size;
 879         output_buffer_size = args->response_packet_size;
 880
 881         if (0 != control_packet_size) {
 882                 u8 control_type;
 883                 u8 *control_packet_data;
 884                 u8 *response_packet_data;
 885                 u32 version_length = sizeof(utgard_setup_version) - 1;
 886
 887                 control_packet_data = (u8 *)(uintptr_t)args->control_packet_data;
 888                 MALI_DEBUG_ASSERT_POINTER(control_packet_data);
 889                 response_packet_data = (u8 *)(uintptr_t)args->response_packet_data;
 890                 MALI_DEBUG_ASSERT_POINTER(response_packet_data);
 891
 892                 /*Decide if need to ignore Utgard setup version.*/
 893                 if (control_packet_size >= version_length) {
 894                         if (0 == memcmp(control_packet_data, utgard_setup_version, version_length)) {
 895                                 if (control_packet_size == version_length) {
 896                                         args->response_packet_size = 0;
 897                                         return _MALI_OSK_ERR_OK;
 898                                 } else {
 899                                         control_packet_data += version_length;
 900                                         control_packet_size -= version_length;
 901                                 }
 902                         }
 903                 }
 904
 905                 current_profiling_pid = _mali_osk_get_pid();
 906
 907                 control_type = control_packet_data[0];
 908                 switch (control_type) {
 909                 case PACKET_HEADER_COUNTERS_REQUEST: {
 910                         int i;
 911
 912                         if (PACKET_HEADER_SIZE > control_packet_size ||
 913                             control_packet_size !=  _mali_profiling_get_packet_size(control_packet_data + 1)) {
 914                                 MALI_PRINT_ERROR(("Wrong control packet  size, type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size));
 915                                 return _MALI_OSK_ERR_FAULT;
 916                         }
 917
 918                         /* Send supported counters */
 919                         if (PACKET_HEADER_SIZE > output_buffer_size)
 920                                 return _MALI_OSK_ERR_FAULT;
 921
 922                         *response_packet_data = PACKET_HEADER_COUNTERS_ACK;
 923                         args->response_packet_size = PACKET_HEADER_SIZE;
 924
 925                         for (i = 0; i < num_global_mali_profiling_counters; ++i) {
 926                                 u32 name_size = strlen(global_mali_profiling_counters[i].counter_name);
 927
 928                                 if ((args->response_packet_size + name_size + 1) > output_buffer_size) {
 929                                         MALI_PRINT_ERROR(("Response packet data is too large..\n"));
 930                                         return _MALI_OSK_ERR_FAULT;
 931                                 }
 932
 933                                 memcpy(response_packet_data + args->response_packet_size,
 934                                        global_mali_profiling_counters[i].counter_name, name_size + 1);
 935
 936                                 args->response_packet_size += (name_size + 1);
 937
 938                                 if (global_mali_profiling_counters[i].counter_id == COUNTER_VP_ACTIVITY) {
 939                                         args->response_packet_size += _mali_profiling_pack_int(response_packet_data,
 940                                                                       output_buffer_size, args->response_packet_size, (s32)1);
 941                                 } else if (global_mali_profiling_counters[i].counter_id == COUNTER_FP_ACTIVITY) {
 942                                         args->response_packet_size += _mali_profiling_pack_int(response_packet_data,
 943                                                                       output_buffer_size, args->response_packet_size, (s32)mali_pp_get_glob_num_pp_cores());
 944                                 } else {
 945                                         args->response_packet_size += _mali_profiling_pack_int(response_packet_data,
 946                                                                       output_buffer_size, args->response_packet_size, (s32) - 1);
 947                                 }
 948                         }
 949
 950                         _mali_profiling_set_packet_size(response_packet_data + 1, args->response_packet_size);
 951                         break;
 952                 }
 953
 954                 case PACKET_HEADER_COUNTERS_ENABLE: {
 955                         int i;
 956                         u32 request_pos = PACKET_HEADER_SIZE;
 957                         mali_bool sw_counter_if_enabled = MALI_FALSE;
 958
 959                         if (PACKET_HEADER_SIZE > control_packet_size ||
 960                             control_packet_size !=  _mali_profiling_get_packet_size(control_packet_data + 1)) {
 961                                 MALI_PRINT_ERROR(("Wrong control packet  size , type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size));
 962                                 return _MALI_OSK_ERR_FAULT;
 963                         }
 964
 965                         /* Init all counter states before enable requested counters.*/
 966                         for (i = 0; i < num_global_mali_profiling_counters; ++i) {
 967                                 _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id, MALI_HW_CORE_NO_COUNTER);
 968                                 global_mali_profiling_counters[i].enabled = 0;
 969                                 global_mali_profiling_counters[i].prev_counter_value = 0;
 970                                 global_mali_profiling_counters[i].current_counter_value = 0;
 971
 972                                 if (global_mali_profiling_counters[i].counter_id >= FIRST_MEM_COUNTER &&
 973                                     global_mali_profiling_counters[i].counter_id <= LAST_MEM_COUNTER) {
 974                                         _mali_profiling_notification_mem_counter(session, global_mali_profiling_counters[i].counter_id, 0, 0);
 975                                 }
 976                         }
 977
 978                         l2_cache_counter_if_enabled = MALI_FALSE;
 979                         num_counters_enabled = 0;
 980                         mem_counters_enabled = 0;
 981                         _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 0);
 982                         _mali_profiling_control(SW_COUNTER_ENABLE, 0);
 983                         _mali_profiling_notification_enable(session, 0, 0);
 984
 985                         /* Enable requested counters */
 986                         while (request_pos < control_packet_size) {
 987                                 u32 begin = request_pos;
 988                                 u32 event;
 989                                 u32 key;
 990
 991                                 /* Check the counter name which should be ended with null */
 992                                 while (request_pos < control_packet_size && control_packet_data[request_pos] != '\0') {
 993                                         ++request_pos;
 994                                 }
 995
 996                                 if (request_pos >= control_packet_size)
 997                                         return _MALI_OSK_ERR_FAULT;
 998
 999                                 ++request_pos;
1000                                 event = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size);
1001                                 key = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size);
1002
1003                                 for (i = 0; i < num_global_mali_profiling_counters; ++i) {
1004                                         u32 name_size = strlen((char *)(control_packet_data + begin));
1005
1006                                         if (strncmp(global_mali_profiling_counters[i].counter_name, (char *)(control_packet_data + begin), name_size) == 0) {
1007                                                 if (!sw_counter_if_enabled && (FIRST_SW_COUNTER <= global_mali_profiling_counters[i].counter_id
1008                                                                                && global_mali_profiling_counters[i].counter_id <= LAST_SW_COUNTER)) {
1009                                                         sw_counter_if_enabled = MALI_TRUE;
1010                                                         _mali_profiling_control(SW_COUNTER_ENABLE, 1);
1011                                                 }
1012
1013                                                 if (COUNTER_FILMSTRIP == global_mali_profiling_counters[i].counter_id) {
1014                                                         _mali_profiling_control(FBDUMP_CONTROL_ENABLE, 1);
1015                                                         _mali_profiling_control(FBDUMP_CONTROL_RATE, event & 0xff);
1016                                                         _mali_profiling_control(FBDUMP_CONTROL_RESIZE_FACTOR, (event >> 8) & 0xff);
1017                                                 }
1018
1019                                                 if (global_mali_profiling_counters[i].counter_id >= FIRST_MEM_COUNTER &&
1020                                                     global_mali_profiling_counters[i].counter_id <= LAST_MEM_COUNTER) {
1021                                                         _mali_profiling_notification_mem_counter(session, global_mali_profiling_counters[i].counter_id,
1022                                                                         key, 1);
1023                                                         mem_counters_enabled++;
1024                                                 }
1025
1026                                                 global_mali_profiling_counters[i].counter_event = event;
1027                                                 global_mali_profiling_counters[i].key = key;
1028                                                 global_mali_profiling_counters[i].enabled = 1;
1029
1030                                                 _mali_profiling_set_event(global_mali_profiling_counters[i].counter_id,
1031                                                                           global_mali_profiling_counters[i].counter_event);
1032                                                 num_counters_enabled++;
1033                                                 break;
1034                                         }
1035                                 }
1036
1037                                 if (i == num_global_mali_profiling_counters) {
1038                                         MALI_PRINT_ERROR(("Counter name does not match for type %u.\n", control_type));
1039                                         return _MALI_OSK_ERR_FAULT;
1040                                 }
1041                         }
1042
1043                         if (PACKET_HEADER_SIZE <= output_buffer_size) {
1044                                 *response_packet_data = PACKET_HEADER_ACK;
1045                                 _mali_profiling_set_packet_size(response_packet_data + 1, PACKET_HEADER_SIZE);
1046                                 args->response_packet_size = PACKET_HEADER_SIZE;
1047                         } else {
1048                                 return _MALI_OSK_ERR_FAULT;
1049                         }
1050
1051                         break;
1052                 }
1053
1054                 case PACKET_HEADER_START_CAPTURE_VALUE: {
1055                         u32 live_rate;
1056                         u32 request_pos = PACKET_HEADER_SIZE;
1057
1058                         if (PACKET_HEADER_SIZE > control_packet_size ||
1059                             control_packet_size !=  _mali_profiling_get_packet_size(control_packet_data + 1)) {
1060                                 MALI_PRINT_ERROR(("Wrong control packet  size , type 0x%x,size 0x%x.\n", control_packet_data[0], control_packet_size));
1061                                 return _MALI_OSK_ERR_FAULT;
1062                         }
1063
1064                         /* Read samping rate in nanoseconds and live rate, start capture.*/
1065                         profiling_sample_rate =  _mali_profiling_read_packet_int(control_packet_data,
1066                                                  &request_pos, control_packet_size);
1067
1068                         live_rate = _mali_profiling_read_packet_int(control_packet_data, &request_pos, control_packet_size);
1069
1070                         if (PACKET_HEADER_SIZE <= output_buffer_size) {
1071                                 *response_packet_data = PACKET_HEADER_ACK;
1072                                 _mali_profiling_set_packet_size(response_packet_data + 1, PACKET_HEADER_SIZE);
1073                                 args->response_packet_size = PACKET_HEADER_SIZE;
1074                         } else {
1075                                 return _MALI_OSK_ERR_FAULT;
1076                         }
1077
1078                         if (0 != num_counters_enabled && 0 != profiling_sample_rate) {
1079                                 _mali_profiling_global_stream_list_free();
1080                                 if (mem_counters_enabled > 0) {
1081                                         _mali_profiling_notification_enable(session, profiling_sample_rate, 1);
1082                                 }
1083                                 hrtimer_start(&profiling_sampling_timer,
1084                                               ktime_set(profiling_sample_rate / 1000000000, profiling_sample_rate % 1000000000),
1085                                               HRTIMER_MODE_REL_PINNED);
1086                         }
1087
1088                         break;
1089                 }
1090                 default:
1091                         MALI_PRINT_ERROR(("Unsupported  profiling packet header type %u.\n", control_type));
1092                         args->response_packet_size  = 0;
1093                         return _MALI_OSK_ERR_FAULT;
1094                 }
1095         } else {
1096                 _mali_osk_profiling_stop_sampling(current_profiling_pid);
1097                 _mali_profiling_notification_enable(session, 0, 0);
1098         }
1099
1100         return _MALI_OSK_ERR_OK;
1101 }
1102
1103 /**
1104  * Called by gator.ko to set HW counters
1105  *
1106  * @param counter_id The counter ID.
1107  * @param event_id Event ID that the counter should count (HW counter value from TRM).
1108  *
1109  * @return 1 on success, 0 on failure.
1110  */
1111 int _mali_profiling_set_event(u32 counter_id, s32 event_id)
1112 {
1113         if (COUNTER_VP_0_C0 == counter_id) {
1114                 mali_gp_job_set_gp_counter_src0(event_id);
1115         } else if (COUNTER_VP_0_C1 == counter_id) {
1116                 mali_gp_job_set_gp_counter_src1(event_id);
1117         } else if (COUNTER_FP_0_C0 <= counter_id && COUNTER_FP_7_C1 >= counter_id) {
1118                 /*
1119                  * Two compatibility notes for this function:
1120                  *
1121                  * 1) Previously the DDK allowed per core counters.
1122                  *
1123                  *    This did not make much sense on Mali-450 with the "virtual PP core" concept,
1124                  *    so this option was removed, and only the same pair of HW counters was allowed on all cores,
1125                  *    beginning with r3p2 release.
1126                  *
1127                  *    Starting with r4p0, it is now possible to set different HW counters for the different sub jobs.
1128                  *    This should be almost the same, since sub job 0 is designed to run on core 0,
1129                  *    sub job 1 on core 1, and so on.
1130                  *
1131                  *    The scheduling of PP sub jobs is not predictable, and this often led to situations where core 0 ran 2
1132                  *    sub jobs, while for instance core 1 ran zero. Having the counters set per sub job would thus increase
1133                  *    the predictability of the returned data (as you would be guaranteed data for all the selected HW counters).
1134                  *
1135                  *    PS: Core scaling needs to be disabled in order to use this reliably (goes for both solutions).
1136                  *
1137                  *    The framework/#defines with Gator still indicates that the counter is for a particular core,
1138                  *    but this is internally used as a sub job ID instead (no translation needed).
1139                  *
1140                  *  2) Global/default vs per sub job counters
1141                  *
1142                  *     Releases before r3p2 had only per PP core counters.
1143                  *     r3p2 releases had only one set of default/global counters which applied to all PP cores
1144                  *     Starting with r4p0, we have both a set of default/global counters,
1145                  *     and individual counters per sub job (equal to per core).
1146                  *
1147                  *     To keep compatibility with Gator/DS-5/streamline, the following scheme is used:
1148                  *
1149                  *     r3p2 release; only counters set for core 0 is handled,
1150                  *     this is applied as the default/global set of counters, and will thus affect all cores.
1151                  *
1152                  *     r4p0 release; counters set for core 0 is applied as both the global/default set of counters,
1153                  *     and counters for sub job 0.
1154                  *     Counters set for core 1-7 is only applied for the corresponding sub job.
1155                  *
1156                  *     This should allow the DS-5/Streamline GUI to have a simple mode where it only allows setting the
1157                  *     values for core 0, and thus this will be applied to all PP sub jobs/cores.
1158                  *     Advanced mode will also be supported, where individual pairs of HW counters can be selected.
1159                  *
1160                  *     The GUI will (until it is updated) still refer to cores instead of sub jobs, but this is probably
1161                  *     something we can live with!
1162                  *
1163                  *     Mali-450 note: Each job is not divided into a deterministic number of sub jobs, as the HW DLBU
1164                  *     automatically distributes the load between whatever number of cores is available at this particular time.
1165                  *     A normal PP job on Mali-450 is thus considered a single (virtual) job, and it will thus only be possible
1166                  *     to use a single pair of HW counters (even if the job ran on multiple PP cores).
1167                  *     In other words, only the global/default pair of PP HW counters will be used for normal Mali-450 jobs.
1168                  */
1169                 u32 sub_job = (counter_id - COUNTER_FP_0_C0) >> 1;
1170                 u32 counter_src = (counter_id - COUNTER_FP_0_C0) & 1;
1171                 if (0 == counter_src) {
1172                         mali_pp_job_set_pp_counter_sub_job_src0(sub_job, event_id);
1173                         if (0 == sub_job) {
1174                                 mali_pp_job_set_pp_counter_global_src0(event_id);
1175                         }
1176                 } else {
1177                         mali_pp_job_set_pp_counter_sub_job_src1(sub_job, event_id);
1178                         if (0 == sub_job) {
1179                                 mali_pp_job_set_pp_counter_global_src1(event_id);
1180                         }
1181                 }
1182         } else if (COUNTER_L2_0_C0 <= counter_id && COUNTER_L2_2_C1 >= counter_id) {
1183                 u32 core_id = (counter_id - COUNTER_L2_0_C0) >> 1;
1184                 struct mali_l2_cache_core *l2_cache_core = mali_l2_cache_core_get_glob_l2_core(core_id);
1185
1186                 if (NULL != l2_cache_core) {
1187                         u32 counter_src = (counter_id - COUNTER_L2_0_C0) & 1;
1188                         mali_l2_cache_core_set_counter_src(l2_cache_core,
1189                                                            counter_src, event_id);
1190                         l2_cache_counter_if_enabled = MALI_TRUE;
1191                 }
1192         } else {
1193                 return 0; /* Failure, unknown event */
1194         }
1195
1196         return 1; /* success */
1197 }
1198
1199 /**
1200  * Called by gator.ko to retrieve the L2 cache counter values for all L2 cache cores.
1201  * The L2 cache counters are unique in that they are polled by gator, rather than being
1202  * transmitted via the tracepoint mechanism.
1203  *
1204  * @param values Pointer to a _mali_profiling_l2_counter_values structure where
1205  *               the counter sources and values will be output
1206  * @return 0 if all went well; otherwise, return the mask with the bits set for the powered off cores
1207  */
1208 u32 _mali_profiling_get_l2_counters(_mali_profiling_l2_counter_values *values)
1209 {
1210         u32 l2_cores_num = mali_l2_cache_core_get_glob_num_l2_cores();
1211         u32 i;
1212
1213         MALI_DEBUG_ASSERT(l2_cores_num <= 3);
1214
1215         for (i = 0; i < l2_cores_num; i++) {
1216                 struct mali_l2_cache_core *l2_cache = mali_l2_cache_core_get_glob_l2_core(i);
1217
1218                 if (NULL == l2_cache) {
1219                         continue;
1220                 }
1221
1222                 mali_l2_cache_core_get_counter_values(l2_cache,
1223                                                       &values->cores[i].source0,
1224                                                       &values->cores[i].value0,
1225                                                       &values->cores[i].source1,
1226                                                       &values->cores[i].value1);
1227         }
1228
1229         return 0;
1230 }
1231
1232 /**
1233  * Called by gator to control the production of profiling information at runtime.
1234  */
1235 void _mali_profiling_control(u32 action, u32 value)
1236 {
1237         switch (action) {
1238         case FBDUMP_CONTROL_ENABLE:
1239                 mali_set_user_setting(_MALI_UK_USER_SETTING_COLORBUFFER_CAPTURE_ENABLED, (value == 0 ? MALI_FALSE : MALI_TRUE));
1240                 break;
1241         case FBDUMP_CONTROL_RATE:
1242                 mali_set_user_setting(_MALI_UK_USER_SETTING_BUFFER_CAPTURE_N_FRAMES, value);
1243                 break;
1244         case SW_COUNTER_ENABLE:
1245                 mali_set_user_setting(_MALI_UK_USER_SETTING_SW_COUNTER_ENABLED, value);
1246                 break;
1247         case FBDUMP_CONTROL_RESIZE_FACTOR:
1248                 mali_set_user_setting(_MALI_UK_USER_SETTING_BUFFER_CAPTURE_RESIZE_FACTOR, value);
1249                 break;
1250         default:
1251                 break;  /* Ignore unimplemented actions */
1252         }
1253 }
1254
1255 /**
1256  * Called by gator to get mali api version.
1257  */
1258 u32 _mali_profiling_get_api_version(void)
1259 {
1260         return MALI_PROFILING_API_VERSION;
1261 }
1262
1263 /**
1264 * Called by gator to get the data about Mali instance in use:
1265 * product id, version, number of cores
1266 */
1267 void _mali_profiling_get_mali_version(struct _mali_profiling_mali_version *values)
1268 {
1269         values->mali_product_id = (u32)mali_kernel_core_get_product_id();
1270         values->mali_version_major = mali_kernel_core_get_gpu_major_version();
1271         values->mali_version_minor = mali_kernel_core_get_gpu_minor_version();
1272         values->num_of_l2_cores = mali_l2_cache_core_get_glob_num_l2_cores();
1273         values->num_of_fp_cores = mali_executor_get_num_cores_total();
1274         values->num_of_vp_cores = 1;
1275 }
1276
1277
1278 EXPORT_SYMBOL(_mali_profiling_set_event);
1279 EXPORT_SYMBOL(_mali_profiling_get_l2_counters);
1280 EXPORT_SYMBOL(_mali_profiling_control);
1281 EXPORT_SYMBOL(_mali_profiling_get_api_version);
1282 EXPORT_SYMBOL(_mali_profiling_get_mali_version);