drivers/gpu/arm/midgard_for_linux/mali_kbase_vinstr.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18 #include <linux/anon_inodes.h>
  19 #include <linux/atomic.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/jiffies.h>
  22 #include <linux/kthread.h>
  23 #include <linux/list.h>
  24 #include <linux/mm.h>
  25 #include <linux/poll.h>
  26 #include <linux/preempt.h>
  27 #include <linux/slab.h>
  28 #include <linux/wait.h>
  29
  30 #include <mali_kbase.h>
  31 #include <mali_kbase_hwaccess_instr.h>
  32 #include <mali_kbase_hwcnt_reader.h>
  33 #include <mali_kbase_mem_linux.h>
  34 #include <mali_kbase_tlstream.h>
  35
  36 /*****************************************************************************/
  37
  38 /* Hwcnt reader API version */
  39 #define HWCNT_READER_API        1
  40
  41 /* The number of nanoseconds in a second. */
  42 #define NSECS_IN_SEC            1000000000ull /* ns */
  43
  44 /* The time resolution of dumping service. */
  45 #define DUMPING_RESOLUTION      500000ull /* ns */
  46
  47 /* The maximal supported number of dumping buffers. */
  48 #define MAX_BUFFER_COUNT        32
  49
  50 /* Size and number of hw counters blocks. */
  51 #define NR_CNT_BLOCKS_PER_GROUP 8
  52 #define NR_CNT_PER_BLOCK        64
  53 #define NR_BYTES_PER_CNT        4
  54 #define NR_BYTES_PER_HDR        16
  55 #define PRFCNT_EN_MASK_OFFSET   0x8
  56
  57 /*****************************************************************************/
  58
  59 enum {
  60         SHADER_HWCNT_BM,
  61         TILER_HWCNT_BM,
  62         MMU_L2_HWCNT_BM,
  63         JM_HWCNT_BM
  64 };
  65
  66 enum vinstr_state {
  67         VINSTR_IDLE,
  68         VINSTR_DUMPING,
  69         VINSTR_SUSPENDING,
  70         VINSTR_SUSPENDED,
  71         VINSTR_RESUMING
  72 };
  73
  74 /**
  75  * struct kbase_vinstr_context - vinstr context per device
  76  * @lock:              protects the entire vinstr context
  77  * @kbdev:             pointer to kbase device
  78  * @kctx:              pointer to kbase context
  79  * @vmap:              vinstr vmap for mapping hwcnt dump buffer
  80  * @gpu_va:            GPU hwcnt dump buffer address
  81  * @cpu_va:            the CPU side mapping of the hwcnt dump buffer
  82  * @dump_size:         size of the dump buffer in bytes
  83  * @bitmap:            current set of counters monitored, not always in sync
  84  *                     with hardware
  85  * @reprogram:         when true, reprogram hwcnt block with the new set of
  86  *                     counters
  87  * @state:             vinstr state
  88  * @state_lock:        protects information about vinstr state
  89  * @suspend_waitq:     notification queue to trigger state re-validation
  90  * @suspend_cnt:       reference counter of vinstr's suspend state
  91  * @suspend_work:      worker to execute on entering suspended state
  92  * @resume_work:       worker to execute on leaving suspended state
  93  * @nclients:          number of attached clients, pending or otherwise
  94  * @waiting_clients:   head of list of clients being periodically sampled
  95  * @idle_clients:      head of list of clients being idle
  96  * @suspended_clients: head of list of clients being suspended
  97  * @thread:            periodic sampling thread
  98  * @waitq:             notification queue of sampling thread
  99  * @request_pending:   request for action for sampling thread
 100  */
 101 struct kbase_vinstr_context {
 102         struct mutex             lock;
 103         struct kbase_device      *kbdev;
 104         struct kbase_context     *kctx;
 105
 106         struct kbase_vmap_struct vmap;
 107         u64                      gpu_va;
 108         void                     *cpu_va;
 109         size_t                   dump_size;
 110         u32                      bitmap[4];
 111         bool                     reprogram;
 112
 113         enum vinstr_state        state;
 114         struct spinlock          state_lock;
 115         wait_queue_head_t        suspend_waitq;
 116         unsigned int             suspend_cnt;
 117         struct work_struct       suspend_work;
 118         struct work_struct       resume_work;
 119
 120         u32                      nclients;
 121         struct list_head         waiting_clients;
 122         struct list_head         idle_clients;
 123         struct list_head         suspended_clients;
 124
 125         struct task_struct       *thread;
 126         wait_queue_head_t        waitq;
 127         atomic_t                 request_pending;
 128 };
 129
 130 /**
 131  * struct kbase_vinstr_client - a vinstr client attached to a vinstr context
 132  * @vinstr_ctx:    vinstr context client is attached to
 133  * @list:          node used to attach this client to list in vinstr context
 134  * @buffer_count:  number of buffers this client is using
 135  * @event_mask:    events this client reacts to
 136  * @dump_size:     size of one dump buffer in bytes
 137  * @bitmap:        bitmap request for JM, TILER, SHADER and MMU counters
 138  * @legacy_buffer: userspace hwcnt dump buffer (legacy interface)
 139  * @kernel_buffer: kernel hwcnt dump buffer (kernel client interface)
 140  * @accum_buffer:  temporary accumulation buffer for preserving counters
 141  * @dump_time:     next time this clients shall request hwcnt dump
 142  * @dump_interval: interval between periodic hwcnt dumps
 143  * @dump_buffers:  kernel hwcnt dump buffers allocated by this client
 144  * @dump_buffers_meta: metadata of dump buffers
 145  * @meta_idx:      index of metadata being accessed by userspace
 146  * @read_idx:      index of buffer read by userspace
 147  * @write_idx:     index of buffer being written by dumping service
 148  * @waitq:         client's notification queue
 149  * @pending:       when true, client has attached but hwcnt not yet updated
 150  */
 151 struct kbase_vinstr_client {
 152         struct kbase_vinstr_context        *vinstr_ctx;
 153         struct list_head                   list;
 154         unsigned int                       buffer_count;
 155         u32                                event_mask;
 156         size_t                             dump_size;
 157         u32                                bitmap[4];
 158         void __user                        *legacy_buffer;
 159         void                               *kernel_buffer;
 160         void                               *accum_buffer;
 161         u64                                dump_time;
 162         u32                                dump_interval;
 163         char                               *dump_buffers;
 164         struct kbase_hwcnt_reader_metadata *dump_buffers_meta;
 165         atomic_t                           meta_idx;
 166         atomic_t                           read_idx;
 167         atomic_t                           write_idx;
 168         wait_queue_head_t                  waitq;
 169         bool                               pending;
 170 };
 171
 172 /**
 173  * struct kbasep_vinstr_wake_up_timer - vinstr service thread wake up timer
 174  * @hrtimer:    high resolution timer
 175  * @vinstr_ctx: vinstr context
 176  */
 177 struct kbasep_vinstr_wake_up_timer {
 178         struct hrtimer              hrtimer;
 179         struct kbase_vinstr_context *vinstr_ctx;
 180 };
 181
 182 /*****************************************************************************/
 183
 184 static int kbasep_vinstr_service_task(void *data);
 185
 186 static unsigned int kbasep_vinstr_hwcnt_reader_poll(
 187                 struct file *filp,
 188                 poll_table  *wait);
 189 static long kbasep_vinstr_hwcnt_reader_ioctl(
 190                 struct file   *filp,
 191                 unsigned int  cmd,
 192                 unsigned long arg);
 193 static int kbasep_vinstr_hwcnt_reader_mmap(
 194                 struct file           *filp,
 195                 struct vm_area_struct *vma);
 196 static int kbasep_vinstr_hwcnt_reader_release(
 197                 struct inode *inode,
 198                 struct file  *filp);
 199
 200 /* The timeline stream file operations structure. */
 201 static const struct file_operations vinstr_client_fops = {
 202         .poll           = kbasep_vinstr_hwcnt_reader_poll,
 203         .unlocked_ioctl = kbasep_vinstr_hwcnt_reader_ioctl,
 204         .compat_ioctl   = kbasep_vinstr_hwcnt_reader_ioctl,
 205         .mmap           = kbasep_vinstr_hwcnt_reader_mmap,
 206         .release        = kbasep_vinstr_hwcnt_reader_release,
 207 };
 208
 209 /*****************************************************************************/
 210
 211 static int enable_hwcnt(struct kbase_vinstr_context *vinstr_ctx)
 212 {
 213         struct kbase_context *kctx = vinstr_ctx->kctx;
 214         struct kbase_device *kbdev = kctx->kbdev;
 215         struct kbase_uk_hwcnt_setup setup;
 216         int err;
 217
 218         setup.dump_buffer = vinstr_ctx->gpu_va;
 219         setup.jm_bm       = vinstr_ctx->bitmap[JM_HWCNT_BM];
 220         setup.tiler_bm    = vinstr_ctx->bitmap[TILER_HWCNT_BM];
 221         setup.shader_bm   = vinstr_ctx->bitmap[SHADER_HWCNT_BM];
 222         setup.mmu_l2_bm   = vinstr_ctx->bitmap[MMU_L2_HWCNT_BM];
 223
 224         /* Mark the context as active so the GPU is kept turned on */
 225         /* A suspend won't happen here, because we're in a syscall from a
 226          * userspace thread. */
 227         kbase_pm_context_active(kbdev);
 228
 229         /* Schedule the context in */
 230         kbasep_js_schedule_privileged_ctx(kbdev, kctx);
 231         err = kbase_instr_hwcnt_enable_internal(kbdev, kctx, &setup);
 232         if (err) {
 233                 /* Release the context. This had its own Power Manager Active
 234                  * reference */
 235                 kbasep_js_release_privileged_ctx(kbdev, kctx);
 236
 237                 /* Also release our Power Manager Active reference */
 238                 kbase_pm_context_idle(kbdev);
 239         }
 240
 241         return err;
 242 }
 243
 244 static void disable_hwcnt(struct kbase_vinstr_context *vinstr_ctx)
 245 {
 246         struct kbase_context *kctx = vinstr_ctx->kctx;
 247         struct kbase_device *kbdev = kctx->kbdev;
 248         int err;
 249
 250         err = kbase_instr_hwcnt_disable_internal(kctx);
 251         if (err) {
 252                 dev_warn(kbdev->dev, "Failed to disable HW counters (ctx:%p)",
 253                                 kctx);
 254                 return;
 255         }
 256
 257         /* Release the context. This had its own Power Manager Active reference. */
 258         kbasep_js_release_privileged_ctx(kbdev, kctx);
 259
 260         /* Also release our Power Manager Active reference. */
 261         kbase_pm_context_idle(kbdev);
 262
 263         dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", kctx);
 264 }
 265
 266 static int reprogram_hwcnt(struct kbase_vinstr_context *vinstr_ctx)
 267 {
 268         disable_hwcnt(vinstr_ctx);
 269         return enable_hwcnt(vinstr_ctx);
 270 }
 271
 272 static void hwcnt_bitmap_set(u32 dst[4], u32 src[4])
 273 {
 274         dst[JM_HWCNT_BM]     = src[JM_HWCNT_BM];
 275         dst[TILER_HWCNT_BM]  = src[TILER_HWCNT_BM];
 276         dst[SHADER_HWCNT_BM] = src[SHADER_HWCNT_BM];
 277         dst[MMU_L2_HWCNT_BM] = src[MMU_L2_HWCNT_BM];
 278 }
 279
 280 static void hwcnt_bitmap_union(u32 dst[4], u32 src[4])
 281 {
 282         dst[JM_HWCNT_BM]     |= src[JM_HWCNT_BM];
 283         dst[TILER_HWCNT_BM]  |= src[TILER_HWCNT_BM];
 284         dst[SHADER_HWCNT_BM] |= src[SHADER_HWCNT_BM];
 285         dst[MMU_L2_HWCNT_BM] |= src[MMU_L2_HWCNT_BM];
 286 }
 287
 288 size_t kbase_vinstr_dump_size(struct kbase_device *kbdev)
 289 {
 290         size_t dump_size;
 291
 292 #ifndef CONFIG_MALI_NO_MALI
 293         if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_V4)) {
 294                 u32 nr_cg;
 295
 296                 nr_cg = kbdev->gpu_props.num_core_groups;
 297                 dump_size = nr_cg * NR_CNT_BLOCKS_PER_GROUP *
 298                                 NR_CNT_PER_BLOCK *
 299                                 NR_BYTES_PER_CNT;
 300         } else
 301 #endif /* CONFIG_MALI_NO_MALI */
 302         {
 303                 /* assume v5 for now */
 304                 base_gpu_props *props = &kbdev->gpu_props.props;
 305                 u32 nr_l2 = props->l2_props.num_l2_slices;
 306                 u64 core_mask = props->coherency_info.group[0].core_mask;
 307                 u32 nr_blocks = fls64(core_mask);
 308
 309                 /* JM and tiler counter blocks are always present */
 310                 dump_size = (2 + nr_l2 + nr_blocks) *
 311                                 NR_CNT_PER_BLOCK *
 312                                 NR_BYTES_PER_CNT;
 313         }
 314         return dump_size;
 315 }
 316 KBASE_EXPORT_TEST_API(kbase_vinstr_dump_size);
 317
 318 static size_t kbasep_vinstr_dump_size_ctx(
 319                 struct kbase_vinstr_context *vinstr_ctx)
 320 {
 321         return kbase_vinstr_dump_size(vinstr_ctx->kctx->kbdev);
 322 }
 323
 324 static int kbasep_vinstr_map_kernel_dump_buffer(
 325                 struct kbase_vinstr_context *vinstr_ctx)
 326 {
 327         struct kbase_va_region *reg;
 328         struct kbase_context *kctx = vinstr_ctx->kctx;
 329         u64 flags, nr_pages;
 330         u16 va_align = 0;
 331
 332         flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_WR;
 333         vinstr_ctx->dump_size = kbasep_vinstr_dump_size_ctx(vinstr_ctx);
 334         nr_pages = PFN_UP(vinstr_ctx->dump_size);
 335
 336         reg = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags,
 337                         &vinstr_ctx->gpu_va, &va_align);
 338         if (!reg)
 339                 return -ENOMEM;
 340
 341         vinstr_ctx->cpu_va = kbase_vmap(
 342                         kctx,
 343                         vinstr_ctx->gpu_va,
 344                         vinstr_ctx->dump_size,
 345                         &vinstr_ctx->vmap);
 346         if (!vinstr_ctx->cpu_va) {
 347                 kbase_mem_free(kctx, vinstr_ctx->gpu_va);
 348                 return -ENOMEM;
 349         }
 350
 351         return 0;
 352 }
 353
 354 static void kbasep_vinstr_unmap_kernel_dump_buffer(
 355                 struct kbase_vinstr_context *vinstr_ctx)
 356 {
 357         struct kbase_context *kctx = vinstr_ctx->kctx;
 358
 359         kbase_vunmap(kctx, &vinstr_ctx->vmap);
 360         kbase_mem_free(kctx, vinstr_ctx->gpu_va);
 361 }
 362
 363 /**
 364  * kbasep_vinstr_create_kctx - create kernel context for vinstr
 365  * @vinstr_ctx: vinstr context
 366  * Return: zero on success
 367  */
 368 static int kbasep_vinstr_create_kctx(struct kbase_vinstr_context *vinstr_ctx)
 369 {
 370         struct kbase_device *kbdev = vinstr_ctx->kbdev;
 371         struct kbasep_kctx_list_element *element;
 372         unsigned long flags;
 373         bool enable_backend = false;
 374         int err;
 375
 376         vinstr_ctx->kctx = kbase_create_context(vinstr_ctx->kbdev, true);
 377         if (!vinstr_ctx->kctx)
 378                 return -ENOMEM;
 379
 380         /* Map the master kernel dump buffer.  The HW dumps the counters
 381          * into this memory region. */
 382         err = kbasep_vinstr_map_kernel_dump_buffer(vinstr_ctx);
 383         if (err) {
 384                 kbase_destroy_context(vinstr_ctx->kctx);
 385                 vinstr_ctx->kctx = NULL;
 386                 return err;
 387         }
 388
 389         /* Add kernel context to list of contexts associated with device. */
 390         element = kzalloc(sizeof(*element), GFP_KERNEL);
 391         if (element) {
 392                 element->kctx = vinstr_ctx->kctx;
 393                 mutex_lock(&kbdev->kctx_list_lock);
 394                 list_add(&element->link, &kbdev->kctx_list);
 395
 396                 /* Inform timeline client about new context.
 397                  * Do this while holding the lock to avoid tracepoint
 398                  * being created in both body and summary stream. */
 399                 kbase_tlstream_tl_new_ctx(
 400                                 vinstr_ctx->kctx,
 401                                 (u32)(vinstr_ctx->kctx->id),
 402                                 (u32)(vinstr_ctx->kctx->tgid));
 403
 404                 mutex_unlock(&kbdev->kctx_list_lock);
 405         } else {
 406                 /* Don't treat this as a fail - just warn about it. */
 407                 dev_warn(kbdev->dev,
 408                                 "couldn't add kctx to kctx_list\n");
 409         }
 410
 411         /* Don't enable hardware counters if vinstr is suspended.
 412          * Note that vinstr resume code is run under vinstr context lock,
 413          * lower layer will be enabled as needed on resume. */
 414         spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
 415         if (VINSTR_IDLE == vinstr_ctx->state)
 416                 enable_backend = true;
 417         spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
 418         if (enable_backend)
 419                 err = enable_hwcnt(vinstr_ctx);
 420
 421         if (err) {
 422                 kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx);
 423                 kbase_destroy_context(vinstr_ctx->kctx);
 424                 if (element) {
 425                         mutex_lock(&kbdev->kctx_list_lock);
 426                         list_del(&element->link);
 427                         kfree(element);
 428                         mutex_unlock(&kbdev->kctx_list_lock);
 429                 }
 430                 kbase_tlstream_tl_del_ctx(vinstr_ctx->kctx);
 431                 vinstr_ctx->kctx = NULL;
 432                 return err;
 433         }
 434
 435         vinstr_ctx->thread = kthread_run(
 436                         kbasep_vinstr_service_task,
 437                         vinstr_ctx,
 438                         "mali_vinstr_service");
 439         if (!vinstr_ctx->thread) {
 440                 disable_hwcnt(vinstr_ctx);
 441                 kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx);
 442                 kbase_destroy_context(vinstr_ctx->kctx);
 443                 if (element) {
 444                         mutex_lock(&kbdev->kctx_list_lock);
 445                         list_del(&element->link);
 446                         kfree(element);
 447                         mutex_unlock(&kbdev->kctx_list_lock);
 448                 }
 449                 kbase_tlstream_tl_del_ctx(vinstr_ctx->kctx);
 450                 vinstr_ctx->kctx = NULL;
 451                 return -EFAULT;
 452         }
 453
 454         return 0;
 455 }
 456
 457 /**
 458  * kbasep_vinstr_destroy_kctx - destroy vinstr's kernel context
 459  * @vinstr_ctx: vinstr context
 460  */
 461 static void kbasep_vinstr_destroy_kctx(struct kbase_vinstr_context *vinstr_ctx)
 462 {
 463         struct kbase_device             *kbdev = vinstr_ctx->kbdev;
 464         struct kbasep_kctx_list_element *element;
 465         struct kbasep_kctx_list_element *tmp;
 466         bool                            found = false;
 467
 468         /* Release hw counters dumping resources. */
 469         vinstr_ctx->thread = NULL;
 470         disable_hwcnt(vinstr_ctx);
 471         kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx);
 472         kbase_destroy_context(vinstr_ctx->kctx);
 473
 474         /* Remove kernel context from the device's contexts list. */
 475         mutex_lock(&kbdev->kctx_list_lock);
 476         list_for_each_entry_safe(element, tmp, &kbdev->kctx_list, link) {
 477                 if (element->kctx == vinstr_ctx->kctx) {
 478                         list_del(&element->link);
 479                         kfree(element);
 480                         found = true;
 481                 }
 482         }
 483         mutex_unlock(&kbdev->kctx_list_lock);
 484
 485         if (!found)
 486                 dev_warn(kbdev->dev, "kctx not in kctx_list\n");
 487
 488         /* Inform timeline client about context destruction. */
 489         kbase_tlstream_tl_del_ctx(vinstr_ctx->kctx);
 490
 491         vinstr_ctx->kctx = NULL;
 492 }
 493
 494 /**
 495  * kbasep_vinstr_attach_client - Attach a client to the vinstr core
 496  * @vinstr_ctx:    vinstr context
 497  * @buffer_count:  requested number of dump buffers
 498  * @bitmap:        bitmaps describing which counters should be enabled
 499  * @argp:          pointer where notification descriptor shall be stored
 500  * @kernel_buffer: pointer to kernel side buffer
 501  *
 502  * Return: vinstr opaque client handle or NULL on failure
 503  */
 504 static struct kbase_vinstr_client *kbasep_vinstr_attach_client(
 505                 struct kbase_vinstr_context *vinstr_ctx, u32 buffer_count,
 506                 u32 bitmap[4], void *argp, void *kernel_buffer)
 507 {
 508         struct task_struct         *thread = NULL;
 509         struct kbase_vinstr_client *cli;
 510
 511         KBASE_DEBUG_ASSERT(vinstr_ctx);
 512
 513         if (buffer_count > MAX_BUFFER_COUNT
 514             || (buffer_count & (buffer_count - 1)))
 515                 return NULL;
 516
 517         cli = kzalloc(sizeof(*cli), GFP_KERNEL);
 518         if (!cli)
 519                 return NULL;
 520
 521         cli->vinstr_ctx   = vinstr_ctx;
 522         cli->buffer_count = buffer_count;
 523         cli->event_mask   =
 524                 (1 << BASE_HWCNT_READER_EVENT_MANUAL) |
 525                 (1 << BASE_HWCNT_READER_EVENT_PERIODIC);
 526         cli->pending      = true;
 527
 528         hwcnt_bitmap_set(cli->bitmap, bitmap);
 529
 530         mutex_lock(&vinstr_ctx->lock);
 531
 532         hwcnt_bitmap_union(vinstr_ctx->bitmap, cli->bitmap);
 533         vinstr_ctx->reprogram = true;
 534
 535         /* If this is the first client, create the vinstr kbase
 536          * context. This context is permanently resident until the
 537          * last client exits. */
 538         if (!vinstr_ctx->nclients) {
 539                 hwcnt_bitmap_set(vinstr_ctx->bitmap, cli->bitmap);
 540                 if (kbasep_vinstr_create_kctx(vinstr_ctx) < 0)
 541                         goto error;
 542
 543                 vinstr_ctx->reprogram = false;
 544                 cli->pending = false;
 545         }
 546
 547         /* The GPU resets the counter block every time there is a request
 548          * to dump it. We need a per client kernel buffer for accumulating
 549          * the counters. */
 550         cli->dump_size    = kbasep_vinstr_dump_size_ctx(vinstr_ctx);
 551         cli->accum_buffer = kzalloc(cli->dump_size, GFP_KERNEL);
 552         if (!cli->accum_buffer)
 553                 goto error;
 554
 555         /* Prepare buffers. */
 556         if (cli->buffer_count) {
 557                 int *fd = (int *)argp;
 558                 size_t tmp;
 559
 560                 /* Allocate area for buffers metadata storage. */
 561                 tmp = sizeof(struct kbase_hwcnt_reader_metadata) *
 562                         cli->buffer_count;
 563                 cli->dump_buffers_meta = kmalloc(tmp, GFP_KERNEL);
 564                 if (!cli->dump_buffers_meta)
 565                         goto error;
 566
 567                 /* Allocate required number of dumping buffers. */
 568                 cli->dump_buffers = (char *)__get_free_pages(
 569                                 GFP_KERNEL | __GFP_ZERO,
 570                                 get_order(cli->dump_size * cli->buffer_count));
 571                 if (!cli->dump_buffers)
 572                         goto error;
 573
 574                 /* Create descriptor for user-kernel data exchange. */
 575                 *fd = anon_inode_getfd(
 576                                 "[mali_vinstr_desc]",
 577                                 &vinstr_client_fops,
 578                                 cli,
 579                                 O_RDONLY | O_CLOEXEC);
 580                 if (0 > *fd)
 581                         goto error;
 582         } else if (kernel_buffer) {
 583                 cli->kernel_buffer = kernel_buffer;
 584         } else {
 585                 cli->legacy_buffer = (void __user *)argp;
 586         }
 587
 588         atomic_set(&cli->read_idx, 0);
 589         atomic_set(&cli->meta_idx, 0);
 590         atomic_set(&cli->write_idx, 0);
 591         init_waitqueue_head(&cli->waitq);
 592
 593         vinstr_ctx->nclients++;
 594         list_add(&cli->list, &vinstr_ctx->idle_clients);
 595
 596         mutex_unlock(&vinstr_ctx->lock);
 597
 598         return cli;
 599
 600 error:
 601         kfree(cli->dump_buffers_meta);
 602         if (cli->dump_buffers)
 603                 free_pages(
 604                                 (unsigned long)cli->dump_buffers,
 605                                 get_order(cli->dump_size * cli->buffer_count));
 606         kfree(cli->accum_buffer);
 607         if (!vinstr_ctx->nclients && vinstr_ctx->kctx) {
 608                 thread = vinstr_ctx->thread;
 609                 kbasep_vinstr_destroy_kctx(vinstr_ctx);
 610         }
 611         kfree(cli);
 612
 613         mutex_unlock(&vinstr_ctx->lock);
 614
 615         /* Thread must be stopped after lock is released. */
 616         if (thread)
 617                 kthread_stop(thread);
 618
 619         return NULL;
 620 }
 621
 622 void kbase_vinstr_detach_client(struct kbase_vinstr_client *cli)
 623 {
 624         struct kbase_vinstr_context *vinstr_ctx;
 625         struct kbase_vinstr_client  *iter, *tmp;
 626         struct task_struct          *thread = NULL;
 627         u32 zerobitmap[4] = { 0 };
 628         int cli_found = 0;
 629
 630         KBASE_DEBUG_ASSERT(cli);
 631         vinstr_ctx = cli->vinstr_ctx;
 632         KBASE_DEBUG_ASSERT(vinstr_ctx);
 633
 634         mutex_lock(&vinstr_ctx->lock);
 635
 636         list_for_each_entry_safe(iter, tmp, &vinstr_ctx->idle_clients, list) {
 637                 if (iter == cli) {
 638                         vinstr_ctx->reprogram = true;
 639                         cli_found = 1;
 640                         list_del(&iter->list);
 641                         break;
 642                 }
 643         }
 644         if (!cli_found) {
 645                 list_for_each_entry_safe(
 646                                 iter, tmp, &vinstr_ctx->waiting_clients, list) {
 647                         if (iter == cli) {
 648                                 vinstr_ctx->reprogram = true;
 649                                 cli_found = 1;
 650                                 list_del(&iter->list);
 651                                 break;
 652                         }
 653                 }
 654         }
 655         KBASE_DEBUG_ASSERT(cli_found);
 656
 657         kfree(cli->dump_buffers_meta);
 658         free_pages(
 659                         (unsigned long)cli->dump_buffers,
 660                         get_order(cli->dump_size * cli->buffer_count));
 661         kfree(cli->accum_buffer);
 662         kfree(cli);
 663
 664         vinstr_ctx->nclients--;
 665         if (!vinstr_ctx->nclients) {
 666                 thread = vinstr_ctx->thread;
 667                 kbasep_vinstr_destroy_kctx(vinstr_ctx);
 668         }
 669
 670         /* Rebuild context bitmap now that the client has detached */
 671         hwcnt_bitmap_set(vinstr_ctx->bitmap, zerobitmap);
 672         list_for_each_entry(iter, &vinstr_ctx->idle_clients, list)
 673                 hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap);
 674         list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list)
 675                 hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap);
 676
 677         mutex_unlock(&vinstr_ctx->lock);
 678
 679         /* Thread must be stopped after lock is released. */
 680         if (thread)
 681                 kthread_stop(thread);
 682 }
 683 KBASE_EXPORT_TEST_API(kbase_vinstr_detach_client);
 684
 685 /* Accumulate counters in the dump buffer */
 686 static void accum_dump_buffer(void *dst, void *src, size_t dump_size)
 687 {
 688         size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT;
 689         u32 *d = dst;
 690         u32 *s = src;
 691         size_t i, j;
 692
 693         for (i = 0; i < dump_size; i += block_size) {
 694                 /* skip over the header block */
 695                 d += NR_BYTES_PER_HDR / sizeof(u32);
 696                 s += NR_BYTES_PER_HDR / sizeof(u32);
 697                 for (j = 0; j < (block_size - NR_BYTES_PER_HDR) / sizeof(u32); j++) {
 698                         /* saturate result if addition would result in wraparound */
 699                         if (U32_MAX - *d < *s)
 700                                 *d = U32_MAX;
 701                         else
 702                                 *d += *s;
 703                         d++;
 704                         s++;
 705                 }
 706         }
 707 }
 708
 709 /* This is the Midgard v4 patch function.  It copies the headers for each
 710  * of the defined blocks from the master kernel buffer and then patches up
 711  * the performance counter enable mask for each of the blocks to exclude
 712  * counters that were not requested by the client. */
 713 static void patch_dump_buffer_hdr_v4(
 714                 struct kbase_vinstr_context *vinstr_ctx,
 715                 struct kbase_vinstr_client *cli)
 716 {
 717         u32 *mask;
 718         u8 *dst = cli->accum_buffer;
 719         u8 *src = vinstr_ctx->cpu_va;
 720         u32 nr_cg = vinstr_ctx->kctx->kbdev->gpu_props.num_core_groups;
 721         size_t i, group_size, group;
 722         enum {
 723                 SC0_BASE    = 0 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
 724                 SC1_BASE    = 1 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
 725                 SC2_BASE    = 2 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
 726                 SC3_BASE    = 3 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
 727                 TILER_BASE  = 4 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
 728                 MMU_L2_BASE = 5 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
 729                 JM_BASE     = 7 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT
 730         };
 731
 732         group_size = NR_CNT_BLOCKS_PER_GROUP *
 733                         NR_CNT_PER_BLOCK *
 734                         NR_BYTES_PER_CNT;
 735         for (i = 0; i < nr_cg; i++) {
 736                 group = i * group_size;
 737                 /* copy shader core headers */
 738                 memcpy(&dst[group + SC0_BASE], &src[group + SC0_BASE],
 739                        NR_BYTES_PER_HDR);
 740                 memcpy(&dst[group + SC1_BASE], &src[group + SC1_BASE],
 741                        NR_BYTES_PER_HDR);
 742                 memcpy(&dst[group + SC2_BASE], &src[group + SC2_BASE],
 743                       NR_BYTES_PER_HDR);
 744                 memcpy(&dst[group + SC3_BASE], &src[group + SC3_BASE],
 745                       NR_BYTES_PER_HDR);
 746
 747                 /* copy tiler header */
 748                 memcpy(&dst[group + TILER_BASE], &src[group + TILER_BASE],
 749                       NR_BYTES_PER_HDR);
 750
 751                 /* copy mmu header */
 752                 memcpy(&dst[group + MMU_L2_BASE], &src[group + MMU_L2_BASE],
 753                       NR_BYTES_PER_HDR);
 754
 755                 /* copy job manager header */
 756                 memcpy(&dst[group + JM_BASE], &src[group + JM_BASE],
 757                       NR_BYTES_PER_HDR);
 758
 759                 /* patch the shader core enable mask */
 760                 mask = (u32 *)&dst[group + SC0_BASE + PRFCNT_EN_MASK_OFFSET];
 761                 *mask &= cli->bitmap[SHADER_HWCNT_BM];
 762                 mask = (u32 *)&dst[group + SC1_BASE + PRFCNT_EN_MASK_OFFSET];
 763                 *mask &= cli->bitmap[SHADER_HWCNT_BM];
 764                 mask = (u32 *)&dst[group + SC2_BASE + PRFCNT_EN_MASK_OFFSET];
 765                 *mask &= cli->bitmap[SHADER_HWCNT_BM];
 766                 mask = (u32 *)&dst[group + SC3_BASE + PRFCNT_EN_MASK_OFFSET];
 767                 *mask &= cli->bitmap[SHADER_HWCNT_BM];
 768
 769                 /* patch the tiler core enable mask */
 770                 mask = (u32 *)&dst[group + TILER_BASE + PRFCNT_EN_MASK_OFFSET];
 771                 *mask &= cli->bitmap[TILER_HWCNT_BM];
 772
 773                 /* patch the mmu core enable mask */
 774                 mask = (u32 *)&dst[group + MMU_L2_BASE + PRFCNT_EN_MASK_OFFSET];
 775                 *mask &= cli->bitmap[MMU_L2_HWCNT_BM];
 776
 777                 /* patch the job manager enable mask */
 778                 mask = (u32 *)&dst[group + JM_BASE + PRFCNT_EN_MASK_OFFSET];
 779                 *mask &= cli->bitmap[JM_HWCNT_BM];
 780         }
 781 }
 782
 783 /* This is the Midgard v5 patch function.  It copies the headers for each
 784  * of the defined blocks from the master kernel buffer and then patches up
 785  * the performance counter enable mask for each of the blocks to exclude
 786  * counters that were not requested by the client. */
 787 static void patch_dump_buffer_hdr_v5(
 788                 struct kbase_vinstr_context *vinstr_ctx,
 789                 struct kbase_vinstr_client *cli)
 790 {
 791         struct kbase_device *kbdev = vinstr_ctx->kctx->kbdev;
 792         u32 i, nr_l2;
 793         u64 core_mask;
 794         u32 *mask;
 795         u8 *dst = cli->accum_buffer;
 796         u8 *src = vinstr_ctx->cpu_va;
 797         size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT;
 798
 799         /* copy and patch job manager header */
 800         memcpy(dst, src, NR_BYTES_PER_HDR);
 801         mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
 802         *mask &= cli->bitmap[JM_HWCNT_BM];
 803         dst += block_size;
 804         src += block_size;
 805
 806         /* copy and patch tiler header */
 807         memcpy(dst, src, NR_BYTES_PER_HDR);
 808         mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
 809         *mask &= cli->bitmap[TILER_HWCNT_BM];
 810         dst += block_size;
 811         src += block_size;
 812
 813         /* copy and patch MMU/L2C headers */
 814         nr_l2 = kbdev->gpu_props.props.l2_props.num_l2_slices;
 815         for (i = 0; i < nr_l2; i++) {
 816                 memcpy(dst, src, NR_BYTES_PER_HDR);
 817                 mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
 818                 *mask &= cli->bitmap[MMU_L2_HWCNT_BM];
 819                 dst += block_size;
 820                 src += block_size;
 821         }
 822
 823         /* copy and patch shader core headers */
 824         core_mask = kbdev->gpu_props.props.coherency_info.group[0].core_mask;
 825         while (0ull != core_mask) {
 826                 memcpy(dst, src, NR_BYTES_PER_HDR);
 827                 if (0ull != (core_mask & 1ull)) {
 828                         /* if block is not reserved update header */
 829                         mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
 830                         *mask &= cli->bitmap[SHADER_HWCNT_BM];
 831                 }
 832                 dst += block_size;
 833                 src += block_size;
 834
 835                 core_mask >>= 1;
 836         }
 837 }
 838
 839 /**
 840  * accum_clients - accumulate dumped hw counters for all known clients
 841  * @vinstr_ctx: vinstr context
 842  */
 843 static void accum_clients(struct kbase_vinstr_context *vinstr_ctx)
 844 {
 845         struct kbase_vinstr_client *iter;
 846         int v4 = 0;
 847
 848 #ifndef CONFIG_MALI_NO_MALI
 849         v4 = kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4);
 850 #endif
 851
 852         list_for_each_entry(iter, &vinstr_ctx->idle_clients, list) {
 853                 /* Don't bother accumulating clients whose hwcnt requests
 854                  * have not yet been honoured. */
 855                 if (iter->pending)
 856                         continue;
 857                 if (v4)
 858                         patch_dump_buffer_hdr_v4(vinstr_ctx, iter);
 859                 else
 860                         patch_dump_buffer_hdr_v5(vinstr_ctx, iter);
 861                 accum_dump_buffer(
 862                                 iter->accum_buffer,
 863                                 vinstr_ctx->cpu_va,
 864                                 iter->dump_size);
 865         }
 866         list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list) {
 867                 /* Don't bother accumulating clients whose hwcnt requests
 868                  * have not yet been honoured. */
 869                 if (iter->pending)
 870                         continue;
 871                 if (v4)
 872                         patch_dump_buffer_hdr_v4(vinstr_ctx, iter);
 873                 else
 874                         patch_dump_buffer_hdr_v5(vinstr_ctx, iter);
 875                 accum_dump_buffer(
 876                                 iter->accum_buffer,
 877                                 vinstr_ctx->cpu_va,
 878                                 iter->dump_size);
 879         }
 880 }
 881
 882 /*****************************************************************************/
 883
 884 /**
 885  * kbasep_vinstr_get_timestamp - return timestamp
 886  *
 887  * Function returns timestamp value based on raw monotonic timer. Value will
 888  * wrap around zero in case of overflow.
 889  *
 890  * Return: timestamp value
 891  */
 892 static u64 kbasep_vinstr_get_timestamp(void)
 893 {
 894         struct timespec ts;
 895
 896         getrawmonotonic(&ts);
 897         return (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec;
 898 }
 899
 900 /**
 901  * kbasep_vinstr_add_dump_request - register client's dumping request
 902  * @cli:             requesting client
 903  * @waiting_clients: list of pending dumping requests
 904  */
 905 static void kbasep_vinstr_add_dump_request(
 906                 struct kbase_vinstr_client *cli,
 907                 struct list_head *waiting_clients)
 908 {
 909         struct kbase_vinstr_client *tmp;
 910
 911         if (list_empty(waiting_clients)) {
 912                 list_add(&cli->list, waiting_clients);
 913                 return;
 914         }
 915         list_for_each_entry(tmp, waiting_clients, list) {
 916                 if (tmp->dump_time > cli->dump_time) {
 917                         list_add_tail(&cli->list, &tmp->list);
 918                         return;
 919                 }
 920         }
 921         list_add_tail(&cli->list, waiting_clients);
 922 }
 923
 924 /**
 925  * kbasep_vinstr_collect_and_accumulate - collect hw counters via low level
 926  *                                        dump and accumulate them for known
 927  *                                        clients
 928  * @vinstr_ctx: vinstr context
 929  * @timestamp: pointer where collection timestamp will be recorded
 930  *
 931  * Return: zero on success
 932  */
 933 static int kbasep_vinstr_collect_and_accumulate(
 934                 struct kbase_vinstr_context *vinstr_ctx, u64 *timestamp)
 935 {
 936         unsigned long flags;
 937         int rcode;
 938
 939 #ifdef CONFIG_MALI_NO_MALI
 940         /* The dummy model needs the CPU mapping. */
 941         gpu_model_set_dummy_prfcnt_base_cpu(vinstr_ctx->cpu_va);
 942 #endif
 943
 944         spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
 945         if (VINSTR_IDLE != vinstr_ctx->state) {
 946                 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
 947                 return -EAGAIN;
 948         } else {
 949                 vinstr_ctx->state = VINSTR_DUMPING;
 950         }
 951         spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
 952
 953         /* Request HW counters dump.
 954          * Disable preemption to make dump timestamp more accurate. */
 955         preempt_disable();
 956         *timestamp = kbasep_vinstr_get_timestamp();
 957         rcode = kbase_instr_hwcnt_request_dump(vinstr_ctx->kctx);
 958         preempt_enable();
 959
 960         if (!rcode)
 961                 rcode = kbase_instr_hwcnt_wait_for_dump(vinstr_ctx->kctx);
 962         WARN_ON(rcode);
 963
 964         spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
 965         switch (vinstr_ctx->state)
 966         {
 967         case VINSTR_SUSPENDING:
 968                 schedule_work(&vinstr_ctx->suspend_work);
 969                 break;
 970         case VINSTR_DUMPING:
 971                 vinstr_ctx->state = VINSTR_IDLE;
 972                 wake_up_all(&vinstr_ctx->suspend_waitq);
 973                 break;
 974         default:
 975                 break;
 976         }
 977         spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
 978
 979         /* Accumulate values of collected counters. */
 980         if (!rcode)
 981                 accum_clients(vinstr_ctx);
 982
 983         return rcode;
 984 }
 985
 986 /**
 987  * kbasep_vinstr_fill_dump_buffer - copy accumulated counters to empty kernel
 988  *                                  buffer
 989  * @cli:       requesting client
 990  * @timestamp: timestamp when counters were collected
 991  * @event_id:  id of event that caused triggered counters collection
 992  *
 993  * Return: zero on success
 994  */
 995 static int kbasep_vinstr_fill_dump_buffer(
 996                 struct kbase_vinstr_client *cli, u64 timestamp,
 997                 enum base_hwcnt_reader_event event_id)
 998 {
 999         unsigned int write_idx = atomic_read(&cli->write_idx);
1000         unsigned int read_idx  = atomic_read(&cli->read_idx);
1001
1002         struct kbase_hwcnt_reader_metadata *meta;
1003         void                               *buffer;
1004
1005         /* Check if there is a place to copy HWC block into. */
1006         if (write_idx - read_idx == cli->buffer_count)
1007                 return -1;
1008         write_idx %= cli->buffer_count;
1009
1010         /* Fill in dump buffer and its metadata. */
1011         buffer = &cli->dump_buffers[write_idx * cli->dump_size];
1012         meta   = &cli->dump_buffers_meta[write_idx];
1013         meta->timestamp  = timestamp;
1014         meta->event_id   = event_id;
1015         meta->buffer_idx = write_idx;
1016         memcpy(buffer, cli->accum_buffer, cli->dump_size);
1017         return 0;
1018 }
1019
1020 /**
1021  * kbasep_vinstr_fill_dump_buffer_legacy - copy accumulated counters to buffer
1022  *                                         allocated in userspace
1023  * @cli: requesting client
1024  *
1025  * Return: zero on success
1026  *
1027  * This is part of legacy ioctl interface.
1028  */
1029 static int kbasep_vinstr_fill_dump_buffer_legacy(
1030                 struct kbase_vinstr_client *cli)
1031 {
1032         void __user  *buffer = cli->legacy_buffer;
1033         int          rcode;
1034
1035         /* Copy data to user buffer. */
1036         rcode = copy_to_user(buffer, cli->accum_buffer, cli->dump_size);
1037         if (rcode)
1038                 pr_warn("error while copying buffer to user\n");
1039         return rcode;
1040 }
1041
1042 /**
1043  * kbasep_vinstr_fill_dump_buffer_kernel - copy accumulated counters to buffer
1044  *                                         allocated in kernel space
1045  * @cli: requesting client
1046  *
1047  * Return: zero on success
1048  *
1049  * This is part of the kernel client interface.
1050  */
1051 static int kbasep_vinstr_fill_dump_buffer_kernel(
1052                 struct kbase_vinstr_client *cli)
1053 {
1054         memcpy(cli->kernel_buffer, cli->accum_buffer, cli->dump_size);
1055
1056         return 0;
1057 }
1058
1059 /**
1060  * kbasep_vinstr_reprogram - reprogram hwcnt set collected by inst
1061  * @vinstr_ctx: vinstr context
1062  */
1063 static void kbasep_vinstr_reprogram(
1064                 struct kbase_vinstr_context *vinstr_ctx)
1065 {
1066         unsigned long flags;
1067         bool suspended = false;
1068
1069         /* Don't enable hardware counters if vinstr is suspended. */
1070         spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
1071         if (VINSTR_IDLE != vinstr_ctx->state)
1072                 suspended = true;
1073         spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
1074         if (suspended)
1075                 return;
1076
1077         /* Change to suspended state is done while holding vinstr context
1078          * lock. Below code will then no re-enable the instrumentation. */
1079
1080         if (vinstr_ctx->reprogram) {
1081                 struct kbase_vinstr_client *iter;
1082
1083                 if (!reprogram_hwcnt(vinstr_ctx)) {
1084                         vinstr_ctx->reprogram = false;
1085                         list_for_each_entry(
1086                                         iter,
1087                                         &vinstr_ctx->idle_clients,
1088                                         list)
1089                                 iter->pending = false;
1090                         list_for_each_entry(
1091                                         iter,
1092                                         &vinstr_ctx->waiting_clients,
1093                                         list)
1094                                 iter->pending = false;
1095                 }
1096         }
1097 }
1098
1099 /**
1100  * kbasep_vinstr_update_client - copy accumulated counters to user readable
1101  *                               buffer and notify the user
1102  * @cli:       requesting client
1103  * @timestamp: timestamp when counters were collected
1104  * @event_id:  id of event that caused triggered counters collection
1105  *
1106  * Return: zero on success
1107  */
1108 static int kbasep_vinstr_update_client(
1109                 struct kbase_vinstr_client *cli, u64 timestamp,
1110                 enum base_hwcnt_reader_event event_id)
1111 {
1112         int rcode = 0;
1113
1114         /* Copy collected counters to user readable buffer. */
1115         if (cli->buffer_count)
1116                 rcode = kbasep_vinstr_fill_dump_buffer(
1117                                 cli, timestamp, event_id);
1118         else if (cli->kernel_buffer)
1119                 rcode = kbasep_vinstr_fill_dump_buffer_kernel(cli);
1120         else
1121                 rcode = kbasep_vinstr_fill_dump_buffer_legacy(cli);
1122
1123         if (rcode)
1124                 goto exit;
1125
1126
1127         /* Notify client. Make sure all changes to memory are visible. */
1128         wmb();
1129         atomic_inc(&cli->write_idx);
1130         wake_up_interruptible(&cli->waitq);
1131
1132         /* Prepare for next request. */
1133         memset(cli->accum_buffer, 0, cli->dump_size);
1134
1135 exit:
1136         return rcode;
1137 }
1138
1139 /**
1140  * kbasep_vinstr_wake_up_callback - vinstr wake up timer wake up function
1141  *
1142  * @hrtimer: high resolution timer
1143  *
1144  * Return: High resolution timer restart enum.
1145  */
1146 static enum hrtimer_restart kbasep_vinstr_wake_up_callback(
1147                 struct hrtimer *hrtimer)
1148 {
1149         struct kbasep_vinstr_wake_up_timer *timer =
1150                 container_of(
1151                         hrtimer,
1152                         struct kbasep_vinstr_wake_up_timer,
1153                         hrtimer);
1154
1155         KBASE_DEBUG_ASSERT(timer);
1156
1157         atomic_set(&timer->vinstr_ctx->request_pending, 1);
1158         wake_up_all(&timer->vinstr_ctx->waitq);
1159
1160         return HRTIMER_NORESTART;
1161 }
1162
1163 /**
1164  * kbasep_vinstr_service_task - HWC dumping service thread
1165  *
1166  * @data: Pointer to vinstr context structure.
1167  *
1168  * Return: Always returns zero.
1169  */
1170 static int kbasep_vinstr_service_task(void *data)
1171 {
1172         struct kbase_vinstr_context        *vinstr_ctx = data;
1173         struct kbasep_vinstr_wake_up_timer timer;
1174
1175         KBASE_DEBUG_ASSERT(vinstr_ctx);
1176
1177         hrtimer_init(&timer.hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1178         timer.hrtimer.function = kbasep_vinstr_wake_up_callback;
1179         timer.vinstr_ctx       = vinstr_ctx;
1180
1181         while (!kthread_should_stop()) {
1182                 struct kbase_vinstr_client *cli = NULL;
1183                 struct kbase_vinstr_client *tmp;
1184                 int                        rcode;
1185
1186                 u64              timestamp = kbasep_vinstr_get_timestamp();
1187                 u64              dump_time = 0;
1188                 struct list_head expired_requests;
1189
1190                 /* Hold lock while performing operations on lists of clients. */
1191                 mutex_lock(&vinstr_ctx->lock);
1192
1193                 /* Closing thread must not interact with client requests. */
1194                 if (current == vinstr_ctx->thread) {
1195                         atomic_set(&vinstr_ctx->request_pending, 0);
1196
1197                         if (!list_empty(&vinstr_ctx->waiting_clients)) {
1198                                 cli = list_first_entry(
1199                                                 &vinstr_ctx->waiting_clients,
1200                                                 struct kbase_vinstr_client,
1201                                                 list);
1202                                 dump_time = cli->dump_time;
1203                         }
1204                 }
1205
1206                 if (!cli || ((s64)timestamp - (s64)dump_time < 0ll)) {
1207                         mutex_unlock(&vinstr_ctx->lock);
1208
1209                         /* Sleep until next dumping event or service request. */
1210                         if (cli) {
1211                                 u64 diff = dump_time - timestamp;
1212
1213                                 hrtimer_start(
1214                                                 &timer.hrtimer,
1215                                                 ns_to_ktime(diff),
1216                                                 HRTIMER_MODE_REL);
1217                         }
1218                         wait_event(
1219                                         vinstr_ctx->waitq,
1220                                         atomic_read(
1221                                                 &vinstr_ctx->request_pending) ||
1222                                         kthread_should_stop());
1223                         hrtimer_cancel(&timer.hrtimer);
1224                         continue;
1225                 }
1226
1227                 rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx,
1228                                 &timestamp);
1229
1230                 INIT_LIST_HEAD(&expired_requests);
1231
1232                 /* Find all expired requests. */
1233                 list_for_each_entry_safe(
1234                                 cli,
1235                                 tmp,
1236                                 &vinstr_ctx->waiting_clients,
1237                                 list) {
1238                         s64 tdiff =
1239                                 (s64)(timestamp + DUMPING_RESOLUTION) -
1240                                 (s64)cli->dump_time;
1241                         if (tdiff >= 0ll) {
1242                                 list_del(&cli->list);
1243                                 list_add(&cli->list, &expired_requests);
1244                         } else {
1245                                 break;
1246                         }
1247                 }
1248
1249                 /* Fill data for each request found. */
1250                 list_for_each_entry_safe(cli, tmp, &expired_requests, list) {
1251                         /* Ensure that legacy buffer will not be used from
1252                          * this kthread context. */
1253                         BUG_ON(0 == cli->buffer_count);
1254                         /* Expect only periodically sampled clients. */
1255                         BUG_ON(0 == cli->dump_interval);
1256
1257                         if (!rcode)
1258                                 kbasep_vinstr_update_client(
1259                                                 cli,
1260                                                 timestamp,
1261                                                 BASE_HWCNT_READER_EVENT_PERIODIC);
1262
1263                         /* Set new dumping time. Drop missed probing times. */
1264                         do {
1265                                 cli->dump_time += cli->dump_interval;
1266                         } while (cli->dump_time < timestamp);
1267
1268                         list_del(&cli->list);
1269                         kbasep_vinstr_add_dump_request(
1270                                         cli,
1271                                         &vinstr_ctx->waiting_clients);
1272                 }
1273
1274                 /* Reprogram counters set if required. */
1275                 kbasep_vinstr_reprogram(vinstr_ctx);
1276
1277                 mutex_unlock(&vinstr_ctx->lock);
1278         }
1279
1280         return 0;
1281 }
1282
1283 /*****************************************************************************/
1284
1285 /**
1286  * kbasep_vinstr_hwcnt_reader_buffer_ready - check if client has ready buffers
1287  * @cli: pointer to vinstr client structure
1288  *
1289  * Return: non-zero if client has at least one dumping buffer filled that was
1290  *         not notified to user yet
1291  */
1292 static int kbasep_vinstr_hwcnt_reader_buffer_ready(
1293                 struct kbase_vinstr_client *cli)
1294 {
1295         KBASE_DEBUG_ASSERT(cli);
1296         return atomic_read(&cli->write_idx) != atomic_read(&cli->meta_idx);
1297 }
1298
1299 /**
1300  * kbasep_vinstr_hwcnt_reader_ioctl_get_buffer - hwcnt reader's ioctl command
1301  * @cli:    pointer to vinstr client structure
1302  * @buffer: pointer to userspace buffer
1303  * @size:   size of buffer
1304  *
1305  * Return: zero on success
1306  */
1307 static long kbasep_vinstr_hwcnt_reader_ioctl_get_buffer(
1308                 struct kbase_vinstr_client *cli, void __user *buffer,
1309                 size_t size)
1310 {
1311         unsigned int meta_idx = atomic_read(&cli->meta_idx);
1312         unsigned int idx = meta_idx % cli->buffer_count;
1313
1314         struct kbase_hwcnt_reader_metadata *meta = &cli->dump_buffers_meta[idx];
1315
1316         /* Metadata sanity check. */
1317         KBASE_DEBUG_ASSERT(idx == meta->buffer_idx);
1318
1319         if (sizeof(struct kbase_hwcnt_reader_metadata) != size)
1320                 return -EINVAL;
1321
1322         /* Check if there is any buffer available. */
1323         if (atomic_read(&cli->write_idx) == meta_idx)
1324                 return -EAGAIN;
1325
1326         /* Check if previously taken buffer was put back. */
1327         if (atomic_read(&cli->read_idx) != meta_idx)
1328                 return -EBUSY;
1329
1330         /* Copy next available buffer's metadata to user. */
1331         if (copy_to_user(buffer, meta, size))
1332                 return -EFAULT;
1333
1334         atomic_inc(&cli->meta_idx);
1335
1336         return 0;
1337 }
1338
1339 /**
1340  * kbasep_vinstr_hwcnt_reader_ioctl_put_buffer - hwcnt reader's ioctl command
1341  * @cli:    pointer to vinstr client structure
1342  * @buffer: pointer to userspace buffer
1343  * @size:   size of buffer
1344  *
1345  * Return: zero on success
1346  */
1347 static long kbasep_vinstr_hwcnt_reader_ioctl_put_buffer(
1348                 struct kbase_vinstr_client *cli, void __user *buffer,
1349                 size_t size)
1350 {
1351         unsigned int read_idx = atomic_read(&cli->read_idx);
1352         unsigned int idx = read_idx % cli->buffer_count;
1353
1354         struct kbase_hwcnt_reader_metadata meta;
1355
1356         if (sizeof(struct kbase_hwcnt_reader_metadata) != size)
1357                 return -EINVAL;
1358
1359         /* Check if any buffer was taken. */
1360         if (atomic_read(&cli->meta_idx) == read_idx)
1361                 return -EPERM;
1362
1363         /* Check if correct buffer is put back. */
1364         if (copy_from_user(&meta, buffer, size))
1365                 return -EFAULT;
1366         if (idx != meta.buffer_idx)
1367                 return -EINVAL;
1368
1369         atomic_inc(&cli->read_idx);
1370
1371         return 0;
1372 }
1373
1374 /**
1375  * kbasep_vinstr_hwcnt_reader_ioctl_set_interval - hwcnt reader's ioctl command
1376  * @cli:      pointer to vinstr client structure
1377  * @interval: periodic dumping interval (disable periodic dumping if zero)
1378  *
1379  * Return: zero on success
1380  */
1381 static long kbasep_vinstr_hwcnt_reader_ioctl_set_interval(
1382                 struct kbase_vinstr_client *cli, u32 interval)
1383 {
1384         struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1385
1386         KBASE_DEBUG_ASSERT(vinstr_ctx);
1387
1388         mutex_lock(&vinstr_ctx->lock);
1389
1390         list_del(&cli->list);
1391
1392         cli->dump_interval = interval;
1393
1394         /* If interval is non-zero, enable periodic dumping for this client. */
1395         if (cli->dump_interval) {
1396                 if (DUMPING_RESOLUTION > cli->dump_interval)
1397                         cli->dump_interval = DUMPING_RESOLUTION;
1398                 cli->dump_time =
1399                         kbasep_vinstr_get_timestamp() + cli->dump_interval;
1400
1401                 kbasep_vinstr_add_dump_request(
1402                                 cli, &vinstr_ctx->waiting_clients);
1403
1404                 atomic_set(&vinstr_ctx->request_pending, 1);
1405                 wake_up_all(&vinstr_ctx->waitq);
1406         } else {
1407                 list_add(&cli->list, &vinstr_ctx->idle_clients);
1408         }
1409
1410         mutex_unlock(&vinstr_ctx->lock);
1411
1412         return 0;
1413 }
1414
1415 /**
1416  * kbasep_vinstr_hwcnt_reader_event_mask - return event mask for event id
1417  * @event_id: id of event
1418  * Return: event_mask or zero if event is not supported or maskable
1419  */
1420 static u32 kbasep_vinstr_hwcnt_reader_event_mask(
1421                 enum base_hwcnt_reader_event event_id)
1422 {
1423         u32 event_mask = 0;
1424
1425         switch (event_id) {
1426         case BASE_HWCNT_READER_EVENT_PREJOB:
1427         case BASE_HWCNT_READER_EVENT_POSTJOB:
1428                 /* These event are maskable. */
1429                 event_mask = (1 << event_id);
1430                 break;
1431
1432         case BASE_HWCNT_READER_EVENT_MANUAL:
1433         case BASE_HWCNT_READER_EVENT_PERIODIC:
1434                 /* These event are non-maskable. */
1435         default:
1436                 /* These event are not supported. */
1437                 break;
1438         }
1439
1440         return event_mask;
1441 }
1442
1443 /**
1444  * kbasep_vinstr_hwcnt_reader_ioctl_enable_event - hwcnt reader's ioctl command
1445  * @cli:      pointer to vinstr client structure
1446  * @event_id: id of event to enable
1447  *
1448  * Return: zero on success
1449  */
1450 static long kbasep_vinstr_hwcnt_reader_ioctl_enable_event(
1451                 struct kbase_vinstr_client *cli,
1452                 enum base_hwcnt_reader_event event_id)
1453 {
1454         struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1455         u32                         event_mask;
1456
1457         KBASE_DEBUG_ASSERT(vinstr_ctx);
1458
1459         event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id);
1460         if (!event_mask)
1461                 return -EINVAL;
1462
1463         mutex_lock(&vinstr_ctx->lock);
1464         cli->event_mask |= event_mask;
1465         mutex_unlock(&vinstr_ctx->lock);
1466
1467         return 0;
1468 }
1469
1470 /**
1471  * kbasep_vinstr_hwcnt_reader_ioctl_disable_event - hwcnt reader's ioctl command
1472  * @cli:      pointer to vinstr client structure
1473  * @event_id: id of event to disable
1474  *
1475  * Return: zero on success
1476  */
1477 static long kbasep_vinstr_hwcnt_reader_ioctl_disable_event(
1478                 struct kbase_vinstr_client *cli,
1479                 enum base_hwcnt_reader_event event_id)
1480 {
1481         struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1482         u32                         event_mask;
1483
1484         KBASE_DEBUG_ASSERT(vinstr_ctx);
1485
1486         event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id);
1487         if (!event_mask)
1488                 return -EINVAL;
1489
1490         mutex_lock(&vinstr_ctx->lock);
1491         cli->event_mask &= ~event_mask;
1492         mutex_unlock(&vinstr_ctx->lock);
1493
1494         return 0;
1495 }
1496
1497 /**
1498  * kbasep_vinstr_hwcnt_reader_ioctl_get_hwver - hwcnt reader's ioctl command
1499  * @cli:   pointer to vinstr client structure
1500  * @hwver: pointer to user buffer where hw version will be stored
1501  *
1502  * Return: zero on success
1503  */
1504 static long kbasep_vinstr_hwcnt_reader_ioctl_get_hwver(
1505                 struct kbase_vinstr_client *cli, u32 __user *hwver)
1506 {
1507 #ifndef CONFIG_MALI_NO_MALI
1508         struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1509 #endif
1510
1511         u32                         ver = 5;
1512
1513 #ifndef CONFIG_MALI_NO_MALI
1514         KBASE_DEBUG_ASSERT(vinstr_ctx);
1515         if (kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4))
1516                 ver = 4;
1517 #endif
1518
1519         return put_user(ver, hwver);
1520 }
1521
1522 /**
1523  * kbasep_vinstr_hwcnt_reader_ioctl - hwcnt reader's ioctl
1524  * @filp:   pointer to file structure
1525  * @cmd:    user command
1526  * @arg:    command's argument
1527  *
1528  * Return: zero on success
1529  */
1530 static long kbasep_vinstr_hwcnt_reader_ioctl(struct file *filp,
1531                 unsigned int cmd, unsigned long arg)
1532 {
1533         long                       rcode = 0;
1534         struct kbase_vinstr_client *cli;
1535
1536         KBASE_DEBUG_ASSERT(filp);
1537
1538         cli = filp->private_data;
1539         KBASE_DEBUG_ASSERT(cli);
1540
1541         if (unlikely(KBASE_HWCNT_READER != _IOC_TYPE(cmd)))
1542                 return -EINVAL;
1543
1544         switch (cmd) {
1545         case KBASE_HWCNT_READER_GET_API_VERSION:
1546                 rcode = put_user(HWCNT_READER_API, (u32 __user *)arg);
1547                 break;
1548         case KBASE_HWCNT_READER_GET_HWVER:
1549                 rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_hwver(
1550                                 cli, (u32 __user *)arg);
1551                 break;
1552         case KBASE_HWCNT_READER_GET_BUFFER_SIZE:
1553                 KBASE_DEBUG_ASSERT(cli->vinstr_ctx);
1554                 rcode = put_user(
1555                                 (u32)cli->vinstr_ctx->dump_size,
1556                                 (u32 __user *)arg);
1557                 break;
1558         case KBASE_HWCNT_READER_DUMP:
1559                 rcode = kbase_vinstr_hwc_dump(
1560                                 cli, BASE_HWCNT_READER_EVENT_MANUAL);
1561                 break;
1562         case KBASE_HWCNT_READER_CLEAR:
1563                 rcode = kbase_vinstr_hwc_clear(cli);
1564                 break;
1565         case KBASE_HWCNT_READER_GET_BUFFER:
1566                 rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_buffer(
1567                                 cli, (void __user *)arg, _IOC_SIZE(cmd));
1568                 break;
1569         case KBASE_HWCNT_READER_PUT_BUFFER:
1570                 rcode = kbasep_vinstr_hwcnt_reader_ioctl_put_buffer(
1571                                 cli, (void __user *)arg, _IOC_SIZE(cmd));
1572                 break;
1573         case KBASE_HWCNT_READER_SET_INTERVAL:
1574                 rcode = kbasep_vinstr_hwcnt_reader_ioctl_set_interval(
1575                                 cli, (u32)arg);
1576                 break;
1577         case KBASE_HWCNT_READER_ENABLE_EVENT:
1578                 rcode = kbasep_vinstr_hwcnt_reader_ioctl_enable_event(
1579                                 cli, (enum base_hwcnt_reader_event)arg);
1580                 break;
1581         case KBASE_HWCNT_READER_DISABLE_EVENT:
1582                 rcode = kbasep_vinstr_hwcnt_reader_ioctl_disable_event(
1583                                 cli, (enum base_hwcnt_reader_event)arg);
1584                 break;
1585         default:
1586                 rcode = -EINVAL;
1587                 break;
1588         }
1589
1590         return rcode;
1591 }
1592
1593 /**
1594  * kbasep_vinstr_hwcnt_reader_poll - hwcnt reader's poll
1595  * @filp: pointer to file structure
1596  * @wait: pointer to poll table
1597  * Return: POLLIN if data can be read without blocking, otherwise zero
1598  */
1599 static unsigned int kbasep_vinstr_hwcnt_reader_poll(struct file *filp,
1600                 poll_table *wait)
1601 {
1602         struct kbase_vinstr_client *cli;
1603
1604         KBASE_DEBUG_ASSERT(filp);
1605         KBASE_DEBUG_ASSERT(wait);
1606
1607         cli = filp->private_data;
1608         KBASE_DEBUG_ASSERT(cli);
1609
1610         poll_wait(filp, &cli->waitq, wait);
1611         if (kbasep_vinstr_hwcnt_reader_buffer_ready(cli))
1612                 return POLLIN;
1613         return 0;
1614 }
1615
1616 /**
1617  * kbasep_vinstr_hwcnt_reader_mmap - hwcnt reader's mmap
1618  * @filp: pointer to file structure
1619  * @vma:  pointer to vma structure
1620  * Return: zero on success
1621  */
1622 static int kbasep_vinstr_hwcnt_reader_mmap(struct file *filp,
1623                 struct vm_area_struct *vma)
1624 {
1625         struct kbase_vinstr_client *cli;
1626         unsigned long size, addr, pfn, offset;
1627         unsigned long vm_size = vma->vm_end - vma->vm_start;
1628
1629         KBASE_DEBUG_ASSERT(filp);
1630         KBASE_DEBUG_ASSERT(vma);
1631
1632         cli = filp->private_data;
1633         KBASE_DEBUG_ASSERT(cli);
1634
1635         size = cli->buffer_count * cli->dump_size;
1636
1637         if (vma->vm_pgoff > (size >> PAGE_SHIFT))
1638                 return -EINVAL;
1639         if (vm_size > size)
1640                 return -EINVAL;
1641
1642         offset = vma->vm_pgoff << PAGE_SHIFT;
1643         if ((vm_size + offset) > size)
1644                 return -EINVAL;
1645
1646         addr = __pa((unsigned long)cli->dump_buffers + offset);
1647         pfn = addr >> PAGE_SHIFT;
1648
1649         return remap_pfn_range(
1650                         vma,
1651                         vma->vm_start,
1652                         pfn,
1653                         vm_size,
1654                         vma->vm_page_prot);
1655 }
1656
1657 /**
1658  * kbasep_vinstr_hwcnt_reader_release - hwcnt reader's release
1659  * @inode: pointer to inode structure
1660  * @filp:  pointer to file structure
1661  * Return always return zero
1662  */
1663 static int kbasep_vinstr_hwcnt_reader_release(struct inode *inode,
1664                 struct file *filp)
1665 {
1666         struct kbase_vinstr_client *cli;
1667
1668         KBASE_DEBUG_ASSERT(inode);
1669         KBASE_DEBUG_ASSERT(filp);
1670
1671         cli = filp->private_data;
1672         KBASE_DEBUG_ASSERT(cli);
1673
1674         kbase_vinstr_detach_client(cli);
1675         return 0;
1676 }
1677
1678 /*****************************************************************************/
1679
1680 /**
1681  * kbasep_vinstr_kick_scheduler - trigger scheduler cycle
1682  * @kbdev: pointer to kbase device structure
1683  */
1684 static void kbasep_vinstr_kick_scheduler(struct kbase_device *kbdev)
1685 {
1686         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1687         unsigned long flags;
1688
1689         down(&js_devdata->schedule_sem);
1690         spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1691         kbase_jm_kick_all(kbdev);
1692         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1693         up(&js_devdata->schedule_sem);
1694 }
1695
1696 /**
1697  * kbasep_vinstr_suspend_worker - worker suspending vinstr module
1698  * @data: pointer to work structure
1699  */
1700 static void kbasep_vinstr_suspend_worker(struct work_struct *data)
1701 {
1702         struct kbase_vinstr_context *vinstr_ctx;
1703         unsigned long flags;
1704
1705         vinstr_ctx = container_of(data, struct kbase_vinstr_context,
1706                         suspend_work);
1707
1708         mutex_lock(&vinstr_ctx->lock);
1709
1710         if (vinstr_ctx->kctx)
1711                 disable_hwcnt(vinstr_ctx);
1712
1713         spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
1714         vinstr_ctx->state = VINSTR_SUSPENDED;
1715         wake_up_all(&vinstr_ctx->suspend_waitq);
1716         spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
1717
1718         mutex_unlock(&vinstr_ctx->lock);
1719
1720         /* Kick GPU scheduler to allow entering protected mode.
1721          * This must happen after vinstr was suspended. */
1722         kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev);
1723 }
1724
1725 /**
1726  * kbasep_vinstr_suspend_worker - worker resuming vinstr module
1727  * @data: pointer to work structure
1728  */
1729 static void kbasep_vinstr_resume_worker(struct work_struct *data)
1730 {
1731         struct kbase_vinstr_context *vinstr_ctx;
1732         unsigned long flags;
1733
1734         vinstr_ctx = container_of(data, struct kbase_vinstr_context,
1735                         resume_work);
1736
1737         mutex_lock(&vinstr_ctx->lock);
1738
1739         if (vinstr_ctx->kctx)
1740                 enable_hwcnt(vinstr_ctx);
1741
1742         spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
1743         vinstr_ctx->state = VINSTR_IDLE;
1744         wake_up_all(&vinstr_ctx->suspend_waitq);
1745         spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
1746
1747         mutex_unlock(&vinstr_ctx->lock);
1748
1749         /* Kick GPU scheduler to allow entering protected mode.
1750          * Note that scheduler state machine might requested re-entry to
1751          * protected mode before vinstr was resumed.
1752          * This must happen after vinstr was release. */
1753         kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev);
1754 }
1755
1756 /*****************************************************************************/
1757
1758 struct kbase_vinstr_context *kbase_vinstr_init(struct kbase_device *kbdev)
1759 {
1760         struct kbase_vinstr_context *vinstr_ctx;
1761
1762         vinstr_ctx = kzalloc(sizeof(*vinstr_ctx), GFP_KERNEL);
1763         if (!vinstr_ctx)
1764                 return NULL;
1765
1766         INIT_LIST_HEAD(&vinstr_ctx->idle_clients);
1767         INIT_LIST_HEAD(&vinstr_ctx->waiting_clients);
1768         mutex_init(&vinstr_ctx->lock);
1769         spin_lock_init(&vinstr_ctx->state_lock);
1770         vinstr_ctx->kbdev = kbdev;
1771         vinstr_ctx->thread = NULL;
1772         vinstr_ctx->state = VINSTR_IDLE;
1773         vinstr_ctx->suspend_cnt = 0;
1774         INIT_WORK(&vinstr_ctx->suspend_work, kbasep_vinstr_suspend_worker);
1775         INIT_WORK(&vinstr_ctx->resume_work, kbasep_vinstr_resume_worker);
1776         init_waitqueue_head(&vinstr_ctx->suspend_waitq);
1777
1778         atomic_set(&vinstr_ctx->request_pending, 0);
1779         init_waitqueue_head(&vinstr_ctx->waitq);
1780
1781         return vinstr_ctx;
1782 }
1783
1784 void kbase_vinstr_term(struct kbase_vinstr_context *vinstr_ctx)
1785 {
1786         struct kbase_vinstr_client *cli;
1787
1788         /* Stop service thread first. */
1789         if (vinstr_ctx->thread)
1790                 kthread_stop(vinstr_ctx->thread);
1791
1792         /* Wait for workers. */
1793         flush_work(&vinstr_ctx->suspend_work);
1794         flush_work(&vinstr_ctx->resume_work);
1795
1796         while (1) {
1797                 struct list_head *list = &vinstr_ctx->idle_clients;
1798
1799                 if (list_empty(list)) {
1800                         list = &vinstr_ctx->waiting_clients;
1801                         if (list_empty(list))
1802                                 break;
1803                 }
1804
1805                 cli = list_first_entry(list, struct kbase_vinstr_client, list);
1806                 list_del(&cli->list);
1807                 kfree(cli->accum_buffer);
1808                 kfree(cli);
1809                 vinstr_ctx->nclients--;
1810         }
1811         KBASE_DEBUG_ASSERT(!vinstr_ctx->nclients);
1812         if (vinstr_ctx->kctx)
1813                 kbasep_vinstr_destroy_kctx(vinstr_ctx);
1814         kfree(vinstr_ctx);
1815 }
1816
1817 int kbase_vinstr_hwcnt_reader_setup(struct kbase_vinstr_context *vinstr_ctx,
1818                 struct kbase_uk_hwcnt_reader_setup *setup)
1819 {
1820         struct kbase_vinstr_client  *cli;
1821         u32                         bitmap[4];
1822
1823         KBASE_DEBUG_ASSERT(vinstr_ctx);
1824         KBASE_DEBUG_ASSERT(setup);
1825         KBASE_DEBUG_ASSERT(setup->buffer_count);
1826
1827         bitmap[SHADER_HWCNT_BM] = setup->shader_bm;
1828         bitmap[TILER_HWCNT_BM]  = setup->tiler_bm;
1829         bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm;
1830         bitmap[JM_HWCNT_BM]     = setup->jm_bm;
1831
1832         cli = kbasep_vinstr_attach_client(
1833                         vinstr_ctx,
1834                         setup->buffer_count,
1835                         bitmap,
1836                         &setup->fd,
1837                         NULL);
1838
1839         if (!cli)
1840                 return -ENOMEM;
1841
1842         return 0;
1843 }
1844
1845 int kbase_vinstr_legacy_hwc_setup(
1846                 struct kbase_vinstr_context *vinstr_ctx,
1847                 struct kbase_vinstr_client  **cli,
1848                 struct kbase_uk_hwcnt_setup *setup)
1849 {
1850         KBASE_DEBUG_ASSERT(vinstr_ctx);
1851         KBASE_DEBUG_ASSERT(setup);
1852         KBASE_DEBUG_ASSERT(cli);
1853
1854         if (setup->dump_buffer) {
1855                 u32 bitmap[4];
1856
1857                 bitmap[SHADER_HWCNT_BM] = setup->shader_bm;
1858                 bitmap[TILER_HWCNT_BM]  = setup->tiler_bm;
1859                 bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm;
1860                 bitmap[JM_HWCNT_BM]     = setup->jm_bm;
1861
1862                 if (*cli)
1863                         return -EBUSY;
1864
1865                 *cli = kbasep_vinstr_attach_client(
1866                                 vinstr_ctx,
1867                                 0,
1868                                 bitmap,
1869                                 (void *)(long)setup->dump_buffer,
1870                                 NULL);
1871
1872                 if (!(*cli))
1873                         return -ENOMEM;
1874         } else {
1875                 if (!*cli)
1876                         return -EINVAL;
1877
1878                 kbase_vinstr_detach_client(*cli);
1879                 *cli = NULL;
1880         }
1881
1882         return 0;
1883 }
1884
1885 struct kbase_vinstr_client *kbase_vinstr_hwcnt_kernel_setup(
1886                 struct kbase_vinstr_context *vinstr_ctx,
1887                 struct kbase_uk_hwcnt_reader_setup *setup,
1888                 void *kernel_buffer)
1889 {
1890         u32 bitmap[4];
1891
1892         if (!vinstr_ctx || !setup || !kernel_buffer)
1893                 return NULL;
1894
1895         bitmap[SHADER_HWCNT_BM] = setup->shader_bm;
1896         bitmap[TILER_HWCNT_BM]  = setup->tiler_bm;
1897         bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm;
1898         bitmap[JM_HWCNT_BM]     = setup->jm_bm;
1899
1900         return kbasep_vinstr_attach_client(
1901                         vinstr_ctx,
1902                         0,
1903                         bitmap,
1904                         NULL,
1905                         kernel_buffer);
1906 }
1907 KBASE_EXPORT_TEST_API(kbase_vinstr_hwcnt_kernel_setup);
1908
1909 int kbase_vinstr_hwc_dump(struct kbase_vinstr_client *cli,
1910                 enum base_hwcnt_reader_event event_id)
1911 {
1912         int                         rcode = 0;
1913         struct kbase_vinstr_context *vinstr_ctx;
1914         u64                         timestamp;
1915         u32                         event_mask;
1916
1917         if (!cli)
1918                 return -EINVAL;
1919
1920         vinstr_ctx = cli->vinstr_ctx;
1921         KBASE_DEBUG_ASSERT(vinstr_ctx);
1922
1923         KBASE_DEBUG_ASSERT(event_id < BASE_HWCNT_READER_EVENT_COUNT);
1924         event_mask = 1 << event_id;
1925
1926         mutex_lock(&vinstr_ctx->lock);
1927
1928         if (event_mask & cli->event_mask) {
1929                 rcode = kbasep_vinstr_collect_and_accumulate(
1930                                 vinstr_ctx,
1931                                 &timestamp);
1932                 if (rcode)
1933                         goto exit;
1934
1935                 rcode = kbasep_vinstr_update_client(cli, timestamp, event_id);
1936                 if (rcode)
1937                         goto exit;
1938
1939                 kbasep_vinstr_reprogram(vinstr_ctx);
1940         }
1941
1942 exit:
1943         mutex_unlock(&vinstr_ctx->lock);
1944
1945         return rcode;
1946 }
1947 KBASE_EXPORT_TEST_API(kbase_vinstr_hwc_dump);
1948
1949 int kbase_vinstr_hwc_clear(struct kbase_vinstr_client *cli)
1950 {
1951         struct kbase_vinstr_context *vinstr_ctx;
1952         int                         rcode;
1953         u64                         unused;
1954
1955         if (!cli)
1956                 return -EINVAL;
1957
1958         vinstr_ctx = cli->vinstr_ctx;
1959         KBASE_DEBUG_ASSERT(vinstr_ctx);
1960
1961         mutex_lock(&vinstr_ctx->lock);
1962
1963         rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx, &unused);
1964         if (rcode)
1965                 goto exit;
1966         rcode = kbase_instr_hwcnt_clear(vinstr_ctx->kctx);
1967         if (rcode)
1968                 goto exit;
1969         memset(cli->accum_buffer, 0, cli->dump_size);
1970
1971         kbasep_vinstr_reprogram(vinstr_ctx);
1972
1973 exit:
1974         mutex_unlock(&vinstr_ctx->lock);
1975
1976         return rcode;
1977 }
1978
1979 int kbase_vinstr_try_suspend(struct kbase_vinstr_context *vinstr_ctx)
1980 {
1981         unsigned long flags;
1982         int ret = -EAGAIN;
1983
1984         KBASE_DEBUG_ASSERT(vinstr_ctx);
1985
1986         spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
1987         switch (vinstr_ctx->state) {
1988         case VINSTR_SUSPENDED:
1989                 vinstr_ctx->suspend_cnt++;
1990                 /* overflow shall not happen */
1991                 BUG_ON(0 == vinstr_ctx->suspend_cnt);
1992                 ret = 0;
1993                 break;
1994
1995         case VINSTR_IDLE:
1996                 vinstr_ctx->state = VINSTR_SUSPENDING;
1997                 schedule_work(&vinstr_ctx->suspend_work);
1998                 break;
1999
2000         case VINSTR_DUMPING:
2001                 vinstr_ctx->state = VINSTR_SUSPENDING;
2002                 break;
2003
2004         case VINSTR_SUSPENDING:
2005                 /* fall through */
2006         case VINSTR_RESUMING:
2007                 break;
2008
2009         default:
2010                 BUG();
2011                 break;
2012         }
2013         spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
2014
2015         return ret;
2016 }
2017
2018 void kbase_vinstr_suspend(struct kbase_vinstr_context *vinstr_ctx)
2019 {
2020         wait_event(vinstr_ctx->suspend_waitq,
2021                         (0 == kbase_vinstr_try_suspend(vinstr_ctx)));
2022 }
2023
2024 void kbase_vinstr_resume(struct kbase_vinstr_context *vinstr_ctx)
2025 {
2026         unsigned long flags;
2027
2028         KBASE_DEBUG_ASSERT(vinstr_ctx);
2029
2030         spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
2031         BUG_ON(VINSTR_SUSPENDING == vinstr_ctx->state);
2032         if (VINSTR_SUSPENDED == vinstr_ctx->state) {
2033                 BUG_ON(0 == vinstr_ctx->suspend_cnt);
2034                 vinstr_ctx->suspend_cnt--;
2035                 if (0 == vinstr_ctx->suspend_cnt) {
2036                         vinstr_ctx->state = VINSTR_RESUMING;
2037                         schedule_work(&vinstr_ctx->resume_work);
2038                 }
2039         }
2040         spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
2041 }