drivers/gpu/arm/midgard/mali_kbase_vinstr.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18 #include <linux/anon_inodes.h>
  19 #include <linux/atomic.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/jiffies.h>
  22 #include <linux/kthread.h>
  23 #include <linux/list.h>
  24 #include <linux/mm.h>
  25 #include <linux/poll.h>
  26 #include <linux/preempt.h>
  27 #include <linux/slab.h>
  28 #include <linux/wait.h>
  29
  30 #include <mali_kbase.h>
  31 #include <mali_kbase_hwaccess_instr.h>
  32 #include <mali_kbase_hwaccess_jm.h>
  33 #include <mali_kbase_hwcnt_reader.h>
  34 #include <mali_kbase_mem_linux.h>
  35 #include <mali_kbase_tlstream.h>
  36
  37 /*****************************************************************************/
  38
  39 /* Hwcnt reader API version */
  40 #define HWCNT_READER_API        1
  41
  42 /* The number of nanoseconds in a second. */
  43 #define NSECS_IN_SEC            1000000000ull /* ns */
  44
  45 /* The time resolution of dumping service. */
  46 #define DUMPING_RESOLUTION      500000ull /* ns */
  47
  48 /* The maximal supported number of dumping buffers. */
  49 #define MAX_BUFFER_COUNT        32
  50
  51 /* Size and number of hw counters blocks. */
  52 #define NR_CNT_BLOCKS_PER_GROUP 8
  53 #define NR_CNT_PER_BLOCK        64
  54 #define NR_BYTES_PER_CNT        4
  55 #define NR_BYTES_PER_HDR        16
  56 #define PRFCNT_EN_MASK_OFFSET   0x8
  57
  58 /*****************************************************************************/
  59
  60 enum {
  61         SHADER_HWCNT_BM,
  62         TILER_HWCNT_BM,
  63         MMU_L2_HWCNT_BM,
  64         JM_HWCNT_BM
  65 };
  66
  67 enum vinstr_state {
  68         VINSTR_IDLE,
  69         VINSTR_DUMPING,
  70         VINSTR_SUSPENDING,
  71         VINSTR_SUSPENDED,
  72         VINSTR_RESUMING
  73 };
  74
  75 /**
  76  * struct kbase_vinstr_context - vinstr context per device
  77  * @lock:              protects the entire vinstr context
  78  * @kbdev:             pointer to kbase device
  79  * @kctx:              pointer to kbase context
  80  * @vmap:              vinstr vmap for mapping hwcnt dump buffer
  81  * @gpu_va:            GPU hwcnt dump buffer address
  82  * @cpu_va:            the CPU side mapping of the hwcnt dump buffer
  83  * @dump_size:         size of the dump buffer in bytes
  84  * @bitmap:            current set of counters monitored, not always in sync
  85  *                     with hardware
  86  * @reprogram:         when true, reprogram hwcnt block with the new set of
  87  *                     counters
  88  * @state:             vinstr state
  89  * @state_lock:        protects information about vinstr state
  90  * @suspend_waitq:     notification queue to trigger state re-validation
  91  * @suspend_cnt:       reference counter of vinstr's suspend state
  92  * @suspend_work:      worker to execute on entering suspended state
  93  * @resume_work:       worker to execute on leaving suspended state
  94  * @nclients:          number of attached clients, pending or otherwise
  95  * @waiting_clients:   head of list of clients being periodically sampled
  96  * @idle_clients:      head of list of clients being idle
  97  * @suspended_clients: head of list of clients being suspended
  98  * @thread:            periodic sampling thread
  99  * @waitq:             notification queue of sampling thread
 100  * @request_pending:   request for action for sampling thread
 101  */
 102 struct kbase_vinstr_context {
 103         struct mutex             lock;
 104         struct kbase_device      *kbdev;
 105         struct kbase_context     *kctx;
 106
 107         struct kbase_vmap_struct vmap;
 108         u64                      gpu_va;
 109         void                     *cpu_va;
 110         size_t                   dump_size;
 111         u32                      bitmap[4];
 112         bool                     reprogram;
 113
 114         enum vinstr_state        state;
 115         struct spinlock          state_lock;
 116         wait_queue_head_t        suspend_waitq;
 117         unsigned int             suspend_cnt;
 118         struct work_struct       suspend_work;
 119         struct work_struct       resume_work;
 120
 121         u32                      nclients;
 122         struct list_head         waiting_clients;
 123         struct list_head         idle_clients;
 124         struct list_head         suspended_clients;
 125
 126         struct task_struct       *thread;
 127         wait_queue_head_t        waitq;
 128         atomic_t                 request_pending;
 129 };
 130
 131 /**
 132  * struct kbase_vinstr_client - a vinstr client attached to a vinstr context
 133  * @vinstr_ctx:    vinstr context client is attached to
 134  * @list:          node used to attach this client to list in vinstr context
 135  * @buffer_count:  number of buffers this client is using
 136  * @event_mask:    events this client reacts to
 137  * @dump_size:     size of one dump buffer in bytes
 138  * @bitmap:        bitmap request for JM, TILER, SHADER and MMU counters
 139  * @legacy_buffer: userspace hwcnt dump buffer (legacy interface)
 140  * @kernel_buffer: kernel hwcnt dump buffer (kernel client interface)
 141  * @accum_buffer:  temporary accumulation buffer for preserving counters
 142  * @dump_time:     next time this clients shall request hwcnt dump
 143  * @dump_interval: interval between periodic hwcnt dumps
 144  * @dump_buffers:  kernel hwcnt dump buffers allocated by this client
 145  * @dump_buffers_meta: metadata of dump buffers
 146  * @meta_idx:      index of metadata being accessed by userspace
 147  * @read_idx:      index of buffer read by userspace
 148  * @write_idx:     index of buffer being written by dumping service
 149  * @waitq:         client's notification queue
 150  * @pending:       when true, client has attached but hwcnt not yet updated
 151  */
 152 struct kbase_vinstr_client {
 153         struct kbase_vinstr_context        *vinstr_ctx;
 154         struct list_head                   list;
 155         unsigned int                       buffer_count;
 156         u32                                event_mask;
 157         size_t                             dump_size;
 158         u32                                bitmap[4];
 159         void __user                        *legacy_buffer;
 160         void                               *kernel_buffer;
 161         void                               *accum_buffer;
 162         u64                                dump_time;
 163         u32                                dump_interval;
 164         char                               *dump_buffers;
 165         struct kbase_hwcnt_reader_metadata *dump_buffers_meta;
 166         atomic_t                           meta_idx;
 167         atomic_t                           read_idx;
 168         atomic_t                           write_idx;
 169         wait_queue_head_t                  waitq;
 170         bool                               pending;
 171 };
 172
 173 /**
 174  * struct kbasep_vinstr_wake_up_timer - vinstr service thread wake up timer
 175  * @hrtimer:    high resolution timer
 176  * @vinstr_ctx: vinstr context
 177  */
 178 struct kbasep_vinstr_wake_up_timer {
 179         struct hrtimer              hrtimer;
 180         struct kbase_vinstr_context *vinstr_ctx;
 181 };
 182
 183 /*****************************************************************************/
 184
 185 static int kbasep_vinstr_service_task(void *data);
 186
 187 static unsigned int kbasep_vinstr_hwcnt_reader_poll(
 188                 struct file *filp,
 189                 poll_table  *wait);
 190 static long kbasep_vinstr_hwcnt_reader_ioctl(
 191                 struct file   *filp,
 192                 unsigned int  cmd,
 193                 unsigned long arg);
 194 static int kbasep_vinstr_hwcnt_reader_mmap(
 195                 struct file           *filp,
 196                 struct vm_area_struct *vma);
 197 static int kbasep_vinstr_hwcnt_reader_release(
 198                 struct inode *inode,
 199                 struct file  *filp);
 200
 201 /* The timeline stream file operations structure. */
 202 static const struct file_operations vinstr_client_fops = {
 203         .poll           = kbasep_vinstr_hwcnt_reader_poll,
 204         .unlocked_ioctl = kbasep_vinstr_hwcnt_reader_ioctl,
 205         .compat_ioctl   = kbasep_vinstr_hwcnt_reader_ioctl,
 206         .mmap           = kbasep_vinstr_hwcnt_reader_mmap,
 207         .release        = kbasep_vinstr_hwcnt_reader_release,
 208 };
 209
 210 /*****************************************************************************/
 211
 212 static int enable_hwcnt(struct kbase_vinstr_context *vinstr_ctx)
 213 {
 214         struct kbase_context *kctx = vinstr_ctx->kctx;
 215         struct kbase_device *kbdev = kctx->kbdev;
 216         struct kbase_uk_hwcnt_setup setup;
 217         int err;
 218
 219         setup.dump_buffer = vinstr_ctx->gpu_va;
 220         setup.jm_bm       = vinstr_ctx->bitmap[JM_HWCNT_BM];
 221         setup.tiler_bm    = vinstr_ctx->bitmap[TILER_HWCNT_BM];
 222         setup.shader_bm   = vinstr_ctx->bitmap[SHADER_HWCNT_BM];
 223         setup.mmu_l2_bm   = vinstr_ctx->bitmap[MMU_L2_HWCNT_BM];
 224
 225         /* Mark the context as active so the GPU is kept turned on */
 226         /* A suspend won't happen here, because we're in a syscall from a
 227          * userspace thread. */
 228         kbase_pm_context_active(kbdev);
 229
 230         /* Schedule the context in */
 231         kbasep_js_schedule_privileged_ctx(kbdev, kctx);
 232         err = kbase_instr_hwcnt_enable_internal(kbdev, kctx, &setup);
 233         if (err) {
 234                 /* Release the context. This had its own Power Manager Active
 235                  * reference */
 236                 kbasep_js_release_privileged_ctx(kbdev, kctx);
 237
 238                 /* Also release our Power Manager Active reference */
 239                 kbase_pm_context_idle(kbdev);
 240         }
 241
 242         return err;
 243 }
 244
 245 static void disable_hwcnt(struct kbase_vinstr_context *vinstr_ctx)
 246 {
 247         struct kbase_context *kctx = vinstr_ctx->kctx;
 248         struct kbase_device *kbdev = kctx->kbdev;
 249         int err;
 250
 251         err = kbase_instr_hwcnt_disable_internal(kctx);
 252         if (err) {
 253                 dev_warn(kbdev->dev, "Failed to disable HW counters (ctx:%p)",
 254                                 kctx);
 255                 return;
 256         }
 257
 258         /* Release the context. This had its own Power Manager Active reference. */
 259         kbasep_js_release_privileged_ctx(kbdev, kctx);
 260
 261         /* Also release our Power Manager Active reference. */
 262         kbase_pm_context_idle(kbdev);
 263
 264         dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", kctx);
 265 }
 266
 267 static int reprogram_hwcnt(struct kbase_vinstr_context *vinstr_ctx)
 268 {
 269         disable_hwcnt(vinstr_ctx);
 270         return enable_hwcnt(vinstr_ctx);
 271 }
 272
 273 static void hwcnt_bitmap_set(u32 dst[4], u32 src[4])
 274 {
 275         dst[JM_HWCNT_BM]     = src[JM_HWCNT_BM];
 276         dst[TILER_HWCNT_BM]  = src[TILER_HWCNT_BM];
 277         dst[SHADER_HWCNT_BM] = src[SHADER_HWCNT_BM];
 278         dst[MMU_L2_HWCNT_BM] = src[MMU_L2_HWCNT_BM];
 279 }
 280
 281 static void hwcnt_bitmap_union(u32 dst[4], u32 src[4])
 282 {
 283         dst[JM_HWCNT_BM]     |= src[JM_HWCNT_BM];
 284         dst[TILER_HWCNT_BM]  |= src[TILER_HWCNT_BM];
 285         dst[SHADER_HWCNT_BM] |= src[SHADER_HWCNT_BM];
 286         dst[MMU_L2_HWCNT_BM] |= src[MMU_L2_HWCNT_BM];
 287 }
 288
 289 size_t kbase_vinstr_dump_size(struct kbase_device *kbdev)
 290 {
 291         size_t dump_size;
 292
 293 #ifndef CONFIG_MALI_NO_MALI
 294         if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_V4)) {
 295                 u32 nr_cg;
 296
 297                 nr_cg = kbdev->gpu_props.num_core_groups;
 298                 dump_size = nr_cg * NR_CNT_BLOCKS_PER_GROUP *
 299                                 NR_CNT_PER_BLOCK *
 300                                 NR_BYTES_PER_CNT;
 301         } else
 302 #endif /* CONFIG_MALI_NO_MALI */
 303         {
 304                 /* assume v5 for now */
 305                 base_gpu_props *props = &kbdev->gpu_props.props;
 306                 u32 nr_l2 = props->l2_props.num_l2_slices;
 307                 u64 core_mask = props->coherency_info.group[0].core_mask;
 308                 u32 nr_blocks = fls64(core_mask);
 309
 310                 /* JM and tiler counter blocks are always present */
 311                 dump_size = (2 + nr_l2 + nr_blocks) *
 312                                 NR_CNT_PER_BLOCK *
 313                                 NR_BYTES_PER_CNT;
 314         }
 315         return dump_size;
 316 }
 317 KBASE_EXPORT_TEST_API(kbase_vinstr_dump_size);
 318
 319 static size_t kbasep_vinstr_dump_size_ctx(
 320                 struct kbase_vinstr_context *vinstr_ctx)
 321 {
 322         return kbase_vinstr_dump_size(vinstr_ctx->kctx->kbdev);
 323 }
 324
 325 static int kbasep_vinstr_map_kernel_dump_buffer(
 326                 struct kbase_vinstr_context *vinstr_ctx)
 327 {
 328         struct kbase_va_region *reg;
 329         struct kbase_context *kctx = vinstr_ctx->kctx;
 330         u64 flags, nr_pages;
 331         u16 va_align = 0;
 332
 333         flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_WR;
 334         vinstr_ctx->dump_size = kbasep_vinstr_dump_size_ctx(vinstr_ctx);
 335         nr_pages = PFN_UP(vinstr_ctx->dump_size);
 336
 337         reg = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags,
 338                         &vinstr_ctx->gpu_va, &va_align);
 339         if (!reg)
 340                 return -ENOMEM;
 341
 342         vinstr_ctx->cpu_va = kbase_vmap(
 343                         kctx,
 344                         vinstr_ctx->gpu_va,
 345                         vinstr_ctx->dump_size,
 346                         &vinstr_ctx->vmap);
 347         if (!vinstr_ctx->cpu_va) {
 348                 kbase_mem_free(kctx, vinstr_ctx->gpu_va);
 349                 return -ENOMEM;
 350         }
 351
 352         return 0;
 353 }
 354
 355 static void kbasep_vinstr_unmap_kernel_dump_buffer(
 356                 struct kbase_vinstr_context *vinstr_ctx)
 357 {
 358         struct kbase_context *kctx = vinstr_ctx->kctx;
 359
 360         kbase_vunmap(kctx, &vinstr_ctx->vmap);
 361         kbase_mem_free(kctx, vinstr_ctx->gpu_va);
 362 }
 363
 364 /**
 365  * kbasep_vinstr_create_kctx - create kernel context for vinstr
 366  * @vinstr_ctx: vinstr context
 367  * Return: zero on success
 368  */
 369 static int kbasep_vinstr_create_kctx(struct kbase_vinstr_context *vinstr_ctx)
 370 {
 371         struct kbase_device *kbdev = vinstr_ctx->kbdev;
 372         struct kbasep_kctx_list_element *element;
 373         unsigned long flags;
 374         bool enable_backend = false;
 375         int err;
 376
 377         vinstr_ctx->kctx = kbase_create_context(vinstr_ctx->kbdev, true);
 378         if (!vinstr_ctx->kctx)
 379                 return -ENOMEM;
 380
 381         /* Map the master kernel dump buffer.  The HW dumps the counters
 382          * into this memory region. */
 383         err = kbasep_vinstr_map_kernel_dump_buffer(vinstr_ctx);
 384         if (err) {
 385                 kbase_destroy_context(vinstr_ctx->kctx);
 386                 vinstr_ctx->kctx = NULL;
 387                 return err;
 388         }
 389
 390         /* Add kernel context to list of contexts associated with device. */
 391         element = kzalloc(sizeof(*element), GFP_KERNEL);
 392         if (element) {
 393                 element->kctx = vinstr_ctx->kctx;
 394                 mutex_lock(&kbdev->kctx_list_lock);
 395                 list_add(&element->link, &kbdev->kctx_list);
 396
 397                 /* Inform timeline client about new context.
 398                  * Do this while holding the lock to avoid tracepoint
 399                  * being created in both body and summary stream. */
 400                 kbase_tlstream_tl_new_ctx(
 401                                 vinstr_ctx->kctx,
 402                                 (u32)(vinstr_ctx->kctx->id),
 403                                 (u32)(vinstr_ctx->kctx->tgid));
 404
 405                 mutex_unlock(&kbdev->kctx_list_lock);
 406         } else {
 407                 /* Don't treat this as a fail - just warn about it. */
 408                 dev_warn(kbdev->dev,
 409                                 "couldn't add kctx to kctx_list\n");
 410         }
 411
 412         /* Don't enable hardware counters if vinstr is suspended.
 413          * Note that vinstr resume code is run under vinstr context lock,
 414          * lower layer will be enabled as needed on resume. */
 415         spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
 416         if (VINSTR_IDLE == vinstr_ctx->state)
 417                 enable_backend = true;
 418         spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
 419         if (enable_backend)
 420                 err = enable_hwcnt(vinstr_ctx);
 421
 422         if (err) {
 423                 kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx);
 424                 kbase_destroy_context(vinstr_ctx->kctx);
 425                 if (element) {
 426                         mutex_lock(&kbdev->kctx_list_lock);
 427                         list_del(&element->link);
 428                         kfree(element);
 429                         mutex_unlock(&kbdev->kctx_list_lock);
 430                 }
 431                 kbase_tlstream_tl_del_ctx(vinstr_ctx->kctx);
 432                 vinstr_ctx->kctx = NULL;
 433                 return err;
 434         }
 435
 436         vinstr_ctx->thread = kthread_run(
 437                         kbasep_vinstr_service_task,
 438                         vinstr_ctx,
 439                         "mali_vinstr_service");
 440         if (!vinstr_ctx->thread) {
 441                 disable_hwcnt(vinstr_ctx);
 442                 kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx);
 443                 kbase_destroy_context(vinstr_ctx->kctx);
 444                 if (element) {
 445                         mutex_lock(&kbdev->kctx_list_lock);
 446                         list_del(&element->link);
 447                         kfree(element);
 448                         mutex_unlock(&kbdev->kctx_list_lock);
 449                 }
 450                 kbase_tlstream_tl_del_ctx(vinstr_ctx->kctx);
 451                 vinstr_ctx->kctx = NULL;
 452                 return -EFAULT;
 453         }
 454
 455         return 0;
 456 }
 457
 458 /**
 459  * kbasep_vinstr_destroy_kctx - destroy vinstr's kernel context
 460  * @vinstr_ctx: vinstr context
 461  */
 462 static void kbasep_vinstr_destroy_kctx(struct kbase_vinstr_context *vinstr_ctx)
 463 {
 464         struct kbase_device             *kbdev = vinstr_ctx->kbdev;
 465         struct kbasep_kctx_list_element *element;
 466         struct kbasep_kctx_list_element *tmp;
 467         bool                            found = false;
 468
 469         /* Release hw counters dumping resources. */
 470         vinstr_ctx->thread = NULL;
 471         disable_hwcnt(vinstr_ctx);
 472         kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx);
 473         kbase_destroy_context(vinstr_ctx->kctx);
 474
 475         /* Remove kernel context from the device's contexts list. */
 476         mutex_lock(&kbdev->kctx_list_lock);
 477         list_for_each_entry_safe(element, tmp, &kbdev->kctx_list, link) {
 478                 if (element->kctx == vinstr_ctx->kctx) {
 479                         list_del(&element->link);
 480                         kfree(element);
 481                         found = true;
 482                 }
 483         }
 484         mutex_unlock(&kbdev->kctx_list_lock);
 485
 486         if (!found)
 487                 dev_warn(kbdev->dev, "kctx not in kctx_list\n");
 488
 489         /* Inform timeline client about context destruction. */
 490         kbase_tlstream_tl_del_ctx(vinstr_ctx->kctx);
 491
 492         vinstr_ctx->kctx = NULL;
 493 }
 494
 495 /**
 496  * kbasep_vinstr_attach_client - Attach a client to the vinstr core
 497  * @vinstr_ctx:    vinstr context
 498  * @buffer_count:  requested number of dump buffers
 499  * @bitmap:        bitmaps describing which counters should be enabled
 500  * @argp:          pointer where notification descriptor shall be stored
 501  * @kernel_buffer: pointer to kernel side buffer
 502  *
 503  * Return: vinstr opaque client handle or NULL on failure
 504  */
 505 static struct kbase_vinstr_client *kbasep_vinstr_attach_client(
 506                 struct kbase_vinstr_context *vinstr_ctx, u32 buffer_count,
 507                 u32 bitmap[4], void *argp, void *kernel_buffer)
 508 {
 509         struct task_struct         *thread = NULL;
 510         struct kbase_vinstr_client *cli;
 511
 512         KBASE_DEBUG_ASSERT(vinstr_ctx);
 513
 514         if (buffer_count > MAX_BUFFER_COUNT
 515             || (buffer_count & (buffer_count - 1)))
 516                 return NULL;
 517
 518         cli = kzalloc(sizeof(*cli), GFP_KERNEL);
 519         if (!cli)
 520                 return NULL;
 521
 522         cli->vinstr_ctx   = vinstr_ctx;
 523         cli->buffer_count = buffer_count;
 524         cli->event_mask   =
 525                 (1 << BASE_HWCNT_READER_EVENT_MANUAL) |
 526                 (1 << BASE_HWCNT_READER_EVENT_PERIODIC);
 527         cli->pending      = true;
 528
 529         hwcnt_bitmap_set(cli->bitmap, bitmap);
 530
 531         mutex_lock(&vinstr_ctx->lock);
 532
 533         hwcnt_bitmap_union(vinstr_ctx->bitmap, cli->bitmap);
 534         vinstr_ctx->reprogram = true;
 535
 536         /* If this is the first client, create the vinstr kbase
 537          * context. This context is permanently resident until the
 538          * last client exits. */
 539         if (!vinstr_ctx->nclients) {
 540                 hwcnt_bitmap_set(vinstr_ctx->bitmap, cli->bitmap);
 541                 if (kbasep_vinstr_create_kctx(vinstr_ctx) < 0)
 542                         goto error;
 543
 544                 vinstr_ctx->reprogram = false;
 545                 cli->pending = false;
 546         }
 547
 548         /* The GPU resets the counter block every time there is a request
 549          * to dump it. We need a per client kernel buffer for accumulating
 550          * the counters. */
 551         cli->dump_size    = kbasep_vinstr_dump_size_ctx(vinstr_ctx);
 552         cli->accum_buffer = kzalloc(cli->dump_size, GFP_KERNEL);
 553         if (!cli->accum_buffer)
 554                 goto error;
 555
 556         /* Prepare buffers. */
 557         if (cli->buffer_count) {
 558                 int *fd = (int *)argp;
 559                 size_t tmp;
 560
 561                 /* Allocate area for buffers metadata storage. */
 562                 tmp = sizeof(struct kbase_hwcnt_reader_metadata) *
 563                         cli->buffer_count;
 564                 cli->dump_buffers_meta = kmalloc(tmp, GFP_KERNEL);
 565                 if (!cli->dump_buffers_meta)
 566                         goto error;
 567
 568                 /* Allocate required number of dumping buffers. */
 569                 cli->dump_buffers = (char *)__get_free_pages(
 570                                 GFP_KERNEL | __GFP_ZERO,
 571                                 get_order(cli->dump_size * cli->buffer_count));
 572                 if (!cli->dump_buffers)
 573                         goto error;
 574
 575                 /* Create descriptor for user-kernel data exchange. */
 576                 *fd = anon_inode_getfd(
 577                                 "[mali_vinstr_desc]",
 578                                 &vinstr_client_fops,
 579                                 cli,
 580                                 O_RDONLY | O_CLOEXEC);
 581                 if (0 > *fd)
 582                         goto error;
 583         } else if (kernel_buffer) {
 584                 cli->kernel_buffer = kernel_buffer;
 585         } else {
 586                 cli->legacy_buffer = (void __user *)argp;
 587         }
 588
 589         atomic_set(&cli->read_idx, 0);
 590         atomic_set(&cli->meta_idx, 0);
 591         atomic_set(&cli->write_idx, 0);
 592         init_waitqueue_head(&cli->waitq);
 593
 594         vinstr_ctx->nclients++;
 595         list_add(&cli->list, &vinstr_ctx->idle_clients);
 596
 597         mutex_unlock(&vinstr_ctx->lock);
 598
 599         return cli;
 600
 601 error:
 602         kfree(cli->dump_buffers_meta);
 603         if (cli->dump_buffers)
 604                 free_pages(
 605                                 (unsigned long)cli->dump_buffers,
 606                                 get_order(cli->dump_size * cli->buffer_count));
 607         kfree(cli->accum_buffer);
 608         if (!vinstr_ctx->nclients && vinstr_ctx->kctx) {
 609                 thread = vinstr_ctx->thread;
 610                 kbasep_vinstr_destroy_kctx(vinstr_ctx);
 611         }
 612         kfree(cli);
 613
 614         mutex_unlock(&vinstr_ctx->lock);
 615
 616         /* Thread must be stopped after lock is released. */
 617         if (thread)
 618                 kthread_stop(thread);
 619
 620         return NULL;
 621 }
 622
 623 void kbase_vinstr_detach_client(struct kbase_vinstr_client *cli)
 624 {
 625         struct kbase_vinstr_context *vinstr_ctx;
 626         struct kbase_vinstr_client  *iter, *tmp;
 627         struct task_struct          *thread = NULL;
 628         u32 zerobitmap[4] = { 0 };
 629         int cli_found = 0;
 630
 631         KBASE_DEBUG_ASSERT(cli);
 632         vinstr_ctx = cli->vinstr_ctx;
 633         KBASE_DEBUG_ASSERT(vinstr_ctx);
 634
 635         mutex_lock(&vinstr_ctx->lock);
 636
 637         list_for_each_entry_safe(iter, tmp, &vinstr_ctx->idle_clients, list) {
 638                 if (iter == cli) {
 639                         vinstr_ctx->reprogram = true;
 640                         cli_found = 1;
 641                         list_del(&iter->list);
 642                         break;
 643                 }
 644         }
 645         if (!cli_found) {
 646                 list_for_each_entry_safe(
 647                                 iter, tmp, &vinstr_ctx->waiting_clients, list) {
 648                         if (iter == cli) {
 649                                 vinstr_ctx->reprogram = true;
 650                                 cli_found = 1;
 651                                 list_del(&iter->list);
 652                                 break;
 653                         }
 654                 }
 655         }
 656         KBASE_DEBUG_ASSERT(cli_found);
 657
 658         kfree(cli->dump_buffers_meta);
 659         free_pages(
 660                         (unsigned long)cli->dump_buffers,
 661                         get_order(cli->dump_size * cli->buffer_count));
 662         kfree(cli->accum_buffer);
 663         kfree(cli);
 664
 665         vinstr_ctx->nclients--;
 666         if (!vinstr_ctx->nclients) {
 667                 thread = vinstr_ctx->thread;
 668                 kbasep_vinstr_destroy_kctx(vinstr_ctx);
 669         }
 670
 671         /* Rebuild context bitmap now that the client has detached */
 672         hwcnt_bitmap_set(vinstr_ctx->bitmap, zerobitmap);
 673         list_for_each_entry(iter, &vinstr_ctx->idle_clients, list)
 674                 hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap);
 675         list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list)
 676                 hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap);
 677
 678         mutex_unlock(&vinstr_ctx->lock);
 679
 680         /* Thread must be stopped after lock is released. */
 681         if (thread)
 682                 kthread_stop(thread);
 683 }
 684 KBASE_EXPORT_TEST_API(kbase_vinstr_detach_client);
 685
 686 /* Accumulate counters in the dump buffer */
 687 static void accum_dump_buffer(void *dst, void *src, size_t dump_size)
 688 {
 689         size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT;
 690         u32 *d = dst;
 691         u32 *s = src;
 692         size_t i, j;
 693
 694         for (i = 0; i < dump_size; i += block_size) {
 695                 /* skip over the header block */
 696                 d += NR_BYTES_PER_HDR / sizeof(u32);
 697                 s += NR_BYTES_PER_HDR / sizeof(u32);
 698                 for (j = 0; j < (block_size - NR_BYTES_PER_HDR) / sizeof(u32); j++) {
 699                         /* saturate result if addition would result in wraparound */
 700                         if (U32_MAX - *d < *s)
 701                                 *d = U32_MAX;
 702                         else
 703                                 *d += *s;
 704                         d++;
 705                         s++;
 706                 }
 707         }
 708 }
 709
 710 /* This is the Midgard v4 patch function.  It copies the headers for each
 711  * of the defined blocks from the master kernel buffer and then patches up
 712  * the performance counter enable mask for each of the blocks to exclude
 713  * counters that were not requested by the client. */
 714 static void patch_dump_buffer_hdr_v4(
 715                 struct kbase_vinstr_context *vinstr_ctx,
 716                 struct kbase_vinstr_client *cli)
 717 {
 718         u32 *mask;
 719         u8 *dst = cli->accum_buffer;
 720         u8 *src = vinstr_ctx->cpu_va;
 721         u32 nr_cg = vinstr_ctx->kctx->kbdev->gpu_props.num_core_groups;
 722         size_t i, group_size, group;
 723         enum {
 724                 SC0_BASE    = 0 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
 725                 SC1_BASE    = 1 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
 726                 SC2_BASE    = 2 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
 727                 SC3_BASE    = 3 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
 728                 TILER_BASE  = 4 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
 729                 MMU_L2_BASE = 5 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
 730                 JM_BASE     = 7 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT
 731         };
 732
 733         group_size = NR_CNT_BLOCKS_PER_GROUP *
 734                         NR_CNT_PER_BLOCK *
 735                         NR_BYTES_PER_CNT;
 736         for (i = 0; i < nr_cg; i++) {
 737                 group = i * group_size;
 738                 /* copy shader core headers */
 739                 memcpy(&dst[group + SC0_BASE], &src[group + SC0_BASE],
 740                        NR_BYTES_PER_HDR);
 741                 memcpy(&dst[group + SC1_BASE], &src[group + SC1_BASE],
 742                        NR_BYTES_PER_HDR);
 743                 memcpy(&dst[group + SC2_BASE], &src[group + SC2_BASE],
 744                       NR_BYTES_PER_HDR);
 745                 memcpy(&dst[group + SC3_BASE], &src[group + SC3_BASE],
 746                       NR_BYTES_PER_HDR);
 747
 748                 /* copy tiler header */
 749                 memcpy(&dst[group + TILER_BASE], &src[group + TILER_BASE],
 750                       NR_BYTES_PER_HDR);
 751
 752                 /* copy mmu header */
 753                 memcpy(&dst[group + MMU_L2_BASE], &src[group + MMU_L2_BASE],
 754                       NR_BYTES_PER_HDR);
 755
 756                 /* copy job manager header */
 757                 memcpy(&dst[group + JM_BASE], &src[group + JM_BASE],
 758                       NR_BYTES_PER_HDR);
 759
 760                 /* patch the shader core enable mask */
 761                 mask = (u32 *)&dst[group + SC0_BASE + PRFCNT_EN_MASK_OFFSET];
 762                 *mask &= cli->bitmap[SHADER_HWCNT_BM];
 763                 mask = (u32 *)&dst[group + SC1_BASE + PRFCNT_EN_MASK_OFFSET];
 764                 *mask &= cli->bitmap[SHADER_HWCNT_BM];
 765                 mask = (u32 *)&dst[group + SC2_BASE + PRFCNT_EN_MASK_OFFSET];
 766                 *mask &= cli->bitmap[SHADER_HWCNT_BM];
 767                 mask = (u32 *)&dst[group + SC3_BASE + PRFCNT_EN_MASK_OFFSET];
 768                 *mask &= cli->bitmap[SHADER_HWCNT_BM];
 769
 770                 /* patch the tiler core enable mask */
 771                 mask = (u32 *)&dst[group + TILER_BASE + PRFCNT_EN_MASK_OFFSET];
 772                 *mask &= cli->bitmap[TILER_HWCNT_BM];
 773
 774                 /* patch the mmu core enable mask */
 775                 mask = (u32 *)&dst[group + MMU_L2_BASE + PRFCNT_EN_MASK_OFFSET];
 776                 *mask &= cli->bitmap[MMU_L2_HWCNT_BM];
 777
 778                 /* patch the job manager enable mask */
 779                 mask = (u32 *)&dst[group + JM_BASE + PRFCNT_EN_MASK_OFFSET];
 780                 *mask &= cli->bitmap[JM_HWCNT_BM];
 781         }
 782 }
 783
 784 /* This is the Midgard v5 patch function.  It copies the headers for each
 785  * of the defined blocks from the master kernel buffer and then patches up
 786  * the performance counter enable mask for each of the blocks to exclude
 787  * counters that were not requested by the client. */
 788 static void patch_dump_buffer_hdr_v5(
 789                 struct kbase_vinstr_context *vinstr_ctx,
 790                 struct kbase_vinstr_client *cli)
 791 {
 792         struct kbase_device *kbdev = vinstr_ctx->kctx->kbdev;
 793         u32 i, nr_l2;
 794         u64 core_mask;
 795         u32 *mask;
 796         u8 *dst = cli->accum_buffer;
 797         u8 *src = vinstr_ctx->cpu_va;
 798         size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT;
 799
 800         /* copy and patch job manager header */
 801         memcpy(dst, src, NR_BYTES_PER_HDR);
 802         mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
 803         *mask &= cli->bitmap[JM_HWCNT_BM];
 804         dst += block_size;
 805         src += block_size;
 806
 807         /* copy and patch tiler header */
 808         memcpy(dst, src, NR_BYTES_PER_HDR);
 809         mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
 810         *mask &= cli->bitmap[TILER_HWCNT_BM];
 811         dst += block_size;
 812         src += block_size;
 813
 814         /* copy and patch MMU/L2C headers */
 815         nr_l2 = kbdev->gpu_props.props.l2_props.num_l2_slices;
 816         for (i = 0; i < nr_l2; i++) {
 817                 memcpy(dst, src, NR_BYTES_PER_HDR);
 818                 mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
 819                 *mask &= cli->bitmap[MMU_L2_HWCNT_BM];
 820                 dst += block_size;
 821                 src += block_size;
 822         }
 823
 824         /* copy and patch shader core headers */
 825         core_mask = kbdev->gpu_props.props.coherency_info.group[0].core_mask;
 826         while (0ull != core_mask) {
 827                 memcpy(dst, src, NR_BYTES_PER_HDR);
 828                 if (0ull != (core_mask & 1ull)) {
 829                         /* if block is not reserved update header */
 830                         mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
 831                         *mask &= cli->bitmap[SHADER_HWCNT_BM];
 832                 }
 833                 dst += block_size;
 834                 src += block_size;
 835
 836                 core_mask >>= 1;
 837         }
 838 }
 839
 840 /**
 841  * accum_clients - accumulate dumped hw counters for all known clients
 842  * @vinstr_ctx: vinstr context
 843  */
 844 static void accum_clients(struct kbase_vinstr_context *vinstr_ctx)
 845 {
 846         struct kbase_vinstr_client *iter;
 847         int v4 = 0;
 848
 849 #ifndef CONFIG_MALI_NO_MALI
 850         v4 = kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4);
 851 #endif
 852
 853         list_for_each_entry(iter, &vinstr_ctx->idle_clients, list) {
 854                 /* Don't bother accumulating clients whose hwcnt requests
 855                  * have not yet been honoured. */
 856                 if (iter->pending)
 857                         continue;
 858                 if (v4)
 859                         patch_dump_buffer_hdr_v4(vinstr_ctx, iter);
 860                 else
 861                         patch_dump_buffer_hdr_v5(vinstr_ctx, iter);
 862                 accum_dump_buffer(
 863                                 iter->accum_buffer,
 864                                 vinstr_ctx->cpu_va,
 865                                 iter->dump_size);
 866         }
 867         list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list) {
 868                 /* Don't bother accumulating clients whose hwcnt requests
 869                  * have not yet been honoured. */
 870                 if (iter->pending)
 871                         continue;
 872                 if (v4)
 873                         patch_dump_buffer_hdr_v4(vinstr_ctx, iter);
 874                 else
 875                         patch_dump_buffer_hdr_v5(vinstr_ctx, iter);
 876                 accum_dump_buffer(
 877                                 iter->accum_buffer,
 878                                 vinstr_ctx->cpu_va,
 879                                 iter->dump_size);
 880         }
 881 }
 882
 883 /*****************************************************************************/
 884
 885 /**
 886  * kbasep_vinstr_get_timestamp - return timestamp
 887  *
 888  * Function returns timestamp value based on raw monotonic timer. Value will
 889  * wrap around zero in case of overflow.
 890  *
 891  * Return: timestamp value
 892  */
 893 static u64 kbasep_vinstr_get_timestamp(void)
 894 {
 895         struct timespec ts;
 896
 897         getrawmonotonic(&ts);
 898         return (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec;
 899 }
 900
 901 /**
 902  * kbasep_vinstr_add_dump_request - register client's dumping request
 903  * @cli:             requesting client
 904  * @waiting_clients: list of pending dumping requests
 905  */
 906 static void kbasep_vinstr_add_dump_request(
 907                 struct kbase_vinstr_client *cli,
 908                 struct list_head *waiting_clients)
 909 {
 910         struct kbase_vinstr_client *tmp;
 911
 912         if (list_empty(waiting_clients)) {
 913                 list_add(&cli->list, waiting_clients);
 914                 return;
 915         }
 916         list_for_each_entry(tmp, waiting_clients, list) {
 917                 if (tmp->dump_time > cli->dump_time) {
 918                         list_add_tail(&cli->list, &tmp->list);
 919                         return;
 920                 }
 921         }
 922         list_add_tail(&cli->list, waiting_clients);
 923 }
 924
 925 /**
 926  * kbasep_vinstr_collect_and_accumulate - collect hw counters via low level
 927  *                                        dump and accumulate them for known
 928  *                                        clients
 929  * @vinstr_ctx: vinstr context
 930  * @timestamp: pointer where collection timestamp will be recorded
 931  *
 932  * Return: zero on success
 933  */
 934 static int kbasep_vinstr_collect_and_accumulate(
 935                 struct kbase_vinstr_context *vinstr_ctx, u64 *timestamp)
 936 {
 937         unsigned long flags;
 938         int rcode;
 939
 940 #ifdef CONFIG_MALI_NO_MALI
 941         /* The dummy model needs the CPU mapping. */
 942         gpu_model_set_dummy_prfcnt_base_cpu(vinstr_ctx->cpu_va);
 943 #endif
 944
 945         spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
 946         if (VINSTR_IDLE != vinstr_ctx->state) {
 947                 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
 948                 return -EAGAIN;
 949         } else {
 950                 vinstr_ctx->state = VINSTR_DUMPING;
 951         }
 952         spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
 953
 954         /* Request HW counters dump.
 955          * Disable preemption to make dump timestamp more accurate. */
 956         preempt_disable();
 957         *timestamp = kbasep_vinstr_get_timestamp();
 958         rcode = kbase_instr_hwcnt_request_dump(vinstr_ctx->kctx);
 959         preempt_enable();
 960
 961         if (!rcode)
 962                 rcode = kbase_instr_hwcnt_wait_for_dump(vinstr_ctx->kctx);
 963         WARN_ON(rcode);
 964
 965         spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
 966         switch (vinstr_ctx->state)
 967         {
 968         case VINSTR_SUSPENDING:
 969                 schedule_work(&vinstr_ctx->suspend_work);
 970                 break;
 971         case VINSTR_DUMPING:
 972                 vinstr_ctx->state = VINSTR_IDLE;
 973                 wake_up_all(&vinstr_ctx->suspend_waitq);
 974                 break;
 975         default:
 976                 break;
 977         }
 978         spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
 979
 980         /* Accumulate values of collected counters. */
 981         if (!rcode)
 982                 accum_clients(vinstr_ctx);
 983
 984         return rcode;
 985 }
 986
 987 /**
 988  * kbasep_vinstr_fill_dump_buffer - copy accumulated counters to empty kernel
 989  *                                  buffer
 990  * @cli:       requesting client
 991  * @timestamp: timestamp when counters were collected
 992  * @event_id:  id of event that caused triggered counters collection
 993  *
 994  * Return: zero on success
 995  */
 996 static int kbasep_vinstr_fill_dump_buffer(
 997                 struct kbase_vinstr_client *cli, u64 timestamp,
 998                 enum base_hwcnt_reader_event event_id)
 999 {
1000         unsigned int write_idx = atomic_read(&cli->write_idx);
1001         unsigned int read_idx  = atomic_read(&cli->read_idx);
1002
1003         struct kbase_hwcnt_reader_metadata *meta;
1004         void                               *buffer;
1005
1006         /* Check if there is a place to copy HWC block into. */
1007         if (write_idx - read_idx == cli->buffer_count)
1008                 return -1;
1009         write_idx %= cli->buffer_count;
1010
1011         /* Fill in dump buffer and its metadata. */
1012         buffer = &cli->dump_buffers[write_idx * cli->dump_size];
1013         meta   = &cli->dump_buffers_meta[write_idx];
1014         meta->timestamp  = timestamp;
1015         meta->event_id   = event_id;
1016         meta->buffer_idx = write_idx;
1017         memcpy(buffer, cli->accum_buffer, cli->dump_size);
1018         return 0;
1019 }
1020
1021 /**
1022  * kbasep_vinstr_fill_dump_buffer_legacy - copy accumulated counters to buffer
1023  *                                         allocated in userspace
1024  * @cli: requesting client
1025  *
1026  * Return: zero on success
1027  *
1028  * This is part of legacy ioctl interface.
1029  */
1030 static int kbasep_vinstr_fill_dump_buffer_legacy(
1031                 struct kbase_vinstr_client *cli)
1032 {
1033         void __user  *buffer = cli->legacy_buffer;
1034         int          rcode;
1035
1036         /* Copy data to user buffer. */
1037         rcode = copy_to_user(buffer, cli->accum_buffer, cli->dump_size);
1038         if (rcode)
1039                 pr_warn("error while copying buffer to user\n");
1040         return rcode;
1041 }
1042
1043 /**
1044  * kbasep_vinstr_fill_dump_buffer_kernel - copy accumulated counters to buffer
1045  *                                         allocated in kernel space
1046  * @cli: requesting client
1047  *
1048  * Return: zero on success
1049  *
1050  * This is part of the kernel client interface.
1051  */
1052 static int kbasep_vinstr_fill_dump_buffer_kernel(
1053                 struct kbase_vinstr_client *cli)
1054 {
1055         memcpy(cli->kernel_buffer, cli->accum_buffer, cli->dump_size);
1056
1057         return 0;
1058 }
1059
1060 /**
1061  * kbasep_vinstr_reprogram - reprogram hwcnt set collected by inst
1062  * @vinstr_ctx: vinstr context
1063  */
1064 static void kbasep_vinstr_reprogram(
1065                 struct kbase_vinstr_context *vinstr_ctx)
1066 {
1067         unsigned long flags;
1068         bool suspended = false;
1069
1070         /* Don't enable hardware counters if vinstr is suspended. */
1071         spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
1072         if (VINSTR_IDLE != vinstr_ctx->state)
1073                 suspended = true;
1074         spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
1075         if (suspended)
1076                 return;
1077
1078         /* Change to suspended state is done while holding vinstr context
1079          * lock. Below code will then no re-enable the instrumentation. */
1080
1081         if (vinstr_ctx->reprogram) {
1082                 struct kbase_vinstr_client *iter;
1083
1084                 if (!reprogram_hwcnt(vinstr_ctx)) {
1085                         vinstr_ctx->reprogram = false;
1086                         list_for_each_entry(
1087                                         iter,
1088                                         &vinstr_ctx->idle_clients,
1089                                         list)
1090                                 iter->pending = false;
1091                         list_for_each_entry(
1092                                         iter,
1093                                         &vinstr_ctx->waiting_clients,
1094                                         list)
1095                                 iter->pending = false;
1096                 }
1097         }
1098 }
1099
1100 /**
1101  * kbasep_vinstr_update_client - copy accumulated counters to user readable
1102  *                               buffer and notify the user
1103  * @cli:       requesting client
1104  * @timestamp: timestamp when counters were collected
1105  * @event_id:  id of event that caused triggered counters collection
1106  *
1107  * Return: zero on success
1108  */
1109 static int kbasep_vinstr_update_client(
1110                 struct kbase_vinstr_client *cli, u64 timestamp,
1111                 enum base_hwcnt_reader_event event_id)
1112 {
1113         int rcode = 0;
1114
1115         /* Copy collected counters to user readable buffer. */
1116         if (cli->buffer_count)
1117                 rcode = kbasep_vinstr_fill_dump_buffer(
1118                                 cli, timestamp, event_id);
1119         else if (cli->kernel_buffer)
1120                 rcode = kbasep_vinstr_fill_dump_buffer_kernel(cli);
1121         else
1122                 rcode = kbasep_vinstr_fill_dump_buffer_legacy(cli);
1123
1124         if (rcode)
1125                 goto exit;
1126
1127
1128         /* Notify client. Make sure all changes to memory are visible. */
1129         wmb();
1130         atomic_inc(&cli->write_idx);
1131         wake_up_interruptible(&cli->waitq);
1132
1133         /* Prepare for next request. */
1134         memset(cli->accum_buffer, 0, cli->dump_size);
1135
1136 exit:
1137         return rcode;
1138 }
1139
1140 /**
1141  * kbasep_vinstr_wake_up_callback - vinstr wake up timer wake up function
1142  *
1143  * @hrtimer: high resolution timer
1144  *
1145  * Return: High resolution timer restart enum.
1146  */
1147 static enum hrtimer_restart kbasep_vinstr_wake_up_callback(
1148                 struct hrtimer *hrtimer)
1149 {
1150         struct kbasep_vinstr_wake_up_timer *timer =
1151                 container_of(
1152                         hrtimer,
1153                         struct kbasep_vinstr_wake_up_timer,
1154                         hrtimer);
1155
1156         KBASE_DEBUG_ASSERT(timer);
1157
1158         atomic_set(&timer->vinstr_ctx->request_pending, 1);
1159         wake_up_all(&timer->vinstr_ctx->waitq);
1160
1161         return HRTIMER_NORESTART;
1162 }
1163
1164 /**
1165  * kbasep_vinstr_service_task - HWC dumping service thread
1166  *
1167  * @data: Pointer to vinstr context structure.
1168  *
1169  * Return: Always returns zero.
1170  */
1171 static int kbasep_vinstr_service_task(void *data)
1172 {
1173         struct kbase_vinstr_context        *vinstr_ctx = data;
1174         struct kbasep_vinstr_wake_up_timer timer;
1175
1176         KBASE_DEBUG_ASSERT(vinstr_ctx);
1177
1178         hrtimer_init(&timer.hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1179         timer.hrtimer.function = kbasep_vinstr_wake_up_callback;
1180         timer.vinstr_ctx       = vinstr_ctx;
1181
1182         while (!kthread_should_stop()) {
1183                 struct kbase_vinstr_client *cli = NULL;
1184                 struct kbase_vinstr_client *tmp;
1185                 int                        rcode;
1186
1187                 u64              timestamp = kbasep_vinstr_get_timestamp();
1188                 u64              dump_time = 0;
1189                 struct list_head expired_requests;
1190
1191                 /* Hold lock while performing operations on lists of clients. */
1192                 mutex_lock(&vinstr_ctx->lock);
1193
1194                 /* Closing thread must not interact with client requests. */
1195                 if (current == vinstr_ctx->thread) {
1196                         atomic_set(&vinstr_ctx->request_pending, 0);
1197
1198                         if (!list_empty(&vinstr_ctx->waiting_clients)) {
1199                                 cli = list_first_entry(
1200                                                 &vinstr_ctx->waiting_clients,
1201                                                 struct kbase_vinstr_client,
1202                                                 list);
1203                                 dump_time = cli->dump_time;
1204                         }
1205                 }
1206
1207                 if (!cli || ((s64)timestamp - (s64)dump_time < 0ll)) {
1208                         mutex_unlock(&vinstr_ctx->lock);
1209
1210                         /* Sleep until next dumping event or service request. */
1211                         if (cli) {
1212                                 u64 diff = dump_time - timestamp;
1213
1214                                 hrtimer_start(
1215                                                 &timer.hrtimer,
1216                                                 ns_to_ktime(diff),
1217                                                 HRTIMER_MODE_REL);
1218                         }
1219                         wait_event(
1220                                         vinstr_ctx->waitq,
1221                                         atomic_read(
1222                                                 &vinstr_ctx->request_pending) ||
1223                                         kthread_should_stop());
1224                         hrtimer_cancel(&timer.hrtimer);
1225                         continue;
1226                 }
1227
1228                 rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx,
1229                                 &timestamp);
1230
1231                 INIT_LIST_HEAD(&expired_requests);
1232
1233                 /* Find all expired requests. */
1234                 list_for_each_entry_safe(
1235                                 cli,
1236                                 tmp,
1237                                 &vinstr_ctx->waiting_clients,
1238                                 list) {
1239                         s64 tdiff =
1240                                 (s64)(timestamp + DUMPING_RESOLUTION) -
1241                                 (s64)cli->dump_time;
1242                         if (tdiff >= 0ll) {
1243                                 list_del(&cli->list);
1244                                 list_add(&cli->list, &expired_requests);
1245                         } else {
1246                                 break;
1247                         }
1248                 }
1249
1250                 /* Fill data for each request found. */
1251                 list_for_each_entry_safe(cli, tmp, &expired_requests, list) {
1252                         /* Ensure that legacy buffer will not be used from
1253                          * this kthread context. */
1254                         BUG_ON(0 == cli->buffer_count);
1255                         /* Expect only periodically sampled clients. */
1256                         BUG_ON(0 == cli->dump_interval);
1257
1258                         if (!rcode)
1259                                 kbasep_vinstr_update_client(
1260                                                 cli,
1261                                                 timestamp,
1262                                                 BASE_HWCNT_READER_EVENT_PERIODIC);
1263
1264                         /* Set new dumping time. Drop missed probing times. */
1265                         do {
1266                                 cli->dump_time += cli->dump_interval;
1267                         } while (cli->dump_time < timestamp);
1268
1269                         list_del(&cli->list);
1270                         kbasep_vinstr_add_dump_request(
1271                                         cli,
1272                                         &vinstr_ctx->waiting_clients);
1273                 }
1274
1275                 /* Reprogram counters set if required. */
1276                 kbasep_vinstr_reprogram(vinstr_ctx);
1277
1278                 mutex_unlock(&vinstr_ctx->lock);
1279         }
1280
1281         return 0;
1282 }
1283
1284 /*****************************************************************************/
1285
1286 /**
1287  * kbasep_vinstr_hwcnt_reader_buffer_ready - check if client has ready buffers
1288  * @cli: pointer to vinstr client structure
1289  *
1290  * Return: non-zero if client has at least one dumping buffer filled that was
1291  *         not notified to user yet
1292  */
1293 static int kbasep_vinstr_hwcnt_reader_buffer_ready(
1294                 struct kbase_vinstr_client *cli)
1295 {
1296         KBASE_DEBUG_ASSERT(cli);
1297         return atomic_read(&cli->write_idx) != atomic_read(&cli->meta_idx);
1298 }
1299
1300 /**
1301  * kbasep_vinstr_hwcnt_reader_ioctl_get_buffer - hwcnt reader's ioctl command
1302  * @cli:    pointer to vinstr client structure
1303  * @buffer: pointer to userspace buffer
1304  * @size:   size of buffer
1305  *
1306  * Return: zero on success
1307  */
1308 static long kbasep_vinstr_hwcnt_reader_ioctl_get_buffer(
1309                 struct kbase_vinstr_client *cli, void __user *buffer,
1310                 size_t size)
1311 {
1312         unsigned int meta_idx = atomic_read(&cli->meta_idx);
1313         unsigned int idx = meta_idx % cli->buffer_count;
1314
1315         struct kbase_hwcnt_reader_metadata *meta = &cli->dump_buffers_meta[idx];
1316
1317         /* Metadata sanity check. */
1318         KBASE_DEBUG_ASSERT(idx == meta->buffer_idx);
1319
1320         if (sizeof(struct kbase_hwcnt_reader_metadata) != size)
1321                 return -EINVAL;
1322
1323         /* Check if there is any buffer available. */
1324         if (atomic_read(&cli->write_idx) == meta_idx)
1325                 return -EAGAIN;
1326
1327         /* Check if previously taken buffer was put back. */
1328         if (atomic_read(&cli->read_idx) != meta_idx)
1329                 return -EBUSY;
1330
1331         /* Copy next available buffer's metadata to user. */
1332         if (copy_to_user(buffer, meta, size))
1333                 return -EFAULT;
1334
1335         atomic_inc(&cli->meta_idx);
1336
1337         return 0;
1338 }
1339
1340 /**
1341  * kbasep_vinstr_hwcnt_reader_ioctl_put_buffer - hwcnt reader's ioctl command
1342  * @cli:    pointer to vinstr client structure
1343  * @buffer: pointer to userspace buffer
1344  * @size:   size of buffer
1345  *
1346  * Return: zero on success
1347  */
1348 static long kbasep_vinstr_hwcnt_reader_ioctl_put_buffer(
1349                 struct kbase_vinstr_client *cli, void __user *buffer,
1350                 size_t size)
1351 {
1352         unsigned int read_idx = atomic_read(&cli->read_idx);
1353         unsigned int idx = read_idx % cli->buffer_count;
1354
1355         struct kbase_hwcnt_reader_metadata meta;
1356
1357         if (sizeof(struct kbase_hwcnt_reader_metadata) != size)
1358                 return -EINVAL;
1359
1360         /* Check if any buffer was taken. */
1361         if (atomic_read(&cli->meta_idx) == read_idx)
1362                 return -EPERM;
1363
1364         /* Check if correct buffer is put back. */
1365         if (copy_from_user(&meta, buffer, size))
1366                 return -EFAULT;
1367         if (idx != meta.buffer_idx)
1368                 return -EINVAL;
1369
1370         atomic_inc(&cli->read_idx);
1371
1372         return 0;
1373 }
1374
1375 /**
1376  * kbasep_vinstr_hwcnt_reader_ioctl_set_interval - hwcnt reader's ioctl command
1377  * @cli:      pointer to vinstr client structure
1378  * @interval: periodic dumping interval (disable periodic dumping if zero)
1379  *
1380  * Return: zero on success
1381  */
1382 static long kbasep_vinstr_hwcnt_reader_ioctl_set_interval(
1383                 struct kbase_vinstr_client *cli, u32 interval)
1384 {
1385         struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1386
1387         KBASE_DEBUG_ASSERT(vinstr_ctx);
1388
1389         mutex_lock(&vinstr_ctx->lock);
1390
1391         list_del(&cli->list);
1392
1393         cli->dump_interval = interval;
1394
1395         /* If interval is non-zero, enable periodic dumping for this client. */
1396         if (cli->dump_interval) {
1397                 if (DUMPING_RESOLUTION > cli->dump_interval)
1398                         cli->dump_interval = DUMPING_RESOLUTION;
1399                 cli->dump_time =
1400                         kbasep_vinstr_get_timestamp() + cli->dump_interval;
1401
1402                 kbasep_vinstr_add_dump_request(
1403                                 cli, &vinstr_ctx->waiting_clients);
1404
1405                 atomic_set(&vinstr_ctx->request_pending, 1);
1406                 wake_up_all(&vinstr_ctx->waitq);
1407         } else {
1408                 list_add(&cli->list, &vinstr_ctx->idle_clients);
1409         }
1410
1411         mutex_unlock(&vinstr_ctx->lock);
1412
1413         return 0;
1414 }
1415
1416 /**
1417  * kbasep_vinstr_hwcnt_reader_event_mask - return event mask for event id
1418  * @event_id: id of event
1419  * Return: event_mask or zero if event is not supported or maskable
1420  */
1421 static u32 kbasep_vinstr_hwcnt_reader_event_mask(
1422                 enum base_hwcnt_reader_event event_id)
1423 {
1424         u32 event_mask = 0;
1425
1426         switch (event_id) {
1427         case BASE_HWCNT_READER_EVENT_PREJOB:
1428         case BASE_HWCNT_READER_EVENT_POSTJOB:
1429                 /* These event are maskable. */
1430                 event_mask = (1 << event_id);
1431                 break;
1432
1433         case BASE_HWCNT_READER_EVENT_MANUAL:
1434         case BASE_HWCNT_READER_EVENT_PERIODIC:
1435                 /* These event are non-maskable. */
1436         default:
1437                 /* These event are not supported. */
1438                 break;
1439         }
1440
1441         return event_mask;
1442 }
1443
1444 /**
1445  * kbasep_vinstr_hwcnt_reader_ioctl_enable_event - hwcnt reader's ioctl command
1446  * @cli:      pointer to vinstr client structure
1447  * @event_id: id of event to enable
1448  *
1449  * Return: zero on success
1450  */
1451 static long kbasep_vinstr_hwcnt_reader_ioctl_enable_event(
1452                 struct kbase_vinstr_client *cli,
1453                 enum base_hwcnt_reader_event event_id)
1454 {
1455         struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1456         u32                         event_mask;
1457
1458         KBASE_DEBUG_ASSERT(vinstr_ctx);
1459
1460         event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id);
1461         if (!event_mask)
1462                 return -EINVAL;
1463
1464         mutex_lock(&vinstr_ctx->lock);
1465         cli->event_mask |= event_mask;
1466         mutex_unlock(&vinstr_ctx->lock);
1467
1468         return 0;
1469 }
1470
1471 /**
1472  * kbasep_vinstr_hwcnt_reader_ioctl_disable_event - hwcnt reader's ioctl command
1473  * @cli:      pointer to vinstr client structure
1474  * @event_id: id of event to disable
1475  *
1476  * Return: zero on success
1477  */
1478 static long kbasep_vinstr_hwcnt_reader_ioctl_disable_event(
1479                 struct kbase_vinstr_client *cli,
1480                 enum base_hwcnt_reader_event event_id)
1481 {
1482         struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1483         u32                         event_mask;
1484
1485         KBASE_DEBUG_ASSERT(vinstr_ctx);
1486
1487         event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id);
1488         if (!event_mask)
1489                 return -EINVAL;
1490
1491         mutex_lock(&vinstr_ctx->lock);
1492         cli->event_mask &= ~event_mask;
1493         mutex_unlock(&vinstr_ctx->lock);
1494
1495         return 0;
1496 }
1497
1498 /**
1499  * kbasep_vinstr_hwcnt_reader_ioctl_get_hwver - hwcnt reader's ioctl command
1500  * @cli:   pointer to vinstr client structure
1501  * @hwver: pointer to user buffer where hw version will be stored
1502  *
1503  * Return: zero on success
1504  */
1505 static long kbasep_vinstr_hwcnt_reader_ioctl_get_hwver(
1506                 struct kbase_vinstr_client *cli, u32 __user *hwver)
1507 {
1508 #ifndef CONFIG_MALI_NO_MALI
1509         struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1510 #endif
1511
1512         u32                         ver = 5;
1513
1514 #ifndef CONFIG_MALI_NO_MALI
1515         KBASE_DEBUG_ASSERT(vinstr_ctx);
1516         if (kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4))
1517                 ver = 4;
1518 #endif
1519
1520         return put_user(ver, hwver);
1521 }
1522
1523 /**
1524  * kbasep_vinstr_hwcnt_reader_ioctl - hwcnt reader's ioctl
1525  * @filp:   pointer to file structure
1526  * @cmd:    user command
1527  * @arg:    command's argument
1528  *
1529  * Return: zero on success
1530  */
1531 static long kbasep_vinstr_hwcnt_reader_ioctl(struct file *filp,
1532                 unsigned int cmd, unsigned long arg)
1533 {
1534         long                       rcode = 0;
1535         struct kbase_vinstr_client *cli;
1536
1537         KBASE_DEBUG_ASSERT(filp);
1538
1539         cli = filp->private_data;
1540         KBASE_DEBUG_ASSERT(cli);
1541
1542         if (unlikely(KBASE_HWCNT_READER != _IOC_TYPE(cmd)))
1543                 return -EINVAL;
1544
1545         switch (cmd) {
1546         case KBASE_HWCNT_READER_GET_API_VERSION:
1547                 rcode = put_user(HWCNT_READER_API, (u32 __user *)arg);
1548                 break;
1549         case KBASE_HWCNT_READER_GET_HWVER:
1550                 rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_hwver(
1551                                 cli, (u32 __user *)arg);
1552                 break;
1553         case KBASE_HWCNT_READER_GET_BUFFER_SIZE:
1554                 KBASE_DEBUG_ASSERT(cli->vinstr_ctx);
1555                 rcode = put_user(
1556                                 (u32)cli->vinstr_ctx->dump_size,
1557                                 (u32 __user *)arg);
1558                 break;
1559         case KBASE_HWCNT_READER_DUMP:
1560                 rcode = kbase_vinstr_hwc_dump(
1561                                 cli, BASE_HWCNT_READER_EVENT_MANUAL);
1562                 break;
1563         case KBASE_HWCNT_READER_CLEAR:
1564                 rcode = kbase_vinstr_hwc_clear(cli);
1565                 break;
1566         case KBASE_HWCNT_READER_GET_BUFFER:
1567                 rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_buffer(
1568                                 cli, (void __user *)arg, _IOC_SIZE(cmd));
1569                 break;
1570         case KBASE_HWCNT_READER_PUT_BUFFER:
1571                 rcode = kbasep_vinstr_hwcnt_reader_ioctl_put_buffer(
1572                                 cli, (void __user *)arg, _IOC_SIZE(cmd));
1573                 break;
1574         case KBASE_HWCNT_READER_SET_INTERVAL:
1575                 rcode = kbasep_vinstr_hwcnt_reader_ioctl_set_interval(
1576                                 cli, (u32)arg);
1577                 break;
1578         case KBASE_HWCNT_READER_ENABLE_EVENT:
1579                 rcode = kbasep_vinstr_hwcnt_reader_ioctl_enable_event(
1580                                 cli, (enum base_hwcnt_reader_event)arg);
1581                 break;
1582         case KBASE_HWCNT_READER_DISABLE_EVENT:
1583                 rcode = kbasep_vinstr_hwcnt_reader_ioctl_disable_event(
1584                                 cli, (enum base_hwcnt_reader_event)arg);
1585                 break;
1586         default:
1587                 rcode = -EINVAL;
1588                 break;
1589         }
1590
1591         return rcode;
1592 }
1593
1594 /**
1595  * kbasep_vinstr_hwcnt_reader_poll - hwcnt reader's poll
1596  * @filp: pointer to file structure
1597  * @wait: pointer to poll table
1598  * Return: POLLIN if data can be read without blocking, otherwise zero
1599  */
1600 static unsigned int kbasep_vinstr_hwcnt_reader_poll(struct file *filp,
1601                 poll_table *wait)
1602 {
1603         struct kbase_vinstr_client *cli;
1604
1605         KBASE_DEBUG_ASSERT(filp);
1606         KBASE_DEBUG_ASSERT(wait);
1607
1608         cli = filp->private_data;
1609         KBASE_DEBUG_ASSERT(cli);
1610
1611         poll_wait(filp, &cli->waitq, wait);
1612         if (kbasep_vinstr_hwcnt_reader_buffer_ready(cli))
1613                 return POLLIN;
1614         return 0;
1615 }
1616
1617 /**
1618  * kbasep_vinstr_hwcnt_reader_mmap - hwcnt reader's mmap
1619  * @filp: pointer to file structure
1620  * @vma:  pointer to vma structure
1621  * Return: zero on success
1622  */
1623 static int kbasep_vinstr_hwcnt_reader_mmap(struct file *filp,
1624                 struct vm_area_struct *vma)
1625 {
1626         struct kbase_vinstr_client *cli;
1627         unsigned long size, addr, pfn, offset;
1628         unsigned long vm_size = vma->vm_end - vma->vm_start;
1629
1630         KBASE_DEBUG_ASSERT(filp);
1631         KBASE_DEBUG_ASSERT(vma);
1632
1633         cli = filp->private_data;
1634         KBASE_DEBUG_ASSERT(cli);
1635
1636         size = cli->buffer_count * cli->dump_size;
1637
1638         if (vma->vm_pgoff > (size >> PAGE_SHIFT))
1639                 return -EINVAL;
1640
1641         offset = vma->vm_pgoff << PAGE_SHIFT;
1642         if (vm_size > size - offset)
1643                 return -EINVAL;
1644
1645         addr = __pa((unsigned long)cli->dump_buffers + offset);
1646         pfn = addr >> PAGE_SHIFT;
1647
1648         return remap_pfn_range(
1649                         vma,
1650                         vma->vm_start,
1651                         pfn,
1652                         vm_size,
1653                         vma->vm_page_prot);
1654 }
1655
1656 /**
1657  * kbasep_vinstr_hwcnt_reader_release - hwcnt reader's release
1658  * @inode: pointer to inode structure
1659  * @filp:  pointer to file structure
1660  * Return always return zero
1661  */
1662 static int kbasep_vinstr_hwcnt_reader_release(struct inode *inode,
1663                 struct file *filp)
1664 {
1665         struct kbase_vinstr_client *cli;
1666
1667         KBASE_DEBUG_ASSERT(inode);
1668         KBASE_DEBUG_ASSERT(filp);
1669
1670         cli = filp->private_data;
1671         KBASE_DEBUG_ASSERT(cli);
1672
1673         kbase_vinstr_detach_client(cli);
1674         return 0;
1675 }
1676
1677 /*****************************************************************************/
1678
1679 /**
1680  * kbasep_vinstr_kick_scheduler - trigger scheduler cycle
1681  * @kbdev: pointer to kbase device structure
1682  */
1683 static void kbasep_vinstr_kick_scheduler(struct kbase_device *kbdev)
1684 {
1685         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1686         unsigned long flags;
1687
1688         down(&js_devdata->schedule_sem);
1689         spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1690         kbase_backend_slot_update(kbdev);
1691         spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1692         up(&js_devdata->schedule_sem);
1693 }
1694
1695 /**
1696  * kbasep_vinstr_suspend_worker - worker suspending vinstr module
1697  * @data: pointer to work structure
1698  */
1699 static void kbasep_vinstr_suspend_worker(struct work_struct *data)
1700 {
1701         struct kbase_vinstr_context *vinstr_ctx;
1702         unsigned long flags;
1703
1704         vinstr_ctx = container_of(data, struct kbase_vinstr_context,
1705                         suspend_work);
1706
1707         mutex_lock(&vinstr_ctx->lock);
1708
1709         if (vinstr_ctx->kctx)
1710                 disable_hwcnt(vinstr_ctx);
1711
1712         spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
1713         vinstr_ctx->state = VINSTR_SUSPENDED;
1714         wake_up_all(&vinstr_ctx->suspend_waitq);
1715         spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
1716
1717         mutex_unlock(&vinstr_ctx->lock);
1718
1719         /* Kick GPU scheduler to allow entering protected mode.
1720          * This must happen after vinstr was suspended. */
1721         kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev);
1722 }
1723
1724 /**
1725  * kbasep_vinstr_suspend_worker - worker resuming vinstr module
1726  * @data: pointer to work structure
1727  */
1728 static void kbasep_vinstr_resume_worker(struct work_struct *data)
1729 {
1730         struct kbase_vinstr_context *vinstr_ctx;
1731         unsigned long flags;
1732
1733         vinstr_ctx = container_of(data, struct kbase_vinstr_context,
1734                         resume_work);
1735
1736         mutex_lock(&vinstr_ctx->lock);
1737
1738         if (vinstr_ctx->kctx)
1739                 enable_hwcnt(vinstr_ctx);
1740
1741         spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
1742         vinstr_ctx->state = VINSTR_IDLE;
1743         wake_up_all(&vinstr_ctx->suspend_waitq);
1744         spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
1745
1746         mutex_unlock(&vinstr_ctx->lock);
1747
1748         /* Kick GPU scheduler to allow entering protected mode.
1749          * Note that scheduler state machine might requested re-entry to
1750          * protected mode before vinstr was resumed.
1751          * This must happen after vinstr was release. */
1752         kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev);
1753 }
1754
1755 /*****************************************************************************/
1756
1757 struct kbase_vinstr_context *kbase_vinstr_init(struct kbase_device *kbdev)
1758 {
1759         struct kbase_vinstr_context *vinstr_ctx;
1760
1761         vinstr_ctx = kzalloc(sizeof(*vinstr_ctx), GFP_KERNEL);
1762         if (!vinstr_ctx)
1763                 return NULL;
1764
1765         INIT_LIST_HEAD(&vinstr_ctx->idle_clients);
1766         INIT_LIST_HEAD(&vinstr_ctx->waiting_clients);
1767         mutex_init(&vinstr_ctx->lock);
1768         spin_lock_init(&vinstr_ctx->state_lock);
1769         vinstr_ctx->kbdev = kbdev;
1770         vinstr_ctx->thread = NULL;
1771         vinstr_ctx->state = VINSTR_IDLE;
1772         vinstr_ctx->suspend_cnt = 0;
1773         INIT_WORK(&vinstr_ctx->suspend_work, kbasep_vinstr_suspend_worker);
1774         INIT_WORK(&vinstr_ctx->resume_work, kbasep_vinstr_resume_worker);
1775         init_waitqueue_head(&vinstr_ctx->suspend_waitq);
1776
1777         atomic_set(&vinstr_ctx->request_pending, 0);
1778         init_waitqueue_head(&vinstr_ctx->waitq);
1779
1780         return vinstr_ctx;
1781 }
1782
1783 void kbase_vinstr_term(struct kbase_vinstr_context *vinstr_ctx)
1784 {
1785         struct kbase_vinstr_client *cli;
1786
1787         /* Stop service thread first. */
1788         if (vinstr_ctx->thread)
1789                 kthread_stop(vinstr_ctx->thread);
1790
1791         /* Wait for workers. */
1792         flush_work(&vinstr_ctx->suspend_work);
1793         flush_work(&vinstr_ctx->resume_work);
1794
1795         while (1) {
1796                 struct list_head *list = &vinstr_ctx->idle_clients;
1797
1798                 if (list_empty(list)) {
1799                         list = &vinstr_ctx->waiting_clients;
1800                         if (list_empty(list))
1801                                 break;
1802                 }
1803
1804                 cli = list_first_entry(list, struct kbase_vinstr_client, list);
1805                 list_del(&cli->list);
1806                 kfree(cli->accum_buffer);
1807                 kfree(cli);
1808                 vinstr_ctx->nclients--;
1809         }
1810         KBASE_DEBUG_ASSERT(!vinstr_ctx->nclients);
1811         if (vinstr_ctx->kctx)
1812                 kbasep_vinstr_destroy_kctx(vinstr_ctx);
1813         kfree(vinstr_ctx);
1814 }
1815
1816 int kbase_vinstr_hwcnt_reader_setup(struct kbase_vinstr_context *vinstr_ctx,
1817                 struct kbase_uk_hwcnt_reader_setup *setup)
1818 {
1819         struct kbase_vinstr_client  *cli;
1820         u32                         bitmap[4];
1821
1822         KBASE_DEBUG_ASSERT(vinstr_ctx);
1823         KBASE_DEBUG_ASSERT(setup);
1824         KBASE_DEBUG_ASSERT(setup->buffer_count);
1825
1826         bitmap[SHADER_HWCNT_BM] = setup->shader_bm;
1827         bitmap[TILER_HWCNT_BM]  = setup->tiler_bm;
1828         bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm;
1829         bitmap[JM_HWCNT_BM]     = setup->jm_bm;
1830
1831         cli = kbasep_vinstr_attach_client(
1832                         vinstr_ctx,
1833                         setup->buffer_count,
1834                         bitmap,
1835                         &setup->fd,
1836                         NULL);
1837
1838         if (!cli)
1839                 return -ENOMEM;
1840
1841         return 0;
1842 }
1843
1844 int kbase_vinstr_legacy_hwc_setup(
1845                 struct kbase_vinstr_context *vinstr_ctx,
1846                 struct kbase_vinstr_client  **cli,
1847                 struct kbase_uk_hwcnt_setup *setup)
1848 {
1849         KBASE_DEBUG_ASSERT(vinstr_ctx);
1850         KBASE_DEBUG_ASSERT(setup);
1851         KBASE_DEBUG_ASSERT(cli);
1852
1853         if (setup->dump_buffer) {
1854                 u32 bitmap[4];
1855
1856                 bitmap[SHADER_HWCNT_BM] = setup->shader_bm;
1857                 bitmap[TILER_HWCNT_BM]  = setup->tiler_bm;
1858                 bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm;
1859                 bitmap[JM_HWCNT_BM]     = setup->jm_bm;
1860
1861                 if (*cli)
1862                         return -EBUSY;
1863
1864                 *cli = kbasep_vinstr_attach_client(
1865                                 vinstr_ctx,
1866                                 0,
1867                                 bitmap,
1868                                 (void *)(long)setup->dump_buffer,
1869                                 NULL);
1870
1871                 if (!(*cli))
1872                         return -ENOMEM;
1873         } else {
1874                 if (!*cli)
1875                         return -EINVAL;
1876
1877                 kbase_vinstr_detach_client(*cli);
1878                 *cli = NULL;
1879         }
1880
1881         return 0;
1882 }
1883
1884 struct kbase_vinstr_client *kbase_vinstr_hwcnt_kernel_setup(
1885                 struct kbase_vinstr_context *vinstr_ctx,
1886                 struct kbase_uk_hwcnt_reader_setup *setup,
1887                 void *kernel_buffer)
1888 {
1889         u32 bitmap[4];
1890
1891         if (!vinstr_ctx || !setup || !kernel_buffer)
1892                 return NULL;
1893
1894         bitmap[SHADER_HWCNT_BM] = setup->shader_bm;
1895         bitmap[TILER_HWCNT_BM]  = setup->tiler_bm;
1896         bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm;
1897         bitmap[JM_HWCNT_BM]     = setup->jm_bm;
1898
1899         return kbasep_vinstr_attach_client(
1900                         vinstr_ctx,
1901                         0,
1902                         bitmap,
1903                         NULL,
1904                         kernel_buffer);
1905 }
1906 KBASE_EXPORT_TEST_API(kbase_vinstr_hwcnt_kernel_setup);
1907
1908 int kbase_vinstr_hwc_dump(struct kbase_vinstr_client *cli,
1909                 enum base_hwcnt_reader_event event_id)
1910 {
1911         int                         rcode = 0;
1912         struct kbase_vinstr_context *vinstr_ctx;
1913         u64                         timestamp;
1914         u32                         event_mask;
1915
1916         if (!cli)
1917                 return -EINVAL;
1918
1919         vinstr_ctx = cli->vinstr_ctx;
1920         KBASE_DEBUG_ASSERT(vinstr_ctx);
1921
1922         KBASE_DEBUG_ASSERT(event_id < BASE_HWCNT_READER_EVENT_COUNT);
1923         event_mask = 1 << event_id;
1924
1925         mutex_lock(&vinstr_ctx->lock);
1926
1927         if (event_mask & cli->event_mask) {
1928                 rcode = kbasep_vinstr_collect_and_accumulate(
1929                                 vinstr_ctx,
1930                                 &timestamp);
1931                 if (rcode)
1932                         goto exit;
1933
1934                 rcode = kbasep_vinstr_update_client(cli, timestamp, event_id);
1935                 if (rcode)
1936                         goto exit;
1937
1938                 kbasep_vinstr_reprogram(vinstr_ctx);
1939         }
1940
1941 exit:
1942         mutex_unlock(&vinstr_ctx->lock);
1943
1944         return rcode;
1945 }
1946 KBASE_EXPORT_TEST_API(kbase_vinstr_hwc_dump);
1947
1948 int kbase_vinstr_hwc_clear(struct kbase_vinstr_client *cli)
1949 {
1950         struct kbase_vinstr_context *vinstr_ctx;
1951         int                         rcode;
1952         u64                         unused;
1953
1954         if (!cli)
1955                 return -EINVAL;
1956
1957         vinstr_ctx = cli->vinstr_ctx;
1958         KBASE_DEBUG_ASSERT(vinstr_ctx);
1959
1960         mutex_lock(&vinstr_ctx->lock);
1961
1962         rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx, &unused);
1963         if (rcode)
1964                 goto exit;
1965         rcode = kbase_instr_hwcnt_clear(vinstr_ctx->kctx);
1966         if (rcode)
1967                 goto exit;
1968         memset(cli->accum_buffer, 0, cli->dump_size);
1969
1970         kbasep_vinstr_reprogram(vinstr_ctx);
1971
1972 exit:
1973         mutex_unlock(&vinstr_ctx->lock);
1974
1975         return rcode;
1976 }
1977
1978 int kbase_vinstr_try_suspend(struct kbase_vinstr_context *vinstr_ctx)
1979 {
1980         unsigned long flags;
1981         int ret = -EAGAIN;
1982
1983         KBASE_DEBUG_ASSERT(vinstr_ctx);
1984
1985         spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
1986         switch (vinstr_ctx->state) {
1987         case VINSTR_SUSPENDED:
1988                 vinstr_ctx->suspend_cnt++;
1989                 /* overflow shall not happen */
1990                 BUG_ON(0 == vinstr_ctx->suspend_cnt);
1991                 ret = 0;
1992                 break;
1993
1994         case VINSTR_IDLE:
1995                 vinstr_ctx->state = VINSTR_SUSPENDING;
1996                 schedule_work(&vinstr_ctx->suspend_work);
1997                 break;
1998
1999         case VINSTR_DUMPING:
2000                 vinstr_ctx->state = VINSTR_SUSPENDING;
2001                 break;
2002
2003         case VINSTR_SUSPENDING:
2004                 /* fall through */
2005         case VINSTR_RESUMING:
2006                 break;
2007
2008         default:
2009                 BUG();
2010                 break;
2011         }
2012         spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
2013
2014         return ret;
2015 }
2016
2017 void kbase_vinstr_suspend(struct kbase_vinstr_context *vinstr_ctx)
2018 {
2019         wait_event(vinstr_ctx->suspend_waitq,
2020                         (0 == kbase_vinstr_try_suspend(vinstr_ctx)));
2021 }
2022
2023 void kbase_vinstr_resume(struct kbase_vinstr_context *vinstr_ctx)
2024 {
2025         unsigned long flags;
2026
2027         KBASE_DEBUG_ASSERT(vinstr_ctx);
2028
2029         spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
2030         BUG_ON(VINSTR_SUSPENDING == vinstr_ctx->state);
2031         if (VINSTR_SUSPENDED == vinstr_ctx->state) {
2032                 BUG_ON(0 == vinstr_ctx->suspend_cnt);
2033                 vinstr_ctx->suspend_cnt--;
2034                 if (0 == vinstr_ctx->suspend_cnt) {
2035                         vinstr_ctx->state = VINSTR_RESUMING;
2036                         schedule_work(&vinstr_ctx->resume_work);
2037                 }
2038         }
2039         spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
2040 }