drivers/gpu/arm/midgard_for_linux/mali_kbase_vinstr.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2011-2015 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18 #include <linux/anon_inodes.h>
  19 #include <linux/atomic.h>
  20 #include <linux/hrtimer.h>
  21 #include <linux/jiffies.h>
  22 #include <linux/kthread.h>
  23 #include <linux/list.h>
  24 #include <linux/mm.h>
  25 #include <linux/poll.h>
  26 #include <linux/preempt.h>
  27 #include <linux/slab.h>
  28 #include <linux/wait.h>
  29
  30 #include <mali_kbase.h>
  31 #include <mali_kbase_hwcnt_reader.h>
  32 #include <mali_kbase_mem_linux.h>
  33
  34 /*****************************************************************************/
  35
  36 /* Hwcnt reader API version */
  37 #define HWCNT_READER_API        1
  38
  39 /* The number of nanoseconds in a second. */
  40 #define NSECS_IN_SEC            1000000000ull /* ns */
  41
  42 /* The time resolution of dumping service. */
  43 #define DUMPING_RESOLUTION      500000ull /* ns */
  44
  45 /* The maximal supported number of dumping buffers. */
  46 #define MAX_BUFFER_COUNT        32
  47
  48 /* Size and number of hw counters blocks. */
  49 #define NR_CNT_BLOCKS_PER_GROUP 8
  50 #define NR_CNT_PER_BLOCK        64
  51 #define NR_BYTES_PER_CNT        4
  52 #define NR_BYTES_PER_HDR        16
  53 #define PRFCNT_EN_MASK_OFFSET   0x8
  54
  55 /*****************************************************************************/
  56
  57 enum {
  58         SHADER_HWCNT_BM,
  59         TILER_HWCNT_BM,
  60         MMU_L2_HWCNT_BM,
  61         JM_HWCNT_BM
  62 };
  63
  64 /**
  65  * struct kbase_vinstr_context - vinstr context per device
  66  * @lock:              protects the entire vinstr context
  67  * @kbdev:             pointer to kbase device
  68  * @kctx:              pointer to kbase context
  69  * @vmap:              vinstr vmap for mapping hwcnt dump buffer
  70  * @gpu_va:            GPU hwcnt dump buffer address
  71  * @cpu_va:            the CPU side mapping of the hwcnt dump buffer
  72  * @dump_size:         size of the dump buffer in bytes
  73  * @bitmap:            current set of counters monitored, not always in sync
  74  *                     with hardware
  75  * @reprogram:         when true, reprogram hwcnt block with the new set of
  76  *                     counters
  77  * @suspended:         when true, the context has been suspended
  78  * @nclients:          number of attached clients, pending or otherwise
  79  * @waiting_clients:   head of list of clients being periodically sampled
  80  * @idle_clients:      head of list of clients being idle
  81  * @suspended_clients: head of list of clients being suspended
  82  * @thread:            periodic sampling thread
  83  * @waitq:             notification queue of sampling thread
  84  * @request_pending:   request for action for sampling thread
  85  */
  86 struct kbase_vinstr_context {
  87         struct mutex             lock;
  88         struct kbase_device      *kbdev;
  89         struct kbase_context     *kctx;
  90
  91         struct kbase_vmap_struct vmap;
  92         u64                      gpu_va;
  93         void                     *cpu_va;
  94         size_t                   dump_size;
  95         u32                      bitmap[4];
  96         bool                     reprogram;
  97         bool                     suspended;
  98
  99         u32                      nclients;
 100         struct list_head         waiting_clients;
 101         struct list_head         idle_clients;
 102         struct list_head         suspended_clients;
 103
 104         struct task_struct       *thread;
 105         wait_queue_head_t        waitq;
 106         atomic_t                 request_pending;
 107 };
 108
 109 /**
 110  * struct kbase_vinstr_client - a vinstr client attached to a vinstr context
 111  * @vinstr_ctx:    vinstr context client is attached to
 112  * @list:          node used to attach this client to list in vinstr context
 113  * @buffer_count:  number of buffers this client is using
 114  * @event_mask:    events this client reacts to
 115  * @dump_size:     size of one dump buffer in bytes
 116  * @bitmap:        bitmap request for JM, TILER, SHADER and MMU counters
 117  * @legacy_buffer: userspace hwcnt dump buffer (legacy interface)
 118  * @kernel_buffer: kernel hwcnt dump buffer (kernel client interface)
 119  * @accum_buffer:  temporary accumulation buffer for preserving counters
 120  * @dump_time:     next time this clients shall request hwcnt dump
 121  * @dump_interval: interval between periodic hwcnt dumps
 122  * @dump_buffers:  kernel hwcnt dump buffers allocated by this client
 123  * @dump_buffers_meta: metadata of dump buffers
 124  * @meta_idx:      index of metadata being accessed by userspace
 125  * @read_idx:      index of buffer read by userspace
 126  * @write_idx:     index of buffer being written by dumping service
 127  * @waitq:         client's notification queue
 128  * @pending:       when true, client has attached but hwcnt not yet updated
 129  */
 130 struct kbase_vinstr_client {
 131         struct kbase_vinstr_context        *vinstr_ctx;
 132         struct list_head                   list;
 133         unsigned int                       buffer_count;
 134         u32                                event_mask;
 135         size_t                             dump_size;
 136         u32                                bitmap[4];
 137         void __user                        *legacy_buffer;
 138         void                               *kernel_buffer;
 139         void                               *accum_buffer;
 140         u64                                dump_time;
 141         u32                                dump_interval;
 142         char                               *dump_buffers;
 143         struct kbase_hwcnt_reader_metadata *dump_buffers_meta;
 144         atomic_t                           meta_idx;
 145         atomic_t                           read_idx;
 146         atomic_t                           write_idx;
 147         wait_queue_head_t                  waitq;
 148         bool                               pending;
 149 };
 150
 151 /**
 152  * struct kbasep_vinstr_wake_up_timer - vinstr service thread wake up timer
 153  * @hrtimer:    high resolution timer
 154  * @vinstr_ctx: vinstr context
 155  */
 156 struct kbasep_vinstr_wake_up_timer {
 157         struct hrtimer              hrtimer;
 158         struct kbase_vinstr_context *vinstr_ctx;
 159 };
 160
 161 /*****************************************************************************/
 162
 163 static int kbasep_vinstr_service_task(void *data);
 164
 165 static unsigned int kbasep_vinstr_hwcnt_reader_poll(
 166                 struct file *filp,
 167                 poll_table  *wait);
 168 static long kbasep_vinstr_hwcnt_reader_ioctl(
 169                 struct file   *filp,
 170                 unsigned int  cmd,
 171                 unsigned long arg);
 172 static int kbasep_vinstr_hwcnt_reader_mmap(
 173                 struct file           *filp,
 174                 struct vm_area_struct *vma);
 175 static int kbasep_vinstr_hwcnt_reader_release(
 176                 struct inode *inode,
 177                 struct file  *filp);
 178
 179 /* The timeline stream file operations structure. */
 180 static const struct file_operations vinstr_client_fops = {
 181         .poll           = kbasep_vinstr_hwcnt_reader_poll,
 182         .unlocked_ioctl = kbasep_vinstr_hwcnt_reader_ioctl,
 183         .compat_ioctl   = kbasep_vinstr_hwcnt_reader_ioctl,
 184         .mmap           = kbasep_vinstr_hwcnt_reader_mmap,
 185         .release        = kbasep_vinstr_hwcnt_reader_release,
 186 };
 187
 188 /*****************************************************************************/
 189
 190 static int enable_hwcnt(struct kbase_vinstr_context *vinstr_ctx)
 191 {
 192         struct kbase_uk_hwcnt_setup setup;
 193
 194         setup.dump_buffer = vinstr_ctx->gpu_va;
 195         setup.jm_bm       = vinstr_ctx->bitmap[JM_HWCNT_BM];
 196         setup.tiler_bm    = vinstr_ctx->bitmap[TILER_HWCNT_BM];
 197         setup.shader_bm   = vinstr_ctx->bitmap[SHADER_HWCNT_BM];
 198         setup.mmu_l2_bm   = vinstr_ctx->bitmap[MMU_L2_HWCNT_BM];
 199
 200         return kbase_instr_hwcnt_enable(vinstr_ctx->kctx, &setup);
 201 }
 202
 203 static void disable_hwcnt(struct kbase_vinstr_context *vinstr_ctx)
 204 {
 205         kbase_instr_hwcnt_disable(vinstr_ctx->kctx);
 206 }
 207
 208 static int reprogram_hwcnt(struct kbase_vinstr_context *vinstr_ctx)
 209 {
 210         disable_hwcnt(vinstr_ctx);
 211         return enable_hwcnt(vinstr_ctx);
 212 }
 213
 214 static void hwcnt_bitmap_set(u32 dst[4], u32 src[4])
 215 {
 216         dst[JM_HWCNT_BM]     = src[JM_HWCNT_BM];
 217         dst[TILER_HWCNT_BM]  = src[TILER_HWCNT_BM];
 218         dst[SHADER_HWCNT_BM] = src[SHADER_HWCNT_BM];
 219         dst[MMU_L2_HWCNT_BM] = src[MMU_L2_HWCNT_BM];
 220 }
 221
 222 static void hwcnt_bitmap_union(u32 dst[4], u32 src[4])
 223 {
 224         dst[JM_HWCNT_BM]     |= src[JM_HWCNT_BM];
 225         dst[TILER_HWCNT_BM]  |= src[TILER_HWCNT_BM];
 226         dst[SHADER_HWCNT_BM] |= src[SHADER_HWCNT_BM];
 227         dst[MMU_L2_HWCNT_BM] |= src[MMU_L2_HWCNT_BM];
 228 }
 229
 230 size_t kbase_vinstr_dump_size(struct kbase_device *kbdev)
 231 {
 232         size_t dump_size;
 233
 234 #ifndef CONFIG_MALI_NO_MALI
 235         if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_V4)) {
 236                 u32 nr_cg;
 237
 238                 nr_cg = kbdev->gpu_props.num_core_groups;
 239                 dump_size = nr_cg * NR_CNT_BLOCKS_PER_GROUP *
 240                                 NR_CNT_PER_BLOCK *
 241                                 NR_BYTES_PER_CNT;
 242         } else
 243 #endif /* CONFIG_MALI_NO_MALI */
 244         {
 245                 /* assume v5 for now */
 246                 base_gpu_props *props = &kbdev->gpu_props.props;
 247                 u32 nr_l2 = props->l2_props.num_l2_slices;
 248                 u64 core_mask = props->coherency_info.group[0].core_mask;
 249                 u32 nr_blocks = fls64(core_mask);
 250
 251                 /* JM and tiler counter blocks are always present */
 252                 dump_size = (2 + nr_l2 + nr_blocks) *
 253                                 NR_CNT_PER_BLOCK *
 254                                 NR_BYTES_PER_CNT;
 255         }
 256         return dump_size;
 257 }
 258 KBASE_EXPORT_TEST_API(kbase_vinstr_dump_size);
 259
 260 static size_t kbasep_vinstr_dump_size_ctx(
 261                 struct kbase_vinstr_context *vinstr_ctx)
 262 {
 263         return kbase_vinstr_dump_size(vinstr_ctx->kctx->kbdev);
 264 }
 265
 266 static int kbasep_vinstr_map_kernel_dump_buffer(
 267                 struct kbase_vinstr_context *vinstr_ctx)
 268 {
 269         struct kbase_va_region *reg;
 270         struct kbase_context *kctx = vinstr_ctx->kctx;
 271         u64 flags, nr_pages;
 272         u16 va_align = 0;
 273
 274         flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_WR;
 275         vinstr_ctx->dump_size = kbasep_vinstr_dump_size_ctx(vinstr_ctx);
 276         nr_pages = PFN_UP(vinstr_ctx->dump_size);
 277
 278         reg = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags,
 279                         &vinstr_ctx->gpu_va, &va_align);
 280         if (!reg)
 281                 return -ENOMEM;
 282
 283         vinstr_ctx->cpu_va = kbase_vmap(
 284                         kctx,
 285                         vinstr_ctx->gpu_va,
 286                         vinstr_ctx->dump_size,
 287                         &vinstr_ctx->vmap);
 288         if (!vinstr_ctx->cpu_va) {
 289                 kbase_mem_free(kctx, vinstr_ctx->gpu_va);
 290                 return -ENOMEM;
 291         }
 292
 293         return 0;
 294 }
 295
 296 static void kbasep_vinstr_unmap_kernel_dump_buffer(
 297                 struct kbase_vinstr_context *vinstr_ctx)
 298 {
 299         struct kbase_context *kctx = vinstr_ctx->kctx;
 300
 301         kbase_vunmap(kctx, &vinstr_ctx->vmap);
 302         kbase_mem_free(kctx, vinstr_ctx->gpu_va);
 303 }
 304
 305 /**
 306  * kbasep_vinstr_create_kctx - create kernel context for vinstr
 307  * @vinstr_ctx: vinstr context
 308  * Return: zero on success
 309  */
 310 static int kbasep_vinstr_create_kctx(struct kbase_vinstr_context *vinstr_ctx)
 311 {
 312         int err;
 313
 314         vinstr_ctx->kctx = kbase_create_context(vinstr_ctx->kbdev, true);
 315         if (!vinstr_ctx->kctx)
 316                 return -ENOMEM;
 317
 318         /* Map the master kernel dump buffer.  The HW dumps the counters
 319          * into this memory region. */
 320         err = kbasep_vinstr_map_kernel_dump_buffer(vinstr_ctx);
 321         if (err) {
 322                 kbase_destroy_context(vinstr_ctx->kctx);
 323                 vinstr_ctx->kctx = NULL;
 324                 return err;
 325         }
 326
 327         err = enable_hwcnt(vinstr_ctx);
 328         if (err) {
 329                 kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx);
 330                 kbase_destroy_context(vinstr_ctx->kctx);
 331                 vinstr_ctx->kctx = NULL;
 332                 return err;
 333         }
 334
 335         vinstr_ctx->thread = kthread_run(
 336                         kbasep_vinstr_service_task,
 337                         vinstr_ctx,
 338                         "mali_vinstr_service");
 339         if (!vinstr_ctx->thread) {
 340                 disable_hwcnt(vinstr_ctx);
 341                 kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx);
 342                 kbase_destroy_context(vinstr_ctx->kctx);
 343                 vinstr_ctx->kctx = NULL;
 344                 return -EFAULT;
 345         }
 346
 347         return 0;
 348 }
 349
 350 /**
 351  * kbasep_vinstr_destroy_kctx - destroy vinstr's kernel context
 352  * @vinstr_ctx: vinstr context
 353  */
 354 static void kbasep_vinstr_destroy_kctx(struct kbase_vinstr_context *vinstr_ctx)
 355 {
 356         /* Release hw counters dumping resources. */
 357         vinstr_ctx->thread = NULL;
 358         disable_hwcnt(vinstr_ctx);
 359         kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx);
 360         kbase_destroy_context(vinstr_ctx->kctx);
 361         vinstr_ctx->kctx = NULL;
 362 }
 363
 364 /**
 365  * kbasep_vinstr_attach_client - Attach a client to the vinstr core
 366  * @vinstr_ctx:    vinstr context
 367  * @buffer_count:  requested number of dump buffers
 368  * @bitmap:        bitmaps describing which counters should be enabled
 369  * @argp:          pointer where notification descriptor shall be stored
 370  * @kernel_buffer: pointer to kernel side buffer
 371  *
 372  * Return: vinstr opaque client handle or NULL on failure
 373  */
 374 static struct kbase_vinstr_client *kbasep_vinstr_attach_client(
 375                 struct kbase_vinstr_context *vinstr_ctx, u32 buffer_count,
 376                 u32 bitmap[4], void *argp, void *kernel_buffer)
 377 {
 378         struct task_struct         *thread = NULL;
 379         struct kbase_vinstr_client *cli;
 380
 381         KBASE_DEBUG_ASSERT(vinstr_ctx);
 382         KBASE_DEBUG_ASSERT(buffer_count >= 0);
 383         KBASE_DEBUG_ASSERT(buffer_count <= MAX_BUFFER_COUNT);
 384         KBASE_DEBUG_ASSERT(!(buffer_count & (buffer_count - 1)));
 385
 386         cli = kzalloc(sizeof(*cli), GFP_KERNEL);
 387         if (!cli)
 388                 return NULL;
 389
 390         cli->vinstr_ctx   = vinstr_ctx;
 391         cli->buffer_count = buffer_count;
 392         cli->event_mask   =
 393                 (1 << BASE_HWCNT_READER_EVENT_MANUAL) |
 394                 (1 << BASE_HWCNT_READER_EVENT_PERIODIC);
 395         cli->pending      = true;
 396
 397         hwcnt_bitmap_set(cli->bitmap, bitmap);
 398
 399         mutex_lock(&vinstr_ctx->lock);
 400
 401         hwcnt_bitmap_union(vinstr_ctx->bitmap, cli->bitmap);
 402         vinstr_ctx->reprogram = true;
 403
 404         /* If this is the first client, create the vinstr kbase
 405          * context. This context is permanently resident until the
 406          * last client exits. */
 407         if (!vinstr_ctx->nclients) {
 408                 hwcnt_bitmap_set(vinstr_ctx->bitmap, cli->bitmap);
 409                 if (kbasep_vinstr_create_kctx(vinstr_ctx) < 0)
 410                         goto error;
 411
 412                 vinstr_ctx->reprogram = false;
 413                 cli->pending = false;
 414         }
 415
 416         /* The GPU resets the counter block every time there is a request
 417          * to dump it. We need a per client kernel buffer for accumulating
 418          * the counters. */
 419         cli->dump_size    = kbasep_vinstr_dump_size_ctx(vinstr_ctx);
 420         cli->accum_buffer = kzalloc(cli->dump_size, GFP_KERNEL);
 421         if (!cli->accum_buffer)
 422                 goto error;
 423
 424         /* Prepare buffers. */
 425         if (cli->buffer_count) {
 426                 int *fd = (int *)argp;
 427                 size_t tmp;
 428
 429                 /* Allocate area for buffers metadata storage. */
 430                 tmp = sizeof(struct kbase_hwcnt_reader_metadata) *
 431                         cli->buffer_count;
 432                 cli->dump_buffers_meta = kmalloc(tmp, GFP_KERNEL);
 433                 if (!cli->dump_buffers_meta)
 434                         goto error;
 435
 436                 /* Allocate required number of dumping buffers. */
 437                 cli->dump_buffers = (char *)__get_free_pages(
 438                                 GFP_KERNEL,
 439                                 get_order(cli->dump_size * cli->buffer_count));
 440                 if (!cli->dump_buffers)
 441                         goto error;
 442
 443                 /* Create descriptor for user-kernel data exchange. */
 444                 *fd = anon_inode_getfd(
 445                                 "[mali_vinstr_desc]",
 446                                 &vinstr_client_fops,
 447                                 cli,
 448                                 O_RDONLY | O_CLOEXEC);
 449                 if (0 > *fd)
 450                         goto error;
 451         } else if (kernel_buffer) {
 452                 cli->kernel_buffer = kernel_buffer;
 453         } else {
 454                 cli->legacy_buffer = (void __user *)argp;
 455         }
 456
 457         atomic_set(&cli->read_idx, 0);
 458         atomic_set(&cli->meta_idx, 0);
 459         atomic_set(&cli->write_idx, 0);
 460         init_waitqueue_head(&cli->waitq);
 461
 462         vinstr_ctx->nclients++;
 463         list_add(&cli->list, &vinstr_ctx->idle_clients);
 464
 465         mutex_unlock(&vinstr_ctx->lock);
 466
 467         return cli;
 468
 469 error:
 470         kfree(cli->dump_buffers_meta);
 471         if (cli->dump_buffers)
 472                 free_pages(
 473                                 (unsigned long)cli->dump_buffers,
 474                                 get_order(cli->dump_size * cli->buffer_count));
 475         kfree(cli->accum_buffer);
 476         if (!vinstr_ctx->nclients && vinstr_ctx->kctx) {
 477                 thread = vinstr_ctx->thread;
 478                 kbasep_vinstr_destroy_kctx(vinstr_ctx);
 479         }
 480         kfree(cli);
 481
 482         mutex_unlock(&vinstr_ctx->lock);
 483
 484         /* Thread must be stopped after lock is released. */
 485         if (thread)
 486                 kthread_stop(thread);
 487
 488         return NULL;
 489 }
 490
 491 void kbase_vinstr_detach_client(struct kbase_vinstr_client *cli)
 492 {
 493         struct kbase_vinstr_context *vinstr_ctx;
 494         struct kbase_vinstr_client  *iter, *tmp;
 495         struct task_struct          *thread = NULL;
 496         u32 zerobitmap[4] = { 0 };
 497         int cli_found = 0;
 498
 499         KBASE_DEBUG_ASSERT(cli);
 500         vinstr_ctx = cli->vinstr_ctx;
 501         KBASE_DEBUG_ASSERT(vinstr_ctx);
 502
 503         mutex_lock(&vinstr_ctx->lock);
 504
 505         list_for_each_entry_safe(iter, tmp, &vinstr_ctx->idle_clients, list) {
 506                 if (iter == cli) {
 507                         vinstr_ctx->reprogram = true;
 508                         cli_found = 1;
 509                         list_del(&iter->list);
 510                         break;
 511                 }
 512         }
 513         if (!cli_found) {
 514                 list_for_each_entry_safe(
 515                                 iter, tmp, &vinstr_ctx->waiting_clients, list) {
 516                         if (iter == cli) {
 517                                 vinstr_ctx->reprogram = true;
 518                                 cli_found = 1;
 519                                 list_del(&iter->list);
 520                                 break;
 521                         }
 522                 }
 523         }
 524         KBASE_DEBUG_ASSERT(cli_found);
 525
 526         kfree(cli->dump_buffers_meta);
 527         free_pages(
 528                         (unsigned long)cli->dump_buffers,
 529                         get_order(cli->dump_size * cli->buffer_count));
 530         kfree(cli->accum_buffer);
 531         kfree(cli);
 532
 533         vinstr_ctx->nclients--;
 534         if (!vinstr_ctx->nclients) {
 535                 thread = vinstr_ctx->thread;
 536                 kbasep_vinstr_destroy_kctx(vinstr_ctx);
 537         }
 538
 539         /* Rebuild context bitmap now that the client has detached */
 540         hwcnt_bitmap_set(vinstr_ctx->bitmap, zerobitmap);
 541         list_for_each_entry(iter, &vinstr_ctx->idle_clients, list)
 542                 hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap);
 543         list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list)
 544                 hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap);
 545
 546         mutex_unlock(&vinstr_ctx->lock);
 547
 548         /* Thread must be stopped after lock is released. */
 549         if (thread)
 550                 kthread_stop(thread);
 551 }
 552 KBASE_EXPORT_TEST_API(kbase_vinstr_detach_client);
 553
 554 /* Accumulate counters in the dump buffer */
 555 static void accum_dump_buffer(void *dst, void *src, size_t dump_size)
 556 {
 557         size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT;
 558         u32 *d = dst;
 559         u32 *s = src;
 560         size_t i, j;
 561
 562         for (i = 0; i < dump_size; i += block_size) {
 563                 /* skip over the header block */
 564                 d += NR_BYTES_PER_HDR / sizeof(u32);
 565                 s += NR_BYTES_PER_HDR / sizeof(u32);
 566                 for (j = 0; j < (block_size - NR_BYTES_PER_HDR) / sizeof(u32); j++) {
 567                         /* saturate result if addition would result in wraparound */
 568                         if (U32_MAX - *d < *s)
 569                                 *d = U32_MAX;
 570                         else
 571                                 *d += *s;
 572                         d++;
 573                         s++;
 574                 }
 575         }
 576 }
 577
 578 /* This is the Midgard v4 patch function.  It copies the headers for each
 579  * of the defined blocks from the master kernel buffer and then patches up
 580  * the performance counter enable mask for each of the blocks to exclude
 581  * counters that were not requested by the client. */
 582 static void patch_dump_buffer_hdr_v4(
 583                 struct kbase_vinstr_context *vinstr_ctx,
 584                 struct kbase_vinstr_client *cli)
 585 {
 586         u32 *mask;
 587         u8 *dst = cli->accum_buffer;
 588         u8 *src = vinstr_ctx->cpu_va;
 589         u32 nr_cg = vinstr_ctx->kctx->kbdev->gpu_props.num_core_groups;
 590         size_t i, group_size, group;
 591         enum {
 592                 SC0_BASE    = 0 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
 593                 SC1_BASE    = 1 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
 594                 SC2_BASE    = 2 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
 595                 SC3_BASE    = 3 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
 596                 TILER_BASE  = 4 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
 597                 MMU_L2_BASE = 5 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
 598                 JM_BASE     = 7 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT
 599         };
 600
 601         group_size = NR_CNT_BLOCKS_PER_GROUP *
 602                         NR_CNT_PER_BLOCK *
 603                         NR_BYTES_PER_CNT;
 604         for (i = 0; i < nr_cg; i++) {
 605                 group = i * group_size;
 606                 /* copy shader core headers */
 607                 memcpy(&dst[group + SC0_BASE], &src[group + SC0_BASE],
 608                        NR_BYTES_PER_HDR);
 609                 memcpy(&dst[group + SC1_BASE], &src[group + SC1_BASE],
 610                        NR_BYTES_PER_HDR);
 611                 memcpy(&dst[group + SC2_BASE], &src[group + SC2_BASE],
 612                       NR_BYTES_PER_HDR);
 613                 memcpy(&dst[group + SC3_BASE], &src[group + SC3_BASE],
 614                       NR_BYTES_PER_HDR);
 615
 616                 /* copy tiler header */
 617                 memcpy(&dst[group + TILER_BASE], &src[group + TILER_BASE],
 618                       NR_BYTES_PER_HDR);
 619
 620                 /* copy mmu header */
 621                 memcpy(&dst[group + MMU_L2_BASE], &src[group + MMU_L2_BASE],
 622                       NR_BYTES_PER_HDR);
 623
 624                 /* copy job manager header */
 625                 memcpy(&dst[group + JM_BASE], &src[group + JM_BASE],
 626                       NR_BYTES_PER_HDR);
 627
 628                 /* patch the shader core enable mask */
 629                 mask = (u32 *)&dst[group + SC0_BASE + PRFCNT_EN_MASK_OFFSET];
 630                 *mask &= cli->bitmap[SHADER_HWCNT_BM];
 631                 mask = (u32 *)&dst[group + SC1_BASE + PRFCNT_EN_MASK_OFFSET];
 632                 *mask &= cli->bitmap[SHADER_HWCNT_BM];
 633                 mask = (u32 *)&dst[group + SC2_BASE + PRFCNT_EN_MASK_OFFSET];
 634                 *mask &= cli->bitmap[SHADER_HWCNT_BM];
 635                 mask = (u32 *)&dst[group + SC3_BASE + PRFCNT_EN_MASK_OFFSET];
 636                 *mask &= cli->bitmap[SHADER_HWCNT_BM];
 637
 638                 /* patch the tiler core enable mask */
 639                 mask = (u32 *)&dst[group + TILER_BASE + PRFCNT_EN_MASK_OFFSET];
 640                 *mask &= cli->bitmap[TILER_HWCNT_BM];
 641
 642                 /* patch the mmu core enable mask */
 643                 mask = (u32 *)&dst[group + MMU_L2_BASE + PRFCNT_EN_MASK_OFFSET];
 644                 *mask &= cli->bitmap[MMU_L2_HWCNT_BM];
 645
 646                 /* patch the job manager enable mask */
 647                 mask = (u32 *)&dst[group + JM_BASE + PRFCNT_EN_MASK_OFFSET];
 648                 *mask &= cli->bitmap[JM_HWCNT_BM];
 649         }
 650 }
 651
 652 /* This is the Midgard v5 patch function.  It copies the headers for each
 653  * of the defined blocks from the master kernel buffer and then patches up
 654  * the performance counter enable mask for each of the blocks to exclude
 655  * counters that were not requested by the client. */
 656 static void patch_dump_buffer_hdr_v5(
 657                 struct kbase_vinstr_context *vinstr_ctx,
 658                 struct kbase_vinstr_client *cli)
 659 {
 660         struct kbase_device *kbdev = vinstr_ctx->kctx->kbdev;
 661         u32 i, nr_l2;
 662         u64 core_mask;
 663         u32 *mask;
 664         u8 *dst = cli->accum_buffer;
 665         u8 *src = vinstr_ctx->cpu_va;
 666         size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT;
 667
 668         /* copy and patch job manager header */
 669         memcpy(dst, src, NR_BYTES_PER_HDR);
 670         mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
 671         *mask &= cli->bitmap[JM_HWCNT_BM];
 672         dst += block_size;
 673         src += block_size;
 674
 675         /* copy and patch tiler header */
 676         memcpy(dst, src, NR_BYTES_PER_HDR);
 677         mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
 678         *mask &= cli->bitmap[TILER_HWCNT_BM];
 679         dst += block_size;
 680         src += block_size;
 681
 682         /* copy and patch MMU/L2C headers */
 683         nr_l2 = kbdev->gpu_props.props.l2_props.num_l2_slices;
 684         for (i = 0; i < nr_l2; i++) {
 685                 memcpy(dst, src, NR_BYTES_PER_HDR);
 686                 mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
 687                 *mask &= cli->bitmap[MMU_L2_HWCNT_BM];
 688                 dst += block_size;
 689                 src += block_size;
 690         }
 691
 692         /* copy and patch shader core headers */
 693         core_mask = kbdev->gpu_props.props.coherency_info.group[0].core_mask;
 694         while (0ull != core_mask) {
 695                 memcpy(dst, src, NR_BYTES_PER_HDR);
 696                 if (0ull != (core_mask & 1ull)) {
 697                         /* if block is not reserved update header */
 698                         mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
 699                         *mask &= cli->bitmap[SHADER_HWCNT_BM];
 700                 }
 701                 dst += block_size;
 702                 src += block_size;
 703
 704                 core_mask >>= 1;
 705         }
 706 }
 707
 708 /**
 709  * accum_clients - accumulate dumped hw counters for all known clients
 710  * @vinstr_ctx: vinstr context
 711  */
 712 static void accum_clients(struct kbase_vinstr_context *vinstr_ctx)
 713 {
 714         struct kbase_vinstr_client *iter;
 715         int v4 = 0;
 716
 717 #ifndef CONFIG_MALI_NO_MALI
 718         v4 = kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4);
 719 #endif
 720
 721         list_for_each_entry(iter, &vinstr_ctx->idle_clients, list) {
 722                 /* Don't bother accumulating clients whose hwcnt requests
 723                  * have not yet been honoured. */
 724                 if (iter->pending)
 725                         continue;
 726                 if (v4)
 727                         patch_dump_buffer_hdr_v4(vinstr_ctx, iter);
 728                 else
 729                         patch_dump_buffer_hdr_v5(vinstr_ctx, iter);
 730                 accum_dump_buffer(
 731                                 iter->accum_buffer,
 732                                 vinstr_ctx->cpu_va,
 733                                 iter->dump_size);
 734         }
 735         list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list) {
 736                 /* Don't bother accumulating clients whose hwcnt requests
 737                  * have not yet been honoured. */
 738                 if (iter->pending)
 739                         continue;
 740                 if (v4)
 741                         patch_dump_buffer_hdr_v4(vinstr_ctx, iter);
 742                 else
 743                         patch_dump_buffer_hdr_v5(vinstr_ctx, iter);
 744                 accum_dump_buffer(
 745                                 iter->accum_buffer,
 746                                 vinstr_ctx->cpu_va,
 747                                 iter->dump_size);
 748         }
 749 }
 750
 751 /*****************************************************************************/
 752
 753 /**
 754  * kbasep_vinstr_get_timestamp - return timestamp
 755  *
 756  * Function returns timestamp value based on raw monotonic timer. Value will
 757  * wrap around zero in case of overflow.
 758  *
 759  * Return: timestamp value
 760  */
 761 static u64 kbasep_vinstr_get_timestamp(void)
 762 {
 763         struct timespec ts;
 764
 765         getrawmonotonic(&ts);
 766         return (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec;
 767 }
 768
 769 /**
 770  * kbasep_vinstr_add_dump_request - register client's dumping request
 771  * @cli:             requesting client
 772  * @waiting_clients: list of pending dumping requests
 773  */
 774 static void kbasep_vinstr_add_dump_request(
 775                 struct kbase_vinstr_client *cli,
 776                 struct list_head *waiting_clients)
 777 {
 778         struct kbase_vinstr_client *tmp;
 779
 780         if (list_empty(waiting_clients)) {
 781                 list_add(&cli->list, waiting_clients);
 782                 return;
 783         }
 784         list_for_each_entry(tmp, waiting_clients, list) {
 785                 if (tmp->dump_time > cli->dump_time) {
 786                         list_add_tail(&cli->list, &tmp->list);
 787                         return;
 788                 }
 789         }
 790         list_add_tail(&cli->list, waiting_clients);
 791 }
 792
 793 /**
 794  * kbasep_vinstr_collect_and_accumulate - collect hw counters via low level
 795  *                                        dump and accumulate them for known
 796  *                                        clients
 797  * @vinstr_ctx: vinstr context
 798  * @timestamp: pointer where collection timestamp will be recorded
 799  *
 800  * Return: zero on success
 801  */
 802 static int kbasep_vinstr_collect_and_accumulate(
 803                 struct kbase_vinstr_context *vinstr_ctx, u64 *timestamp)
 804 {
 805         int rcode;
 806
 807 #ifdef CONFIG_MALI_NO_MALI
 808         /* The dummy model needs the CPU mapping. */
 809         gpu_model_set_dummy_prfcnt_base_cpu(vinstr_ctx->cpu_va);
 810 #endif
 811
 812         /* Request HW counters dump.
 813          * Disable preemption to make dump timestamp more accurate. */
 814         preempt_disable();
 815         *timestamp = kbasep_vinstr_get_timestamp();
 816         rcode = kbase_instr_hwcnt_request_dump(vinstr_ctx->kctx);
 817         preempt_enable();
 818
 819         if (!rcode)
 820                 rcode = kbase_instr_hwcnt_wait_for_dump(vinstr_ctx->kctx);
 821         WARN_ON(rcode);
 822
 823         /* Accumulate values of collected counters. */
 824         if (!rcode)
 825                 accum_clients(vinstr_ctx);
 826
 827         return rcode;
 828 }
 829
 830 /**
 831  * kbasep_vinstr_fill_dump_buffer - copy accumulated counters to empty kernel
 832  *                                  buffer
 833  * @cli:       requesting client
 834  * @timestamp: timestamp when counters were collected
 835  * @event_id:  id of event that caused triggered counters collection
 836  *
 837  * Return: zero on success
 838  */
 839 static int kbasep_vinstr_fill_dump_buffer(
 840                 struct kbase_vinstr_client *cli, u64 timestamp,
 841                 enum base_hwcnt_reader_event event_id)
 842 {
 843         unsigned int write_idx = atomic_read(&cli->write_idx);
 844         unsigned int read_idx  = atomic_read(&cli->read_idx);
 845
 846         struct kbase_hwcnt_reader_metadata *meta;
 847         void                               *buffer;
 848
 849         /* Check if there is a place to copy HWC block into. */
 850         if (write_idx - read_idx == cli->buffer_count)
 851                 return -1;
 852         write_idx %= cli->buffer_count;
 853
 854         /* Fill in dump buffer and its metadata. */
 855         buffer = &cli->dump_buffers[write_idx * cli->dump_size];
 856         meta   = &cli->dump_buffers_meta[write_idx];
 857         meta->timestamp  = timestamp;
 858         meta->event_id   = event_id;
 859         meta->buffer_idx = write_idx;
 860         memcpy(buffer, cli->accum_buffer, cli->dump_size);
 861         return 0;
 862 }
 863
 864 /**
 865  * kbasep_vinstr_fill_dump_buffer_legacy - copy accumulated counters to buffer
 866  *                                         allocated in userspace
 867  * @cli: requesting client
 868  *
 869  * Return: zero on success
 870  *
 871  * This is part of legacy ioctl interface.
 872  */
 873 static int kbasep_vinstr_fill_dump_buffer_legacy(
 874                 struct kbase_vinstr_client *cli)
 875 {
 876         void __user  *buffer = cli->legacy_buffer;
 877         int          rcode;
 878
 879         /* Copy data to user buffer. */
 880         rcode = copy_to_user(buffer, cli->accum_buffer, cli->dump_size);
 881         if (rcode)
 882                 pr_warn("error while copying buffer to user\n");
 883         return rcode;
 884 }
 885
 886 /**
 887  * kbasep_vinstr_fill_dump_buffer_kernel - copy accumulated counters to buffer
 888  *                                         allocated in kernel space
 889  * @cli: requesting client
 890  *
 891  * Return: zero on success
 892  *
 893  * This is part of the kernel client interface.
 894  */
 895 static int kbasep_vinstr_fill_dump_buffer_kernel(
 896                 struct kbase_vinstr_client *cli)
 897 {
 898         memcpy(cli->kernel_buffer, cli->accum_buffer, cli->dump_size);
 899
 900         return 0;
 901 }
 902
 903 /**
 904  * kbasep_vinstr_reprogram - reprogram hwcnt set collected by inst
 905  * @vinstr_ctx: vinstr context
 906  */
 907 static void kbasep_vinstr_reprogram(
 908                 struct kbase_vinstr_context *vinstr_ctx)
 909 {
 910         if (vinstr_ctx->reprogram) {
 911                 struct kbase_vinstr_client *iter;
 912
 913                 if (!reprogram_hwcnt(vinstr_ctx)) {
 914                         vinstr_ctx->reprogram = false;
 915                         list_for_each_entry(
 916                                         iter,
 917                                         &vinstr_ctx->idle_clients,
 918                                         list)
 919                                 iter->pending = false;
 920                         list_for_each_entry(
 921                                         iter,
 922                                         &vinstr_ctx->waiting_clients,
 923                                         list)
 924                                 iter->pending = false;
 925                 }
 926         }
 927 }
 928
 929 /**
 930  * kbasep_vinstr_update_client - copy accumulated counters to user readable
 931  *                               buffer and notify the user
 932  * @cli:       requesting client
 933  * @timestamp: timestamp when counters were collected
 934  * @event_id:  id of event that caused triggered counters collection
 935  *
 936  * Return: zero on success
 937  */
 938 static int kbasep_vinstr_update_client(
 939                 struct kbase_vinstr_client *cli, u64 timestamp,
 940                 enum base_hwcnt_reader_event event_id)
 941 {
 942         int rcode = 0;
 943
 944         /* Copy collected counters to user readable buffer. */
 945         if (cli->buffer_count)
 946                 rcode = kbasep_vinstr_fill_dump_buffer(
 947                                 cli, timestamp, event_id);
 948         else if (cli->kernel_buffer)
 949                 rcode = kbasep_vinstr_fill_dump_buffer_kernel(cli);
 950         else
 951                 rcode = kbasep_vinstr_fill_dump_buffer_legacy(cli);
 952
 953         if (rcode)
 954                 goto exit;
 955
 956
 957         /* Notify client. Make sure all changes to memory are visible. */
 958         wmb();
 959         atomic_inc(&cli->write_idx);
 960         wake_up_interruptible(&cli->waitq);
 961
 962         /* Prepare for next request. */
 963         memset(cli->accum_buffer, 0, cli->dump_size);
 964
 965 exit:
 966         return rcode;
 967 }
 968
 969 /**
 970  * kbasep_vinstr_wake_up_callback - vinstr wake up timer wake up function
 971  *
 972  * @hrtimer: high resolution timer
 973  *
 974  * Return: High resolution timer restart enum.
 975  */
 976 static enum hrtimer_restart kbasep_vinstr_wake_up_callback(
 977                 struct hrtimer *hrtimer)
 978 {
 979         struct kbasep_vinstr_wake_up_timer *timer =
 980                 container_of(
 981                         hrtimer,
 982                         struct kbasep_vinstr_wake_up_timer,
 983                         hrtimer);
 984
 985         KBASE_DEBUG_ASSERT(timer);
 986
 987         atomic_set(&timer->vinstr_ctx->request_pending, 1);
 988         wake_up_all(&timer->vinstr_ctx->waitq);
 989
 990         return HRTIMER_NORESTART;
 991 }
 992
 993 /**
 994  * kbasep_vinstr_service_task - HWC dumping service thread
 995  *
 996  * @data: Pointer to vinstr context structure.
 997  *
 998  * Return: Always returns zero.
 999  */
1000 static int kbasep_vinstr_service_task(void *data)
1001 {
1002         struct kbase_vinstr_context        *vinstr_ctx = data;
1003         struct kbasep_vinstr_wake_up_timer timer;
1004
1005         KBASE_DEBUG_ASSERT(vinstr_ctx);
1006
1007         hrtimer_init(&timer.hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1008         timer.hrtimer.function = kbasep_vinstr_wake_up_callback;
1009         timer.vinstr_ctx       = vinstr_ctx;
1010
1011         while (!kthread_should_stop()) {
1012                 struct kbase_vinstr_client *cli = NULL;
1013                 struct kbase_vinstr_client *tmp;
1014
1015                 u64              timestamp = kbasep_vinstr_get_timestamp();
1016                 u64              dump_time = 0;
1017                 struct list_head expired_requests;
1018
1019                 /* Hold lock while performing operations on lists of clients. */
1020                 mutex_lock(&vinstr_ctx->lock);
1021
1022                 /* Closing thread must not interact with client requests. */
1023                 if (current == vinstr_ctx->thread) {
1024                         atomic_set(&vinstr_ctx->request_pending, 0);
1025
1026                         if (!list_empty(&vinstr_ctx->waiting_clients)) {
1027                                 cli = list_first_entry(
1028                                                 &vinstr_ctx->waiting_clients,
1029                                                 struct kbase_vinstr_client,
1030                                                 list);
1031                                 dump_time = cli->dump_time;
1032                         }
1033                 }
1034
1035                 if (!cli || ((s64)timestamp - (s64)dump_time < 0ll)) {
1036                         mutex_unlock(&vinstr_ctx->lock);
1037
1038                         /* Sleep until next dumping event or service request. */
1039                         if (cli) {
1040                                 u64 diff = dump_time - timestamp;
1041
1042                                 hrtimer_start(
1043                                                 &timer.hrtimer,
1044                                                 ns_to_ktime(diff),
1045                                                 HRTIMER_MODE_REL);
1046                         }
1047                         wait_event(
1048                                         vinstr_ctx->waitq,
1049                                         atomic_read(
1050                                                 &vinstr_ctx->request_pending) ||
1051                                         kthread_should_stop());
1052                         hrtimer_cancel(&timer.hrtimer);
1053                         continue;
1054                 }
1055
1056                 kbasep_vinstr_collect_and_accumulate(vinstr_ctx, &timestamp);
1057
1058                 INIT_LIST_HEAD(&expired_requests);
1059
1060                 /* Find all expired requests. */
1061                 list_for_each_entry_safe(
1062                                 cli,
1063                                 tmp,
1064                                 &vinstr_ctx->waiting_clients,
1065                                 list) {
1066                         s64 tdiff =
1067                                 (s64)(timestamp + DUMPING_RESOLUTION) -
1068                                 (s64)cli->dump_time;
1069                         if (tdiff >= 0ll) {
1070                                 list_del(&cli->list);
1071                                 list_add(&cli->list, &expired_requests);
1072                         } else {
1073                                 break;
1074                         }
1075                 }
1076
1077                 /* Fill data for each request found. */
1078                 list_for_each_entry_safe(cli, tmp, &expired_requests, list) {
1079                         /* Ensure that legacy buffer will not be used from
1080                          * this kthread context. */
1081                         BUG_ON(0 == cli->buffer_count);
1082                         /* Expect only periodically sampled clients. */
1083                         BUG_ON(0 == cli->dump_interval);
1084
1085                         kbasep_vinstr_update_client(
1086                                         cli,
1087                                         timestamp,
1088                                         BASE_HWCNT_READER_EVENT_PERIODIC);
1089
1090                         /* Set new dumping time. Drop missed probing times. */
1091                         do {
1092                                 cli->dump_time += cli->dump_interval;
1093                         } while (cli->dump_time < timestamp);
1094
1095                         list_del(&cli->list);
1096                         kbasep_vinstr_add_dump_request(
1097                                         cli,
1098                                         &vinstr_ctx->waiting_clients);
1099                 }
1100
1101                 /* Reprogram counters set if required. */
1102                 kbasep_vinstr_reprogram(vinstr_ctx);
1103
1104                 mutex_unlock(&vinstr_ctx->lock);
1105         }
1106
1107         return 0;
1108 }
1109
1110 /*****************************************************************************/
1111
1112 /**
1113  * kbasep_vinstr_hwcnt_reader_buffer_ready - check if client has ready buffers
1114  * @cli: pointer to vinstr client structure
1115  *
1116  * Return: non-zero if client has at least one dumping buffer filled that was
1117  *         not notified to user yet
1118  */
1119 static int kbasep_vinstr_hwcnt_reader_buffer_ready(
1120                 struct kbase_vinstr_client *cli)
1121 {
1122         KBASE_DEBUG_ASSERT(cli);
1123         return atomic_read(&cli->write_idx) != atomic_read(&cli->meta_idx);
1124 }
1125
1126 /**
1127  * kbasep_vinstr_hwcnt_reader_ioctl_get_buffer - hwcnt reader's ioctl command
1128  * @cli:    pointer to vinstr client structure
1129  * @buffer: pointer to userspace buffer
1130  * @size:   size of buffer
1131  *
1132  * Return: zero on success
1133  */
1134 static long kbasep_vinstr_hwcnt_reader_ioctl_get_buffer(
1135                 struct kbase_vinstr_client *cli, void __user *buffer,
1136                 size_t size)
1137 {
1138         unsigned int meta_idx = atomic_read(&cli->meta_idx);
1139         unsigned int idx = meta_idx % cli->buffer_count;
1140
1141         struct kbase_hwcnt_reader_metadata *meta = &cli->dump_buffers_meta[idx];
1142
1143         /* Metadata sanity check. */
1144         KBASE_DEBUG_ASSERT(idx == meta->buffer_idx);
1145
1146         if (sizeof(struct kbase_hwcnt_reader_metadata) != size)
1147                 return -EINVAL;
1148
1149         /* Check if there is any buffer available. */
1150         if (atomic_read(&cli->write_idx) == meta_idx)
1151                 return -EAGAIN;
1152
1153         /* Check if previously taken buffer was put back. */
1154         if (atomic_read(&cli->read_idx) != meta_idx)
1155                 return -EBUSY;
1156
1157         /* Copy next available buffer's metadata to user. */
1158         if (copy_to_user(buffer, meta, size))
1159                 return -EFAULT;
1160
1161         atomic_inc(&cli->meta_idx);
1162
1163         return 0;
1164 }
1165
1166 /**
1167  * kbasep_vinstr_hwcnt_reader_ioctl_put_buffer - hwcnt reader's ioctl command
1168  * @cli:    pointer to vinstr client structure
1169  * @buffer: pointer to userspace buffer
1170  * @size:   size of buffer
1171  *
1172  * Return: zero on success
1173  */
1174 static long kbasep_vinstr_hwcnt_reader_ioctl_put_buffer(
1175                 struct kbase_vinstr_client *cli, void __user *buffer,
1176                 size_t size)
1177 {
1178         unsigned int read_idx = atomic_read(&cli->read_idx);
1179         unsigned int idx = read_idx % cli->buffer_count;
1180
1181         struct kbase_hwcnt_reader_metadata meta;
1182
1183         if (sizeof(struct kbase_hwcnt_reader_metadata) != size)
1184                 return -EINVAL;
1185
1186         /* Check if any buffer was taken. */
1187         if (atomic_read(&cli->meta_idx) == read_idx)
1188                 return -EPERM;
1189
1190         /* Check if correct buffer is put back. */
1191         if (copy_from_user(&meta, buffer, size))
1192                 return -EFAULT;
1193         if (idx != meta.buffer_idx)
1194                 return -EINVAL;
1195
1196         atomic_inc(&cli->read_idx);
1197
1198         return 0;
1199 }
1200
1201 /**
1202  * kbasep_vinstr_hwcnt_reader_ioctl_set_interval - hwcnt reader's ioctl command
1203  * @cli:      pointer to vinstr client structure
1204  * @interval: periodic dumping interval (disable periodic dumping if zero)
1205  *
1206  * Return: zero on success
1207  */
1208 static long kbasep_vinstr_hwcnt_reader_ioctl_set_interval(
1209                 struct kbase_vinstr_client *cli, u32 interval)
1210 {
1211         struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1212
1213         KBASE_DEBUG_ASSERT(vinstr_ctx);
1214
1215         mutex_lock(&vinstr_ctx->lock);
1216
1217         if (vinstr_ctx->suspended) {
1218                 mutex_unlock(&vinstr_ctx->lock);
1219                 return -EBUSY;
1220         }
1221
1222         list_del(&cli->list);
1223
1224         cli->dump_interval = interval;
1225
1226         /* If interval is non-zero, enable periodic dumping for this client. */
1227         if (cli->dump_interval) {
1228                 if (DUMPING_RESOLUTION > cli->dump_interval)
1229                         cli->dump_interval = DUMPING_RESOLUTION;
1230                 cli->dump_time =
1231                         kbasep_vinstr_get_timestamp() + cli->dump_interval;
1232
1233                 kbasep_vinstr_add_dump_request(
1234                                 cli, &vinstr_ctx->waiting_clients);
1235
1236                 atomic_set(&vinstr_ctx->request_pending, 1);
1237                 wake_up_all(&vinstr_ctx->waitq);
1238         } else {
1239                 list_add(&cli->list, &vinstr_ctx->idle_clients);
1240         }
1241
1242         mutex_unlock(&vinstr_ctx->lock);
1243
1244         return 0;
1245 }
1246
1247 /**
1248  * kbasep_vinstr_hwcnt_reader_event_mask - return event mask for event id
1249  * @event_id: id of event
1250  * Return: event_mask or zero if event is not supported or maskable
1251  */
1252 static u32 kbasep_vinstr_hwcnt_reader_event_mask(
1253                 enum base_hwcnt_reader_event event_id)
1254 {
1255         u32 event_mask = 0;
1256
1257         switch (event_id) {
1258         case BASE_HWCNT_READER_EVENT_PREJOB:
1259         case BASE_HWCNT_READER_EVENT_POSTJOB:
1260                 /* These event are maskable. */
1261                 event_mask = (1 << event_id);
1262                 break;
1263
1264         case BASE_HWCNT_READER_EVENT_MANUAL:
1265         case BASE_HWCNT_READER_EVENT_PERIODIC:
1266                 /* These event are non-maskable. */
1267         default:
1268                 /* These event are not supported. */
1269                 break;
1270         }
1271
1272         return event_mask;
1273 }
1274
1275 /**
1276  * kbasep_vinstr_hwcnt_reader_ioctl_enable_event - hwcnt reader's ioctl command
1277  * @cli:      pointer to vinstr client structure
1278  * @event_id: id of event to enable
1279  *
1280  * Return: zero on success
1281  */
1282 static long kbasep_vinstr_hwcnt_reader_ioctl_enable_event(
1283                 struct kbase_vinstr_client *cli,
1284                 enum base_hwcnt_reader_event event_id)
1285 {
1286         struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1287         u32                         event_mask;
1288
1289         KBASE_DEBUG_ASSERT(vinstr_ctx);
1290
1291         event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id);
1292         if (!event_mask)
1293                 return -EINVAL;
1294
1295         mutex_lock(&vinstr_ctx->lock);
1296         cli->event_mask |= event_mask;
1297         mutex_unlock(&vinstr_ctx->lock);
1298
1299         return 0;
1300 }
1301
1302 /**
1303  * kbasep_vinstr_hwcnt_reader_ioctl_disable_event - hwcnt reader's ioctl command
1304  * @cli:      pointer to vinstr client structure
1305  * @event_id: id of event to disable
1306  *
1307  * Return: zero on success
1308  */
1309 static long kbasep_vinstr_hwcnt_reader_ioctl_disable_event(
1310                 struct kbase_vinstr_client *cli,
1311                 enum base_hwcnt_reader_event event_id)
1312 {
1313         struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1314         u32                         event_mask;
1315
1316         KBASE_DEBUG_ASSERT(vinstr_ctx);
1317
1318         event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id);
1319         if (!event_mask)
1320                 return -EINVAL;
1321
1322         mutex_lock(&vinstr_ctx->lock);
1323         cli->event_mask &= ~event_mask;
1324         mutex_unlock(&vinstr_ctx->lock);
1325
1326         return 0;
1327 }
1328
1329 /**
1330  * kbasep_vinstr_hwcnt_reader_ioctl_get_hwver - hwcnt reader's ioctl command
1331  * @cli:   pointer to vinstr client structure
1332  * @hwver: pointer to user buffer where hw version will be stored
1333  *
1334  * Return: zero on success
1335  */
1336 static long kbasep_vinstr_hwcnt_reader_ioctl_get_hwver(
1337                 struct kbase_vinstr_client *cli, u32 __user *hwver)
1338 {
1339 #ifndef CONFIG_MALI_NO_MALI
1340         struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1341 #endif
1342
1343         u32                         ver = 5;
1344
1345 #ifndef CONFIG_MALI_NO_MALI
1346         KBASE_DEBUG_ASSERT(vinstr_ctx);
1347         if (kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4))
1348                 ver = 4;
1349 #endif
1350
1351         return put_user(ver, hwver);
1352 }
1353
1354 /**
1355  * kbasep_vinstr_hwcnt_reader_ioctl - hwcnt reader's ioctl
1356  * @filp:   pointer to file structure
1357  * @cmd:    user command
1358  * @arg:    command's argument
1359  *
1360  * Return: zero on success
1361  */
1362 static long kbasep_vinstr_hwcnt_reader_ioctl(struct file *filp,
1363                 unsigned int cmd, unsigned long arg)
1364 {
1365         long                       rcode = 0;
1366         struct kbase_vinstr_client *cli;
1367
1368         KBASE_DEBUG_ASSERT(filp);
1369
1370         cli = filp->private_data;
1371         KBASE_DEBUG_ASSERT(cli);
1372
1373         if (unlikely(KBASE_HWCNT_READER != _IOC_TYPE(cmd)))
1374                 return -EINVAL;
1375
1376         switch (cmd) {
1377         case KBASE_HWCNT_READER_GET_API_VERSION:
1378                 rcode = put_user(HWCNT_READER_API, (u32 __user *)arg);
1379                 break;
1380         case KBASE_HWCNT_READER_GET_HWVER:
1381                 rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_hwver(
1382                                 cli, (u32 __user *)arg);
1383                 break;
1384         case KBASE_HWCNT_READER_GET_BUFFER_SIZE:
1385                 KBASE_DEBUG_ASSERT(cli->vinstr_ctx);
1386                 rcode = put_user(
1387                                 (u32)cli->vinstr_ctx->dump_size,
1388                                 (u32 __user *)arg);
1389                 break;
1390         case KBASE_HWCNT_READER_DUMP:
1391                 rcode = kbase_vinstr_hwc_dump(
1392                                 cli, BASE_HWCNT_READER_EVENT_MANUAL);
1393                 break;
1394         case KBASE_HWCNT_READER_CLEAR:
1395                 rcode = kbase_vinstr_hwc_clear(cli);
1396                 break;
1397         case KBASE_HWCNT_READER_GET_BUFFER:
1398                 rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_buffer(
1399                                 cli, (void __user *)arg, _IOC_SIZE(cmd));
1400                 break;
1401         case KBASE_HWCNT_READER_PUT_BUFFER:
1402                 rcode = kbasep_vinstr_hwcnt_reader_ioctl_put_buffer(
1403                                 cli, (void __user *)arg, _IOC_SIZE(cmd));
1404                 break;
1405         case KBASE_HWCNT_READER_SET_INTERVAL:
1406                 rcode = kbasep_vinstr_hwcnt_reader_ioctl_set_interval(
1407                                 cli, (u32)arg);
1408                 break;
1409         case KBASE_HWCNT_READER_ENABLE_EVENT:
1410                 rcode = kbasep_vinstr_hwcnt_reader_ioctl_enable_event(
1411                                 cli, (enum base_hwcnt_reader_event)arg);
1412                 break;
1413         case KBASE_HWCNT_READER_DISABLE_EVENT:
1414                 rcode = kbasep_vinstr_hwcnt_reader_ioctl_disable_event(
1415                                 cli, (enum base_hwcnt_reader_event)arg);
1416                 break;
1417         default:
1418                 rcode = -EINVAL;
1419                 break;
1420         }
1421
1422         return rcode;
1423 }
1424
1425 /**
1426  * kbasep_vinstr_hwcnt_reader_poll - hwcnt reader's poll
1427  * @filp: pointer to file structure
1428  * @wait: pointer to poll table
1429  * Return: POLLIN if data can be read without blocking, otherwise zero
1430  */
1431 static unsigned int kbasep_vinstr_hwcnt_reader_poll(struct file *filp,
1432                 poll_table *wait)
1433 {
1434         struct kbase_vinstr_client *cli;
1435
1436         KBASE_DEBUG_ASSERT(filp);
1437         KBASE_DEBUG_ASSERT(wait);
1438
1439         cli = filp->private_data;
1440         KBASE_DEBUG_ASSERT(cli);
1441
1442         poll_wait(filp, &cli->waitq, wait);
1443         if (kbasep_vinstr_hwcnt_reader_buffer_ready(cli))
1444                 return POLLIN;
1445         return 0;
1446 }
1447
1448 /**
1449  * kbasep_vinstr_hwcnt_reader_mmap - hwcnt reader's mmap
1450  * @filp: pointer to file structure
1451  * @vma:  pointer to vma structure
1452  * Return: zero on success
1453  */
1454 static int kbasep_vinstr_hwcnt_reader_mmap(struct file *filp,
1455                 struct vm_area_struct *vma)
1456 {
1457         struct kbase_vinstr_client *cli;
1458         size_t                     size;
1459
1460         KBASE_DEBUG_ASSERT(filp);
1461         KBASE_DEBUG_ASSERT(vma);
1462
1463         cli = filp->private_data;
1464         KBASE_DEBUG_ASSERT(cli);
1465
1466         size = cli->buffer_count * cli->dump_size;
1467         if (vma->vm_end - vma->vm_start > size)
1468                 return -ENOMEM;
1469
1470         return remap_pfn_range(
1471                         vma,
1472                         vma->vm_start,
1473                         __pa((unsigned long)cli->dump_buffers) >> PAGE_SHIFT,
1474                         size,
1475                         vma->vm_page_prot);
1476 }
1477
1478 /**
1479  * kbasep_vinstr_hwcnt_reader_release - hwcnt reader's release
1480  * @inode: pointer to inode structure
1481  * @filp:  pointer to file structure
1482  * Return always return zero
1483  */
1484 static int kbasep_vinstr_hwcnt_reader_release(struct inode *inode,
1485                 struct file *filp)
1486 {
1487         struct kbase_vinstr_client *cli;
1488
1489         KBASE_DEBUG_ASSERT(inode);
1490         KBASE_DEBUG_ASSERT(filp);
1491
1492         cli = filp->private_data;
1493         KBASE_DEBUG_ASSERT(cli);
1494
1495         kbase_vinstr_detach_client(cli);
1496         return 0;
1497 }
1498
1499 /*****************************************************************************/
1500
1501 struct kbase_vinstr_context *kbase_vinstr_init(struct kbase_device *kbdev)
1502 {
1503         struct kbase_vinstr_context *vinstr_ctx;
1504
1505         vinstr_ctx = kzalloc(sizeof(*vinstr_ctx), GFP_KERNEL);
1506         if (!vinstr_ctx)
1507                 return NULL;
1508
1509         INIT_LIST_HEAD(&vinstr_ctx->idle_clients);
1510         INIT_LIST_HEAD(&vinstr_ctx->waiting_clients);
1511         mutex_init(&vinstr_ctx->lock);
1512         vinstr_ctx->kbdev = kbdev;
1513         vinstr_ctx->thread = NULL;
1514
1515         atomic_set(&vinstr_ctx->request_pending, 0);
1516         init_waitqueue_head(&vinstr_ctx->waitq);
1517
1518         return vinstr_ctx;
1519 }
1520
1521 void kbase_vinstr_term(struct kbase_vinstr_context *vinstr_ctx)
1522 {
1523         struct kbase_vinstr_client *cli;
1524
1525         /* Stop service thread first. */
1526         if (vinstr_ctx->thread)
1527                 kthread_stop(vinstr_ctx->thread);
1528
1529         while (1) {
1530                 struct list_head *list = &vinstr_ctx->idle_clients;
1531
1532                 if (list_empty(list)) {
1533                         list = &vinstr_ctx->waiting_clients;
1534                         if (list_empty(list))
1535                                 break;
1536                 }
1537
1538                 cli = list_first_entry(list, struct kbase_vinstr_client, list);
1539                 list_del(&cli->list);
1540                 kfree(cli->accum_buffer);
1541                 kfree(cli);
1542                 vinstr_ctx->nclients--;
1543         }
1544         KBASE_DEBUG_ASSERT(!vinstr_ctx->nclients);
1545         if (vinstr_ctx->kctx)
1546                 kbasep_vinstr_destroy_kctx(vinstr_ctx);
1547         kfree(vinstr_ctx);
1548 }
1549
1550 int kbase_vinstr_hwcnt_reader_setup(struct kbase_vinstr_context *vinstr_ctx,
1551                 struct kbase_uk_hwcnt_reader_setup *setup)
1552 {
1553         struct kbase_vinstr_client  *cli;
1554         u32                         bitmap[4];
1555
1556         KBASE_DEBUG_ASSERT(vinstr_ctx);
1557         KBASE_DEBUG_ASSERT(setup);
1558         KBASE_DEBUG_ASSERT(setup->buffer_count);
1559
1560         bitmap[SHADER_HWCNT_BM] = setup->shader_bm;
1561         bitmap[TILER_HWCNT_BM]  = setup->tiler_bm;
1562         bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm;
1563         bitmap[JM_HWCNT_BM]     = setup->jm_bm;
1564
1565         cli = kbasep_vinstr_attach_client(
1566                         vinstr_ctx,
1567                         setup->buffer_count,
1568                         bitmap,
1569                         &setup->fd,
1570                         NULL);
1571
1572         if (!cli)
1573                 return -ENOMEM;
1574
1575         return 0;
1576 }
1577
1578 int kbase_vinstr_legacy_hwc_setup(
1579                 struct kbase_vinstr_context *vinstr_ctx,
1580                 struct kbase_vinstr_client  **cli,
1581                 struct kbase_uk_hwcnt_setup *setup)
1582 {
1583         KBASE_DEBUG_ASSERT(vinstr_ctx);
1584         KBASE_DEBUG_ASSERT(setup);
1585         KBASE_DEBUG_ASSERT(cli);
1586
1587         if (setup->dump_buffer) {
1588                 u32 bitmap[4];
1589
1590                 bitmap[SHADER_HWCNT_BM] = setup->shader_bm;
1591                 bitmap[TILER_HWCNT_BM]  = setup->tiler_bm;
1592                 bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm;
1593                 bitmap[JM_HWCNT_BM]     = setup->jm_bm;
1594
1595                 if (*cli)
1596                         return -EBUSY;
1597
1598                 *cli = kbasep_vinstr_attach_client(
1599                                 vinstr_ctx,
1600                                 0,
1601                                 bitmap,
1602                                 (void *)(long)setup->dump_buffer,
1603                                 NULL);
1604
1605                 if (!(*cli))
1606                         return -ENOMEM;
1607         } else {
1608                 if (!*cli)
1609                         return -EINVAL;
1610
1611                 kbase_vinstr_detach_client(*cli);
1612                 *cli = NULL;
1613         }
1614
1615         return 0;
1616 }
1617
1618 struct kbase_vinstr_client *kbase_vinstr_hwcnt_kernel_setup(
1619                 struct kbase_vinstr_context *vinstr_ctx,
1620                 struct kbase_uk_hwcnt_reader_setup *setup,
1621                 void *kernel_buffer)
1622 {
1623         u32 bitmap[4];
1624
1625         if (!vinstr_ctx || !setup || !kernel_buffer)
1626                 return NULL;
1627
1628         bitmap[SHADER_HWCNT_BM] = setup->shader_bm;
1629         bitmap[TILER_HWCNT_BM]  = setup->tiler_bm;
1630         bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm;
1631         bitmap[JM_HWCNT_BM]     = setup->jm_bm;
1632
1633         return kbasep_vinstr_attach_client(
1634                         vinstr_ctx,
1635                         0,
1636                         bitmap,
1637                         NULL,
1638                         kernel_buffer);
1639 }
1640 KBASE_EXPORT_TEST_API(kbase_vinstr_hwcnt_kernel_setup);
1641
1642 int kbase_vinstr_hwc_dump(struct kbase_vinstr_client *cli,
1643                 enum base_hwcnt_reader_event event_id)
1644 {
1645         int                         rcode = 0;
1646         struct kbase_vinstr_context *vinstr_ctx;
1647         u64                         timestamp;
1648         u32                         event_mask;
1649
1650         if (!cli)
1651                 return -EINVAL;
1652
1653         vinstr_ctx = cli->vinstr_ctx;
1654         KBASE_DEBUG_ASSERT(vinstr_ctx);
1655
1656         KBASE_DEBUG_ASSERT(event_id < BASE_HWCNT_READER_EVENT_COUNT);
1657         event_mask = 1 << event_id;
1658
1659         mutex_lock(&vinstr_ctx->lock);
1660
1661         if (vinstr_ctx->suspended) {
1662                 rcode = -EBUSY;
1663                 goto exit;
1664         }
1665
1666         if (event_mask & cli->event_mask) {
1667                 rcode = kbasep_vinstr_collect_and_accumulate(
1668                                 vinstr_ctx,
1669                                 &timestamp);
1670                 if (rcode)
1671                         goto exit;
1672
1673                 rcode = kbasep_vinstr_update_client(cli, timestamp, event_id);
1674                 if (rcode)
1675                         goto exit;
1676
1677                 kbasep_vinstr_reprogram(vinstr_ctx);
1678         }
1679
1680 exit:
1681         mutex_unlock(&vinstr_ctx->lock);
1682
1683         return rcode;
1684 }
1685 KBASE_EXPORT_TEST_API(kbase_vinstr_hwc_dump);
1686
1687 int kbase_vinstr_hwc_clear(struct kbase_vinstr_client *cli)
1688 {
1689         struct kbase_vinstr_context *vinstr_ctx;
1690         int                         rcode;
1691         u64                         unused;
1692
1693         if (!cli)
1694                 return -EINVAL;
1695
1696         vinstr_ctx = cli->vinstr_ctx;
1697         KBASE_DEBUG_ASSERT(vinstr_ctx);
1698
1699         mutex_lock(&vinstr_ctx->lock);
1700
1701         if (vinstr_ctx->suspended) {
1702                 rcode = -EBUSY;
1703                 goto exit;
1704         }
1705
1706         rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx, &unused);
1707         if (rcode)
1708                 goto exit;
1709         rcode = kbase_instr_hwcnt_clear(vinstr_ctx->kctx);
1710         if (rcode)
1711                 goto exit;
1712         memset(cli->accum_buffer, 0, cli->dump_size);
1713
1714         kbasep_vinstr_reprogram(vinstr_ctx);
1715
1716 exit:
1717         mutex_unlock(&vinstr_ctx->lock);
1718
1719         return rcode;
1720 }
1721
1722 void kbase_vinstr_hwc_suspend(struct kbase_vinstr_context *vinstr_ctx)
1723 {
1724         u64 unused;
1725
1726         KBASE_DEBUG_ASSERT(vinstr_ctx);
1727
1728         mutex_lock(&vinstr_ctx->lock);
1729         if (!vinstr_ctx->nclients || vinstr_ctx->suspended) {
1730                 mutex_unlock(&vinstr_ctx->lock);
1731                 return;
1732         }
1733
1734         kbasep_vinstr_collect_and_accumulate(vinstr_ctx, &unused);
1735         vinstr_ctx->suspended = true;
1736         vinstr_ctx->suspended_clients = vinstr_ctx->waiting_clients;
1737         INIT_LIST_HEAD(&vinstr_ctx->waiting_clients);
1738         mutex_unlock(&vinstr_ctx->lock);
1739 }
1740
1741 void kbase_vinstr_hwc_resume(struct kbase_vinstr_context *vinstr_ctx)
1742 {
1743         KBASE_DEBUG_ASSERT(vinstr_ctx);
1744
1745         mutex_lock(&vinstr_ctx->lock);
1746         if (!vinstr_ctx->nclients || !vinstr_ctx->suspended) {
1747                 mutex_unlock(&vinstr_ctx->lock);
1748                 return;
1749         }
1750
1751         vinstr_ctx->suspended = false;
1752         vinstr_ctx->waiting_clients = vinstr_ctx->suspended_clients;
1753         vinstr_ctx->reprogram = true;
1754         kbasep_vinstr_reprogram(vinstr_ctx);
1755         atomic_set(&vinstr_ctx->request_pending, 1);
1756         wake_up_all(&vinstr_ctx->waitq);
1757         mutex_unlock(&vinstr_ctx->lock);
1758 }