3 * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved.
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
18 #include <linux/anon_inodes.h>
19 #include <linux/atomic.h>
20 #include <linux/hrtimer.h>
21 #include <linux/jiffies.h>
22 #include <linux/kthread.h>
23 #include <linux/list.h>
25 #include <linux/poll.h>
26 #include <linux/preempt.h>
27 #include <linux/slab.h>
28 #include <linux/wait.h>
30 #include <mali_kbase.h>
31 #include <mali_kbase_hwaccess_instr.h>
32 #include <mali_kbase_hwaccess_jm.h>
33 #include <mali_kbase_hwcnt_reader.h>
34 #include <mali_kbase_mem_linux.h>
35 #include <mali_kbase_tlstream.h>
37 /*****************************************************************************/
39 /* Hwcnt reader API version */
40 #define HWCNT_READER_API 1
42 /* The number of nanoseconds in a second. */
43 #define NSECS_IN_SEC 1000000000ull /* ns */
45 /* The time resolution of dumping service. */
46 #define DUMPING_RESOLUTION 500000ull /* ns */
48 /* The maximal supported number of dumping buffers. */
49 #define MAX_BUFFER_COUNT 32
51 /* Size and number of hw counters blocks. */
52 #define NR_CNT_BLOCKS_PER_GROUP 8
53 #define NR_CNT_PER_BLOCK 64
54 #define NR_BYTES_PER_CNT 4
55 #define NR_BYTES_PER_HDR 16
56 #define PRFCNT_EN_MASK_OFFSET 0x8
58 /*****************************************************************************/
76 * struct kbase_vinstr_context - vinstr context per device
77 * @lock: protects the entire vinstr context
78 * @kbdev: pointer to kbase device
79 * @kctx: pointer to kbase context
80 * @vmap: vinstr vmap for mapping hwcnt dump buffer
81 * @gpu_va: GPU hwcnt dump buffer address
82 * @cpu_va: the CPU side mapping of the hwcnt dump buffer
83 * @dump_size: size of the dump buffer in bytes
84 * @bitmap: current set of counters monitored, not always in sync
86 * @reprogram: when true, reprogram hwcnt block with the new set of
88 * @state: vinstr state
89 * @state_lock: protects information about vinstr state
90 * @suspend_waitq: notification queue to trigger state re-validation
91 * @suspend_cnt: reference counter of vinstr's suspend state
92 * @suspend_work: worker to execute on entering suspended state
93 * @resume_work: worker to execute on leaving suspended state
94 * @nclients: number of attached clients, pending or otherwise
95 * @waiting_clients: head of list of clients being periodically sampled
96 * @idle_clients: head of list of clients being idle
97 * @suspended_clients: head of list of clients being suspended
98 * @thread: periodic sampling thread
99 * @waitq: notification queue of sampling thread
100 * @request_pending: request for action for sampling thread
102 struct kbase_vinstr_context {
104 struct kbase_device *kbdev;
105 struct kbase_context *kctx;
107 struct kbase_vmap_struct vmap;
114 enum vinstr_state state;
115 struct spinlock state_lock;
116 wait_queue_head_t suspend_waitq;
117 unsigned int suspend_cnt;
118 struct work_struct suspend_work;
119 struct work_struct resume_work;
122 struct list_head waiting_clients;
123 struct list_head idle_clients;
124 struct list_head suspended_clients;
126 struct task_struct *thread;
127 wait_queue_head_t waitq;
128 atomic_t request_pending;
132 * struct kbase_vinstr_client - a vinstr client attached to a vinstr context
133 * @vinstr_ctx: vinstr context client is attached to
134 * @list: node used to attach this client to list in vinstr context
135 * @buffer_count: number of buffers this client is using
136 * @event_mask: events this client reacts to
137 * @dump_size: size of one dump buffer in bytes
138 * @bitmap: bitmap request for JM, TILER, SHADER and MMU counters
139 * @legacy_buffer: userspace hwcnt dump buffer (legacy interface)
140 * @kernel_buffer: kernel hwcnt dump buffer (kernel client interface)
141 * @accum_buffer: temporary accumulation buffer for preserving counters
142 * @dump_time: next time this clients shall request hwcnt dump
143 * @dump_interval: interval between periodic hwcnt dumps
144 * @dump_buffers: kernel hwcnt dump buffers allocated by this client
145 * @dump_buffers_meta: metadata of dump buffers
146 * @meta_idx: index of metadata being accessed by userspace
147 * @read_idx: index of buffer read by userspace
148 * @write_idx: index of buffer being written by dumping service
149 * @waitq: client's notification queue
150 * @pending: when true, client has attached but hwcnt not yet updated
152 struct kbase_vinstr_client {
153 struct kbase_vinstr_context *vinstr_ctx;
154 struct list_head list;
155 unsigned int buffer_count;
159 void __user *legacy_buffer;
165 struct kbase_hwcnt_reader_metadata *dump_buffers_meta;
169 wait_queue_head_t waitq;
174 * struct kbasep_vinstr_wake_up_timer - vinstr service thread wake up timer
175 * @hrtimer: high resolution timer
176 * @vinstr_ctx: vinstr context
178 struct kbasep_vinstr_wake_up_timer {
179 struct hrtimer hrtimer;
180 struct kbase_vinstr_context *vinstr_ctx;
183 /*****************************************************************************/
185 static int kbasep_vinstr_service_task(void *data);
187 static unsigned int kbasep_vinstr_hwcnt_reader_poll(
190 static long kbasep_vinstr_hwcnt_reader_ioctl(
194 static int kbasep_vinstr_hwcnt_reader_mmap(
196 struct vm_area_struct *vma);
197 static int kbasep_vinstr_hwcnt_reader_release(
201 /* The timeline stream file operations structure. */
202 static const struct file_operations vinstr_client_fops = {
203 .poll = kbasep_vinstr_hwcnt_reader_poll,
204 .unlocked_ioctl = kbasep_vinstr_hwcnt_reader_ioctl,
205 .compat_ioctl = kbasep_vinstr_hwcnt_reader_ioctl,
206 .mmap = kbasep_vinstr_hwcnt_reader_mmap,
207 .release = kbasep_vinstr_hwcnt_reader_release,
210 /*****************************************************************************/
212 static int enable_hwcnt(struct kbase_vinstr_context *vinstr_ctx)
214 struct kbase_context *kctx = vinstr_ctx->kctx;
215 struct kbase_device *kbdev = kctx->kbdev;
216 struct kbase_uk_hwcnt_setup setup;
219 setup.dump_buffer = vinstr_ctx->gpu_va;
220 setup.jm_bm = vinstr_ctx->bitmap[JM_HWCNT_BM];
221 setup.tiler_bm = vinstr_ctx->bitmap[TILER_HWCNT_BM];
222 setup.shader_bm = vinstr_ctx->bitmap[SHADER_HWCNT_BM];
223 setup.mmu_l2_bm = vinstr_ctx->bitmap[MMU_L2_HWCNT_BM];
225 /* Mark the context as active so the GPU is kept turned on */
226 /* A suspend won't happen here, because we're in a syscall from a
227 * userspace thread. */
228 kbase_pm_context_active(kbdev);
230 /* Schedule the context in */
231 kbasep_js_schedule_privileged_ctx(kbdev, kctx);
232 err = kbase_instr_hwcnt_enable_internal(kbdev, kctx, &setup);
234 /* Release the context. This had its own Power Manager Active
236 kbasep_js_release_privileged_ctx(kbdev, kctx);
238 /* Also release our Power Manager Active reference */
239 kbase_pm_context_idle(kbdev);
245 static void disable_hwcnt(struct kbase_vinstr_context *vinstr_ctx)
247 struct kbase_context *kctx = vinstr_ctx->kctx;
248 struct kbase_device *kbdev = kctx->kbdev;
251 err = kbase_instr_hwcnt_disable_internal(kctx);
253 dev_warn(kbdev->dev, "Failed to disable HW counters (ctx:%p)",
258 /* Release the context. This had its own Power Manager Active reference. */
259 kbasep_js_release_privileged_ctx(kbdev, kctx);
261 /* Also release our Power Manager Active reference. */
262 kbase_pm_context_idle(kbdev);
264 dev_dbg(kbdev->dev, "HW counters dumping disabled for context %p", kctx);
267 static int reprogram_hwcnt(struct kbase_vinstr_context *vinstr_ctx)
269 disable_hwcnt(vinstr_ctx);
270 return enable_hwcnt(vinstr_ctx);
273 static void hwcnt_bitmap_set(u32 dst[4], u32 src[4])
275 dst[JM_HWCNT_BM] = src[JM_HWCNT_BM];
276 dst[TILER_HWCNT_BM] = src[TILER_HWCNT_BM];
277 dst[SHADER_HWCNT_BM] = src[SHADER_HWCNT_BM];
278 dst[MMU_L2_HWCNT_BM] = src[MMU_L2_HWCNT_BM];
281 static void hwcnt_bitmap_union(u32 dst[4], u32 src[4])
283 dst[JM_HWCNT_BM] |= src[JM_HWCNT_BM];
284 dst[TILER_HWCNT_BM] |= src[TILER_HWCNT_BM];
285 dst[SHADER_HWCNT_BM] |= src[SHADER_HWCNT_BM];
286 dst[MMU_L2_HWCNT_BM] |= src[MMU_L2_HWCNT_BM];
289 size_t kbase_vinstr_dump_size(struct kbase_device *kbdev)
293 #ifndef CONFIG_MALI_NO_MALI
294 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_V4)) {
297 nr_cg = kbdev->gpu_props.num_core_groups;
298 dump_size = nr_cg * NR_CNT_BLOCKS_PER_GROUP *
302 #endif /* CONFIG_MALI_NO_MALI */
304 /* assume v5 for now */
305 base_gpu_props *props = &kbdev->gpu_props.props;
306 u32 nr_l2 = props->l2_props.num_l2_slices;
307 u64 core_mask = props->coherency_info.group[0].core_mask;
308 u32 nr_blocks = fls64(core_mask);
310 /* JM and tiler counter blocks are always present */
311 dump_size = (2 + nr_l2 + nr_blocks) *
317 KBASE_EXPORT_TEST_API(kbase_vinstr_dump_size);
319 static size_t kbasep_vinstr_dump_size_ctx(
320 struct kbase_vinstr_context *vinstr_ctx)
322 return kbase_vinstr_dump_size(vinstr_ctx->kctx->kbdev);
325 static int kbasep_vinstr_map_kernel_dump_buffer(
326 struct kbase_vinstr_context *vinstr_ctx)
328 struct kbase_va_region *reg;
329 struct kbase_context *kctx = vinstr_ctx->kctx;
333 flags = BASE_MEM_PROT_CPU_RD | BASE_MEM_PROT_GPU_WR;
334 vinstr_ctx->dump_size = kbasep_vinstr_dump_size_ctx(vinstr_ctx);
335 nr_pages = PFN_UP(vinstr_ctx->dump_size);
337 reg = kbase_mem_alloc(kctx, nr_pages, nr_pages, 0, &flags,
338 &vinstr_ctx->gpu_va, &va_align);
342 vinstr_ctx->cpu_va = kbase_vmap(
345 vinstr_ctx->dump_size,
347 if (!vinstr_ctx->cpu_va) {
348 kbase_mem_free(kctx, vinstr_ctx->gpu_va);
355 static void kbasep_vinstr_unmap_kernel_dump_buffer(
356 struct kbase_vinstr_context *vinstr_ctx)
358 struct kbase_context *kctx = vinstr_ctx->kctx;
360 kbase_vunmap(kctx, &vinstr_ctx->vmap);
361 kbase_mem_free(kctx, vinstr_ctx->gpu_va);
365 * kbasep_vinstr_create_kctx - create kernel context for vinstr
366 * @vinstr_ctx: vinstr context
367 * Return: zero on success
369 static int kbasep_vinstr_create_kctx(struct kbase_vinstr_context *vinstr_ctx)
371 struct kbase_device *kbdev = vinstr_ctx->kbdev;
372 struct kbasep_kctx_list_element *element;
374 bool enable_backend = false;
377 vinstr_ctx->kctx = kbase_create_context(vinstr_ctx->kbdev, true);
378 if (!vinstr_ctx->kctx)
381 /* Map the master kernel dump buffer. The HW dumps the counters
382 * into this memory region. */
383 err = kbasep_vinstr_map_kernel_dump_buffer(vinstr_ctx);
385 kbase_destroy_context(vinstr_ctx->kctx);
386 vinstr_ctx->kctx = NULL;
390 /* Add kernel context to list of contexts associated with device. */
391 element = kzalloc(sizeof(*element), GFP_KERNEL);
393 element->kctx = vinstr_ctx->kctx;
394 mutex_lock(&kbdev->kctx_list_lock);
395 list_add(&element->link, &kbdev->kctx_list);
397 /* Inform timeline client about new context.
398 * Do this while holding the lock to avoid tracepoint
399 * being created in both body and summary stream. */
400 kbase_tlstream_tl_new_ctx(
402 (u32)(vinstr_ctx->kctx->id),
403 (u32)(vinstr_ctx->kctx->tgid));
405 mutex_unlock(&kbdev->kctx_list_lock);
407 /* Don't treat this as a fail - just warn about it. */
409 "couldn't add kctx to kctx_list\n");
412 /* Don't enable hardware counters if vinstr is suspended.
413 * Note that vinstr resume code is run under vinstr context lock,
414 * lower layer will be enabled as needed on resume. */
415 spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
416 if (VINSTR_IDLE == vinstr_ctx->state)
417 enable_backend = true;
418 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
420 err = enable_hwcnt(vinstr_ctx);
423 kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx);
424 kbase_destroy_context(vinstr_ctx->kctx);
426 mutex_lock(&kbdev->kctx_list_lock);
427 list_del(&element->link);
429 mutex_unlock(&kbdev->kctx_list_lock);
431 kbase_tlstream_tl_del_ctx(vinstr_ctx->kctx);
432 vinstr_ctx->kctx = NULL;
436 vinstr_ctx->thread = kthread_run(
437 kbasep_vinstr_service_task,
439 "mali_vinstr_service");
440 if (!vinstr_ctx->thread) {
441 disable_hwcnt(vinstr_ctx);
442 kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx);
443 kbase_destroy_context(vinstr_ctx->kctx);
445 mutex_lock(&kbdev->kctx_list_lock);
446 list_del(&element->link);
448 mutex_unlock(&kbdev->kctx_list_lock);
450 kbase_tlstream_tl_del_ctx(vinstr_ctx->kctx);
451 vinstr_ctx->kctx = NULL;
459 * kbasep_vinstr_destroy_kctx - destroy vinstr's kernel context
460 * @vinstr_ctx: vinstr context
462 static void kbasep_vinstr_destroy_kctx(struct kbase_vinstr_context *vinstr_ctx)
464 struct kbase_device *kbdev = vinstr_ctx->kbdev;
465 struct kbasep_kctx_list_element *element;
466 struct kbasep_kctx_list_element *tmp;
469 /* Release hw counters dumping resources. */
470 vinstr_ctx->thread = NULL;
471 disable_hwcnt(vinstr_ctx);
472 kbasep_vinstr_unmap_kernel_dump_buffer(vinstr_ctx);
473 kbase_destroy_context(vinstr_ctx->kctx);
475 /* Remove kernel context from the device's contexts list. */
476 mutex_lock(&kbdev->kctx_list_lock);
477 list_for_each_entry_safe(element, tmp, &kbdev->kctx_list, link) {
478 if (element->kctx == vinstr_ctx->kctx) {
479 list_del(&element->link);
484 mutex_unlock(&kbdev->kctx_list_lock);
487 dev_warn(kbdev->dev, "kctx not in kctx_list\n");
489 /* Inform timeline client about context destruction. */
490 kbase_tlstream_tl_del_ctx(vinstr_ctx->kctx);
492 vinstr_ctx->kctx = NULL;
496 * kbasep_vinstr_attach_client - Attach a client to the vinstr core
497 * @vinstr_ctx: vinstr context
498 * @buffer_count: requested number of dump buffers
499 * @bitmap: bitmaps describing which counters should be enabled
500 * @argp: pointer where notification descriptor shall be stored
501 * @kernel_buffer: pointer to kernel side buffer
503 * Return: vinstr opaque client handle or NULL on failure
505 static struct kbase_vinstr_client *kbasep_vinstr_attach_client(
506 struct kbase_vinstr_context *vinstr_ctx, u32 buffer_count,
507 u32 bitmap[4], void *argp, void *kernel_buffer)
509 struct task_struct *thread = NULL;
510 struct kbase_vinstr_client *cli;
512 KBASE_DEBUG_ASSERT(vinstr_ctx);
514 if (buffer_count > MAX_BUFFER_COUNT
515 || (buffer_count & (buffer_count - 1)))
518 cli = kzalloc(sizeof(*cli), GFP_KERNEL);
522 cli->vinstr_ctx = vinstr_ctx;
523 cli->buffer_count = buffer_count;
525 (1 << BASE_HWCNT_READER_EVENT_MANUAL) |
526 (1 << BASE_HWCNT_READER_EVENT_PERIODIC);
529 hwcnt_bitmap_set(cli->bitmap, bitmap);
531 mutex_lock(&vinstr_ctx->lock);
533 hwcnt_bitmap_union(vinstr_ctx->bitmap, cli->bitmap);
534 vinstr_ctx->reprogram = true;
536 /* If this is the first client, create the vinstr kbase
537 * context. This context is permanently resident until the
538 * last client exits. */
539 if (!vinstr_ctx->nclients) {
540 hwcnt_bitmap_set(vinstr_ctx->bitmap, cli->bitmap);
541 if (kbasep_vinstr_create_kctx(vinstr_ctx) < 0)
544 vinstr_ctx->reprogram = false;
545 cli->pending = false;
548 /* The GPU resets the counter block every time there is a request
549 * to dump it. We need a per client kernel buffer for accumulating
551 cli->dump_size = kbasep_vinstr_dump_size_ctx(vinstr_ctx);
552 cli->accum_buffer = kzalloc(cli->dump_size, GFP_KERNEL);
553 if (!cli->accum_buffer)
556 /* Prepare buffers. */
557 if (cli->buffer_count) {
558 int *fd = (int *)argp;
561 /* Allocate area for buffers metadata storage. */
562 tmp = sizeof(struct kbase_hwcnt_reader_metadata) *
564 cli->dump_buffers_meta = kmalloc(tmp, GFP_KERNEL);
565 if (!cli->dump_buffers_meta)
568 /* Allocate required number of dumping buffers. */
569 cli->dump_buffers = (char *)__get_free_pages(
570 GFP_KERNEL | __GFP_ZERO,
571 get_order(cli->dump_size * cli->buffer_count));
572 if (!cli->dump_buffers)
575 /* Create descriptor for user-kernel data exchange. */
576 *fd = anon_inode_getfd(
577 "[mali_vinstr_desc]",
580 O_RDONLY | O_CLOEXEC);
583 } else if (kernel_buffer) {
584 cli->kernel_buffer = kernel_buffer;
586 cli->legacy_buffer = (void __user *)argp;
589 atomic_set(&cli->read_idx, 0);
590 atomic_set(&cli->meta_idx, 0);
591 atomic_set(&cli->write_idx, 0);
592 init_waitqueue_head(&cli->waitq);
594 vinstr_ctx->nclients++;
595 list_add(&cli->list, &vinstr_ctx->idle_clients);
597 mutex_unlock(&vinstr_ctx->lock);
602 kfree(cli->dump_buffers_meta);
603 if (cli->dump_buffers)
605 (unsigned long)cli->dump_buffers,
606 get_order(cli->dump_size * cli->buffer_count));
607 kfree(cli->accum_buffer);
608 if (!vinstr_ctx->nclients && vinstr_ctx->kctx) {
609 thread = vinstr_ctx->thread;
610 kbasep_vinstr_destroy_kctx(vinstr_ctx);
614 mutex_unlock(&vinstr_ctx->lock);
616 /* Thread must be stopped after lock is released. */
618 kthread_stop(thread);
623 void kbase_vinstr_detach_client(struct kbase_vinstr_client *cli)
625 struct kbase_vinstr_context *vinstr_ctx;
626 struct kbase_vinstr_client *iter, *tmp;
627 struct task_struct *thread = NULL;
628 u32 zerobitmap[4] = { 0 };
631 KBASE_DEBUG_ASSERT(cli);
632 vinstr_ctx = cli->vinstr_ctx;
633 KBASE_DEBUG_ASSERT(vinstr_ctx);
635 mutex_lock(&vinstr_ctx->lock);
637 list_for_each_entry_safe(iter, tmp, &vinstr_ctx->idle_clients, list) {
639 vinstr_ctx->reprogram = true;
641 list_del(&iter->list);
646 list_for_each_entry_safe(
647 iter, tmp, &vinstr_ctx->waiting_clients, list) {
649 vinstr_ctx->reprogram = true;
651 list_del(&iter->list);
656 KBASE_DEBUG_ASSERT(cli_found);
658 kfree(cli->dump_buffers_meta);
660 (unsigned long)cli->dump_buffers,
661 get_order(cli->dump_size * cli->buffer_count));
662 kfree(cli->accum_buffer);
665 vinstr_ctx->nclients--;
666 if (!vinstr_ctx->nclients) {
667 thread = vinstr_ctx->thread;
668 kbasep_vinstr_destroy_kctx(vinstr_ctx);
671 /* Rebuild context bitmap now that the client has detached */
672 hwcnt_bitmap_set(vinstr_ctx->bitmap, zerobitmap);
673 list_for_each_entry(iter, &vinstr_ctx->idle_clients, list)
674 hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap);
675 list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list)
676 hwcnt_bitmap_union(vinstr_ctx->bitmap, iter->bitmap);
678 mutex_unlock(&vinstr_ctx->lock);
680 /* Thread must be stopped after lock is released. */
682 kthread_stop(thread);
684 KBASE_EXPORT_TEST_API(kbase_vinstr_detach_client);
686 /* Accumulate counters in the dump buffer */
687 static void accum_dump_buffer(void *dst, void *src, size_t dump_size)
689 size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT;
694 for (i = 0; i < dump_size; i += block_size) {
695 /* skip over the header block */
696 d += NR_BYTES_PER_HDR / sizeof(u32);
697 s += NR_BYTES_PER_HDR / sizeof(u32);
698 for (j = 0; j < (block_size - NR_BYTES_PER_HDR) / sizeof(u32); j++) {
699 /* saturate result if addition would result in wraparound */
700 if (U32_MAX - *d < *s)
710 /* This is the Midgard v4 patch function. It copies the headers for each
711 * of the defined blocks from the master kernel buffer and then patches up
712 * the performance counter enable mask for each of the blocks to exclude
713 * counters that were not requested by the client. */
714 static void patch_dump_buffer_hdr_v4(
715 struct kbase_vinstr_context *vinstr_ctx,
716 struct kbase_vinstr_client *cli)
719 u8 *dst = cli->accum_buffer;
720 u8 *src = vinstr_ctx->cpu_va;
721 u32 nr_cg = vinstr_ctx->kctx->kbdev->gpu_props.num_core_groups;
722 size_t i, group_size, group;
724 SC0_BASE = 0 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
725 SC1_BASE = 1 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
726 SC2_BASE = 2 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
727 SC3_BASE = 3 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
728 TILER_BASE = 4 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
729 MMU_L2_BASE = 5 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT,
730 JM_BASE = 7 * NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT
733 group_size = NR_CNT_BLOCKS_PER_GROUP *
736 for (i = 0; i < nr_cg; i++) {
737 group = i * group_size;
738 /* copy shader core headers */
739 memcpy(&dst[group + SC0_BASE], &src[group + SC0_BASE],
741 memcpy(&dst[group + SC1_BASE], &src[group + SC1_BASE],
743 memcpy(&dst[group + SC2_BASE], &src[group + SC2_BASE],
745 memcpy(&dst[group + SC3_BASE], &src[group + SC3_BASE],
748 /* copy tiler header */
749 memcpy(&dst[group + TILER_BASE], &src[group + TILER_BASE],
752 /* copy mmu header */
753 memcpy(&dst[group + MMU_L2_BASE], &src[group + MMU_L2_BASE],
756 /* copy job manager header */
757 memcpy(&dst[group + JM_BASE], &src[group + JM_BASE],
760 /* patch the shader core enable mask */
761 mask = (u32 *)&dst[group + SC0_BASE + PRFCNT_EN_MASK_OFFSET];
762 *mask &= cli->bitmap[SHADER_HWCNT_BM];
763 mask = (u32 *)&dst[group + SC1_BASE + PRFCNT_EN_MASK_OFFSET];
764 *mask &= cli->bitmap[SHADER_HWCNT_BM];
765 mask = (u32 *)&dst[group + SC2_BASE + PRFCNT_EN_MASK_OFFSET];
766 *mask &= cli->bitmap[SHADER_HWCNT_BM];
767 mask = (u32 *)&dst[group + SC3_BASE + PRFCNT_EN_MASK_OFFSET];
768 *mask &= cli->bitmap[SHADER_HWCNT_BM];
770 /* patch the tiler core enable mask */
771 mask = (u32 *)&dst[group + TILER_BASE + PRFCNT_EN_MASK_OFFSET];
772 *mask &= cli->bitmap[TILER_HWCNT_BM];
774 /* patch the mmu core enable mask */
775 mask = (u32 *)&dst[group + MMU_L2_BASE + PRFCNT_EN_MASK_OFFSET];
776 *mask &= cli->bitmap[MMU_L2_HWCNT_BM];
778 /* patch the job manager enable mask */
779 mask = (u32 *)&dst[group + JM_BASE + PRFCNT_EN_MASK_OFFSET];
780 *mask &= cli->bitmap[JM_HWCNT_BM];
784 /* This is the Midgard v5 patch function. It copies the headers for each
785 * of the defined blocks from the master kernel buffer and then patches up
786 * the performance counter enable mask for each of the blocks to exclude
787 * counters that were not requested by the client. */
788 static void patch_dump_buffer_hdr_v5(
789 struct kbase_vinstr_context *vinstr_ctx,
790 struct kbase_vinstr_client *cli)
792 struct kbase_device *kbdev = vinstr_ctx->kctx->kbdev;
796 u8 *dst = cli->accum_buffer;
797 u8 *src = vinstr_ctx->cpu_va;
798 size_t block_size = NR_CNT_PER_BLOCK * NR_BYTES_PER_CNT;
800 /* copy and patch job manager header */
801 memcpy(dst, src, NR_BYTES_PER_HDR);
802 mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
803 *mask &= cli->bitmap[JM_HWCNT_BM];
807 /* copy and patch tiler header */
808 memcpy(dst, src, NR_BYTES_PER_HDR);
809 mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
810 *mask &= cli->bitmap[TILER_HWCNT_BM];
814 /* copy and patch MMU/L2C headers */
815 nr_l2 = kbdev->gpu_props.props.l2_props.num_l2_slices;
816 for (i = 0; i < nr_l2; i++) {
817 memcpy(dst, src, NR_BYTES_PER_HDR);
818 mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
819 *mask &= cli->bitmap[MMU_L2_HWCNT_BM];
824 /* copy and patch shader core headers */
825 core_mask = kbdev->gpu_props.props.coherency_info.group[0].core_mask;
826 while (0ull != core_mask) {
827 memcpy(dst, src, NR_BYTES_PER_HDR);
828 if (0ull != (core_mask & 1ull)) {
829 /* if block is not reserved update header */
830 mask = (u32 *)&dst[PRFCNT_EN_MASK_OFFSET];
831 *mask &= cli->bitmap[SHADER_HWCNT_BM];
841 * accum_clients - accumulate dumped hw counters for all known clients
842 * @vinstr_ctx: vinstr context
844 static void accum_clients(struct kbase_vinstr_context *vinstr_ctx)
846 struct kbase_vinstr_client *iter;
849 #ifndef CONFIG_MALI_NO_MALI
850 v4 = kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4);
853 list_for_each_entry(iter, &vinstr_ctx->idle_clients, list) {
854 /* Don't bother accumulating clients whose hwcnt requests
855 * have not yet been honoured. */
859 patch_dump_buffer_hdr_v4(vinstr_ctx, iter);
861 patch_dump_buffer_hdr_v5(vinstr_ctx, iter);
867 list_for_each_entry(iter, &vinstr_ctx->waiting_clients, list) {
868 /* Don't bother accumulating clients whose hwcnt requests
869 * have not yet been honoured. */
873 patch_dump_buffer_hdr_v4(vinstr_ctx, iter);
875 patch_dump_buffer_hdr_v5(vinstr_ctx, iter);
883 /*****************************************************************************/
886 * kbasep_vinstr_get_timestamp - return timestamp
888 * Function returns timestamp value based on raw monotonic timer. Value will
889 * wrap around zero in case of overflow.
891 * Return: timestamp value
893 static u64 kbasep_vinstr_get_timestamp(void)
897 getrawmonotonic(&ts);
898 return (u64)ts.tv_sec * NSECS_IN_SEC + ts.tv_nsec;
902 * kbasep_vinstr_add_dump_request - register client's dumping request
903 * @cli: requesting client
904 * @waiting_clients: list of pending dumping requests
906 static void kbasep_vinstr_add_dump_request(
907 struct kbase_vinstr_client *cli,
908 struct list_head *waiting_clients)
910 struct kbase_vinstr_client *tmp;
912 if (list_empty(waiting_clients)) {
913 list_add(&cli->list, waiting_clients);
916 list_for_each_entry(tmp, waiting_clients, list) {
917 if (tmp->dump_time > cli->dump_time) {
918 list_add_tail(&cli->list, &tmp->list);
922 list_add_tail(&cli->list, waiting_clients);
926 * kbasep_vinstr_collect_and_accumulate - collect hw counters via low level
927 * dump and accumulate them for known
929 * @vinstr_ctx: vinstr context
930 * @timestamp: pointer where collection timestamp will be recorded
932 * Return: zero on success
934 static int kbasep_vinstr_collect_and_accumulate(
935 struct kbase_vinstr_context *vinstr_ctx, u64 *timestamp)
940 #ifdef CONFIG_MALI_NO_MALI
941 /* The dummy model needs the CPU mapping. */
942 gpu_model_set_dummy_prfcnt_base_cpu(vinstr_ctx->cpu_va);
945 spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
946 if (VINSTR_IDLE != vinstr_ctx->state) {
947 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
950 vinstr_ctx->state = VINSTR_DUMPING;
952 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
954 /* Request HW counters dump.
955 * Disable preemption to make dump timestamp more accurate. */
957 *timestamp = kbasep_vinstr_get_timestamp();
958 rcode = kbase_instr_hwcnt_request_dump(vinstr_ctx->kctx);
962 rcode = kbase_instr_hwcnt_wait_for_dump(vinstr_ctx->kctx);
965 spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
966 switch (vinstr_ctx->state)
968 case VINSTR_SUSPENDING:
969 schedule_work(&vinstr_ctx->suspend_work);
972 vinstr_ctx->state = VINSTR_IDLE;
973 wake_up_all(&vinstr_ctx->suspend_waitq);
978 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
980 /* Accumulate values of collected counters. */
982 accum_clients(vinstr_ctx);
988 * kbasep_vinstr_fill_dump_buffer - copy accumulated counters to empty kernel
990 * @cli: requesting client
991 * @timestamp: timestamp when counters were collected
992 * @event_id: id of event that caused triggered counters collection
994 * Return: zero on success
996 static int kbasep_vinstr_fill_dump_buffer(
997 struct kbase_vinstr_client *cli, u64 timestamp,
998 enum base_hwcnt_reader_event event_id)
1000 unsigned int write_idx = atomic_read(&cli->write_idx);
1001 unsigned int read_idx = atomic_read(&cli->read_idx);
1003 struct kbase_hwcnt_reader_metadata *meta;
1006 /* Check if there is a place to copy HWC block into. */
1007 if (write_idx - read_idx == cli->buffer_count)
1009 write_idx %= cli->buffer_count;
1011 /* Fill in dump buffer and its metadata. */
1012 buffer = &cli->dump_buffers[write_idx * cli->dump_size];
1013 meta = &cli->dump_buffers_meta[write_idx];
1014 meta->timestamp = timestamp;
1015 meta->event_id = event_id;
1016 meta->buffer_idx = write_idx;
1017 memcpy(buffer, cli->accum_buffer, cli->dump_size);
1022 * kbasep_vinstr_fill_dump_buffer_legacy - copy accumulated counters to buffer
1023 * allocated in userspace
1024 * @cli: requesting client
1026 * Return: zero on success
1028 * This is part of legacy ioctl interface.
1030 static int kbasep_vinstr_fill_dump_buffer_legacy(
1031 struct kbase_vinstr_client *cli)
1033 void __user *buffer = cli->legacy_buffer;
1036 /* Copy data to user buffer. */
1037 rcode = copy_to_user(buffer, cli->accum_buffer, cli->dump_size);
1039 pr_warn("error while copying buffer to user\n");
1044 * kbasep_vinstr_fill_dump_buffer_kernel - copy accumulated counters to buffer
1045 * allocated in kernel space
1046 * @cli: requesting client
1048 * Return: zero on success
1050 * This is part of the kernel client interface.
1052 static int kbasep_vinstr_fill_dump_buffer_kernel(
1053 struct kbase_vinstr_client *cli)
1055 memcpy(cli->kernel_buffer, cli->accum_buffer, cli->dump_size);
1061 * kbasep_vinstr_reprogram - reprogram hwcnt set collected by inst
1062 * @vinstr_ctx: vinstr context
1064 static void kbasep_vinstr_reprogram(
1065 struct kbase_vinstr_context *vinstr_ctx)
1067 unsigned long flags;
1068 bool suspended = false;
1070 /* Don't enable hardware counters if vinstr is suspended. */
1071 spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
1072 if (VINSTR_IDLE != vinstr_ctx->state)
1074 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
1078 /* Change to suspended state is done while holding vinstr context
1079 * lock. Below code will then no re-enable the instrumentation. */
1081 if (vinstr_ctx->reprogram) {
1082 struct kbase_vinstr_client *iter;
1084 if (!reprogram_hwcnt(vinstr_ctx)) {
1085 vinstr_ctx->reprogram = false;
1086 list_for_each_entry(
1088 &vinstr_ctx->idle_clients,
1090 iter->pending = false;
1091 list_for_each_entry(
1093 &vinstr_ctx->waiting_clients,
1095 iter->pending = false;
1101 * kbasep_vinstr_update_client - copy accumulated counters to user readable
1102 * buffer and notify the user
1103 * @cli: requesting client
1104 * @timestamp: timestamp when counters were collected
1105 * @event_id: id of event that caused triggered counters collection
1107 * Return: zero on success
1109 static int kbasep_vinstr_update_client(
1110 struct kbase_vinstr_client *cli, u64 timestamp,
1111 enum base_hwcnt_reader_event event_id)
1115 /* Copy collected counters to user readable buffer. */
1116 if (cli->buffer_count)
1117 rcode = kbasep_vinstr_fill_dump_buffer(
1118 cli, timestamp, event_id);
1119 else if (cli->kernel_buffer)
1120 rcode = kbasep_vinstr_fill_dump_buffer_kernel(cli);
1122 rcode = kbasep_vinstr_fill_dump_buffer_legacy(cli);
1128 /* Notify client. Make sure all changes to memory are visible. */
1130 atomic_inc(&cli->write_idx);
1131 wake_up_interruptible(&cli->waitq);
1133 /* Prepare for next request. */
1134 memset(cli->accum_buffer, 0, cli->dump_size);
1141 * kbasep_vinstr_wake_up_callback - vinstr wake up timer wake up function
1143 * @hrtimer: high resolution timer
1145 * Return: High resolution timer restart enum.
1147 static enum hrtimer_restart kbasep_vinstr_wake_up_callback(
1148 struct hrtimer *hrtimer)
1150 struct kbasep_vinstr_wake_up_timer *timer =
1153 struct kbasep_vinstr_wake_up_timer,
1156 KBASE_DEBUG_ASSERT(timer);
1158 atomic_set(&timer->vinstr_ctx->request_pending, 1);
1159 wake_up_all(&timer->vinstr_ctx->waitq);
1161 return HRTIMER_NORESTART;
1165 * kbasep_vinstr_service_task - HWC dumping service thread
1167 * @data: Pointer to vinstr context structure.
1169 * Return: Always returns zero.
1171 static int kbasep_vinstr_service_task(void *data)
1173 struct kbase_vinstr_context *vinstr_ctx = data;
1174 struct kbasep_vinstr_wake_up_timer timer;
1176 KBASE_DEBUG_ASSERT(vinstr_ctx);
1178 hrtimer_init(&timer.hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1179 timer.hrtimer.function = kbasep_vinstr_wake_up_callback;
1180 timer.vinstr_ctx = vinstr_ctx;
1182 while (!kthread_should_stop()) {
1183 struct kbase_vinstr_client *cli = NULL;
1184 struct kbase_vinstr_client *tmp;
1187 u64 timestamp = kbasep_vinstr_get_timestamp();
1189 struct list_head expired_requests;
1191 /* Hold lock while performing operations on lists of clients. */
1192 mutex_lock(&vinstr_ctx->lock);
1194 /* Closing thread must not interact with client requests. */
1195 if (current == vinstr_ctx->thread) {
1196 atomic_set(&vinstr_ctx->request_pending, 0);
1198 if (!list_empty(&vinstr_ctx->waiting_clients)) {
1199 cli = list_first_entry(
1200 &vinstr_ctx->waiting_clients,
1201 struct kbase_vinstr_client,
1203 dump_time = cli->dump_time;
1207 if (!cli || ((s64)timestamp - (s64)dump_time < 0ll)) {
1208 mutex_unlock(&vinstr_ctx->lock);
1210 /* Sleep until next dumping event or service request. */
1212 u64 diff = dump_time - timestamp;
1222 &vinstr_ctx->request_pending) ||
1223 kthread_should_stop());
1224 hrtimer_cancel(&timer.hrtimer);
1228 rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx,
1231 INIT_LIST_HEAD(&expired_requests);
1233 /* Find all expired requests. */
1234 list_for_each_entry_safe(
1237 &vinstr_ctx->waiting_clients,
1240 (s64)(timestamp + DUMPING_RESOLUTION) -
1241 (s64)cli->dump_time;
1243 list_del(&cli->list);
1244 list_add(&cli->list, &expired_requests);
1250 /* Fill data for each request found. */
1251 list_for_each_entry_safe(cli, tmp, &expired_requests, list) {
1252 /* Ensure that legacy buffer will not be used from
1253 * this kthread context. */
1254 BUG_ON(0 == cli->buffer_count);
1255 /* Expect only periodically sampled clients. */
1256 BUG_ON(0 == cli->dump_interval);
1259 kbasep_vinstr_update_client(
1262 BASE_HWCNT_READER_EVENT_PERIODIC);
1264 /* Set new dumping time. Drop missed probing times. */
1266 cli->dump_time += cli->dump_interval;
1267 } while (cli->dump_time < timestamp);
1269 list_del(&cli->list);
1270 kbasep_vinstr_add_dump_request(
1272 &vinstr_ctx->waiting_clients);
1275 /* Reprogram counters set if required. */
1276 kbasep_vinstr_reprogram(vinstr_ctx);
1278 mutex_unlock(&vinstr_ctx->lock);
1284 /*****************************************************************************/
1287 * kbasep_vinstr_hwcnt_reader_buffer_ready - check if client has ready buffers
1288 * @cli: pointer to vinstr client structure
1290 * Return: non-zero if client has at least one dumping buffer filled that was
1291 * not notified to user yet
1293 static int kbasep_vinstr_hwcnt_reader_buffer_ready(
1294 struct kbase_vinstr_client *cli)
1296 KBASE_DEBUG_ASSERT(cli);
1297 return atomic_read(&cli->write_idx) != atomic_read(&cli->meta_idx);
1301 * kbasep_vinstr_hwcnt_reader_ioctl_get_buffer - hwcnt reader's ioctl command
1302 * @cli: pointer to vinstr client structure
1303 * @buffer: pointer to userspace buffer
1304 * @size: size of buffer
1306 * Return: zero on success
1308 static long kbasep_vinstr_hwcnt_reader_ioctl_get_buffer(
1309 struct kbase_vinstr_client *cli, void __user *buffer,
1312 unsigned int meta_idx = atomic_read(&cli->meta_idx);
1313 unsigned int idx = meta_idx % cli->buffer_count;
1315 struct kbase_hwcnt_reader_metadata *meta = &cli->dump_buffers_meta[idx];
1317 /* Metadata sanity check. */
1318 KBASE_DEBUG_ASSERT(idx == meta->buffer_idx);
1320 if (sizeof(struct kbase_hwcnt_reader_metadata) != size)
1323 /* Check if there is any buffer available. */
1324 if (atomic_read(&cli->write_idx) == meta_idx)
1327 /* Check if previously taken buffer was put back. */
1328 if (atomic_read(&cli->read_idx) != meta_idx)
1331 /* Copy next available buffer's metadata to user. */
1332 if (copy_to_user(buffer, meta, size))
1335 atomic_inc(&cli->meta_idx);
1341 * kbasep_vinstr_hwcnt_reader_ioctl_put_buffer - hwcnt reader's ioctl command
1342 * @cli: pointer to vinstr client structure
1343 * @buffer: pointer to userspace buffer
1344 * @size: size of buffer
1346 * Return: zero on success
1348 static long kbasep_vinstr_hwcnt_reader_ioctl_put_buffer(
1349 struct kbase_vinstr_client *cli, void __user *buffer,
1352 unsigned int read_idx = atomic_read(&cli->read_idx);
1353 unsigned int idx = read_idx % cli->buffer_count;
1355 struct kbase_hwcnt_reader_metadata meta;
1357 if (sizeof(struct kbase_hwcnt_reader_metadata) != size)
1360 /* Check if any buffer was taken. */
1361 if (atomic_read(&cli->meta_idx) == read_idx)
1364 /* Check if correct buffer is put back. */
1365 if (copy_from_user(&meta, buffer, size))
1367 if (idx != meta.buffer_idx)
1370 atomic_inc(&cli->read_idx);
1376 * kbasep_vinstr_hwcnt_reader_ioctl_set_interval - hwcnt reader's ioctl command
1377 * @cli: pointer to vinstr client structure
1378 * @interval: periodic dumping interval (disable periodic dumping if zero)
1380 * Return: zero on success
1382 static long kbasep_vinstr_hwcnt_reader_ioctl_set_interval(
1383 struct kbase_vinstr_client *cli, u32 interval)
1385 struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1387 KBASE_DEBUG_ASSERT(vinstr_ctx);
1389 mutex_lock(&vinstr_ctx->lock);
1391 list_del(&cli->list);
1393 cli->dump_interval = interval;
1395 /* If interval is non-zero, enable periodic dumping for this client. */
1396 if (cli->dump_interval) {
1397 if (DUMPING_RESOLUTION > cli->dump_interval)
1398 cli->dump_interval = DUMPING_RESOLUTION;
1400 kbasep_vinstr_get_timestamp() + cli->dump_interval;
1402 kbasep_vinstr_add_dump_request(
1403 cli, &vinstr_ctx->waiting_clients);
1405 atomic_set(&vinstr_ctx->request_pending, 1);
1406 wake_up_all(&vinstr_ctx->waitq);
1408 list_add(&cli->list, &vinstr_ctx->idle_clients);
1411 mutex_unlock(&vinstr_ctx->lock);
1417 * kbasep_vinstr_hwcnt_reader_event_mask - return event mask for event id
1418 * @event_id: id of event
1419 * Return: event_mask or zero if event is not supported or maskable
1421 static u32 kbasep_vinstr_hwcnt_reader_event_mask(
1422 enum base_hwcnt_reader_event event_id)
1427 case BASE_HWCNT_READER_EVENT_PREJOB:
1428 case BASE_HWCNT_READER_EVENT_POSTJOB:
1429 /* These event are maskable. */
1430 event_mask = (1 << event_id);
1433 case BASE_HWCNT_READER_EVENT_MANUAL:
1434 case BASE_HWCNT_READER_EVENT_PERIODIC:
1435 /* These event are non-maskable. */
1437 /* These event are not supported. */
1445 * kbasep_vinstr_hwcnt_reader_ioctl_enable_event - hwcnt reader's ioctl command
1446 * @cli: pointer to vinstr client structure
1447 * @event_id: id of event to enable
1449 * Return: zero on success
1451 static long kbasep_vinstr_hwcnt_reader_ioctl_enable_event(
1452 struct kbase_vinstr_client *cli,
1453 enum base_hwcnt_reader_event event_id)
1455 struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1458 KBASE_DEBUG_ASSERT(vinstr_ctx);
1460 event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id);
1464 mutex_lock(&vinstr_ctx->lock);
1465 cli->event_mask |= event_mask;
1466 mutex_unlock(&vinstr_ctx->lock);
1472 * kbasep_vinstr_hwcnt_reader_ioctl_disable_event - hwcnt reader's ioctl command
1473 * @cli: pointer to vinstr client structure
1474 * @event_id: id of event to disable
1476 * Return: zero on success
1478 static long kbasep_vinstr_hwcnt_reader_ioctl_disable_event(
1479 struct kbase_vinstr_client *cli,
1480 enum base_hwcnt_reader_event event_id)
1482 struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1485 KBASE_DEBUG_ASSERT(vinstr_ctx);
1487 event_mask = kbasep_vinstr_hwcnt_reader_event_mask(event_id);
1491 mutex_lock(&vinstr_ctx->lock);
1492 cli->event_mask &= ~event_mask;
1493 mutex_unlock(&vinstr_ctx->lock);
1499 * kbasep_vinstr_hwcnt_reader_ioctl_get_hwver - hwcnt reader's ioctl command
1500 * @cli: pointer to vinstr client structure
1501 * @hwver: pointer to user buffer where hw version will be stored
1503 * Return: zero on success
1505 static long kbasep_vinstr_hwcnt_reader_ioctl_get_hwver(
1506 struct kbase_vinstr_client *cli, u32 __user *hwver)
1508 #ifndef CONFIG_MALI_NO_MALI
1509 struct kbase_vinstr_context *vinstr_ctx = cli->vinstr_ctx;
1514 #ifndef CONFIG_MALI_NO_MALI
1515 KBASE_DEBUG_ASSERT(vinstr_ctx);
1516 if (kbase_hw_has_feature(vinstr_ctx->kbdev, BASE_HW_FEATURE_V4))
1520 return put_user(ver, hwver);
1524 * kbasep_vinstr_hwcnt_reader_ioctl - hwcnt reader's ioctl
1525 * @filp: pointer to file structure
1526 * @cmd: user command
1527 * @arg: command's argument
1529 * Return: zero on success
1531 static long kbasep_vinstr_hwcnt_reader_ioctl(struct file *filp,
1532 unsigned int cmd, unsigned long arg)
1535 struct kbase_vinstr_client *cli;
1537 KBASE_DEBUG_ASSERT(filp);
1539 cli = filp->private_data;
1540 KBASE_DEBUG_ASSERT(cli);
1542 if (unlikely(KBASE_HWCNT_READER != _IOC_TYPE(cmd)))
1546 case KBASE_HWCNT_READER_GET_API_VERSION:
1547 rcode = put_user(HWCNT_READER_API, (u32 __user *)arg);
1549 case KBASE_HWCNT_READER_GET_HWVER:
1550 rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_hwver(
1551 cli, (u32 __user *)arg);
1553 case KBASE_HWCNT_READER_GET_BUFFER_SIZE:
1554 KBASE_DEBUG_ASSERT(cli->vinstr_ctx);
1556 (u32)cli->vinstr_ctx->dump_size,
1559 case KBASE_HWCNT_READER_DUMP:
1560 rcode = kbase_vinstr_hwc_dump(
1561 cli, BASE_HWCNT_READER_EVENT_MANUAL);
1563 case KBASE_HWCNT_READER_CLEAR:
1564 rcode = kbase_vinstr_hwc_clear(cli);
1566 case KBASE_HWCNT_READER_GET_BUFFER:
1567 rcode = kbasep_vinstr_hwcnt_reader_ioctl_get_buffer(
1568 cli, (void __user *)arg, _IOC_SIZE(cmd));
1570 case KBASE_HWCNT_READER_PUT_BUFFER:
1571 rcode = kbasep_vinstr_hwcnt_reader_ioctl_put_buffer(
1572 cli, (void __user *)arg, _IOC_SIZE(cmd));
1574 case KBASE_HWCNT_READER_SET_INTERVAL:
1575 rcode = kbasep_vinstr_hwcnt_reader_ioctl_set_interval(
1578 case KBASE_HWCNT_READER_ENABLE_EVENT:
1579 rcode = kbasep_vinstr_hwcnt_reader_ioctl_enable_event(
1580 cli, (enum base_hwcnt_reader_event)arg);
1582 case KBASE_HWCNT_READER_DISABLE_EVENT:
1583 rcode = kbasep_vinstr_hwcnt_reader_ioctl_disable_event(
1584 cli, (enum base_hwcnt_reader_event)arg);
1595 * kbasep_vinstr_hwcnt_reader_poll - hwcnt reader's poll
1596 * @filp: pointer to file structure
1597 * @wait: pointer to poll table
1598 * Return: POLLIN if data can be read without blocking, otherwise zero
1600 static unsigned int kbasep_vinstr_hwcnt_reader_poll(struct file *filp,
1603 struct kbase_vinstr_client *cli;
1605 KBASE_DEBUG_ASSERT(filp);
1606 KBASE_DEBUG_ASSERT(wait);
1608 cli = filp->private_data;
1609 KBASE_DEBUG_ASSERT(cli);
1611 poll_wait(filp, &cli->waitq, wait);
1612 if (kbasep_vinstr_hwcnt_reader_buffer_ready(cli))
1618 * kbasep_vinstr_hwcnt_reader_mmap - hwcnt reader's mmap
1619 * @filp: pointer to file structure
1620 * @vma: pointer to vma structure
1621 * Return: zero on success
1623 static int kbasep_vinstr_hwcnt_reader_mmap(struct file *filp,
1624 struct vm_area_struct *vma)
1626 struct kbase_vinstr_client *cli;
1627 unsigned long size, addr, pfn, offset;
1628 unsigned long vm_size = vma->vm_end - vma->vm_start;
1630 KBASE_DEBUG_ASSERT(filp);
1631 KBASE_DEBUG_ASSERT(vma);
1633 cli = filp->private_data;
1634 KBASE_DEBUG_ASSERT(cli);
1636 size = cli->buffer_count * cli->dump_size;
1638 if (vma->vm_pgoff > (size >> PAGE_SHIFT))
1641 offset = vma->vm_pgoff << PAGE_SHIFT;
1642 if (vm_size > size - offset)
1645 addr = __pa((unsigned long)cli->dump_buffers + offset);
1646 pfn = addr >> PAGE_SHIFT;
1648 return remap_pfn_range(
1657 * kbasep_vinstr_hwcnt_reader_release - hwcnt reader's release
1658 * @inode: pointer to inode structure
1659 * @filp: pointer to file structure
1660 * Return always return zero
1662 static int kbasep_vinstr_hwcnt_reader_release(struct inode *inode,
1665 struct kbase_vinstr_client *cli;
1667 KBASE_DEBUG_ASSERT(inode);
1668 KBASE_DEBUG_ASSERT(filp);
1670 cli = filp->private_data;
1671 KBASE_DEBUG_ASSERT(cli);
1673 kbase_vinstr_detach_client(cli);
1677 /*****************************************************************************/
1680 * kbasep_vinstr_kick_scheduler - trigger scheduler cycle
1681 * @kbdev: pointer to kbase device structure
1683 static void kbasep_vinstr_kick_scheduler(struct kbase_device *kbdev)
1685 struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1686 unsigned long flags;
1688 down(&js_devdata->schedule_sem);
1689 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1690 kbase_backend_slot_update(kbdev);
1691 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1692 up(&js_devdata->schedule_sem);
1696 * kbasep_vinstr_suspend_worker - worker suspending vinstr module
1697 * @data: pointer to work structure
1699 static void kbasep_vinstr_suspend_worker(struct work_struct *data)
1701 struct kbase_vinstr_context *vinstr_ctx;
1702 unsigned long flags;
1704 vinstr_ctx = container_of(data, struct kbase_vinstr_context,
1707 mutex_lock(&vinstr_ctx->lock);
1709 if (vinstr_ctx->kctx)
1710 disable_hwcnt(vinstr_ctx);
1712 spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
1713 vinstr_ctx->state = VINSTR_SUSPENDED;
1714 wake_up_all(&vinstr_ctx->suspend_waitq);
1715 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
1717 mutex_unlock(&vinstr_ctx->lock);
1719 /* Kick GPU scheduler to allow entering protected mode.
1720 * This must happen after vinstr was suspended. */
1721 kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev);
1725 * kbasep_vinstr_suspend_worker - worker resuming vinstr module
1726 * @data: pointer to work structure
1728 static void kbasep_vinstr_resume_worker(struct work_struct *data)
1730 struct kbase_vinstr_context *vinstr_ctx;
1731 unsigned long flags;
1733 vinstr_ctx = container_of(data, struct kbase_vinstr_context,
1736 mutex_lock(&vinstr_ctx->lock);
1738 if (vinstr_ctx->kctx)
1739 enable_hwcnt(vinstr_ctx);
1741 spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
1742 vinstr_ctx->state = VINSTR_IDLE;
1743 wake_up_all(&vinstr_ctx->suspend_waitq);
1744 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
1746 mutex_unlock(&vinstr_ctx->lock);
1748 /* Kick GPU scheduler to allow entering protected mode.
1749 * Note that scheduler state machine might requested re-entry to
1750 * protected mode before vinstr was resumed.
1751 * This must happen after vinstr was release. */
1752 kbasep_vinstr_kick_scheduler(vinstr_ctx->kbdev);
1755 /*****************************************************************************/
1757 struct kbase_vinstr_context *kbase_vinstr_init(struct kbase_device *kbdev)
1759 struct kbase_vinstr_context *vinstr_ctx;
1761 vinstr_ctx = kzalloc(sizeof(*vinstr_ctx), GFP_KERNEL);
1765 INIT_LIST_HEAD(&vinstr_ctx->idle_clients);
1766 INIT_LIST_HEAD(&vinstr_ctx->waiting_clients);
1767 mutex_init(&vinstr_ctx->lock);
1768 spin_lock_init(&vinstr_ctx->state_lock);
1769 vinstr_ctx->kbdev = kbdev;
1770 vinstr_ctx->thread = NULL;
1771 vinstr_ctx->state = VINSTR_IDLE;
1772 vinstr_ctx->suspend_cnt = 0;
1773 INIT_WORK(&vinstr_ctx->suspend_work, kbasep_vinstr_suspend_worker);
1774 INIT_WORK(&vinstr_ctx->resume_work, kbasep_vinstr_resume_worker);
1775 init_waitqueue_head(&vinstr_ctx->suspend_waitq);
1777 atomic_set(&vinstr_ctx->request_pending, 0);
1778 init_waitqueue_head(&vinstr_ctx->waitq);
1783 void kbase_vinstr_term(struct kbase_vinstr_context *vinstr_ctx)
1785 struct kbase_vinstr_client *cli;
1787 /* Stop service thread first. */
1788 if (vinstr_ctx->thread)
1789 kthread_stop(vinstr_ctx->thread);
1791 /* Wait for workers. */
1792 flush_work(&vinstr_ctx->suspend_work);
1793 flush_work(&vinstr_ctx->resume_work);
1796 struct list_head *list = &vinstr_ctx->idle_clients;
1798 if (list_empty(list)) {
1799 list = &vinstr_ctx->waiting_clients;
1800 if (list_empty(list))
1804 cli = list_first_entry(list, struct kbase_vinstr_client, list);
1805 list_del(&cli->list);
1806 kfree(cli->accum_buffer);
1808 vinstr_ctx->nclients--;
1810 KBASE_DEBUG_ASSERT(!vinstr_ctx->nclients);
1811 if (vinstr_ctx->kctx)
1812 kbasep_vinstr_destroy_kctx(vinstr_ctx);
1816 int kbase_vinstr_hwcnt_reader_setup(struct kbase_vinstr_context *vinstr_ctx,
1817 struct kbase_uk_hwcnt_reader_setup *setup)
1819 struct kbase_vinstr_client *cli;
1822 KBASE_DEBUG_ASSERT(vinstr_ctx);
1823 KBASE_DEBUG_ASSERT(setup);
1824 KBASE_DEBUG_ASSERT(setup->buffer_count);
1826 bitmap[SHADER_HWCNT_BM] = setup->shader_bm;
1827 bitmap[TILER_HWCNT_BM] = setup->tiler_bm;
1828 bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm;
1829 bitmap[JM_HWCNT_BM] = setup->jm_bm;
1831 cli = kbasep_vinstr_attach_client(
1833 setup->buffer_count,
1844 int kbase_vinstr_legacy_hwc_setup(
1845 struct kbase_vinstr_context *vinstr_ctx,
1846 struct kbase_vinstr_client **cli,
1847 struct kbase_uk_hwcnt_setup *setup)
1849 KBASE_DEBUG_ASSERT(vinstr_ctx);
1850 KBASE_DEBUG_ASSERT(setup);
1851 KBASE_DEBUG_ASSERT(cli);
1853 if (setup->dump_buffer) {
1856 bitmap[SHADER_HWCNT_BM] = setup->shader_bm;
1857 bitmap[TILER_HWCNT_BM] = setup->tiler_bm;
1858 bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm;
1859 bitmap[JM_HWCNT_BM] = setup->jm_bm;
1864 *cli = kbasep_vinstr_attach_client(
1868 (void *)(long)setup->dump_buffer,
1877 kbase_vinstr_detach_client(*cli);
1884 struct kbase_vinstr_client *kbase_vinstr_hwcnt_kernel_setup(
1885 struct kbase_vinstr_context *vinstr_ctx,
1886 struct kbase_uk_hwcnt_reader_setup *setup,
1887 void *kernel_buffer)
1891 if (!vinstr_ctx || !setup || !kernel_buffer)
1894 bitmap[SHADER_HWCNT_BM] = setup->shader_bm;
1895 bitmap[TILER_HWCNT_BM] = setup->tiler_bm;
1896 bitmap[MMU_L2_HWCNT_BM] = setup->mmu_l2_bm;
1897 bitmap[JM_HWCNT_BM] = setup->jm_bm;
1899 return kbasep_vinstr_attach_client(
1906 KBASE_EXPORT_TEST_API(kbase_vinstr_hwcnt_kernel_setup);
1908 int kbase_vinstr_hwc_dump(struct kbase_vinstr_client *cli,
1909 enum base_hwcnt_reader_event event_id)
1912 struct kbase_vinstr_context *vinstr_ctx;
1919 vinstr_ctx = cli->vinstr_ctx;
1920 KBASE_DEBUG_ASSERT(vinstr_ctx);
1922 KBASE_DEBUG_ASSERT(event_id < BASE_HWCNT_READER_EVENT_COUNT);
1923 event_mask = 1 << event_id;
1925 mutex_lock(&vinstr_ctx->lock);
1927 if (event_mask & cli->event_mask) {
1928 rcode = kbasep_vinstr_collect_and_accumulate(
1934 rcode = kbasep_vinstr_update_client(cli, timestamp, event_id);
1938 kbasep_vinstr_reprogram(vinstr_ctx);
1942 mutex_unlock(&vinstr_ctx->lock);
1946 KBASE_EXPORT_TEST_API(kbase_vinstr_hwc_dump);
1948 int kbase_vinstr_hwc_clear(struct kbase_vinstr_client *cli)
1950 struct kbase_vinstr_context *vinstr_ctx;
1957 vinstr_ctx = cli->vinstr_ctx;
1958 KBASE_DEBUG_ASSERT(vinstr_ctx);
1960 mutex_lock(&vinstr_ctx->lock);
1962 rcode = kbasep_vinstr_collect_and_accumulate(vinstr_ctx, &unused);
1965 rcode = kbase_instr_hwcnt_clear(vinstr_ctx->kctx);
1968 memset(cli->accum_buffer, 0, cli->dump_size);
1970 kbasep_vinstr_reprogram(vinstr_ctx);
1973 mutex_unlock(&vinstr_ctx->lock);
1978 int kbase_vinstr_try_suspend(struct kbase_vinstr_context *vinstr_ctx)
1980 unsigned long flags;
1983 KBASE_DEBUG_ASSERT(vinstr_ctx);
1985 spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
1986 switch (vinstr_ctx->state) {
1987 case VINSTR_SUSPENDED:
1988 vinstr_ctx->suspend_cnt++;
1989 /* overflow shall not happen */
1990 BUG_ON(0 == vinstr_ctx->suspend_cnt);
1995 vinstr_ctx->state = VINSTR_SUSPENDING;
1996 schedule_work(&vinstr_ctx->suspend_work);
1999 case VINSTR_DUMPING:
2000 vinstr_ctx->state = VINSTR_SUSPENDING;
2003 case VINSTR_SUSPENDING:
2005 case VINSTR_RESUMING:
2012 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);
2017 void kbase_vinstr_suspend(struct kbase_vinstr_context *vinstr_ctx)
2019 wait_event(vinstr_ctx->suspend_waitq,
2020 (0 == kbase_vinstr_try_suspend(vinstr_ctx)));
2023 void kbase_vinstr_resume(struct kbase_vinstr_context *vinstr_ctx)
2025 unsigned long flags;
2027 KBASE_DEBUG_ASSERT(vinstr_ctx);
2029 spin_lock_irqsave(&vinstr_ctx->state_lock, flags);
2030 BUG_ON(VINSTR_SUSPENDING == vinstr_ctx->state);
2031 if (VINSTR_SUSPENDED == vinstr_ctx->state) {
2032 BUG_ON(0 == vinstr_ctx->suspend_cnt);
2033 vinstr_ctx->suspend_cnt--;
2034 if (0 == vinstr_ctx->suspend_cnt) {
2035 vinstr_ctx->state = VINSTR_RESUMING;
2036 schedule_work(&vinstr_ctx->resume_work);
2039 spin_unlock_irqrestore(&vinstr_ctx->state_lock, flags);