2 * Copyright (C) ARM Limited 2012-2014. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
12 #include <linux/module.h>
13 #include <linux/time.h>
14 #include <linux/math64.h>
15 #include <linux/slab.h>
18 /* Mali T6xx DDK includes */
19 #ifdef MALI_DIR_MIDGARD
20 /* New DDK Directory structure with kernel/drivers/gpu/arm/midgard*/
21 #include "mali_linux_trace.h"
22 #include "mali_kbase.h"
23 #include "mali_kbase_mem_linux.h"
25 /* Old DDK Directory structure with kernel/drivers/gpu/arm/t6xx*/
26 #include "linux/mali_linux_trace.h"
27 #include "kbase/src/common/mali_kbase.h"
28 #include "kbase/src/linux/mali_kbase_mem_linux.h"
31 #include "gator_events_mali_common.h"
33 /* If API version is not specified then assume API version 1. */
34 #ifndef MALI_DDK_GATOR_API_VERSION
35 #define MALI_DDK_GATOR_API_VERSION 1
38 #if (MALI_DDK_GATOR_API_VERSION != 1) && (MALI_DDK_GATOR_API_VERSION != 2)
39 #error MALI_DDK_GATOR_API_VERSION is invalid (must be 1 for r1/r2 DDK, or 2 for r3 DDK).
45 typedef struct kbase_device *kbase_find_device_type(int);
46 typedef kbase_context *kbase_create_context_type(kbase_device *);
47 typedef void kbase_destroy_context_type(kbase_context *);
49 #if MALI_DDK_GATOR_API_VERSION == 1
50 typedef void *kbase_va_alloc_type(kbase_context *, u32);
51 typedef void kbase_va_free_type(kbase_context *, void *);
52 #elif MALI_DDK_GATOR_API_VERSION == 2
53 typedef void *kbase_va_alloc_type(kbase_context *, u32, kbase_hwc_dma_mapping * handle);
54 typedef void kbase_va_free_type(kbase_context *, kbase_hwc_dma_mapping * handle);
57 typedef mali_error kbase_instr_hwcnt_enable_type(kbase_context *, kbase_uk_hwcnt_setup *);
58 typedef mali_error kbase_instr_hwcnt_disable_type(kbase_context *);
59 typedef mali_error kbase_instr_hwcnt_clear_type(kbase_context *);
60 typedef mali_error kbase_instr_hwcnt_dump_irq_type(kbase_context *);
61 typedef mali_bool kbase_instr_hwcnt_dump_complete_type(kbase_context *, mali_bool *);
63 static kbase_find_device_type *kbase_find_device_symbol;
64 static kbase_create_context_type *kbase_create_context_symbol;
65 static kbase_va_alloc_type *kbase_va_alloc_symbol;
66 static kbase_instr_hwcnt_enable_type *kbase_instr_hwcnt_enable_symbol;
67 static kbase_instr_hwcnt_clear_type *kbase_instr_hwcnt_clear_symbol;
68 static kbase_instr_hwcnt_dump_irq_type *kbase_instr_hwcnt_dump_irq_symbol;
69 static kbase_instr_hwcnt_dump_complete_type *kbase_instr_hwcnt_dump_complete_symbol;
70 static kbase_instr_hwcnt_disable_type *kbase_instr_hwcnt_disable_symbol;
71 static kbase_va_free_type *kbase_va_free_symbol;
72 static kbase_destroy_context_type *kbase_destroy_context_symbol;
74 static long shader_present_low = 0;
76 /** The interval between reads, in ns.
78 * Earlier we introduced
79 * a 'hold off for 1ms after last read' to resolve MIDBASE-2178 and MALINE-724.
80 * However, the 1ms hold off is too long if no context switches occur as there is a race
81 * between this value and the tick of the read clock in gator which is also 1ms. If we 'miss' the
82 * current read, the counter values are effectively 'spread' over 2ms and the values seen are half
83 * what they should be (since Streamline averages over sample time). In the presence of context switches
84 * this spread can vary and markedly affect the counters. Currently there is no 'proper' solution to
85 * this, but empirically we have found that reducing the minimum read interval to 950us causes the
86 * counts to be much more stable.
88 static const int READ_INTERVAL_NSEC = 950000;
91 #include "gator_events_mali_t6xx_hw_test.c"
94 /* Blocks for HW counters */
102 /* Counters for Mali-T6xx:
104 * - HW counters, 4 blocks
105 * For HW counters we need strings to create /dev/gator/events files.
106 * Enums are not needed because the position of the HW name in the array is the same
107 * of the corresponding value in the received block of memory.
108 * HW counters are requested by calculating a bitmask, passed then to the driver.
109 * Every millisecond a HW counters dump is requested, and if the previous has been completed they are read.
112 /* Hardware Counters */
113 static const char *const hardware_counter_names[] = {
121 "GPU_ACTIVE", /* 6 */
250 "SHADER_CORE_ACTIVE",
253 "FRAG_PRIMATIVES_DROPPED",
257 "FRAG_CYCLES_TRISETUP",
260 "FRAG_DUMMY_THREADS",
262 "FRAG_QUADS_EZS_TEST",
263 "FRAG_QUADS_EZS_KILLED",
264 "FRAG_QUADS_LZS_TEST",
265 "FRAG_QUADS_LZS_KILLED",
266 "FRAG_CYCLE_NO_TILE",
272 "COMPUTE_CYCLES_DESC",
283 "LS_REISSUE_ATTRIB_MISS",
306 "AXI_TLB_TRANSACTION",
365 "L2_EXT_WRITE_SMALL",
369 "L2_EXT_RD_BUF_FULL",
379 #define NUMBER_OF_HARDWARE_COUNTERS (sizeof(hardware_counter_names) / sizeof(hardware_counter_names[0]))
381 #define GET_HW_BLOCK(c) (((c) >> 6) & 0x3)
382 #define GET_COUNTER_OFFSET(c) ((c) & 0x3f)
384 /* Memory to dump hardware counters into */
385 static void *kernel_dump_buffer;
387 #if MALI_DDK_GATOR_API_VERSION == 2
388 /* DMA state used to manage lifetime of the buffer */
389 kbase_hwc_dma_mapping kernel_dump_buffer_handle;
392 /* kbase context and device */
393 static kbase_context *kbcontext = NULL;
394 static struct kbase_device *kbdevice = NULL;
397 * The following function has no external prototype in older DDK revisions. When the DDK
398 * is updated then this should be removed.
400 struct kbase_device *kbase_find_device(int minor);
402 static volatile bool kbase_device_busy = false;
403 static unsigned int num_hardware_counters_enabled;
406 * gatorfs variables for counter enable state
408 static mali_counter counters[NUMBER_OF_HARDWARE_COUNTERS];
410 /* An array used to return the data we recorded
411 * as key,value pairs hence the *2
413 static unsigned long counter_dump[NUMBER_OF_HARDWARE_COUNTERS * 2];
415 #define SYMBOL_GET(FUNCTION, ERROR_COUNT) \
416 if(FUNCTION ## _symbol) \
418 printk("gator: mali " #FUNCTION " symbol was already registered\n"); \
423 FUNCTION ## _symbol = symbol_get(FUNCTION); \
424 if(! FUNCTION ## _symbol) \
426 printk("gator: mali online " #FUNCTION " symbol not found\n"); \
431 #define SYMBOL_CLEANUP(FUNCTION) \
432 if(FUNCTION ## _symbol) \
434 symbol_put(FUNCTION); \
435 FUNCTION ## _symbol = NULL; \
439 * Execute symbol_get for all the Mali symbols and check for success.
440 * @return the number of symbols not loaded.
442 static int init_symbols(void)
445 SYMBOL_GET(kbase_find_device, error_count);
446 SYMBOL_GET(kbase_create_context, error_count);
447 SYMBOL_GET(kbase_va_alloc, error_count);
448 SYMBOL_GET(kbase_instr_hwcnt_enable, error_count);
449 SYMBOL_GET(kbase_instr_hwcnt_clear, error_count);
450 SYMBOL_GET(kbase_instr_hwcnt_dump_irq, error_count);
451 SYMBOL_GET(kbase_instr_hwcnt_dump_complete, error_count);
452 SYMBOL_GET(kbase_instr_hwcnt_disable, error_count);
453 SYMBOL_GET(kbase_va_free, error_count);
454 SYMBOL_GET(kbase_destroy_context, error_count);
460 * Execute symbol_put for all the registered Mali symbols.
462 static void clean_symbols(void)
464 SYMBOL_CLEANUP(kbase_find_device);
465 SYMBOL_CLEANUP(kbase_create_context);
466 SYMBOL_CLEANUP(kbase_va_alloc);
467 SYMBOL_CLEANUP(kbase_instr_hwcnt_enable);
468 SYMBOL_CLEANUP(kbase_instr_hwcnt_clear);
469 SYMBOL_CLEANUP(kbase_instr_hwcnt_dump_irq);
470 SYMBOL_CLEANUP(kbase_instr_hwcnt_dump_complete);
471 SYMBOL_CLEANUP(kbase_instr_hwcnt_disable);
472 SYMBOL_CLEANUP(kbase_va_free);
473 SYMBOL_CLEANUP(kbase_destroy_context);
477 * Determines whether a read should take place
478 * @param current_time The current time, obtained from getnstimeofday()
479 * @param prev_time_s The number of seconds at the previous read attempt.
480 * @param next_read_time_ns The time (in ns) when the next read should be allowed.
482 * Note that this function has been separated out here to allow it to be tested.
484 static int is_read_scheduled(const struct timespec *current_time, u32 *prev_time_s, s32 *next_read_time_ns)
486 /* If the current ns count rolls over a second, roll the next read time too. */
487 if (current_time->tv_sec != *prev_time_s) {
488 *next_read_time_ns = *next_read_time_ns - NSEC_PER_SEC;
491 /* Abort the read if the next read time has not arrived. */
492 if (current_time->tv_nsec < *next_read_time_ns) {
496 /* Set the next read some fixed time after this one, and update the read timestamp. */
497 *next_read_time_ns = current_time->tv_nsec + READ_INTERVAL_NSEC;
499 *prev_time_s = current_time->tv_sec;
503 static int start(void)
505 kbase_uk_hwcnt_setup setup;
508 u16 bitmask[] = { 0, 0, 0, 0 };
509 unsigned long long shadersPresent = 0;
511 /* Setup HW counters */
512 num_hardware_counters_enabled = 0;
514 if (NUMBER_OF_HARDWARE_COUNTERS != 256) {
515 pr_debug("Unexpected number of hardware counters defined: expecting 256, got %d\n", NUMBER_OF_HARDWARE_COUNTERS);
518 /* Calculate enable bitmasks based on counters_enabled array */
519 for (cnt = 0; cnt < NUMBER_OF_HARDWARE_COUNTERS; cnt++) {
520 const mali_counter *counter = &counters[cnt];
521 if (counter->enabled) {
522 int block = GET_HW_BLOCK(cnt);
523 int enable_bit = GET_COUNTER_OFFSET(cnt) / 4;
524 bitmask[block] |= (1 << enable_bit);
525 pr_debug("gator: Mali-T6xx: hardware counter %s selected [%d]\n", hardware_counter_names[cnt], cnt);
526 num_hardware_counters_enabled++;
530 /* Create a kbase context for HW counters */
531 if (num_hardware_counters_enabled > 0) {
532 if (init_symbols() > 0) {
534 /* No Mali driver code entrypoints found - not a fault. */
538 kbdevice = kbase_find_device_symbol(-1);
540 /* If we already got a context, fail */
542 pr_debug("gator: Mali-T6xx: error context already present\n");
546 /* kbcontext will only be valid after all the Mali symbols are loaded successfully */
547 kbcontext = kbase_create_context_symbol(kbdevice);
549 pr_debug("gator: Mali-T6xx: error creating kbase context\n");
554 /* See if we can get the number of shader cores */
555 shadersPresent = kbdevice->shader_present_bitmap;
556 shader_present_low = (unsigned long)shadersPresent;
559 * The amount of memory needed to store the dump (bytes)
560 * DUMP_SIZE = number of core groups
561 * * number of blocks (always 8 for midgard)
562 * * number of counters per block (always 64 for midgard)
563 * * number of bytes per counter (always 4 in midgard)
564 * For a Mali-T6xx with a single core group = 1 * 8 * 64 * 4 = 2048
565 * For a Mali-T6xx with a dual core group = 2 * 8 * 64 * 4 = 4096
567 #if MALI_DDK_GATOR_API_VERSION == 1
568 kernel_dump_buffer = kbase_va_alloc_symbol(kbcontext, 4096);
569 #elif MALI_DDK_GATOR_API_VERSION == 2
570 kernel_dump_buffer = kbase_va_alloc_symbol(kbcontext, 4096, &kernel_dump_buffer_handle);
572 if (!kernel_dump_buffer) {
573 pr_debug("gator: Mali-T6xx: error trying to allocate va\n");
574 goto destroy_context;
577 setup.dump_buffer = (uintptr_t)kernel_dump_buffer;
578 setup.jm_bm = bitmask[JM_BLOCK];
579 setup.tiler_bm = bitmask[TILER_BLOCK];
580 setup.shader_bm = bitmask[SHADER_BLOCK];
581 setup.mmu_l2_bm = bitmask[MMU_BLOCK];
582 /* These counters do not exist on Mali-T60x */
583 setup.l3_cache_bm = 0;
585 /* Use kbase API to enable hardware counters and provide dump buffer */
586 err = kbase_instr_hwcnt_enable_symbol(kbcontext, &setup);
587 if (err != MALI_ERROR_NONE) {
588 pr_debug("gator: Mali-T6xx: can't setup hardware counters\n");
591 pr_debug("gator: Mali-T6xx: hardware counters enabled\n");
592 kbase_instr_hwcnt_clear_symbol(kbcontext);
593 pr_debug("gator: Mali-T6xx: hardware counters cleared \n");
595 kbase_device_busy = false;
601 #if MALI_DDK_GATOR_API_VERSION == 1
602 kbase_va_free_symbol(kbcontext, kernel_dump_buffer);
603 #elif MALI_DDK_GATOR_API_VERSION == 2
604 kbase_va_free_symbol(kbcontext, &kernel_dump_buffer_handle);
608 kbase_destroy_context_symbol(kbcontext);
615 static void stop(void)
618 kbase_context *temp_kbcontext;
620 pr_debug("gator: Mali-T6xx: stop\n");
622 /* Set all counters as disabled */
623 for (cnt = 0; cnt < NUMBER_OF_HARDWARE_COUNTERS; cnt++) {
624 counters[cnt].enabled = 0;
627 /* Destroy the context for HW counters */
628 if (num_hardware_counters_enabled > 0 && kbcontext != NULL) {
630 * Set the global variable to NULL before destroying it, because
631 * other function will check this before using it.
633 temp_kbcontext = kbcontext;
636 kbase_instr_hwcnt_disable_symbol(temp_kbcontext);
638 #if MALI_DDK_GATOR_API_VERSION == 1
639 kbase_va_free_symbol(temp_kbcontext, kernel_dump_buffer);
640 #elif MALI_DDK_GATOR_API_VERSION == 2
641 kbase_va_free_symbol(temp_kbcontext, &kernel_dump_buffer_handle);
644 kbase_destroy_context_symbol(temp_kbcontext);
646 pr_debug("gator: Mali-T6xx: hardware counters stopped\n");
652 static int read(int **buffer)
659 struct timespec current_time;
660 static u32 prev_time_s = 0;
661 static s32 next_read_time_ns = 0;
663 if (!on_primary_core()) {
667 getnstimeofday(¤t_time);
670 * Discard reads unless a respectable time has passed. This reduces the load on the GPU without sacrificing
671 * accuracy on the Streamline display.
673 if (!is_read_scheduled(¤t_time, &prev_time_s, &next_read_time_ns)) {
678 * Report the HW counters
679 * Only process hardware counters if at least one of the hardware counters is enabled.
681 if (num_hardware_counters_enabled > 0) {
682 const unsigned int vithar_blocks[] = {
683 0x700, /* VITHAR_JOB_MANAGER, Block 0 */
684 0x400, /* VITHAR_TILER, Block 1 */
685 0x000, /* VITHAR_SHADER_CORE, Block 2 */
686 0x500 /* VITHAR_MEMORY_SYSTEM, Block 3 */
693 /* Mali symbols can be called safely since a kbcontext is valid */
694 if (kbase_instr_hwcnt_dump_complete_symbol(kbcontext, &success) == MALI_TRUE) {
695 kbase_device_busy = false;
697 if (success == MALI_TRUE) {
698 /* Cycle through hardware counters and accumulate totals */
699 for (cnt = 0; cnt < NUMBER_OF_HARDWARE_COUNTERS; cnt++) {
700 const mali_counter *counter = &counters[cnt];
701 if (counter->enabled) {
702 const int block = GET_HW_BLOCK(cnt);
703 const int counter_offset = GET_COUNTER_OFFSET(cnt);
705 const char* block_base_address = (char*)kernel_dump_buffer + vithar_blocks[block];
707 /* If counter belongs to shader block need to take into account all cores */
708 if (block == SHADER_BLOCK) {
710 int shader_core_count = 0;
713 for (i = 0; i < 4; i++) {
714 if ((shader_present_low >> i) & 1) {
715 value += *((u32*) (block_base_address + (0x100 * i)) + counter_offset);
720 for (i = 0; i < 4; i++) {
721 if((shader_present_low >> (i+4)) & 1) {
722 value += *((u32*)(block_base_address + (0x100 * i) + 0x800) + counter_offset);
727 /* Need to total by number of cores to produce an average */
728 if (shader_core_count != 0) {
729 value /= shader_core_count;
732 value = *((u32*)block_base_address + counter_offset);
735 counter_dump[len++] = counter->key;
736 counter_dump[len++] = value;
742 if (!kbase_device_busy) {
743 kbase_device_busy = true;
744 kbase_instr_hwcnt_dump_irq_symbol(kbcontext);
748 /* Update the buffer */
750 *buffer = (int *)counter_dump;
756 static int create_files(struct super_block *sb, struct dentry *root)
760 * Create the filesystem for all events
762 int counter_index = 0;
763 const char *mali_name = gator_mali_get_mali_name();
765 for (event = 0; event < NUMBER_OF_HARDWARE_COUNTERS; event++) {
766 if (gator_mali_create_file_system(mali_name, hardware_counter_names[counter_index], sb, root, &counters[event], NULL) != 0)
774 static struct gator_interface gator_events_mali_t6xx_interface = {
775 .create_files = create_files,
781 int gator_events_mali_t6xx_hw_init(void)
783 pr_debug("gator: Mali-T6xx: sw_counters init\n");
786 test_all_is_read_scheduled();
789 gator_mali_initialise_counters(counters, NUMBER_OF_HARDWARE_COUNTERS);
791 return gator_events_install(&gator_events_mali_t6xx_interface);