2 * Copyright (C) ARM Limited 2012-2013. All rights reserved.
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
12 #include <linux/module.h>
13 #include <linux/time.h>
14 #include <linux/math64.h>
15 #include <linux/slab.h>
18 /* Mali T6xx DDK includes */
19 #include "linux/mali_linux_trace.h"
20 #include "kbase/src/common/mali_kbase.h"
21 #include "kbase/src/linux/mali_kbase_mem_linux.h"
23 #include "gator_events_mali_common.h"
25 /* If API version is not specified then assume API version 1. */
26 #ifndef MALI_DDK_GATOR_API_VERSION
27 #define MALI_DDK_GATOR_API_VERSION 1
30 #if (MALI_DDK_GATOR_API_VERSION != 1) && (MALI_DDK_GATOR_API_VERSION != 2)
31 #error MALI_DDK_GATOR_API_VERSION is invalid (must be 1 for r1/r2 DDK, or 2 for r3 DDK).
37 typedef struct kbase_device *kbase_find_device_type(int);
38 typedef kbase_context *kbase_create_context_type(kbase_device *);
39 typedef void kbase_destroy_context_type(kbase_context *);
41 #if MALI_DDK_GATOR_API_VERSION == 1
42 typedef void *kbase_va_alloc_type(kbase_context *, u32);
43 typedef void kbase_va_free_type(kbase_context *, void *);
44 #elif MALI_DDK_GATOR_API_VERSION == 2
45 typedef void *kbase_va_alloc_type(kbase_context *, u32, kbase_hwc_dma_mapping * handle);
46 typedef void kbase_va_free_type(kbase_context *, kbase_hwc_dma_mapping * handle);
49 typedef mali_error kbase_instr_hwcnt_enable_type(kbase_context *, kbase_uk_hwcnt_setup *);
50 typedef mali_error kbase_instr_hwcnt_disable_type(kbase_context *);
51 typedef mali_error kbase_instr_hwcnt_clear_type(kbase_context *);
52 typedef mali_error kbase_instr_hwcnt_dump_irq_type(kbase_context *);
53 typedef mali_bool kbase_instr_hwcnt_dump_complete_type(kbase_context *, mali_bool *);
55 static kbase_find_device_type *kbase_find_device_symbol;
56 static kbase_create_context_type *kbase_create_context_symbol;
57 static kbase_va_alloc_type *kbase_va_alloc_symbol;
58 static kbase_instr_hwcnt_enable_type *kbase_instr_hwcnt_enable_symbol;
59 static kbase_instr_hwcnt_clear_type *kbase_instr_hwcnt_clear_symbol;
60 static kbase_instr_hwcnt_dump_irq_type *kbase_instr_hwcnt_dump_irq_symbol;
61 static kbase_instr_hwcnt_dump_complete_type *kbase_instr_hwcnt_dump_complete_symbol;
62 static kbase_instr_hwcnt_disable_type *kbase_instr_hwcnt_disable_symbol;
63 static kbase_va_free_type *kbase_va_free_symbol;
64 static kbase_destroy_context_type *kbase_destroy_context_symbol;
66 /** The interval between reads, in ns.
68 * Earlier we introduced
69 * a 'hold off for 1ms after last read' to resolve MIDBASE-2178 and MALINE-724.
70 * However, the 1ms hold off is too long if no context switches occur as there is a race
71 * between this value and the tick of the read clock in gator which is also 1ms. If we 'miss' the
72 * current read, the counter values are effectively 'spread' over 2ms and the values seen are half
73 * what they should be (since Streamline averages over sample time). In the presence of context switches
74 * this spread can vary and markedly affect the counters. Currently there is no 'proper' solution to
75 * this, but empirically we have found that reducing the minimum read interval to 950us causes the
76 * counts to be much more stable.
78 static const int READ_INTERVAL_NSEC = 950000;
81 #include "gator_events_mali_t6xx_hw_test.c"
84 /* Blocks for HW counters */
92 /* Counters for Mali-T6xx:
94 * - HW counters, 4 blocks
95 * For HW counters we need strings to create /dev/gator/events files.
96 * Enums are not needed because the position of the HW name in the array is the same
97 * of the corresponding value in the received block of memory.
98 * HW counters are requested by calculating a bitmask, passed then to the driver.
99 * Every millisecond a HW counters dump is requested, and if the previous has been completed they are read.
102 /* Hardware Counters */
103 static const char *const hardware_counter_names[] = {
111 "GPU_ACTIVE", /* 6 */
240 "SHADER_CORE_ACTIVE",
243 "FRAG_PRIMATIVES_DROPPED",
247 "FRAG_CYCLES_TRISETUP",
250 "FRAG_DUMMY_THREADS",
252 "FRAG_QUADS_EZS_TEST",
253 "FRAG_QUADS_EZS_KILLED",
254 "FRAG_QUADS_LZS_TEST",
255 "FRAG_QUADS_LZS_KILLED",
256 "FRAG_CYCLE_NO_TILE",
262 "COMPUTE_CYCLES_DESC",
273 "LS_REISSUE_ATTRIB_MISS",
296 "AXI_TLB_TRANSACTION",
355 "L2_EXT_WRITE_SMALL",
359 "L2_EXT_RD_BUF_FULL",
369 #define NUMBER_OF_HARDWARE_COUNTERS (sizeof(hardware_counter_names) / sizeof(hardware_counter_names[0]))
371 #define GET_HW_BLOCK(c) (((c) >> 6) & 0x3)
372 #define GET_COUNTER_OFFSET(c) ((c) & 0x3f)
374 /* Memory to dump hardware counters into */
375 static void *kernel_dump_buffer;
377 #if MALI_DDK_GATOR_API_VERSION == 2
378 /* DMA state used to manage lifetime of the buffer */
379 kbase_hwc_dma_mapping kernel_dump_buffer_handle;
382 /* kbase context and device */
383 static kbase_context *kbcontext = NULL;
384 static struct kbase_device *kbdevice = NULL;
387 * The following function has no external prototype in older DDK revisions. When the DDK
388 * is updated then this should be removed.
390 struct kbase_device *kbase_find_device(int minor);
392 static volatile bool kbase_device_busy = false;
393 static unsigned int num_hardware_counters_enabled;
396 * gatorfs variables for counter enable state
398 static mali_counter counters[NUMBER_OF_HARDWARE_COUNTERS];
400 /* An array used to return the data we recorded
401 * as key,value pairs hence the *2
403 static unsigned long counter_dump[NUMBER_OF_HARDWARE_COUNTERS * 2];
405 #define SYMBOL_GET(FUNCTION, ERROR_COUNT) \
406 if(FUNCTION ## _symbol) \
408 printk("gator: mali " #FUNCTION " symbol was already registered\n"); \
413 FUNCTION ## _symbol = symbol_get(FUNCTION); \
414 if(! FUNCTION ## _symbol) \
416 printk("gator: mali online " #FUNCTION " symbol not found\n"); \
421 #define SYMBOL_CLEANUP(FUNCTION) \
422 if(FUNCTION ## _symbol) \
424 symbol_put(FUNCTION); \
425 FUNCTION ## _symbol = NULL; \
429 * Execute symbol_get for all the Mali symbols and check for success.
430 * @return the number of symbols not loaded.
432 static int init_symbols(void)
435 SYMBOL_GET(kbase_find_device, error_count);
436 SYMBOL_GET(kbase_create_context, error_count);
437 SYMBOL_GET(kbase_va_alloc, error_count);
438 SYMBOL_GET(kbase_instr_hwcnt_enable, error_count);
439 SYMBOL_GET(kbase_instr_hwcnt_clear, error_count);
440 SYMBOL_GET(kbase_instr_hwcnt_dump_irq, error_count);
441 SYMBOL_GET(kbase_instr_hwcnt_dump_complete, error_count);
442 SYMBOL_GET(kbase_instr_hwcnt_disable, error_count);
443 SYMBOL_GET(kbase_va_free, error_count);
444 SYMBOL_GET(kbase_destroy_context, error_count);
450 * Execute symbol_put for all the registered Mali symbols.
452 static void clean_symbols(void)
454 SYMBOL_CLEANUP(kbase_find_device);
455 SYMBOL_CLEANUP(kbase_create_context);
456 SYMBOL_CLEANUP(kbase_va_alloc);
457 SYMBOL_CLEANUP(kbase_instr_hwcnt_enable);
458 SYMBOL_CLEANUP(kbase_instr_hwcnt_clear);
459 SYMBOL_CLEANUP(kbase_instr_hwcnt_dump_irq);
460 SYMBOL_CLEANUP(kbase_instr_hwcnt_dump_complete);
461 SYMBOL_CLEANUP(kbase_instr_hwcnt_disable);
462 SYMBOL_CLEANUP(kbase_va_free);
463 SYMBOL_CLEANUP(kbase_destroy_context);
467 * Determines whether a read should take place
468 * @param current_time The current time, obtained from getnstimeofday()
469 * @param prev_time_s The number of seconds at the previous read attempt.
470 * @param next_read_time_ns The time (in ns) when the next read should be allowed.
472 * Note that this function has been separated out here to allow it to be tested.
474 static int is_read_scheduled(const struct timespec *current_time, u32 *prev_time_s, s32 *next_read_time_ns)
476 /* If the current ns count rolls over a second, roll the next read time too. */
477 if (current_time->tv_sec != *prev_time_s) {
478 *next_read_time_ns = *next_read_time_ns - NSEC_PER_SEC;
481 /* Abort the read if the next read time has not arrived. */
482 if (current_time->tv_nsec < *next_read_time_ns) {
486 /* Set the next read some fixed time after this one, and update the read timestamp. */
487 *next_read_time_ns = current_time->tv_nsec + READ_INTERVAL_NSEC;
489 *prev_time_s = current_time->tv_sec;
493 static int start(void)
495 kbase_uk_hwcnt_setup setup;
498 u16 bitmask[] = { 0, 0, 0, 0 };
500 /* Setup HW counters */
501 num_hardware_counters_enabled = 0;
503 if (NUMBER_OF_HARDWARE_COUNTERS != 256) {
504 pr_debug("Unexpected number of hardware counters defined: expecting 256, got %d\n", NUMBER_OF_HARDWARE_COUNTERS);
507 /* Calculate enable bitmasks based on counters_enabled array */
508 for (cnt = 0; cnt < NUMBER_OF_HARDWARE_COUNTERS; cnt++) {
509 const mali_counter *counter = &counters[cnt];
510 if (counter->enabled) {
511 int block = GET_HW_BLOCK(cnt);
512 int enable_bit = GET_COUNTER_OFFSET(cnt) / 4;
513 bitmask[block] |= (1 << enable_bit);
514 pr_debug("gator: Mali-T6xx: hardware counter %s selected [%d]\n", hardware_counter_names[cnt], cnt);
515 num_hardware_counters_enabled++;
519 /* Create a kbase context for HW counters */
520 if (num_hardware_counters_enabled > 0) {
521 if (init_symbols() > 0) {
523 /* No Mali driver code entrypoints found - not a fault. */
527 kbdevice = kbase_find_device_symbol(-1);
529 /* If we already got a context, fail */
531 pr_debug("gator: Mali-T6xx: error context already present\n");
535 /* kbcontext will only be valid after all the Mali symbols are loaded successfully */
536 kbcontext = kbase_create_context_symbol(kbdevice);
538 pr_debug("gator: Mali-T6xx: error creating kbase context\n");
543 * The amount of memory needed to store the dump (bytes)
544 * DUMP_SIZE = number of core groups
545 * * number of blocks (always 8 for midgard)
546 * * number of counters per block (always 64 for midgard)
547 * * number of bytes per counter (always 4 in midgard)
548 * For a Mali-T6xx with a single core group = 1 * 8 * 64 * 4 = 2048
549 * For a Mali-T6xx with a dual core group = 2 * 8 * 64 * 4 = 4096
551 #if MALI_DDK_GATOR_API_VERSION == 1
552 kernel_dump_buffer = kbase_va_alloc_symbol(kbcontext, 4096);
553 #elif MALI_DDK_GATOR_API_VERSION == 2
554 kernel_dump_buffer = kbase_va_alloc_symbol(kbcontext, 4096, &kernel_dump_buffer_handle);
556 if (!kernel_dump_buffer) {
557 pr_debug("gator: Mali-T6xx: error trying to allocate va\n");
558 goto destroy_context;
561 setup.dump_buffer = (uintptr_t)kernel_dump_buffer;
562 setup.jm_bm = bitmask[JM_BLOCK];
563 setup.tiler_bm = bitmask[TILER_BLOCK];
564 setup.shader_bm = bitmask[SHADER_BLOCK];
565 setup.mmu_l2_bm = bitmask[MMU_BLOCK];
566 /* These counters do not exist on Mali-T60x */
567 setup.l3_cache_bm = 0;
569 /* Use kbase API to enable hardware counters and provide dump buffer */
570 err = kbase_instr_hwcnt_enable_symbol(kbcontext, &setup);
571 if (err != MALI_ERROR_NONE) {
572 pr_debug("gator: Mali-T6xx: can't setup hardware counters\n");
575 pr_debug("gator: Mali-T6xx: hardware counters enabled\n");
576 kbase_instr_hwcnt_clear_symbol(kbcontext);
577 pr_debug("gator: Mali-T6xx: hardware counters cleared \n");
579 kbase_device_busy = false;
585 #if MALI_DDK_GATOR_API_VERSION == 1
586 kbase_va_free_symbol(kbcontext, kernel_dump_buffer);
587 #elif MALI_DDK_GATOR_API_VERSION == 2
588 kbase_va_free_symbol(kbcontext, &kernel_dump_buffer_handle);
592 kbase_destroy_context_symbol(kbcontext);
599 static void stop(void)
602 kbase_context *temp_kbcontext;
604 pr_debug("gator: Mali-T6xx: stop\n");
606 /* Set all counters as disabled */
607 for (cnt = 0; cnt < NUMBER_OF_HARDWARE_COUNTERS; cnt++) {
608 counters[cnt].enabled = 0;
611 /* Destroy the context for HW counters */
612 if (num_hardware_counters_enabled > 0 && kbcontext != NULL) {
614 * Set the global variable to NULL before destroying it, because
615 * other function will check this before using it.
617 temp_kbcontext = kbcontext;
620 kbase_instr_hwcnt_disable_symbol(temp_kbcontext);
622 #if MALI_DDK_GATOR_API_VERSION == 1
623 kbase_va_free_symbol(temp_kbcontext, kernel_dump_buffer);
624 #elif MALI_DDK_GATOR_API_VERSION == 2
625 kbase_va_free_symbol(temp_kbcontext, &kernel_dump_buffer_handle);
628 kbase_destroy_context_symbol(temp_kbcontext);
630 pr_debug("gator: Mali-T6xx: hardware counters stopped\n");
636 static int read(int **buffer)
643 struct timespec current_time;
644 static u32 prev_time_s = 0;
645 static s32 next_read_time_ns = 0;
647 if (!on_primary_core()) {
651 getnstimeofday(¤t_time);
654 * Discard reads unless a respectable time has passed. This reduces the load on the GPU without sacrificing
655 * accuracy on the Streamline display.
657 if (!is_read_scheduled(¤t_time, &prev_time_s, &next_read_time_ns)) {
662 * Report the HW counters
663 * Only process hardware counters if at least one of the hardware counters is enabled.
665 if (num_hardware_counters_enabled > 0) {
666 const unsigned int vithar_blocks[] = {
667 0x700, /* VITHAR_JOB_MANAGER, Block 0 */
668 0x400, /* VITHAR_TILER, Block 1 */
669 0x000, /* VITHAR_SHADER_CORE, Block 2 */
670 0x500 /* VITHAR_MEMORY_SYSTEM, Block 3 */
677 /* Mali symbols can be called safely since a kbcontext is valid */
678 if (kbase_instr_hwcnt_dump_complete_symbol(kbcontext, &success) == MALI_TRUE) {
679 kbase_device_busy = false;
681 if (success == MALI_TRUE) {
682 for (cnt = 0; cnt < NUMBER_OF_HARDWARE_COUNTERS; cnt++) {
683 const mali_counter *counter = &counters[cnt];
684 if (counter->enabled) {
685 const int block = GET_HW_BLOCK(cnt);
686 const int counter_offset = GET_COUNTER_OFFSET(cnt);
687 const u32 *counter_block = (u32 *) ((uintptr_t)kernel_dump_buffer + vithar_blocks[block]);
688 const u32 *counter_address = counter_block + counter_offset;
690 value = *counter_address;
692 if (block == SHADER_BLOCK) {
693 /* (counter_address + 0x000) has already been accounted-for above. */
694 value += *(counter_address + 0x100);
695 value += *(counter_address + 0x200);
696 value += *(counter_address + 0x300);
699 counter_dump[len++] = counter->key;
700 counter_dump[len++] = value;
706 if (!kbase_device_busy) {
707 kbase_device_busy = true;
708 kbase_instr_hwcnt_dump_irq_symbol(kbcontext);
712 /* Update the buffer */
714 *buffer = (int *)counter_dump;
720 static int create_files(struct super_block *sb, struct dentry *root)
724 * Create the filesystem for all events
726 int counter_index = 0;
727 const char *mali_name = gator_mali_get_mali_name();
729 for (event = 0; event < NUMBER_OF_HARDWARE_COUNTERS; event++) {
730 if (gator_mali_create_file_system(mali_name, hardware_counter_names[counter_index], sb, root, &counters[event]) != 0)
738 static struct gator_interface gator_events_mali_t6xx_interface = {
739 .create_files = create_files,
745 int gator_events_mali_t6xx_hw_init(void)
747 pr_debug("gator: Mali-T6xx: sw_counters init\n");
750 test_all_is_read_scheduled();
753 gator_mali_initialise_counters(counters, NUMBER_OF_HARDWARE_COUNTERS);
755 return gator_events_install(&gator_events_mali_t6xx_interface);
758 gator_events_init(gator_events_mali_t6xx_hw_init);