drivers/gator/gator_events_mali_t6xx_hw.c

   1 /**
   2  * Copyright (C) ARM Limited 2012-2014. All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  */
   9
  10 #include "gator.h"
  11
  12 #include <linux/module.h>
  13 #include <linux/time.h>
  14 #include <linux/math64.h>
  15 #include <linux/slab.h>
  16 #include <asm/io.h>
  17
  18 /* Mali T6xx DDK includes */
  19 #ifdef MALI_DIR_MIDGARD
  20 /* New DDK Directory structure with kernel/drivers/gpu/arm/midgard*/
  21 #include "mali_linux_trace.h"
  22 #include "mali_kbase.h"
  23 #include "mali_kbase_mem_linux.h"
  24 #else
  25 /* Old DDK Directory structure with kernel/drivers/gpu/arm/t6xx*/
  26 #include "linux/mali_linux_trace.h"
  27 #include "kbase/src/common/mali_kbase.h"
  28 #include "kbase/src/linux/mali_kbase_mem_linux.h"
  29 #endif
  30
  31 #include "gator_events_mali_common.h"
  32
  33 /* If API version is not specified then assume API version 1. */
  34 #ifndef MALI_DDK_GATOR_API_VERSION
  35 #define MALI_DDK_GATOR_API_VERSION 1
  36 #endif
  37
  38 #if (MALI_DDK_GATOR_API_VERSION != 1) && (MALI_DDK_GATOR_API_VERSION != 2)
  39 #error MALI_DDK_GATOR_API_VERSION is invalid (must be 1 for r1/r2 DDK, or 2 for r3 DDK).
  40 #endif
  41
  42 /*
  43  * Mali-T6xx
  44  */
  45 typedef struct kbase_device *kbase_find_device_type(int);
  46 typedef kbase_context *kbase_create_context_type(kbase_device *);
  47 typedef void kbase_destroy_context_type(kbase_context *);
  48
  49 #if MALI_DDK_GATOR_API_VERSION == 1
  50 typedef void *kbase_va_alloc_type(kbase_context *, u32);
  51 typedef void kbase_va_free_type(kbase_context *, void *);
  52 #elif MALI_DDK_GATOR_API_VERSION == 2
  53 typedef void *kbase_va_alloc_type(kbase_context *, u32, kbase_hwc_dma_mapping * handle);
  54 typedef void kbase_va_free_type(kbase_context *, kbase_hwc_dma_mapping * handle);
  55 #endif
  56
  57 typedef mali_error kbase_instr_hwcnt_enable_type(kbase_context *, kbase_uk_hwcnt_setup *);
  58 typedef mali_error kbase_instr_hwcnt_disable_type(kbase_context *);
  59 typedef mali_error kbase_instr_hwcnt_clear_type(kbase_context *);
  60 typedef mali_error kbase_instr_hwcnt_dump_irq_type(kbase_context *);
  61 typedef mali_bool kbase_instr_hwcnt_dump_complete_type(kbase_context *, mali_bool *);
  62
  63 static kbase_find_device_type *kbase_find_device_symbol;
  64 static kbase_create_context_type *kbase_create_context_symbol;
  65 static kbase_va_alloc_type *kbase_va_alloc_symbol;
  66 static kbase_instr_hwcnt_enable_type *kbase_instr_hwcnt_enable_symbol;
  67 static kbase_instr_hwcnt_clear_type *kbase_instr_hwcnt_clear_symbol;
  68 static kbase_instr_hwcnt_dump_irq_type *kbase_instr_hwcnt_dump_irq_symbol;
  69 static kbase_instr_hwcnt_dump_complete_type *kbase_instr_hwcnt_dump_complete_symbol;
  70 static kbase_instr_hwcnt_disable_type *kbase_instr_hwcnt_disable_symbol;
  71 static kbase_va_free_type *kbase_va_free_symbol;
  72 static kbase_destroy_context_type *kbase_destroy_context_symbol;
  73
  74 static long shader_present_low = 0;
  75
  76 /** The interval between reads, in ns.
  77  *
  78  * Earlier we introduced
  79  * a 'hold off for 1ms after last read' to resolve MIDBASE-2178 and MALINE-724.
  80  * However, the 1ms hold off is too long if no context switches occur as there is a race
  81  * between this value and the tick of the read clock in gator which is also 1ms. If we 'miss' the
  82  * current read, the counter values are effectively 'spread' over 2ms and the values seen are half
  83  * what they should be (since Streamline averages over sample time). In the presence of context switches
  84  * this spread can vary and markedly affect the counters.  Currently there is no 'proper' solution to
  85  * this, but empirically we have found that reducing the minimum read interval to 950us causes the
  86  * counts to be much more stable.
  87  */
  88 static const int READ_INTERVAL_NSEC = 950000;
  89
  90 #if GATOR_TEST
  91 #include "gator_events_mali_t6xx_hw_test.c"
  92 #endif
  93
  94 /* Blocks for HW counters */
  95 enum {
  96         JM_BLOCK = 0,
  97         TILER_BLOCK,
  98         SHADER_BLOCK,
  99         MMU_BLOCK
 100 };
 101
 102 /* Counters for Mali-T6xx:
 103  *
 104  *  - HW counters, 4 blocks
 105  *    For HW counters we need strings to create /dev/gator/events files.
 106  *    Enums are not needed because the position of the HW name in the array is the same
 107  *    of the corresponding value in the received block of memory.
 108  *    HW counters are requested by calculating a bitmask, passed then to the driver.
 109  *    Every millisecond a HW counters dump is requested, and if the previous has been completed they are read.
 110  */
 111
 112 /* Hardware Counters */
 113 static const char *const hardware_counter_names[] = {
 114         /* Job Manager */
 115         "",
 116         "",
 117         "",
 118         "",
 119         "MESSAGES_SENT",
 120         "MESSAGES_RECEIVED",
 121         "GPU_ACTIVE",           /* 6 */
 122         "IRQ_ACTIVE",
 123         "JS0_JOBS",
 124         "JS0_TASKS",
 125         "JS0_ACTIVE",
 126         "",
 127         "JS0_WAIT_READ",
 128         "JS0_WAIT_ISSUE",
 129         "JS0_WAIT_DEPEND",
 130         "JS0_WAIT_FINISH",
 131         "JS1_JOBS",
 132         "JS1_TASKS",
 133         "JS1_ACTIVE",
 134         "",
 135         "JS1_WAIT_READ",
 136         "JS1_WAIT_ISSUE",
 137         "JS1_WAIT_DEPEND",
 138         "JS1_WAIT_FINISH",
 139         "JS2_JOBS",
 140         "JS2_TASKS",
 141         "JS2_ACTIVE",
 142         "",
 143         "JS2_WAIT_READ",
 144         "JS2_WAIT_ISSUE",
 145         "JS2_WAIT_DEPEND",
 146         "JS2_WAIT_FINISH",
 147         "JS3_JOBS",
 148         "JS3_TASKS",
 149         "JS3_ACTIVE",
 150         "",
 151         "JS3_WAIT_READ",
 152         "JS3_WAIT_ISSUE",
 153         "JS3_WAIT_DEPEND",
 154         "JS3_WAIT_FINISH",
 155         "JS4_JOBS",
 156         "JS4_TASKS",
 157         "JS4_ACTIVE",
 158         "",
 159         "JS4_WAIT_READ",
 160         "JS4_WAIT_ISSUE",
 161         "JS4_WAIT_DEPEND",
 162         "JS4_WAIT_FINISH",
 163         "JS5_JOBS",
 164         "JS5_TASKS",
 165         "JS5_ACTIVE",
 166         "",
 167         "JS5_WAIT_READ",
 168         "JS5_WAIT_ISSUE",
 169         "JS5_WAIT_DEPEND",
 170         "JS5_WAIT_FINISH",
 171         "JS6_JOBS",
 172         "JS6_TASKS",
 173         "JS6_ACTIVE",
 174         "",
 175         "JS6_WAIT_READ",
 176         "JS6_WAIT_ISSUE",
 177         "JS6_WAIT_DEPEND",
 178         "JS6_WAIT_FINISH",
 179
 180         /*Tiler */
 181         "",
 182         "",
 183         "",
 184         "JOBS_PROCESSED",
 185         "TRIANGLES",
 186         "QUADS",
 187         "POLYGONS",
 188         "POINTS",
 189         "LINES",
 190         "VCACHE_HIT",
 191         "VCACHE_MISS",
 192         "FRONT_FACING",
 193         "BACK_FACING",
 194         "PRIM_VISIBLE",
 195         "PRIM_CULLED",
 196         "PRIM_CLIPPED",
 197         "LEVEL0",
 198         "LEVEL1",
 199         "LEVEL2",
 200         "LEVEL3",
 201         "LEVEL4",
 202         "LEVEL5",
 203         "LEVEL6",
 204         "LEVEL7",
 205         "COMMAND_1",
 206         "COMMAND_2",
 207         "COMMAND_3",
 208         "COMMAND_4",
 209         "COMMAND_4_7",
 210         "COMMAND_8_15",
 211         "COMMAND_16_63",
 212         "COMMAND_64",
 213         "COMPRESS_IN",
 214         "COMPRESS_OUT",
 215         "COMPRESS_FLUSH",
 216         "TIMESTAMPS",
 217         "PCACHE_HIT",
 218         "PCACHE_MISS",
 219         "PCACHE_LINE",
 220         "PCACHE_STALL",
 221         "WRBUF_HIT",
 222         "WRBUF_MISS",
 223         "WRBUF_LINE",
 224         "WRBUF_PARTIAL",
 225         "WRBUF_STALL",
 226         "ACTIVE",
 227         "LOADING_DESC",
 228         "INDEX_WAIT",
 229         "INDEX_RANGE_WAIT",
 230         "VERTEX_WAIT",
 231         "PCACHE_WAIT",
 232         "WRBUF_WAIT",
 233         "BUS_READ",
 234         "BUS_WRITE",
 235         "",
 236         "",
 237         "",
 238         "",
 239         "",
 240         "UTLB_STALL",
 241         "UTLB_REPLAY_MISS",
 242         "UTLB_REPLAY_FULL",
 243         "UTLB_NEW_MISS",
 244         "UTLB_HIT",
 245
 246         /* Shader Core */
 247         "",
 248         "",
 249         "",
 250         "SHADER_CORE_ACTIVE",
 251         "FRAG_ACTIVE",
 252         "FRAG_PRIMATIVES",
 253         "FRAG_PRIMATIVES_DROPPED",
 254         "FRAG_CYCLE_DESC",
 255         "FRAG_CYCLES_PLR",
 256         "FRAG_CYCLES_VERT",
 257         "FRAG_CYCLES_TRISETUP",
 258         "FRAG_CYCLES_RAST",
 259         "FRAG_THREADS",
 260         "FRAG_DUMMY_THREADS",
 261         "FRAG_QUADS_RAST",
 262         "FRAG_QUADS_EZS_TEST",
 263         "FRAG_QUADS_EZS_KILLED",
 264         "FRAG_QUADS_LZS_TEST",
 265         "FRAG_QUADS_LZS_KILLED",
 266         "FRAG_CYCLE_NO_TILE",
 267         "FRAG_NUM_TILES",
 268         "FRAG_TRANS_ELIM",
 269         "COMPUTE_ACTIVE",
 270         "COMPUTE_TASKS",
 271         "COMPUTE_THREADS",
 272         "COMPUTE_CYCLES_DESC",
 273         "TRIPIPE_ACTIVE",
 274         "ARITH_WORDS",
 275         "ARITH_CYCLES_REG",
 276         "ARITH_CYCLES_L0",
 277         "ARITH_FRAG_DEPEND",
 278         "LS_WORDS",
 279         "LS_ISSUES",
 280         "LS_RESTARTS",
 281         "LS_REISSUES_MISS",
 282         "LS_REISSUES_VD",
 283         "LS_REISSUE_ATTRIB_MISS",
 284         "LS_NO_WB",
 285         "TEX_WORDS",
 286         "TEX_BUBBLES",
 287         "TEX_WORDS_L0",
 288         "TEX_WORDS_DESC",
 289         "TEX_THREADS",
 290         "TEX_RECIRC_FMISS",
 291         "TEX_RECIRC_DESC",
 292         "TEX_RECIRC_MULTI",
 293         "TEX_RECIRC_PMISS",
 294         "TEX_RECIRC_CONF",
 295         "LSC_READ_HITS",
 296         "LSC_READ_MISSES",
 297         "LSC_WRITE_HITS",
 298         "LSC_WRITE_MISSES",
 299         "LSC_ATOMIC_HITS",
 300         "LSC_ATOMIC_MISSES",
 301         "LSC_LINE_FETCHES",
 302         "LSC_DIRTY_LINE",
 303         "LSC_SNOOPS",
 304         "AXI_TLB_STALL",
 305         "AXI_TLB_MIESS",
 306         "AXI_TLB_TRANSACTION",
 307         "LS_TLB_MISS",
 308         "LS_TLB_HIT",
 309         "AXI_BEATS_READ",
 310         "AXI_BEATS_WRITTEN",
 311
 312         /*L2 and MMU */
 313         "",
 314         "",
 315         "",
 316         "",
 317         "MMU_HIT",
 318         "MMU_NEW_MISS",
 319         "MMU_REPLAY_FULL",
 320         "MMU_REPLAY_MISS",
 321         "MMU_TABLE_WALK",
 322         "",
 323         "",
 324         "",
 325         "",
 326         "",
 327         "",
 328         "",
 329         "UTLB_HIT",
 330         "UTLB_NEW_MISS",
 331         "UTLB_REPLAY_FULL",
 332         "UTLB_REPLAY_MISS",
 333         "UTLB_STALL",
 334         "",
 335         "",
 336         "",
 337         "",
 338         "",
 339         "",
 340         "",
 341         "",
 342         "",
 343         "L2_WRITE_BEATS",
 344         "L2_READ_BEATS",
 345         "L2_ANY_LOOKUP",
 346         "L2_READ_LOOKUP",
 347         "L2_SREAD_LOOKUP",
 348         "L2_READ_REPLAY",
 349         "L2_READ_SNOOP",
 350         "L2_READ_HIT",
 351         "L2_CLEAN_MISS",
 352         "L2_WRITE_LOOKUP",
 353         "L2_SWRITE_LOOKUP",
 354         "L2_WRITE_REPLAY",
 355         "L2_WRITE_SNOOP",
 356         "L2_WRITE_HIT",
 357         "L2_EXT_READ_FULL",
 358         "L2_EXT_READ_HALF",
 359         "L2_EXT_WRITE_FULL",
 360         "L2_EXT_WRITE_HALF",
 361         "L2_EXT_READ",
 362         "L2_EXT_READ_LINE",
 363         "L2_EXT_WRITE",
 364         "L2_EXT_WRITE_LINE",
 365         "L2_EXT_WRITE_SMALL",
 366         "L2_EXT_BARRIER",
 367         "L2_EXT_AR_STALL",
 368         "L2_EXT_R_BUF_FULL",
 369         "L2_EXT_RD_BUF_FULL",
 370         "L2_EXT_R_RAW",
 371         "L2_EXT_W_STALL",
 372         "L2_EXT_W_BUF_FULL",
 373         "L2_EXT_R_W_HAZARD",
 374         "L2_TAG_HAZARD",
 375         "L2_SNOOP_FULL",
 376         "L2_REPLAY_FULL"
 377 };
 378
 379 #define NUMBER_OF_HARDWARE_COUNTERS (sizeof(hardware_counter_names) / sizeof(hardware_counter_names[0]))
 380
 381 #define GET_HW_BLOCK(c) (((c) >> 6) & 0x3)
 382 #define GET_COUNTER_OFFSET(c) ((c) & 0x3f)
 383
 384 /* Memory to dump hardware counters into */
 385 static void *kernel_dump_buffer;
 386
 387 #if MALI_DDK_GATOR_API_VERSION == 2
 388 /* DMA state used to manage lifetime of the buffer */
 389 kbase_hwc_dma_mapping kernel_dump_buffer_handle;
 390 #endif
 391
 392 /* kbase context and device */
 393 static kbase_context *kbcontext = NULL;
 394 static struct kbase_device *kbdevice = NULL;
 395
 396 /*
 397  * The following function has no external prototype in older DDK revisions.  When the DDK
 398  * is updated then this should be removed.
 399  */
 400 struct kbase_device *kbase_find_device(int minor);
 401
 402 static volatile bool kbase_device_busy = false;
 403 static unsigned int num_hardware_counters_enabled;
 404
 405 /*
 406  * gatorfs variables for counter enable state
 407  */
 408 static mali_counter counters[NUMBER_OF_HARDWARE_COUNTERS];
 409
 410 /* An array used to return the data we recorded
 411  * as key,value pairs hence the *2
 412  */
 413 static unsigned long counter_dump[NUMBER_OF_HARDWARE_COUNTERS * 2];
 414
 415 #define SYMBOL_GET(FUNCTION, ERROR_COUNT) \
 416         if(FUNCTION ## _symbol) \
 417         { \
 418                 printk("gator: mali " #FUNCTION " symbol was already registered\n"); \
 419                 (ERROR_COUNT)++; \
 420         } \
 421         else \
 422         { \
 423                 FUNCTION ## _symbol = symbol_get(FUNCTION); \
 424                 if(! FUNCTION ## _symbol) \
 425                 { \
 426                         printk("gator: mali online " #FUNCTION " symbol not found\n"); \
 427                         (ERROR_COUNT)++; \
 428                 } \
 429         }
 430
 431 #define SYMBOL_CLEANUP(FUNCTION) \
 432         if(FUNCTION ## _symbol) \
 433         { \
 434         symbol_put(FUNCTION); \
 435         FUNCTION ## _symbol = NULL; \
 436         }
 437
 438 /**
 439  * Execute symbol_get for all the Mali symbols and check for success.
 440  * @return the number of symbols not loaded.
 441  */
 442 static int init_symbols(void)
 443 {
 444         int error_count = 0;
 445         SYMBOL_GET(kbase_find_device, error_count);
 446         SYMBOL_GET(kbase_create_context, error_count);
 447         SYMBOL_GET(kbase_va_alloc, error_count);
 448         SYMBOL_GET(kbase_instr_hwcnt_enable, error_count);
 449         SYMBOL_GET(kbase_instr_hwcnt_clear, error_count);
 450         SYMBOL_GET(kbase_instr_hwcnt_dump_irq, error_count);
 451         SYMBOL_GET(kbase_instr_hwcnt_dump_complete, error_count);
 452         SYMBOL_GET(kbase_instr_hwcnt_disable, error_count);
 453         SYMBOL_GET(kbase_va_free, error_count);
 454         SYMBOL_GET(kbase_destroy_context, error_count);
 455
 456         return error_count;
 457 }
 458
 459 /**
 460  * Execute symbol_put for all the registered Mali symbols.
 461  */
 462 static void clean_symbols(void)
 463 {
 464         SYMBOL_CLEANUP(kbase_find_device);
 465         SYMBOL_CLEANUP(kbase_create_context);
 466         SYMBOL_CLEANUP(kbase_va_alloc);
 467         SYMBOL_CLEANUP(kbase_instr_hwcnt_enable);
 468         SYMBOL_CLEANUP(kbase_instr_hwcnt_clear);
 469         SYMBOL_CLEANUP(kbase_instr_hwcnt_dump_irq);
 470         SYMBOL_CLEANUP(kbase_instr_hwcnt_dump_complete);
 471         SYMBOL_CLEANUP(kbase_instr_hwcnt_disable);
 472         SYMBOL_CLEANUP(kbase_va_free);
 473         SYMBOL_CLEANUP(kbase_destroy_context);
 474 }
 475
 476 /**
 477  * Determines whether a read should take place
 478  * @param current_time The current time, obtained from getnstimeofday()
 479  * @param prev_time_s The number of seconds at the previous read attempt.
 480  * @param next_read_time_ns The time (in ns) when the next read should be allowed.
 481  *
 482  * Note that this function has been separated out here to allow it to be tested.
 483  */
 484 static int is_read_scheduled(const struct timespec *current_time, u32 *prev_time_s, s32 *next_read_time_ns)
 485 {
 486         /* If the current ns count rolls over a second, roll the next read time too. */
 487         if (current_time->tv_sec != *prev_time_s) {
 488                 *next_read_time_ns = *next_read_time_ns - NSEC_PER_SEC;
 489         }
 490
 491         /* Abort the read if the next read time has not arrived. */
 492         if (current_time->tv_nsec < *next_read_time_ns) {
 493                 return 0;
 494         }
 495
 496         /* Set the next read some fixed time after this one, and update the read timestamp. */
 497         *next_read_time_ns = current_time->tv_nsec + READ_INTERVAL_NSEC;
 498
 499         *prev_time_s = current_time->tv_sec;
 500         return 1;
 501 }
 502
 503 static int start(void)
 504 {
 505         kbase_uk_hwcnt_setup setup;
 506         mali_error err;
 507         int cnt;
 508         u16 bitmask[] = { 0, 0, 0, 0 };
 509         unsigned long long shadersPresent = 0;
 510
 511         /* Setup HW counters */
 512         num_hardware_counters_enabled = 0;
 513
 514         if (NUMBER_OF_HARDWARE_COUNTERS != 256) {
 515                 pr_debug("Unexpected number of hardware counters defined: expecting 256, got %d\n", NUMBER_OF_HARDWARE_COUNTERS);
 516         }
 517
 518         /* Calculate enable bitmasks based on counters_enabled array */
 519         for (cnt = 0; cnt < NUMBER_OF_HARDWARE_COUNTERS; cnt++) {
 520                 const mali_counter *counter = &counters[cnt];
 521                 if (counter->enabled) {
 522                         int block = GET_HW_BLOCK(cnt);
 523                         int enable_bit = GET_COUNTER_OFFSET(cnt) / 4;
 524                         bitmask[block] |= (1 << enable_bit);
 525                         pr_debug("gator: Mali-T6xx: hardware counter %s selected [%d]\n", hardware_counter_names[cnt], cnt);
 526                         num_hardware_counters_enabled++;
 527                 }
 528         }
 529
 530         /* Create a kbase context for HW counters */
 531         if (num_hardware_counters_enabled > 0) {
 532                 if (init_symbols() > 0) {
 533                         clean_symbols();
 534                         /* No Mali driver code entrypoints found - not a fault. */
 535                         return 0;
 536                 }
 537
 538                 kbdevice = kbase_find_device_symbol(-1);
 539
 540                 /* If we already got a context, fail */
 541                 if (kbcontext) {
 542                         pr_debug("gator: Mali-T6xx: error context already present\n");
 543                         goto out;
 544                 }
 545
 546                 /* kbcontext will only be valid after all the Mali symbols are loaded successfully */
 547                 kbcontext = kbase_create_context_symbol(kbdevice);
 548                 if (!kbcontext) {
 549                         pr_debug("gator: Mali-T6xx: error creating kbase context\n");
 550                         goto out;
 551                 }
 552
 553
 554                 /* See if we can get the number of shader cores */
 555                 shadersPresent = kbdevice->shader_present_bitmap;
 556                 shader_present_low = (unsigned long)shadersPresent;
 557
 558                 /*
 559                  * The amount of memory needed to store the dump (bytes)
 560                  * DUMP_SIZE = number of core groups
 561                  *             * number of blocks (always 8 for midgard)
 562                  *             * number of counters per block (always 64 for midgard)
 563                  *             * number of bytes per counter (always 4 in midgard)
 564                  * For a Mali-T6xx with a single core group = 1 * 8 * 64 * 4 = 2048
 565                  * For a Mali-T6xx with a dual core group   = 2 * 8 * 64 * 4 = 4096
 566                  */
 567 #if MALI_DDK_GATOR_API_VERSION == 1
 568                 kernel_dump_buffer = kbase_va_alloc_symbol(kbcontext, 4096);
 569 #elif MALI_DDK_GATOR_API_VERSION == 2
 570                 kernel_dump_buffer = kbase_va_alloc_symbol(kbcontext, 4096, &kernel_dump_buffer_handle);
 571 #endif
 572                 if (!kernel_dump_buffer) {
 573                         pr_debug("gator: Mali-T6xx: error trying to allocate va\n");
 574                         goto destroy_context;
 575                 }
 576
 577                 setup.dump_buffer = (uintptr_t)kernel_dump_buffer;
 578                 setup.jm_bm = bitmask[JM_BLOCK];
 579                 setup.tiler_bm = bitmask[TILER_BLOCK];
 580                 setup.shader_bm = bitmask[SHADER_BLOCK];
 581                 setup.mmu_l2_bm = bitmask[MMU_BLOCK];
 582                 /* These counters do not exist on Mali-T60x */
 583                 setup.l3_cache_bm = 0;
 584
 585                 /* Use kbase API to enable hardware counters and provide dump buffer */
 586                 err = kbase_instr_hwcnt_enable_symbol(kbcontext, &setup);
 587                 if (err != MALI_ERROR_NONE) {
 588                         pr_debug("gator: Mali-T6xx: can't setup hardware counters\n");
 589                         goto free_buffer;
 590                 }
 591                 pr_debug("gator: Mali-T6xx: hardware counters enabled\n");
 592                 kbase_instr_hwcnt_clear_symbol(kbcontext);
 593                 pr_debug("gator: Mali-T6xx: hardware counters cleared \n");
 594
 595                 kbase_device_busy = false;
 596         }
 597
 598         return 0;
 599
 600 free_buffer:
 601 #if MALI_DDK_GATOR_API_VERSION == 1
 602         kbase_va_free_symbol(kbcontext, kernel_dump_buffer);
 603 #elif MALI_DDK_GATOR_API_VERSION == 2
 604         kbase_va_free_symbol(kbcontext, &kernel_dump_buffer_handle);
 605 #endif
 606
 607 destroy_context:
 608         kbase_destroy_context_symbol(kbcontext);
 609
 610 out:
 611         clean_symbols();
 612         return -1;
 613 }
 614
 615 static void stop(void)
 616 {
 617         unsigned int cnt;
 618         kbase_context *temp_kbcontext;
 619
 620         pr_debug("gator: Mali-T6xx: stop\n");
 621
 622         /* Set all counters as disabled */
 623         for (cnt = 0; cnt < NUMBER_OF_HARDWARE_COUNTERS; cnt++) {
 624                 counters[cnt].enabled = 0;
 625         }
 626
 627         /* Destroy the context for HW counters */
 628         if (num_hardware_counters_enabled > 0 && kbcontext != NULL) {
 629                 /*
 630                  * Set the global variable to NULL before destroying it, because
 631                  * other function will check this before using it.
 632                  */
 633                 temp_kbcontext = kbcontext;
 634                 kbcontext = NULL;
 635
 636                 kbase_instr_hwcnt_disable_symbol(temp_kbcontext);
 637
 638 #if MALI_DDK_GATOR_API_VERSION == 1
 639                 kbase_va_free_symbol(temp_kbcontext, kernel_dump_buffer);
 640 #elif MALI_DDK_GATOR_API_VERSION == 2
 641                 kbase_va_free_symbol(temp_kbcontext, &kernel_dump_buffer_handle);
 642 #endif
 643
 644                 kbase_destroy_context_symbol(temp_kbcontext);
 645
 646                 pr_debug("gator: Mali-T6xx: hardware counters stopped\n");
 647
 648                 clean_symbols();
 649         }
 650 }
 651
 652 static int read(int **buffer)
 653 {
 654         int cnt;
 655         int len = 0;
 656         u32 value = 0;
 657         mali_bool success;
 658
 659         struct timespec current_time;
 660         static u32 prev_time_s = 0;
 661         static s32 next_read_time_ns = 0;
 662
 663         if (!on_primary_core()) {
 664                 return 0;
 665         }
 666
 667         getnstimeofday(&current_time);
 668
 669         /*
 670          * Discard reads unless a respectable time has passed.  This reduces the load on the GPU without sacrificing
 671          * accuracy on the Streamline display.
 672          */
 673         if (!is_read_scheduled(&current_time, &prev_time_s, &next_read_time_ns)) {
 674                 return 0;
 675         }
 676
 677         /*
 678          * Report the HW counters
 679          * Only process hardware counters if at least one of the hardware counters is enabled.
 680          */
 681         if (num_hardware_counters_enabled > 0) {
 682                 const unsigned int vithar_blocks[] = {
 683                         0x700,  /* VITHAR_JOB_MANAGER,     Block 0 */
 684                         0x400,  /* VITHAR_TILER,           Block 1 */
 685                         0x000,  /* VITHAR_SHADER_CORE,     Block 2 */
 686                         0x500   /* VITHAR_MEMORY_SYSTEM,   Block 3 */
 687                 };
 688
 689                 if (!kbcontext) {
 690                         return -1;
 691                 }
 692
 693                 /* Mali symbols can be called safely since a kbcontext is valid */
 694                 if (kbase_instr_hwcnt_dump_complete_symbol(kbcontext, &success) == MALI_TRUE) {
 695                         kbase_device_busy = false;
 696
 697                         if (success == MALI_TRUE) {
 698                                 /* Cycle through hardware counters and accumulate totals */
 699                                 for (cnt = 0; cnt < NUMBER_OF_HARDWARE_COUNTERS; cnt++) {
 700                                         const mali_counter *counter = &counters[cnt];
 701                                         if (counter->enabled) {
 702                                                 const int block = GET_HW_BLOCK(cnt);
 703                                                 const int counter_offset = GET_COUNTER_OFFSET(cnt);
 704
 705                                                 const char* block_base_address = (char*)kernel_dump_buffer + vithar_blocks[block];
 706
 707                                                 /* If counter belongs to shader block need to take into account all cores */
 708                                                 if (block == SHADER_BLOCK) {
 709                                                         int i = 0;
 710                                                         int shader_core_count = 0;
 711                                                         value = 0;
 712
 713                                                         for (i = 0; i < 4; i++) {
 714                                                                 if ((shader_present_low >> i) & 1) {
 715                                                                         value += *((u32*) (block_base_address + (0x100 * i)) + counter_offset);
 716                                                                         shader_core_count++;
 717                                                                 }
 718                                                         }
 719
 720                                                         for (i = 0; i < 4; i++) {
 721                                                                 if((shader_present_low >> (i+4)) & 1) {
 722                                                                         value += *((u32*)(block_base_address + (0x100 * i) + 0x800) + counter_offset);
 723                                                                         shader_core_count++;
 724                                                                 }
 725                                                         }
 726
 727                                                         /* Need to total by number of cores to produce an average */
 728                                                         if (shader_core_count != 0) {
 729                                                                 value /= shader_core_count;
 730                                                         }
 731                                                 } else {
 732                                                         value = *((u32*)block_base_address + counter_offset);
 733                                                 }
 734
 735                                                 counter_dump[len++] = counter->key;
 736                                                 counter_dump[len++] = value;
 737                                         }
 738                                 }
 739                         }
 740                 }
 741
 742                 if (!kbase_device_busy) {
 743                         kbase_device_busy = true;
 744                         kbase_instr_hwcnt_dump_irq_symbol(kbcontext);
 745                 }
 746         }
 747
 748         /* Update the buffer */
 749         if (buffer) {
 750                 *buffer = (int *)counter_dump;
 751         }
 752
 753         return len;
 754 }
 755
 756 static int create_files(struct super_block *sb, struct dentry *root)
 757 {
 758         unsigned int event;
 759         /*
 760          * Create the filesystem for all events
 761          */
 762         int counter_index = 0;
 763         const char *mali_name = gator_mali_get_mali_name();
 764
 765         for (event = 0; event < NUMBER_OF_HARDWARE_COUNTERS; event++) {
 766                 if (gator_mali_create_file_system(mali_name, hardware_counter_names[counter_index], sb, root, &counters[event], NULL) != 0)
 767                         return -1;
 768                 counter_index++;
 769         }
 770
 771         return 0;
 772 }
 773
 774 static struct gator_interface gator_events_mali_t6xx_interface = {
 775         .create_files = create_files,
 776         .start = start,
 777         .stop = stop,
 778         .read = read
 779 };
 780
 781 int gator_events_mali_t6xx_hw_init(void)
 782 {
 783         pr_debug("gator: Mali-T6xx: sw_counters init\n");
 784
 785 #if GATOR_TEST
 786         test_all_is_read_scheduled();
 787 #endif
 788
 789         gator_mali_initialise_counters(counters, NUMBER_OF_HARDWARE_COUNTERS);
 790
 791         return gator_events_install(&gator_events_mali_t6xx_interface);
 792 }