drivers/gpu/arm/mali400/mali/common/mali_group.c

   1 /*
   2  * Copyright (C) 2011-2015 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10 #include "mali_kernel_common.h"
  11 #include "mali_group.h"
  12 #include "mali_osk.h"
  13 #include "mali_l2_cache.h"
  14 #include "mali_gp.h"
  15 #include "mali_pp.h"
  16 #include "mali_mmu.h"
  17 #include "mali_dlbu.h"
  18 #include "mali_broadcast.h"
  19 #include "mali_scheduler.h"
  20 #include "mali_osk_profiling.h"
  21 #include "mali_pm_domain.h"
  22 #include "mali_pm.h"
  23 #include "mali_executor.h"
  24
  25 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
  26 #include <linux/sched.h>
  27 #include <trace/events/gpu.h>
  28 #endif
  29
  30 #define MALI_MAX_NUM_DOMAIN_REFS (MALI_MAX_NUMBER_OF_GROUPS * 2)
  31
  32 #if defined(CONFIG_MALI400_PROFILING)
  33 static void mali_group_report_l2_cache_counters_per_core(struct mali_group *group, u32 core_num);
  34 #endif /* #if defined(CONFIG_MALI400_PROFILING) */
  35
  36 static struct mali_group *mali_global_groups[MALI_MAX_NUMBER_OF_GROUPS] = { NULL, };
  37 static u32 mali_global_num_groups = 0;
  38
  39 /* SW timer for job execution */
  40 int mali_max_job_runtime = MALI_MAX_JOB_RUNTIME_DEFAULT;
  41
  42 /* local helper functions */
  43 static void mali_group_bottom_half_mmu(void *data);
  44 static void mali_group_bottom_half_gp(void *data);
  45 static void mali_group_bottom_half_pp(void *data);
  46 static void mali_group_timeout(void *data);
  47 static void mali_group_reset_pp(struct mali_group *group);
  48 static void mali_group_reset_mmu(struct mali_group *group);
  49
  50 static void mali_group_activate_page_directory(struct mali_group *group, struct mali_session_data *session);
  51 static void mali_group_recovery_reset(struct mali_group *group);
  52
  53 struct mali_group *mali_group_create(struct mali_l2_cache_core *core,
  54                                      struct mali_dlbu_core *dlbu,
  55                                      struct mali_bcast_unit *bcast,
  56                                      u32 domain_index)
  57 {
  58         struct mali_group *group = NULL;
  59
  60         if (mali_global_num_groups >= MALI_MAX_NUMBER_OF_GROUPS) {
  61                 MALI_PRINT_ERROR(("Mali group: Too many group objects created\n"));
  62                 return NULL;
  63         }
  64
  65         group = _mali_osk_calloc(1, sizeof(struct mali_group));
  66         if (NULL != group) {
  67                 group->timeout_timer = _mali_osk_timer_init();
  68                 if (NULL != group->timeout_timer) {
  69                         _mali_osk_timer_setcallback(group->timeout_timer, mali_group_timeout, (void *)group);
  70
  71                         group->l2_cache_core[0] = core;
  72                         _mali_osk_list_init(&group->group_list);
  73                         _mali_osk_list_init(&group->executor_list);
  74                         _mali_osk_list_init(&group->pm_domain_list);
  75                         group->bcast_core = bcast;
  76                         group->dlbu_core = dlbu;
  77
  78                         /* register this object as a part of the correct power domain */
  79                         if ((NULL != core) || (NULL != dlbu) || (NULL != bcast))
  80                                 group->pm_domain = mali_pm_register_group(domain_index, group);
  81
  82                         mali_global_groups[mali_global_num_groups] = group;
  83                         mali_global_num_groups++;
  84
  85                         return group;
  86                 }
  87                 _mali_osk_free(group);
  88         }
  89
  90         return NULL;
  91 }
  92
  93 void mali_group_delete(struct mali_group *group)
  94 {
  95         u32 i;
  96
  97         MALI_DEBUG_PRINT(4, ("Deleting group %s\n",
  98                              mali_group_core_description(group)));
  99
 100         MALI_DEBUG_ASSERT(NULL == group->parent_group);
 101         MALI_DEBUG_ASSERT((MALI_GROUP_STATE_INACTIVE == group->state) || ((MALI_GROUP_STATE_ACTIVATION_PENDING == group->state)));
 102
 103         /* Delete the resources that this group owns */
 104         if (NULL != group->gp_core) {
 105                 mali_gp_delete(group->gp_core);
 106         }
 107
 108         if (NULL != group->pp_core) {
 109                 mali_pp_delete(group->pp_core);
 110         }
 111
 112         if (NULL != group->mmu) {
 113                 mali_mmu_delete(group->mmu);
 114         }
 115
 116         if (mali_group_is_virtual(group)) {
 117                 /* Remove all groups from virtual group */
 118                 struct mali_group *child;
 119                 struct mali_group *temp;
 120
 121                 _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) {
 122                         child->parent_group = NULL;
 123                         mali_group_delete(child);
 124                 }
 125
 126                 mali_dlbu_delete(group->dlbu_core);
 127
 128                 if (NULL != group->bcast_core) {
 129                         mali_bcast_unit_delete(group->bcast_core);
 130                 }
 131         }
 132
 133         for (i = 0; i < mali_global_num_groups; i++) {
 134                 if (mali_global_groups[i] == group) {
 135                         mali_global_groups[i] = NULL;
 136                         mali_global_num_groups--;
 137
 138                         if (i != mali_global_num_groups) {
 139                                 /* We removed a group from the middle of the array -- move the last
 140                                  * group to the current position to close the gap */
 141                                 mali_global_groups[i] = mali_global_groups[mali_global_num_groups];
 142                                 mali_global_groups[mali_global_num_groups] = NULL;
 143                         }
 144
 145                         break;
 146                 }
 147         }
 148
 149         if (NULL != group->timeout_timer) {
 150                 _mali_osk_timer_del(group->timeout_timer);
 151                 _mali_osk_timer_term(group->timeout_timer);
 152         }
 153
 154         if (NULL != group->bottom_half_work_mmu) {
 155                 _mali_osk_wq_delete_work(group->bottom_half_work_mmu);
 156         }
 157
 158         if (NULL != group->bottom_half_work_gp) {
 159                 _mali_osk_wq_delete_work(group->bottom_half_work_gp);
 160         }
 161
 162         if (NULL != group->bottom_half_work_pp) {
 163                 _mali_osk_wq_delete_work(group->bottom_half_work_pp);
 164         }
 165
 166         _mali_osk_free(group);
 167 }
 168
 169 _mali_osk_errcode_t mali_group_add_mmu_core(struct mali_group *group, struct mali_mmu_core *mmu_core)
 170 {
 171         /* This group object now owns the MMU core object */
 172         group->mmu = mmu_core;
 173         group->bottom_half_work_mmu = _mali_osk_wq_create_work(mali_group_bottom_half_mmu, group);
 174         if (NULL == group->bottom_half_work_mmu) {
 175                 return _MALI_OSK_ERR_FAULT;
 176         }
 177         return _MALI_OSK_ERR_OK;
 178 }
 179
 180 void mali_group_remove_mmu_core(struct mali_group *group)
 181 {
 182         /* This group object no longer owns the MMU core object */
 183         group->mmu = NULL;
 184         if (NULL != group->bottom_half_work_mmu) {
 185                 _mali_osk_wq_delete_work(group->bottom_half_work_mmu);
 186         }
 187 }
 188
 189 _mali_osk_errcode_t mali_group_add_gp_core(struct mali_group *group, struct mali_gp_core *gp_core)
 190 {
 191         /* This group object now owns the GP core object */
 192         group->gp_core = gp_core;
 193         group->bottom_half_work_gp = _mali_osk_wq_create_work(mali_group_bottom_half_gp, group);
 194         if (NULL == group->bottom_half_work_gp) {
 195                 return _MALI_OSK_ERR_FAULT;
 196         }
 197
 198         return _MALI_OSK_ERR_OK;
 199 }
 200
 201 void mali_group_remove_gp_core(struct mali_group *group)
 202 {
 203         /* This group object no longer owns the GP core object */
 204         group->gp_core = NULL;
 205         if (NULL != group->bottom_half_work_gp) {
 206                 _mali_osk_wq_delete_work(group->bottom_half_work_gp);
 207         }
 208 }
 209
 210 _mali_osk_errcode_t mali_group_add_pp_core(struct mali_group *group, struct mali_pp_core *pp_core)
 211 {
 212         /* This group object now owns the PP core object */
 213         group->pp_core = pp_core;
 214         group->bottom_half_work_pp = _mali_osk_wq_create_work(mali_group_bottom_half_pp, group);
 215         if (NULL == group->bottom_half_work_pp) {
 216                 return _MALI_OSK_ERR_FAULT;
 217         }
 218         return _MALI_OSK_ERR_OK;
 219 }
 220
 221 void mali_group_remove_pp_core(struct mali_group *group)
 222 {
 223         /* This group object no longer owns the PP core object */
 224         group->pp_core = NULL;
 225         if (NULL != group->bottom_half_work_pp) {
 226                 _mali_osk_wq_delete_work(group->bottom_half_work_pp);
 227         }
 228 }
 229
 230 enum mali_group_state mali_group_activate(struct mali_group *group)
 231 {
 232         MALI_DEBUG_ASSERT_POINTER(group);
 233         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
 234
 235         MALI_DEBUG_PRINT(4, ("Group: Activating group %s\n",
 236                              mali_group_core_description(group)));
 237
 238         if (MALI_GROUP_STATE_INACTIVE == group->state) {
 239                 /* Group is inactive, get PM refs in order to power up */
 240
 241                 /*
 242                  * We'll take a maximum of 2 power domain references pr group,
 243                  * one for the group itself, and one for it's L2 cache.
 244                  */
 245                 struct mali_pm_domain *domains[MALI_MAX_NUM_DOMAIN_REFS];
 246                 struct mali_group *groups[MALI_MAX_NUM_DOMAIN_REFS];
 247                 u32 num_domains = 0;
 248                 mali_bool all_groups_on;
 249
 250                 /* Deal with child groups first */
 251                 if (mali_group_is_virtual(group)) {
 252                         /*
 253                          * The virtual group might have 0, 1 or 2 L2s in
 254                          * its l2_cache_core array, but we ignore these and
 255                          * let the child groups take the needed L2 cache ref
 256                          * on behalf of the virtual group.
 257                          * In other words; The L2 refs are taken in pair with
 258                          * the physical group which the L2 is attached to.
 259                          */
 260                         struct mali_group *child;
 261                         struct mali_group *temp;
 262
 263                         /*
 264                          * Child group is inactive, get PM
 265                          * refs in order to power up.
 266                          */
 267                         _MALI_OSK_LIST_FOREACHENTRY(child, temp,
 268                                                     &group->group_list,
 269                                                     struct mali_group, group_list) {
 270                                 MALI_DEBUG_ASSERT(MALI_GROUP_STATE_INACTIVE
 271                                                   == child->state);
 272
 273                                 child->state = MALI_GROUP_STATE_ACTIVATION_PENDING;
 274
 275                                 MALI_DEBUG_ASSERT_POINTER(
 276                                         child->pm_domain);
 277                                 domains[num_domains] = child->pm_domain;
 278                                 groups[num_domains] = child;
 279                                 num_domains++;
 280
 281                                 /*
 282                                  * Take L2 domain ref for child group.
 283                                  */
 284                                 MALI_DEBUG_ASSERT(MALI_MAX_NUM_DOMAIN_REFS
 285                                                   > num_domains);
 286                                 domains[num_domains] = mali_l2_cache_get_pm_domain(
 287                                                                child->l2_cache_core[0]);
 288                                 groups[num_domains] = NULL;
 289                                 MALI_DEBUG_ASSERT(NULL ==
 290                                                   child->l2_cache_core[1]);
 291                                 num_domains++;
 292                         }
 293                 } else {
 294                         /* Take L2 domain ref for physical groups. */
 295                         MALI_DEBUG_ASSERT(MALI_MAX_NUM_DOMAIN_REFS >
 296                                           num_domains);
 297
 298                         domains[num_domains] = mali_l2_cache_get_pm_domain(
 299                                                        group->l2_cache_core[0]);
 300                         groups[num_domains] = NULL;
 301                         MALI_DEBUG_ASSERT(NULL == group->l2_cache_core[1]);
 302                         num_domains++;
 303                 }
 304
 305                 /* Do the group itself last (it's dependencies first) */
 306
 307                 group->state = MALI_GROUP_STATE_ACTIVATION_PENDING;
 308
 309                 MALI_DEBUG_ASSERT_POINTER(group->pm_domain);
 310                 domains[num_domains] = group->pm_domain;
 311                 groups[num_domains] = group;
 312                 num_domains++;
 313
 314                 all_groups_on = mali_pm_get_domain_refs(domains, groups,
 315                                                         num_domains);
 316
 317                 /*
 318                  * Complete activation for group, include
 319                  * virtual group or physical group.
 320                  */
 321                 if (MALI_TRUE == all_groups_on) {
 322
 323                         mali_group_set_active(group);
 324                 }
 325         } else if (MALI_GROUP_STATE_ACTIVE == group->state) {
 326                 /* Already active */
 327                 MALI_DEBUG_ASSERT(MALI_TRUE == group->power_is_on);
 328         } else {
 329                 /*
 330                  * Activation already pending, group->power_is_on could
 331                  * be both true or false. We need to wait for power up
 332                  * notification anyway.
 333                  */
 334                 MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVATION_PENDING
 335                                   == group->state);
 336         }
 337
 338         MALI_DEBUG_PRINT(4, ("Group: group %s activation result: %s\n",
 339                              mali_group_core_description(group),
 340                              MALI_GROUP_STATE_ACTIVE == group->state ?
 341                              "ACTIVE" : "PENDING"));
 342
 343         return group->state;
 344 }
 345
 346 mali_bool mali_group_set_active(struct mali_group *group)
 347 {
 348         MALI_DEBUG_ASSERT_POINTER(group);
 349         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
 350         MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVATION_PENDING == group->state);
 351         MALI_DEBUG_ASSERT(MALI_TRUE == group->power_is_on);
 352
 353         MALI_DEBUG_PRINT(4, ("Group: Activation completed for %s\n",
 354                              mali_group_core_description(group)));
 355
 356         if (mali_group_is_virtual(group)) {
 357                 struct mali_group *child;
 358                 struct mali_group *temp;
 359
 360                 _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list,
 361                                             struct mali_group, group_list) {
 362                         if (MALI_TRUE != child->power_is_on) {
 363                                 return MALI_FALSE;
 364                         }
 365
 366                         child->state = MALI_GROUP_STATE_ACTIVE;
 367                 }
 368
 369                 mali_group_reset(group);
 370         }
 371
 372         /* Go to ACTIVE state */
 373         group->state = MALI_GROUP_STATE_ACTIVE;
 374
 375         return MALI_TRUE;
 376 }
 377
 378 mali_bool mali_group_deactivate(struct mali_group *group)
 379 {
 380         struct mali_pm_domain *domains[MALI_MAX_NUM_DOMAIN_REFS];
 381         u32 num_domains = 0;
 382         mali_bool power_down = MALI_FALSE;
 383
 384         MALI_DEBUG_ASSERT_POINTER(group);
 385         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
 386         MALI_DEBUG_ASSERT(MALI_GROUP_STATE_INACTIVE != group->state);
 387
 388         MALI_DEBUG_PRINT(3, ("Group: Deactivating group %s\n",
 389                              mali_group_core_description(group)));
 390
 391         group->state = MALI_GROUP_STATE_INACTIVE;
 392
 393         MALI_DEBUG_ASSERT_POINTER(group->pm_domain);
 394         domains[num_domains] = group->pm_domain;
 395         num_domains++;
 396
 397         if (mali_group_is_virtual(group)) {
 398                 /* Release refs for all child groups */
 399                 struct mali_group *child;
 400                 struct mali_group *temp;
 401
 402                 _MALI_OSK_LIST_FOREACHENTRY(child, temp,
 403                                             &group->group_list,
 404                                             struct mali_group, group_list) {
 405                         child->state = MALI_GROUP_STATE_INACTIVE;
 406
 407                         MALI_DEBUG_ASSERT_POINTER(child->pm_domain);
 408                         domains[num_domains] = child->pm_domain;
 409                         num_domains++;
 410
 411                         /* Release L2 cache domain for child groups */
 412                         MALI_DEBUG_ASSERT(MALI_MAX_NUM_DOMAIN_REFS >
 413                                           num_domains);
 414                         domains[num_domains] = mali_l2_cache_get_pm_domain(
 415                                                        child->l2_cache_core[0]);
 416                         MALI_DEBUG_ASSERT(NULL == child->l2_cache_core[1]);
 417                         num_domains++;
 418                 }
 419
 420                 /*
 421                  * Must do mali_group_power_down() steps right here for
 422                  * virtual group, because virtual group itself is likely to
 423                  * stay powered on, however child groups are now very likely
 424                  * to be powered off (and thus lose their state).
 425                  */
 426
 427                 mali_group_clear_session(group);
 428                 /*
 429                  * Disable the broadcast unit (clear it's mask).
 430                  * This is needed in case the GPU isn't actually
 431                  * powered down at this point and groups are
 432                  * removed from an inactive virtual group.
 433                  * If not, then the broadcast unit will intercept
 434                  * their interrupts!
 435                  */
 436                 mali_bcast_disable(group->bcast_core);
 437         } else {
 438                 /* Release L2 cache domain for physical groups */
 439                 MALI_DEBUG_ASSERT(MALI_MAX_NUM_DOMAIN_REFS >
 440                                   num_domains);
 441                 domains[num_domains] = mali_l2_cache_get_pm_domain(
 442                                                group->l2_cache_core[0]);
 443                 MALI_DEBUG_ASSERT(NULL == group->l2_cache_core[1]);
 444                 num_domains++;
 445         }
 446
 447         power_down = mali_pm_put_domain_refs(domains, num_domains);
 448
 449         return power_down;
 450 }
 451
 452 void mali_group_power_up(struct mali_group *group)
 453 {
 454         MALI_DEBUG_ASSERT_POINTER(group);
 455         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
 456
 457         MALI_DEBUG_PRINT(3, ("Group: Power up for %s\n",
 458                              mali_group_core_description(group)));
 459
 460         group->power_is_on = MALI_TRUE;
 461
 462         if (MALI_FALSE == mali_group_is_virtual(group)
 463             && MALI_FALSE == mali_group_is_in_virtual(group)) {
 464                 mali_group_reset(group);
 465         }
 466
 467         /*
 468          * When we just acquire only one physical group form virt group,
 469          * we should remove the bcast&dlbu mask from virt group and
 470          * reset bcast and dlbu core, although part of pp cores in virt
 471          * group maybe not be powered on.
 472          */
 473         if (MALI_TRUE == mali_group_is_virtual(group)) {
 474                 mali_bcast_reset(group->bcast_core);
 475                 mali_dlbu_update_mask(group->dlbu_core);
 476         }
 477 }
 478
 479 void mali_group_power_down(struct mali_group *group)
 480 {
 481         MALI_DEBUG_ASSERT_POINTER(group);
 482         MALI_DEBUG_ASSERT(MALI_TRUE == group->power_is_on);
 483         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
 484
 485         MALI_DEBUG_PRINT(3, ("Group: Power down for %s\n",
 486                              mali_group_core_description(group)));
 487
 488         group->power_is_on = MALI_FALSE;
 489
 490         if (mali_group_is_virtual(group)) {
 491                 /*
 492                  * What we do for physical jobs in this function should
 493                  * already have been done in mali_group_deactivate()
 494                  * for virtual group.
 495                  */
 496                 MALI_DEBUG_ASSERT(NULL == group->session);
 497         } else {
 498                 mali_group_clear_session(group);
 499         }
 500 }
 501
 502 MALI_DEBUG_CODE(static void mali_group_print_virtual(struct mali_group *vgroup)
 503 {
 504         u32 i;
 505         struct mali_group *group;
 506         struct mali_group *temp;
 507
 508         MALI_DEBUG_PRINT(4, ("Virtual group %s (%p)\n",
 509                              mali_group_core_description(vgroup),
 510                              vgroup));
 511         MALI_DEBUG_PRINT(4, ("l2_cache_core[0] = %p, ref = %d\n", vgroup->l2_cache_core[0], vgroup->l2_cache_core_ref_count[0]));
 512         MALI_DEBUG_PRINT(4, ("l2_cache_core[1] = %p, ref = %d\n", vgroup->l2_cache_core[1], vgroup->l2_cache_core_ref_count[1]));
 513
 514         i = 0;
 515         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &vgroup->group_list, struct mali_group, group_list) {
 516                 MALI_DEBUG_PRINT(4, ("[%d] %s (%p), l2_cache_core[0] = %p\n",
 517                                      i, mali_group_core_description(group),
 518                                      group, group->l2_cache_core[0]));
 519                 i++;
 520         }
 521 })
 522
 523 static void mali_group_dump_core_status(struct mali_group *group)
 524 {
 525         u32 i;
 526
 527         MALI_DEBUG_ASSERT_POINTER(group);
 528         MALI_DEBUG_ASSERT(NULL != group->gp_core || (NULL != group->pp_core && !mali_group_is_virtual(group)));
 529
 530         if (NULL != group->gp_core) {
 531                 MALI_PRINT(("Dump Group %s\n", group->gp_core->hw_core.description));
 532
 533                 for (i = 0; i < 0xA8; i += 0x10) {
 534                         MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->gp_core->hw_core, i),
 535                                     mali_hw_core_register_read(&group->gp_core->hw_core, i + 4),
 536                                     mali_hw_core_register_read(&group->gp_core->hw_core, i + 8),
 537                                     mali_hw_core_register_read(&group->gp_core->hw_core, i + 12)));
 538                 }
 539
 540
 541         } else {
 542                 MALI_PRINT(("Dump Group %s\n", group->pp_core->hw_core.description));
 543
 544                 for (i = 0; i < 0x5c; i += 0x10) {
 545                         MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->pp_core->hw_core, i),
 546                                     mali_hw_core_register_read(&group->pp_core->hw_core, i + 4),
 547                                     mali_hw_core_register_read(&group->pp_core->hw_core, i + 8),
 548                                     mali_hw_core_register_read(&group->pp_core->hw_core, i + 12)));
 549                 }
 550
 551                 /* Ignore some minor registers */
 552                 for (i = 0x1000; i < 0x1068; i += 0x10) {
 553                         MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->pp_core->hw_core, i),
 554                                     mali_hw_core_register_read(&group->pp_core->hw_core, i + 4),
 555                                     mali_hw_core_register_read(&group->pp_core->hw_core, i + 8),
 556                                     mali_hw_core_register_read(&group->pp_core->hw_core, i + 12)));
 557                 }
 558         }
 559
 560         MALI_PRINT(("Dump Group MMU\n"));
 561         for (i = 0; i < 0x24; i += 0x10) {
 562                 MALI_PRINT(("0x%04x: 0x%08x 0x%08x 0x%08x 0x%08x\n", i, mali_hw_core_register_read(&group->mmu->hw_core, i),
 563                             mali_hw_core_register_read(&group->mmu->hw_core, i + 4),
 564                             mali_hw_core_register_read(&group->mmu->hw_core, i + 8),
 565                             mali_hw_core_register_read(&group->mmu->hw_core, i + 12)));
 566         }
 567 }
 568
 569
 570 /**
 571  * @Dump group status
 572  */
 573 void mali_group_dump_status(struct mali_group *group)
 574 {
 575         MALI_DEBUG_ASSERT_POINTER(group);
 576
 577         if (mali_group_is_virtual(group)) {
 578                 struct mali_group *group_c;
 579                 struct mali_group *temp;
 580                 _MALI_OSK_LIST_FOREACHENTRY(group_c, temp, &group->group_list, struct mali_group, group_list) {
 581                         mali_group_dump_core_status(group_c);
 582                 }
 583         } else {
 584                 mali_group_dump_core_status(group);
 585         }
 586 }
 587
 588 /**
 589  * @brief Add child group to virtual group parent
 590  */
 591 void mali_group_add_group(struct mali_group *parent, struct mali_group *child)
 592 {
 593         mali_bool found;
 594         u32 i;
 595
 596         MALI_DEBUG_PRINT(3, ("Adding group %s to virtual group %s\n",
 597                              mali_group_core_description(child),
 598                              mali_group_core_description(parent)));
 599
 600         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
 601         MALI_DEBUG_ASSERT(mali_group_is_virtual(parent));
 602         MALI_DEBUG_ASSERT(!mali_group_is_virtual(child));
 603         MALI_DEBUG_ASSERT(NULL == child->parent_group);
 604
 605         _mali_osk_list_addtail(&child->group_list, &parent->group_list);
 606
 607         child->parent_group = parent;
 608
 609         MALI_DEBUG_ASSERT_POINTER(child->l2_cache_core[0]);
 610
 611         MALI_DEBUG_PRINT(4, ("parent->l2_cache_core: [0] = %p, [1] = %p\n", parent->l2_cache_core[0], parent->l2_cache_core[1]));
 612         MALI_DEBUG_PRINT(4, ("child->l2_cache_core: [0] = %p, [1] = %p\n", child->l2_cache_core[0], child->l2_cache_core[1]));
 613
 614         /* Keep track of the L2 cache cores of child groups */
 615         found = MALI_FALSE;
 616         for (i = 0; i < 2; i++) {
 617                 if (parent->l2_cache_core[i] == child->l2_cache_core[0]) {
 618                         MALI_DEBUG_ASSERT(parent->l2_cache_core_ref_count[i] > 0);
 619                         parent->l2_cache_core_ref_count[i]++;
 620                         found = MALI_TRUE;
 621                 }
 622         }
 623
 624         if (!found) {
 625                 /* First time we see this L2 cache, add it to our list */
 626                 i = (NULL == parent->l2_cache_core[0]) ? 0 : 1;
 627
 628                 MALI_DEBUG_PRINT(4, ("First time we see l2_cache %p. Adding to [%d] = %p\n", child->l2_cache_core[0], i, parent->l2_cache_core[i]));
 629
 630                 MALI_DEBUG_ASSERT(NULL == parent->l2_cache_core[i]);
 631
 632                 parent->l2_cache_core[i] = child->l2_cache_core[0];
 633                 parent->l2_cache_core_ref_count[i]++;
 634         }
 635
 636         /* Update Broadcast Unit and DLBU */
 637         mali_bcast_add_group(parent->bcast_core, child);
 638         mali_dlbu_add_group(parent->dlbu_core, child);
 639
 640         if (MALI_TRUE == parent->power_is_on) {
 641                 mali_bcast_reset(parent->bcast_core);
 642                 mali_dlbu_update_mask(parent->dlbu_core);
 643         }
 644
 645         if (MALI_TRUE == child->power_is_on) {
 646                 if (NULL == parent->session) {
 647                         if (NULL != child->session) {
 648                                 /*
 649                                  * Parent has no session, so clear
 650                                  * child session as well.
 651                                  */
 652                                 mali_mmu_activate_empty_page_directory(child->mmu);
 653                         }
 654                 } else {
 655                         if (parent->session == child->session) {
 656                                 /* We already have same session as parent,
 657                                  * so a simple zap should be enough.
 658                                  */
 659                                 mali_mmu_zap_tlb(child->mmu);
 660                         } else {
 661                                 /*
 662                                  * Parent has a different session, so we must
 663                                  * switch to that sessions page table
 664                                  */
 665                                 mali_mmu_activate_page_directory(child->mmu, mali_session_get_page_directory(parent->session));
 666                         }
 667
 668                         /* It is the parent which keeps the session from now on */
 669                         child->session = NULL;
 670                 }
 671         } else {
 672                 /* should have been cleared when child was powered down */
 673                 MALI_DEBUG_ASSERT(NULL == child->session);
 674         }
 675
 676         /* Start job on child when parent is active */
 677         if (NULL != parent->pp_running_job) {
 678                 struct mali_pp_job *job = parent->pp_running_job;
 679
 680                 MALI_DEBUG_PRINT(3, ("Group %x joining running job %d on virtual group %x\n",
 681                                      child, mali_pp_job_get_id(job), parent));
 682
 683                 /* Only allowed to add active child to an active parent */
 684                 MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVE == parent->state);
 685                 MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVE == child->state);
 686
 687                 mali_pp_job_start(child->pp_core, job, mali_pp_core_get_id(child->pp_core), MALI_TRUE);
 688
 689                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 690                                               MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) |
 691                                               MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH,
 692                                               mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job), 0, 0, 0);
 693
 694                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START |
 695                                               MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) |
 696                                               MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL,
 697                                               mali_pp_job_get_pid(job), mali_pp_job_get_tid(job), 0, 0, 0);
 698 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
 699                 if (child->pp_core) {
 700                         trace_gpu_sched_switch(
 701                                 mali_pp_core_description(child->pp_core),
 702                                 sched_clock(), mali_pp_job_get_tid(job),
 703                                 0, mali_pp_job_get_id(job));
 704                 }
 705 #endif
 706
 707 #if defined(CONFIG_MALI400_PROFILING)
 708                 trace_mali_core_active(mali_pp_job_get_pid(job), 1 /* active */, 0 /* PP */, mali_pp_core_get_id(child->pp_core),
 709                                        mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job));
 710 #endif
 711         }
 712
 713         MALI_DEBUG_CODE(mali_group_print_virtual(parent);)
 714 }
 715
 716 /**
 717  * @brief Remove child group from virtual group parent
 718  */
 719 void mali_group_remove_group(struct mali_group *parent, struct mali_group *child)
 720 {
 721         u32 i;
 722
 723         MALI_DEBUG_PRINT(3, ("Removing group %s from virtual group %s\n",
 724                              mali_group_core_description(child),
 725                              mali_group_core_description(parent)));
 726
 727         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
 728         MALI_DEBUG_ASSERT(mali_group_is_virtual(parent));
 729         MALI_DEBUG_ASSERT(!mali_group_is_virtual(child));
 730         MALI_DEBUG_ASSERT(parent == child->parent_group);
 731
 732         /* Update Broadcast Unit and DLBU */
 733         mali_bcast_remove_group(parent->bcast_core, child);
 734         mali_dlbu_remove_group(parent->dlbu_core, child);
 735
 736         if (MALI_TRUE == parent->power_is_on) {
 737                 mali_bcast_reset(parent->bcast_core);
 738                 mali_dlbu_update_mask(parent->dlbu_core);
 739         }
 740
 741         child->session = parent->session;
 742         child->parent_group = NULL;
 743
 744         _mali_osk_list_delinit(&child->group_list);
 745         if (_mali_osk_list_empty(&parent->group_list)) {
 746                 parent->session = NULL;
 747         }
 748
 749         /* Keep track of the L2 cache cores of child groups */
 750         i = (child->l2_cache_core[0] == parent->l2_cache_core[0]) ? 0 : 1;
 751
 752         MALI_DEBUG_ASSERT(child->l2_cache_core[0] == parent->l2_cache_core[i]);
 753
 754         parent->l2_cache_core_ref_count[i]--;
 755         if (parent->l2_cache_core_ref_count[i] == 0) {
 756                 parent->l2_cache_core[i] = NULL;
 757         }
 758
 759         MALI_DEBUG_CODE(mali_group_print_virtual(parent));
 760 }
 761
 762 struct mali_group *mali_group_acquire_group(struct mali_group *parent)
 763 {
 764         struct mali_group *child = NULL;
 765
 766         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
 767         MALI_DEBUG_ASSERT(mali_group_is_virtual(parent));
 768
 769         if (!_mali_osk_list_empty(&parent->group_list)) {
 770                 child = _MALI_OSK_LIST_ENTRY(parent->group_list.prev, struct mali_group, group_list);
 771                 mali_group_remove_group(parent, child);
 772         }
 773
 774         if (NULL != child) {
 775                 if (MALI_GROUP_STATE_ACTIVE != parent->state
 776                     && MALI_TRUE == child->power_is_on) {
 777                         mali_group_reset(child);
 778                 }
 779         }
 780
 781         return child;
 782 }
 783
 784 void mali_group_reset(struct mali_group *group)
 785 {
 786         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
 787         MALI_DEBUG_ASSERT(NULL == group->gp_running_job);
 788         MALI_DEBUG_ASSERT(NULL == group->pp_running_job);
 789         MALI_DEBUG_ASSERT(NULL == group->session);
 790
 791         MALI_DEBUG_PRINT(3, ("Group: reset of %s\n",
 792                              mali_group_core_description(group)));
 793
 794         if (NULL != group->dlbu_core) {
 795                 mali_dlbu_reset(group->dlbu_core);
 796         }
 797
 798         if (NULL != group->bcast_core) {
 799                 mali_bcast_reset(group->bcast_core);
 800         }
 801
 802         MALI_DEBUG_ASSERT(NULL != group->mmu);
 803         mali_group_reset_mmu(group);
 804
 805         if (NULL != group->gp_core) {
 806                 MALI_DEBUG_ASSERT(NULL == group->pp_core);
 807                 mali_gp_reset(group->gp_core);
 808         } else {
 809                 MALI_DEBUG_ASSERT(NULL != group->pp_core);
 810                 mali_group_reset_pp(group);
 811         }
 812 }
 813
 814 void mali_group_start_gp_job(struct mali_group *group, struct mali_gp_job *job)
 815 {
 816         struct mali_session_data *session;
 817
 818         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
 819
 820         MALI_DEBUG_PRINT(3, ("Group: Starting GP job 0x%08X on group %s\n",
 821                              job,
 822                              mali_group_core_description(group)));
 823
 824         session = mali_gp_job_get_session(job);
 825
 826         MALI_DEBUG_ASSERT_POINTER(group->l2_cache_core[0]);
 827         mali_l2_cache_invalidate_conditional(group->l2_cache_core[0], mali_gp_job_get_cache_order(job));
 828
 829         mali_group_activate_page_directory(group, session);
 830
 831         mali_gp_job_start(group->gp_core, job);
 832
 833         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 834                                       MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0) |
 835                                       MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH,
 836                                       mali_gp_job_get_frame_builder_id(job), mali_gp_job_get_flush_id(job), 0, 0, 0);
 837         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START |
 838                                       MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0),
 839                                       mali_gp_job_get_pid(job), mali_gp_job_get_tid(job), 0, 0, 0);
 840
 841 #if defined(CONFIG_MALI400_PROFILING)
 842         trace_mali_core_active(mali_gp_job_get_pid(job), 1 /* active */, 1 /* GP */,  0 /* core */,
 843                                mali_gp_job_get_frame_builder_id(job), mali_gp_job_get_flush_id(job));
 844 #endif
 845
 846 #if defined(CONFIG_MALI400_PROFILING)
 847         if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) &&
 848             (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) {
 849                 mali_group_report_l2_cache_counters_per_core(group, 0);
 850         }
 851 #endif /* #if defined(CONFIG_MALI400_PROFILING) */
 852
 853 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
 854         if (group->gp_core) {
 855                 trace_gpu_sched_switch(mali_gp_core_description(group->gp_core),
 856                                        sched_clock(), mali_gp_job_get_tid(job),
 857                                        0, mali_gp_job_get_id(job));
 858         }
 859 #endif
 860
 861         group->gp_running_job = job;
 862         group->is_working = MALI_TRUE;
 863
 864         /* Setup SW timer and record start time */
 865         group->start_time = _mali_osk_time_tickcount();
 866         _mali_osk_timer_mod(group->timeout_timer, _mali_osk_time_mstoticks(mali_max_job_runtime));
 867
 868         MALI_DEBUG_PRINT(4, ("Group: Started GP job 0x%08X on group %s at %u\n",
 869                              job,
 870                              mali_group_core_description(group),
 871                              group->start_time));
 872 }
 873
 874 /* Used to set all the registers except frame renderer list address and fragment shader stack address
 875  * It means the caller must set these two registers properly before calling this function
 876  */
 877 void mali_group_start_pp_job(struct mali_group *group, struct mali_pp_job *job, u32 sub_job)
 878 {
 879         struct mali_session_data *session;
 880
 881         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
 882
 883         MALI_DEBUG_PRINT(3, ("Group: Starting PP job 0x%08X part %u/%u on group %s\n",
 884                              job, sub_job + 1,
 885                              mali_pp_job_get_sub_job_count(job),
 886                              mali_group_core_description(group)));
 887
 888         session = mali_pp_job_get_session(job);
 889
 890         if (NULL != group->l2_cache_core[0]) {
 891                 mali_l2_cache_invalidate_conditional(group->l2_cache_core[0], mali_pp_job_get_cache_order(job));
 892         }
 893
 894         if (NULL != group->l2_cache_core[1]) {
 895                 mali_l2_cache_invalidate_conditional(group->l2_cache_core[1], mali_pp_job_get_cache_order(job));
 896         }
 897
 898         mali_group_activate_page_directory(group, session);
 899
 900         if (mali_group_is_virtual(group)) {
 901                 struct mali_group *child;
 902                 struct mali_group *temp;
 903                 u32 core_num = 0;
 904
 905                 MALI_DEBUG_ASSERT(mali_pp_job_is_virtual(job));
 906
 907                 /* Configure DLBU for the job */
 908                 mali_dlbu_config_job(group->dlbu_core, job);
 909
 910                 /* Write stack address for each child group */
 911                 _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) {
 912                         mali_pp_write_addr_stack(child->pp_core, job);
 913                         core_num++;
 914                 }
 915
 916                 mali_pp_job_start(group->pp_core, job, sub_job, MALI_FALSE);
 917         } else {
 918                 mali_pp_job_start(group->pp_core, job, sub_job, MALI_FALSE);
 919         }
 920
 921         /* if the group is virtual, loop through physical groups which belong to this group
 922          * and call profiling events for its cores as virtual */
 923         if (MALI_TRUE == mali_group_is_virtual(group)) {
 924                 struct mali_group *child;
 925                 struct mali_group *temp;
 926
 927                 _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) {
 928                         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 929                                                       MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) |
 930                                                       MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH,
 931                                                       mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job), 0, 0, 0);
 932
 933                         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START |
 934                                                       MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) |
 935                                                       MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL,
 936                                                       mali_pp_job_get_pid(job), mali_pp_job_get_tid(job), 0, 0, 0);
 937
 938 #if defined(CONFIG_MALI400_PROFILING)
 939                         trace_mali_core_active(mali_pp_job_get_pid(job), 1 /* active */, 0 /* PP */, mali_pp_core_get_id(child->pp_core),
 940                                                mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job));
 941 #endif
 942                 }
 943
 944 #if defined(CONFIG_MALI400_PROFILING)
 945                 if (0 != group->l2_cache_core_ref_count[0]) {
 946                         if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) &&
 947                             (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) {
 948                                 mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0]));
 949                         }
 950                 }
 951                 if (0 != group->l2_cache_core_ref_count[1]) {
 952                         if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[1])) &&
 953                             (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[1]))) {
 954                                 mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[1]));
 955                         }
 956                 }
 957 #endif /* #if defined(CONFIG_MALI400_PROFILING) */
 958
 959         } else { /* group is physical - call profiling events for physical cores */
 960                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_SINGLE |
 961                                               MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(group->pp_core)) |
 962                                               MALI_PROFILING_EVENT_REASON_SINGLE_HW_FLUSH,
 963                                               mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job), 0, 0, 0);
 964
 965                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START |
 966                                               MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(group->pp_core)) |
 967                                               MALI_PROFILING_EVENT_REASON_START_STOP_HW_PHYSICAL,
 968                                               mali_pp_job_get_pid(job), mali_pp_job_get_tid(job), 0, 0, 0);
 969
 970 #if defined(CONFIG_MALI400_PROFILING)
 971                 trace_mali_core_active(mali_pp_job_get_pid(job), 1 /* active */, 0 /* PP */, mali_pp_core_get_id(group->pp_core),
 972                                        mali_pp_job_get_frame_builder_id(job), mali_pp_job_get_flush_id(job));
 973 #endif
 974
 975 #if defined(CONFIG_MALI400_PROFILING)
 976                 if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) &&
 977                     (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) {
 978                         mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0]));
 979                 }
 980 #endif /* #if defined(CONFIG_MALI400_PROFILING) */
 981         }
 982
 983 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
 984         if (group->pp_core) {
 985                 trace_gpu_sched_switch(mali_pp_core_description(group->pp_core),
 986                                        sched_clock(), mali_pp_job_get_tid(job),
 987                                        0, mali_pp_job_get_id(job));
 988         }
 989 #endif
 990
 991         group->pp_running_job = job;
 992         group->pp_running_sub_job = sub_job;
 993         group->is_working = MALI_TRUE;
 994
 995         /* Setup SW timer and record start time */
 996         group->start_time = _mali_osk_time_tickcount();
 997         _mali_osk_timer_mod(group->timeout_timer, _mali_osk_time_mstoticks(mali_max_job_runtime));
 998
 999         MALI_DEBUG_PRINT(4, ("Group: Started PP job 0x%08X part %u/%u on group %s at %u\n",
1000                              job, sub_job + 1,
1001                              mali_pp_job_get_sub_job_count(job),
1002                              mali_group_core_description(group),
1003                              group->start_time));
1004
1005 }
1006
1007 void mali_group_resume_gp_with_new_heap(struct mali_group *group, u32 job_id, u32 start_addr, u32 end_addr)
1008 {
1009         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1010
1011         MALI_DEBUG_ASSERT_POINTER(group->l2_cache_core[0]);
1012         mali_l2_cache_invalidate(group->l2_cache_core[0]);
1013
1014         mali_mmu_zap_tlb_without_stall(group->mmu);
1015
1016         mali_gp_resume_with_new_heap(group->gp_core, start_addr, end_addr);
1017
1018         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_RESUME |
1019                                       MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0),
1020                                       0, 0, 0, 0, 0);
1021
1022 #if defined(CONFIG_MALI400_PROFILING)
1023         trace_mali_core_active(mali_gp_job_get_pid(group->gp_running_job), 1 /* active */, 1 /* GP */,  0 /* core */,
1024                                mali_gp_job_get_frame_builder_id(group->gp_running_job), mali_gp_job_get_flush_id(group->gp_running_job));
1025 #endif
1026 }
1027
1028 static void mali_group_reset_mmu(struct mali_group *group)
1029 {
1030         struct mali_group *child;
1031         struct mali_group *temp;
1032         _mali_osk_errcode_t err;
1033
1034         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1035
1036         if (!mali_group_is_virtual(group)) {
1037                 /* This is a physical group or an idle virtual group -- simply wait for
1038                  * the reset to complete. */
1039                 err = mali_mmu_reset(group->mmu);
1040                 MALI_DEBUG_ASSERT(_MALI_OSK_ERR_OK == err);
1041         } else { /* virtual group */
1042                 /* Loop through all members of this virtual group and wait
1043                  * until they are done resetting.
1044                  */
1045                 _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) {
1046                         err = mali_mmu_reset(child->mmu);
1047                         MALI_DEBUG_ASSERT(_MALI_OSK_ERR_OK == err);
1048                 }
1049         }
1050 }
1051
1052 static void mali_group_reset_pp(struct mali_group *group)
1053 {
1054         struct mali_group *child;
1055         struct mali_group *temp;
1056
1057         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1058
1059         mali_pp_reset_async(group->pp_core);
1060
1061         if (!mali_group_is_virtual(group) || NULL == group->pp_running_job) {
1062                 /* This is a physical group or an idle virtual group -- simply wait for
1063                  * the reset to complete. */
1064                 mali_pp_reset_wait(group->pp_core);
1065         } else {
1066                 /* Loop through all members of this virtual group and wait until they
1067                  * are done resetting.
1068                  */
1069                 _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) {
1070                         mali_pp_reset_wait(child->pp_core);
1071                 }
1072         }
1073 }
1074
1075 struct mali_pp_job *mali_group_complete_pp(struct mali_group *group, mali_bool success, u32 *sub_job)
1076 {
1077         struct mali_pp_job *pp_job_to_return;
1078
1079         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1080         MALI_DEBUG_ASSERT_POINTER(group);
1081         MALI_DEBUG_ASSERT_POINTER(group->pp_core);
1082         MALI_DEBUG_ASSERT_POINTER(group->pp_running_job);
1083         MALI_DEBUG_ASSERT_POINTER(sub_job);
1084         MALI_DEBUG_ASSERT(MALI_TRUE == group->is_working);
1085
1086         /* Stop/clear the timeout timer. */
1087         _mali_osk_timer_del_async(group->timeout_timer);
1088
1089         if (NULL != group->pp_running_job) {
1090
1091                 /* Deal with HW counters and profiling */
1092
1093                 if (MALI_TRUE == mali_group_is_virtual(group)) {
1094                         struct mali_group *child;
1095                         struct mali_group *temp;
1096
1097                         /* update performance counters from each physical pp core within this virtual group */
1098                         _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) {
1099                                 mali_pp_update_performance_counters(group->pp_core, child->pp_core, group->pp_running_job, mali_pp_core_get_id(child->pp_core));
1100                         }
1101
1102 #if defined(CONFIG_MALI400_PROFILING)
1103                         /* send profiling data per physical core */
1104                         _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) {
1105                                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1106                                                               MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(child->pp_core)) |
1107                                                               MALI_PROFILING_EVENT_REASON_START_STOP_HW_VIRTUAL,
1108                                                               mali_pp_job_get_perf_counter_value0(group->pp_running_job, mali_pp_core_get_id(child->pp_core)),
1109                                                               mali_pp_job_get_perf_counter_value1(group->pp_running_job, mali_pp_core_get_id(child->pp_core)),
1110                                                               mali_pp_job_get_perf_counter_src0(group->pp_running_job, group->pp_running_sub_job) | (mali_pp_job_get_perf_counter_src1(group->pp_running_job, group->pp_running_sub_job) << 8),
1111                                                               0, 0);
1112
1113                                 trace_mali_core_active(mali_pp_job_get_pid(group->pp_running_job),
1114                                                        0 /* active */, 0 /* PP */, mali_pp_core_get_id(child->pp_core),
1115                                                        mali_pp_job_get_frame_builder_id(group->pp_running_job),
1116                                                        mali_pp_job_get_flush_id(group->pp_running_job));
1117                         }
1118                         if (0 != group->l2_cache_core_ref_count[0]) {
1119                                 if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) &&
1120                                     (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) {
1121                                         mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0]));
1122                                 }
1123                         }
1124                         if (0 != group->l2_cache_core_ref_count[1]) {
1125                                 if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[1])) &&
1126                                     (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[1]))) {
1127                                         mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[1]));
1128                                 }
1129                         }
1130
1131 #endif
1132                 } else {
1133                         /* update performance counters for a physical group's pp core */
1134                         mali_pp_update_performance_counters(group->pp_core, group->pp_core, group->pp_running_job, group->pp_running_sub_job);
1135
1136 #if defined(CONFIG_MALI400_PROFILING)
1137                         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1138                                                       MALI_PROFILING_MAKE_EVENT_CHANNEL_PP(mali_pp_core_get_id(group->pp_core)) |
1139                                                       MALI_PROFILING_EVENT_REASON_START_STOP_HW_PHYSICAL,
1140                                                       mali_pp_job_get_perf_counter_value0(group->pp_running_job, group->pp_running_sub_job),
1141                                                       mali_pp_job_get_perf_counter_value1(group->pp_running_job, group->pp_running_sub_job),
1142                                                       mali_pp_job_get_perf_counter_src0(group->pp_running_job, group->pp_running_sub_job) | (mali_pp_job_get_perf_counter_src1(group->pp_running_job, group->pp_running_sub_job) << 8),
1143                                                       0, 0);
1144
1145                         trace_mali_core_active(mali_pp_job_get_pid(group->pp_running_job),
1146                                                0 /* active */, 0 /* PP */, mali_pp_core_get_id(group->pp_core),
1147                                                mali_pp_job_get_frame_builder_id(group->pp_running_job),
1148                                                mali_pp_job_get_flush_id(group->pp_running_job));
1149
1150                         if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) &&
1151                             (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0]))) {
1152                                 mali_group_report_l2_cache_counters_per_core(group, mali_l2_cache_get_id(group->l2_cache_core[0]));
1153                         }
1154 #endif
1155                 }
1156
1157 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
1158                 if (group->gp_core) {
1159                         trace_gpu_sched_switch(
1160                                 mali_gp_core_description(group->gp_core),
1161                                 sched_clock(), 0, 0, 0);
1162                 }
1163 #endif
1164
1165         }
1166
1167         if (success) {
1168                 /* Only do soft reset for successful jobs, a full recovery
1169                  * reset will be done for failed jobs. */
1170                 mali_pp_reset_async(group->pp_core);
1171         }
1172
1173         pp_job_to_return = group->pp_running_job;
1174         group->pp_running_job = NULL;
1175         group->is_working = MALI_FALSE;
1176         *sub_job = group->pp_running_sub_job;
1177
1178         if (!success) {
1179                 MALI_DEBUG_PRINT(2, ("Mali group: Executing recovery reset due to job failure\n"));
1180                 mali_group_recovery_reset(group);
1181         } else if (_MALI_OSK_ERR_OK != mali_pp_reset_wait(group->pp_core)) {
1182                 MALI_PRINT_ERROR(("Mali group: Executing recovery reset due to reset failure\n"));
1183                 mali_group_recovery_reset(group);
1184         }
1185
1186         return pp_job_to_return;
1187 }
1188
1189 struct mali_gp_job *mali_group_complete_gp(struct mali_group *group, mali_bool success)
1190 {
1191         struct mali_gp_job *gp_job_to_return;
1192
1193         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1194         MALI_DEBUG_ASSERT_POINTER(group);
1195         MALI_DEBUG_ASSERT_POINTER(group->gp_core);
1196         MALI_DEBUG_ASSERT_POINTER(group->gp_running_job);
1197         MALI_DEBUG_ASSERT(MALI_TRUE == group->is_working);
1198
1199         /* Stop/clear the timeout timer. */
1200         _mali_osk_timer_del_async(group->timeout_timer);
1201
1202         if (NULL != group->gp_running_job) {
1203                 mali_gp_update_performance_counters(group->gp_core, group->gp_running_job);
1204
1205 #if defined(CONFIG_MALI400_PROFILING)
1206                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP | MALI_PROFILING_MAKE_EVENT_CHANNEL_GP(0),
1207                                               mali_gp_job_get_perf_counter_value0(group->gp_running_job),
1208                                               mali_gp_job_get_perf_counter_value1(group->gp_running_job),
1209                                               mali_gp_job_get_perf_counter_src0(group->gp_running_job) | (mali_gp_job_get_perf_counter_src1(group->gp_running_job) << 8),
1210                                               0, 0);
1211
1212                 if ((MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src0(group->l2_cache_core[0])) &&
1213                     (MALI_HW_CORE_NO_COUNTER != mali_l2_cache_core_get_counter_src1(group->l2_cache_core[0])))
1214                         mali_group_report_l2_cache_counters_per_core(group, 0);
1215 #endif
1216
1217 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
1218                 if (group->pp_core) {
1219                         trace_gpu_sched_switch(
1220                                 mali_pp_core_description(group->pp_core),
1221                                 sched_clock(), 0, 0, 0);
1222                 }
1223 #endif
1224
1225 #if defined(CONFIG_MALI400_PROFILING)
1226                 trace_mali_core_active(mali_gp_job_get_pid(group->gp_running_job), 0 /* active */, 1 /* GP */,  0 /* core */,
1227                                        mali_gp_job_get_frame_builder_id(group->gp_running_job), mali_gp_job_get_flush_id(group->gp_running_job));
1228 #endif
1229
1230                 mali_gp_job_set_current_heap_addr(group->gp_running_job,
1231                                                   mali_gp_read_plbu_alloc_start_addr(group->gp_core));
1232         }
1233
1234         if (success) {
1235                 /* Only do soft reset for successful jobs, a full recovery
1236                  * reset will be done for failed jobs. */
1237                 mali_gp_reset_async(group->gp_core);
1238         }
1239
1240         gp_job_to_return = group->gp_running_job;
1241         group->gp_running_job = NULL;
1242         group->is_working = MALI_FALSE;
1243
1244         if (!success) {
1245                 MALI_DEBUG_PRINT(2, ("Mali group: Executing recovery reset due to job failure\n"));
1246                 mali_group_recovery_reset(group);
1247         } else if (_MALI_OSK_ERR_OK != mali_gp_reset_wait(group->gp_core)) {
1248                 MALI_PRINT_ERROR(("Mali group: Executing recovery reset due to reset failure\n"));
1249                 mali_group_recovery_reset(group);
1250         }
1251
1252         return gp_job_to_return;
1253 }
1254
1255 struct mali_group *mali_group_get_glob_group(u32 index)
1256 {
1257         if (mali_global_num_groups > index) {
1258                 return mali_global_groups[index];
1259         }
1260
1261         return NULL;
1262 }
1263
1264 u32 mali_group_get_glob_num_groups(void)
1265 {
1266         return mali_global_num_groups;
1267 }
1268
1269 static void mali_group_activate_page_directory(struct mali_group *group, struct mali_session_data *session)
1270 {
1271         MALI_DEBUG_PRINT(5, ("Mali group: Activating page directory 0x%08X from session 0x%08X on group %s\n",
1272                              mali_session_get_page_directory(session), session,
1273                              mali_group_core_description(group)));
1274
1275         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1276
1277         if (group->session != session) {
1278                 /* Different session than last time, so we need to do some work */
1279                 MALI_DEBUG_PRINT(5, ("Mali group: Activate session: %08x previous: %08x on group %s\n",
1280                                      session, group->session,
1281                                      mali_group_core_description(group)));
1282                 mali_mmu_activate_page_directory(group->mmu, mali_session_get_page_directory(session));
1283                 group->session = session;
1284         } else {
1285                 /* Same session as last time, so no work required */
1286                 MALI_DEBUG_PRINT(4, ("Mali group: Activate existing session 0x%08X on group %s\n",
1287                                      session->page_directory,
1288                                      mali_group_core_description(group)));
1289                 mali_mmu_zap_tlb_without_stall(group->mmu);
1290         }
1291 }
1292
1293 static void mali_group_recovery_reset(struct mali_group *group)
1294 {
1295         _mali_osk_errcode_t err;
1296
1297         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1298
1299         /* Stop cores, bus stop */
1300         if (NULL != group->pp_core) {
1301                 mali_pp_stop_bus(group->pp_core);
1302         } else {
1303                 mali_gp_stop_bus(group->gp_core);
1304         }
1305
1306         /* Flush MMU and clear page fault (if any) */
1307         mali_mmu_activate_fault_flush_page_directory(group->mmu);
1308         mali_mmu_page_fault_done(group->mmu);
1309
1310         /* Wait for cores to stop bus, then do a hard reset on them */
1311         if (NULL != group->pp_core) {
1312                 if (mali_group_is_virtual(group)) {
1313                         struct mali_group *child, *temp;
1314
1315                         /* Disable the broadcast unit while we do reset directly on the member cores. */
1316                         mali_bcast_disable(group->bcast_core);
1317
1318                         _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) {
1319                                 mali_pp_stop_bus_wait(child->pp_core);
1320                                 mali_pp_hard_reset(child->pp_core);
1321                         }
1322
1323                         mali_bcast_enable(group->bcast_core);
1324                 } else {
1325                         mali_pp_stop_bus_wait(group->pp_core);
1326                         mali_pp_hard_reset(group->pp_core);
1327                 }
1328         } else {
1329                 mali_gp_stop_bus_wait(group->gp_core);
1330                 mali_gp_hard_reset(group->gp_core);
1331         }
1332
1333         /* Reset MMU */
1334         err = mali_mmu_reset(group->mmu);
1335         MALI_DEBUG_ASSERT(_MALI_OSK_ERR_OK == err);
1336         MALI_IGNORE(err);
1337
1338         group->session = NULL;
1339 }
1340
1341 #if MALI_STATE_TRACKING
1342 u32 mali_group_dump_state(struct mali_group *group, char *buf, u32 size)
1343 {
1344         int n = 0;
1345         int i;
1346         struct mali_group *child;
1347         struct mali_group *temp;
1348
1349         if (mali_group_is_virtual(group)) {
1350                 n += _mali_osk_snprintf(buf + n, size - n,
1351                                         "Virtual PP Group: %p\n", group);
1352         } else if (mali_group_is_in_virtual(group)) {
1353                 n += _mali_osk_snprintf(buf + n, size - n,
1354                                         "Child PP Group: %p\n", group);
1355         } else if (NULL != group->pp_core) {
1356                 n += _mali_osk_snprintf(buf + n, size - n,
1357                                         "Physical PP Group: %p\n", group);
1358         } else {
1359                 MALI_DEBUG_ASSERT_POINTER(group->gp_core);
1360                 n += _mali_osk_snprintf(buf + n, size - n,
1361                                         "GP Group: %p\n", group);
1362         }
1363
1364         switch (group->state) {
1365         case MALI_GROUP_STATE_INACTIVE:
1366                 n += _mali_osk_snprintf(buf + n, size - n,
1367                                         "\tstate: INACTIVE\n");
1368                 break;
1369         case MALI_GROUP_STATE_ACTIVATION_PENDING:
1370                 n += _mali_osk_snprintf(buf + n, size - n,
1371                                         "\tstate: ACTIVATION_PENDING\n");
1372                 break;
1373         case MALI_GROUP_STATE_ACTIVE:
1374                 n += _mali_osk_snprintf(buf + n, size - n,
1375                                         "\tstate: MALI_GROUP_STATE_ACTIVE\n");
1376                 break;
1377         default:
1378                 n += _mali_osk_snprintf(buf + n, size - n,
1379                                         "\tstate: UNKNOWN (%d)\n", group->state);
1380                 MALI_DEBUG_ASSERT(0);
1381                 break;
1382         }
1383
1384         n += _mali_osk_snprintf(buf + n, size - n,
1385                                 "\tSW power: %s\n",
1386                                 group->power_is_on ? "On" : "Off");
1387
1388         n += mali_pm_dump_state_domain(group->pm_domain, buf + n, size - n);
1389
1390         for (i = 0; i < 2; i++) {
1391                 if (NULL != group->l2_cache_core[i]) {
1392                         struct mali_pm_domain *domain;
1393                         domain = mali_l2_cache_get_pm_domain(
1394                                          group->l2_cache_core[i]);
1395                         n += mali_pm_dump_state_domain(domain,
1396                                                        buf + n, size - n);
1397                 }
1398         }
1399
1400         if (group->gp_core) {
1401                 n += mali_gp_dump_state(group->gp_core, buf + n, size - n);
1402                 n += _mali_osk_snprintf(buf + n, size - n,
1403                                         "\tGP running job: %p\n", group->gp_running_job);
1404         }
1405
1406         if (group->pp_core) {
1407                 n += mali_pp_dump_state(group->pp_core, buf + n, size - n);
1408                 n += _mali_osk_snprintf(buf + n, size - n,
1409                                         "\tPP running job: %p, subjob %d \n",
1410                                         group->pp_running_job,
1411                                         group->pp_running_sub_job);
1412         }
1413
1414         _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list,
1415                                     struct mali_group, group_list) {
1416                 n += mali_group_dump_state(child, buf + n, size - n);
1417         }
1418
1419         return n;
1420 }
1421 #endif
1422
1423 _mali_osk_errcode_t mali_group_upper_half_mmu(void *data)
1424 {
1425         struct mali_group *group = (struct mali_group *)data;
1426         _mali_osk_errcode_t ret;
1427
1428         MALI_DEBUG_ASSERT_POINTER(group);
1429         MALI_DEBUG_ASSERT_POINTER(group->mmu);
1430
1431 #if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS)
1432 #if defined(CONFIG_MALI_SHARED_INTERRUPTS)
1433         mali_executor_lock();
1434         if (!mali_group_is_working(group)) {
1435                 /* Not working, so nothing to do */
1436                 mali_executor_unlock();
1437                 return _MALI_OSK_ERR_FAULT;
1438         }
1439 #endif
1440         if (NULL != group->gp_core) {
1441                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START |
1442                                               MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1443                                               MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF,
1444                                               0, 0, /* No pid and tid for interrupt handler */
1445                                               MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(0),
1446                                               mali_mmu_get_rawstat(group->mmu), 0);
1447         } else {
1448                 MALI_DEBUG_ASSERT_POINTER(group->pp_core);
1449                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START |
1450                                               MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1451                                               MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF,
1452                                               0, 0, /* No pid and tid for interrupt handler */
1453                                               MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU(
1454                                                       mali_pp_core_get_id(group->pp_core)),
1455                                               mali_mmu_get_rawstat(group->mmu), 0);
1456         }
1457 #if defined(CONFIG_MALI_SHARED_INTERRUPTS)
1458         mali_executor_unlock();
1459 #endif
1460 #endif
1461
1462         ret = mali_executor_interrupt_mmu(group, MALI_TRUE);
1463
1464 #if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS)
1465 #if defined(CONFIG_MALI_SHARED_INTERRUPTS)
1466         mali_executor_lock();
1467         if (!mali_group_is_working(group) && (!mali_group_power_is_on(group))) {
1468                 /* group complete and on job shedule on it, it already power off */
1469                 if (NULL != group->gp_core) {
1470                         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1471                                                       MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1472                                                       MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF,
1473                                                       0, 0, /* No pid and tid for interrupt handler */
1474                                                       MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(0),
1475                                                       0xFFFFFFFF, 0);
1476                 } else {
1477                         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1478                                                       MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1479                                                       MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF,
1480                                                       0, 0, /* No pid and tid for interrupt handler */
1481                                                       MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU(
1482                                                               mali_pp_core_get_id(group->pp_core)),
1483                                                       0xFFFFFFFF, 0);
1484                 }
1485
1486                 mali_executor_unlock();
1487                 return ret;
1488         }
1489 #endif
1490
1491         if (NULL != group->gp_core) {
1492                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1493                                               MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1494                                               MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF,
1495                                               0, 0, /* No pid and tid for interrupt handler */
1496                                               MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(0),
1497                                               mali_mmu_get_rawstat(group->mmu), 0);
1498         } else {
1499                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1500                                               MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1501                                               MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF,
1502                                               0, 0, /* No pid and tid for interrupt handler */
1503                                               MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU(
1504                                                       mali_pp_core_get_id(group->pp_core)),
1505                                               mali_mmu_get_rawstat(group->mmu), 0);
1506         }
1507 #if defined(CONFIG_MALI_SHARED_INTERRUPTS)
1508         mali_executor_unlock();
1509 #endif
1510 #endif
1511
1512         return ret;
1513 }
1514
1515 static void mali_group_bottom_half_mmu(void *data)
1516 {
1517         struct mali_group *group = (struct mali_group *)data;
1518
1519         MALI_DEBUG_ASSERT_POINTER(group);
1520         MALI_DEBUG_ASSERT_POINTER(group->mmu);
1521
1522         if (NULL != group->gp_core) {
1523                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START |
1524                                               MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1525                                               MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF,
1526                                               0, _mali_osk_get_tid(), /* pid and tid */
1527                                               MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(0),
1528                                               mali_mmu_get_rawstat(group->mmu), 0);
1529         } else {
1530                 MALI_DEBUG_ASSERT_POINTER(group->pp_core);
1531                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START |
1532                                               MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1533                                               MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF,
1534                                               0, _mali_osk_get_tid(), /* pid and tid */
1535                                               MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU(
1536                                                       mali_pp_core_get_id(group->pp_core)),
1537                                               mali_mmu_get_rawstat(group->mmu), 0);
1538         }
1539
1540         mali_executor_interrupt_mmu(group, MALI_FALSE);
1541
1542         if (NULL != group->gp_core) {
1543                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1544                                               MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1545                                               MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF,
1546                                               0, _mali_osk_get_tid(), /* pid and tid */
1547                                               MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP_MMU(0),
1548                                               mali_mmu_get_rawstat(group->mmu), 0);
1549         } else {
1550                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1551                                               MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1552                                               MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF,
1553                                               0, _mali_osk_get_tid(), /* pid and tid */
1554                                               MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP_MMU(
1555                                                       mali_pp_core_get_id(group->pp_core)),
1556                                               mali_mmu_get_rawstat(group->mmu), 0);
1557         }
1558 }
1559
1560 _mali_osk_errcode_t mali_group_upper_half_gp(void *data)
1561 {
1562         struct mali_group *group = (struct mali_group *)data;
1563         _mali_osk_errcode_t ret;
1564
1565         MALI_DEBUG_ASSERT_POINTER(group);
1566         MALI_DEBUG_ASSERT_POINTER(group->gp_core);
1567         MALI_DEBUG_ASSERT_POINTER(group->mmu);
1568
1569 #if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS)
1570 #if defined(CONFIG_MALI_SHARED_INTERRUPTS)
1571         mali_executor_lock();
1572         if (!mali_group_is_working(group)) {
1573                 /* Not working, so nothing to do */
1574                 mali_executor_unlock();
1575                 return _MALI_OSK_ERR_FAULT;
1576         }
1577 #endif
1578         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START |
1579                                       MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1580                                       MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF,
1581                                       0, 0, /* No pid and tid for interrupt handler */
1582                                       MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0),
1583                                       mali_gp_get_rawstat(group->gp_core), 0);
1584
1585         MALI_DEBUG_PRINT(4, ("Group: Interrupt 0x%08X from %s\n",
1586                              mali_gp_get_rawstat(group->gp_core),
1587                              mali_group_core_description(group)));
1588 #if defined(CONFIG_MALI_SHARED_INTERRUPTS)
1589         mali_executor_unlock();
1590 #endif
1591 #endif
1592         ret = mali_executor_interrupt_gp(group, MALI_TRUE);
1593
1594 #if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS)
1595 #if defined(CONFIG_MALI_SHARED_INTERRUPTS)
1596         mali_executor_lock();
1597         if (!mali_group_is_working(group) && (!mali_group_power_is_on(group))) {
1598                 /* group complete and on job shedule on it, it already power off */
1599                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1600                                               MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1601                                               MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF,
1602                                               0, 0, /* No pid and tid for interrupt handler */
1603                                               MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0),
1604                                               0xFFFFFFFF, 0);
1605                 mali_executor_unlock();
1606                 return ret;
1607         }
1608 #endif
1609         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1610                                       MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1611                                       MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF,
1612                                       0, 0, /* No pid and tid for interrupt handler */
1613                                       MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0),
1614                                       mali_gp_get_rawstat(group->gp_core), 0);
1615 #if defined(CONFIG_MALI_SHARED_INTERRUPTS)
1616         mali_executor_unlock();
1617 #endif
1618 #endif
1619         return ret;
1620 }
1621
1622 static void mali_group_bottom_half_gp(void *data)
1623 {
1624         struct mali_group *group = (struct mali_group *)data;
1625
1626         MALI_DEBUG_ASSERT_POINTER(group);
1627         MALI_DEBUG_ASSERT_POINTER(group->gp_core);
1628         MALI_DEBUG_ASSERT_POINTER(group->mmu);
1629
1630         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START |
1631                                       MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1632                                       MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF,
1633                                       0, _mali_osk_get_tid(), /* pid and tid */
1634                                       MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0),
1635                                       mali_gp_get_rawstat(group->gp_core), 0);
1636
1637         mali_executor_interrupt_gp(group, MALI_FALSE);
1638
1639         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1640                                       MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1641                                       MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF,
1642                                       0, _mali_osk_get_tid(), /* pid and tid */
1643                                       MALI_PROFILING_MAKE_EVENT_DATA_CORE_GP(0),
1644                                       mali_gp_get_rawstat(group->gp_core), 0);
1645 }
1646
1647 _mali_osk_errcode_t mali_group_upper_half_pp(void *data)
1648 {
1649         struct mali_group *group = (struct mali_group *)data;
1650         _mali_osk_errcode_t ret;
1651
1652         MALI_DEBUG_ASSERT_POINTER(group);
1653         MALI_DEBUG_ASSERT_POINTER(group->pp_core);
1654         MALI_DEBUG_ASSERT_POINTER(group->mmu);
1655
1656 #if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS)
1657 #if defined(CONFIG_MALI_SHARED_INTERRUPTS)
1658         mali_executor_lock();
1659         if (!mali_group_is_working(group)) {
1660                 /* Not working, so nothing to do */
1661                 mali_executor_unlock();
1662                 return _MALI_OSK_ERR_FAULT;
1663         }
1664 #endif
1665
1666         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START |
1667                                       MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1668                                       MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF,
1669                                       0, 0, /* No pid and tid for interrupt handler */
1670                                       MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP(
1671                                               mali_pp_core_get_id(group->pp_core)),
1672                                       mali_pp_get_rawstat(group->pp_core), 0);
1673
1674         MALI_DEBUG_PRINT(4, ("Group: Interrupt 0x%08X from %s\n",
1675                              mali_pp_get_rawstat(group->pp_core),
1676                              mali_group_core_description(group)));
1677 #if defined(CONFIG_MALI_SHARED_INTERRUPTS)
1678         mali_executor_unlock();
1679 #endif
1680 #endif
1681
1682         ret = mali_executor_interrupt_pp(group, MALI_TRUE);
1683
1684 #if defined(CONFIG_MALI400_PROFILING) && defined (CONFIG_TRACEPOINTS)
1685 #if defined(CONFIG_MALI_SHARED_INTERRUPTS)
1686         mali_executor_lock();
1687         if (!mali_group_is_working(group) && (!mali_group_power_is_on(group))) {
1688                 /* group complete and on job shedule on it, it already power off */
1689                 _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1690                                               MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1691                                               MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF,
1692                                               0, 0, /* No pid and tid for interrupt handler */
1693                                               MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP(
1694                                                       mali_pp_core_get_id(group->pp_core)),
1695                                               0xFFFFFFFF, 0);
1696                 mali_executor_unlock();
1697                 return ret;
1698         }
1699 #endif
1700         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1701                                       MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1702                                       MALI_PROFILING_EVENT_REASON_START_STOP_SW_UPPER_HALF,
1703                                       0, 0, /* No pid and tid for interrupt handler */
1704                                       MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP(
1705                                               mali_pp_core_get_id(group->pp_core)),
1706                                       mali_pp_get_rawstat(group->pp_core), 0);
1707 #if defined(CONFIG_MALI_SHARED_INTERRUPTS)
1708         mali_executor_unlock();
1709 #endif
1710 #endif
1711         return ret;
1712 }
1713
1714 static void mali_group_bottom_half_pp(void *data)
1715 {
1716         struct mali_group *group = (struct mali_group *)data;
1717
1718         MALI_DEBUG_ASSERT_POINTER(group);
1719         MALI_DEBUG_ASSERT_POINTER(group->pp_core);
1720         MALI_DEBUG_ASSERT_POINTER(group->mmu);
1721
1722         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_START |
1723                                       MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1724                                       MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF,
1725                                       0, _mali_osk_get_tid(), /* pid and tid */
1726                                       MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP(
1727                                               mali_pp_core_get_id(group->pp_core)),
1728                                       mali_pp_get_rawstat(group->pp_core), 0);
1729
1730         mali_executor_interrupt_pp(group, MALI_FALSE);
1731
1732         _mali_osk_profiling_add_event(MALI_PROFILING_EVENT_TYPE_STOP |
1733                                       MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1734                                       MALI_PROFILING_EVENT_REASON_START_STOP_SW_BOTTOM_HALF,
1735                                       0, _mali_osk_get_tid(), /* pid and tid */
1736                                       MALI_PROFILING_MAKE_EVENT_DATA_CORE_PP(
1737                                               mali_pp_core_get_id(group->pp_core)),
1738                                       mali_pp_get_rawstat(group->pp_core), 0);
1739 }
1740
1741 static void mali_group_timeout(void *data)
1742 {
1743         struct mali_group *group = (struct mali_group *)data;
1744         MALI_DEBUG_ASSERT_POINTER(group);
1745
1746         MALI_DEBUG_PRINT(2, ("Group: timeout handler for %s at %u\n",
1747                              mali_group_core_description(group),
1748                              _mali_osk_time_tickcount()));
1749
1750         if (NULL != group->gp_core) {
1751                 mali_group_schedule_bottom_half_gp(group);
1752         } else {
1753                 MALI_DEBUG_ASSERT_POINTER(group->pp_core);
1754                 mali_group_schedule_bottom_half_pp(group);
1755         }
1756 }
1757
1758 mali_bool mali_group_zap_session(struct mali_group *group,
1759                                  struct mali_session_data *session)
1760 {
1761         MALI_DEBUG_ASSERT_POINTER(group);
1762         MALI_DEBUG_ASSERT_POINTER(session);
1763         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1764
1765         if (group->session != session) {
1766                 /* not running from this session */
1767                 return MALI_TRUE; /* success */
1768         }
1769
1770         if (group->is_working) {
1771                 /* The Zap also does the stall and disable_stall */
1772                 mali_bool zap_success = mali_mmu_zap_tlb(group->mmu);
1773                 return zap_success;
1774         } else {
1775                 /* Just remove the session instead of zapping */
1776                 mali_group_clear_session(group);
1777                 return MALI_TRUE; /* success */
1778         }
1779 }
1780
1781 #if defined(CONFIG_MALI400_PROFILING)
1782 static void mali_group_report_l2_cache_counters_per_core(struct mali_group *group, u32 core_num)
1783 {
1784         u32 source0 = 0;
1785         u32 value0 = 0;
1786         u32 source1 = 0;
1787         u32 value1 = 0;
1788         u32 profiling_channel = 0;
1789
1790         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1791
1792         switch (core_num) {
1793         case 0:
1794                 profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE |
1795                                     MALI_PROFILING_EVENT_CHANNEL_GPU |
1796                                     MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L20_COUNTERS;
1797                 break;
1798         case 1:
1799                 profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE |
1800                                     MALI_PROFILING_EVENT_CHANNEL_GPU |
1801                                     MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L21_COUNTERS;
1802                 break;
1803         case 2:
1804                 profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE |
1805                                     MALI_PROFILING_EVENT_CHANNEL_GPU |
1806                                     MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L22_COUNTERS;
1807                 break;
1808         default:
1809                 profiling_channel = MALI_PROFILING_EVENT_TYPE_SINGLE |
1810                                     MALI_PROFILING_EVENT_CHANNEL_GPU |
1811                                     MALI_PROFILING_EVENT_REASON_SINGLE_GPU_L20_COUNTERS;
1812                 break;
1813         }
1814
1815         if (0 == core_num) {
1816                 mali_l2_cache_core_get_counter_values(group->l2_cache_core[0], &source0, &value0, &source1, &value1);
1817         }
1818         if (1 == core_num) {
1819                 if (1 == mali_l2_cache_get_id(group->l2_cache_core[0])) {
1820                         mali_l2_cache_core_get_counter_values(group->l2_cache_core[0], &source0, &value0, &source1, &value1);
1821                 } else if (1 == mali_l2_cache_get_id(group->l2_cache_core[1])) {
1822                         mali_l2_cache_core_get_counter_values(group->l2_cache_core[1], &source0, &value0, &source1, &value1);
1823                 }
1824         }
1825         if (2 == core_num) {
1826                 if (2 == mali_l2_cache_get_id(group->l2_cache_core[0])) {
1827                         mali_l2_cache_core_get_counter_values(group->l2_cache_core[0], &source0, &value0, &source1, &value1);
1828                 } else if (2 == mali_l2_cache_get_id(group->l2_cache_core[1])) {
1829                         mali_l2_cache_core_get_counter_values(group->l2_cache_core[1], &source0, &value0, &source1, &value1);
1830                 }
1831         }
1832
1833         _mali_osk_profiling_add_event(profiling_channel, source1 << 8 | source0, value0, value1, 0, 0);
1834 }
1835 #endif /* #if defined(CONFIG_MALI400_PROFILING) */