drivers/gpu/arm/mali400/mali/common/mali_executor.c

   1 /*
   2  * Copyright (C) 2012-2015 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_executor.h"
  12 #include "mali_scheduler.h"
  13 #include "mali_kernel_common.h"
  14 #include "mali_kernel_core.h"
  15 #include "mali_osk.h"
  16 #include "mali_osk_list.h"
  17 #include "mali_pp.h"
  18 #include "mali_pp_job.h"
  19 #include "mali_group.h"
  20 #include "mali_pm.h"
  21 #include "mali_timeline.h"
  22 #include "mali_osk_profiling.h"
  23 #include "mali_session.h"
  24
  25 /*Add for voltage scan function*/
  26 extern u32 mali_group_error;
  27
  28 /*
  29  * If dma_buf with map on demand is used, we defer job deletion and job queue
  30  * if in atomic context, since both might sleep.
  31  */
  32 #if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
  33 #define MALI_EXECUTOR_USE_DEFERRED_PP_JOB_DELETE 1
  34 #define MALI_EXECUTOR_USE_DEFERRED_PP_JOB_QUEUE 1
  35 #endif /* !defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) */
  36
  37 /*
  38  * ---------- static type definitions (structs, enums, etc) ----------
  39  */
  40
  41 enum mali_executor_state_t {
  42         EXEC_STATE_NOT_PRESENT, /* Virtual group on Mali-300/400 (do not use) */
  43         EXEC_STATE_DISABLED,    /* Disabled by core scaling (do not use) */
  44         EXEC_STATE_EMPTY,       /* No child groups for virtual group (do not use) */
  45         EXEC_STATE_INACTIVE,    /* Can be used, but must be activate first */
  46         EXEC_STATE_IDLE,        /* Active and ready to be used */
  47         EXEC_STATE_WORKING,     /* Executing a job */
  48 };
  49
  50 /*
  51  * ---------- global variables (exported due to inline functions) ----------
  52  */
  53
  54 /* Lock for this module (protecting all HW access except L2 caches) */
  55 _mali_osk_spinlock_irq_t *mali_executor_lock_obj = NULL;
  56
  57 mali_bool mali_executor_hints[MALI_EXECUTOR_HINT_MAX];
  58
  59 /*
  60  * ---------- static variables ----------
  61  */
  62
  63 /* Used to defer job scheduling */
  64 static _mali_osk_wq_work_t *executor_wq_high_pri = NULL;
  65
  66 /* Store version from GP and PP (user space wants to know this) */
  67 static u32 pp_version = 0;
  68 static u32 gp_version = 0;
  69
  70 /* List of physical PP groups which are disabled by some external source */
  71 static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_disabled);
  72 static u32 group_list_disabled_count = 0;
  73
  74 /* List of groups which can be used, but activate first */
  75 static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_inactive);
  76 static u32 group_list_inactive_count = 0;
  77
  78 /* List of groups which are active and ready to be used */
  79 static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_idle);
  80 static u32 group_list_idle_count = 0;
  81
  82 /* List of groups which are executing a job */
  83 static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_working);
  84 static u32 group_list_working_count = 0;
  85
  86 /* Virtual group (if any) */
  87 static struct mali_group *virtual_group = NULL;
  88
  89 /* Virtual group state is tracked with a state variable instead of 4 lists */
  90 static enum mali_executor_state_t virtual_group_state = EXEC_STATE_NOT_PRESENT;
  91
  92 /* GP group */
  93 static struct mali_group *gp_group = NULL;
  94
  95 /* GP group state is tracked with a state variable instead of 4 lists */
  96 static enum mali_executor_state_t gp_group_state = EXEC_STATE_NOT_PRESENT;
  97
  98 static u32 gp_returned_cookie = 0;
  99
 100 /* Total number of physical PP cores present */
 101 static u32 num_physical_pp_cores_total = 0;
 102
 103 /* Number of physical cores which are enabled */
 104 static u32 num_physical_pp_cores_enabled = 0;
 105
 106 /* Enable or disable core scaling */
 107 static mali_bool core_scaling_enabled = MALI_TRUE;
 108
 109 /* Variables to allow safe pausing of the scheduler */
 110 static _mali_osk_wait_queue_t *executor_working_wait_queue = NULL;
 111 static u32 pause_count = 0;
 112
 113 /* PP cores haven't been enabled because of some pp cores haven't been disabled. */
 114 static int core_scaling_delay_up_mask[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 };
 115
 116 /* Variables used to implement notify pp core changes to userspace when core scaling
 117  * is finished in mali_executor_complete_group() function. */
 118 static _mali_osk_wq_work_t *executor_wq_notify_core_change = NULL;
 119 static _mali_osk_wait_queue_t *executor_notify_core_change_wait_queue = NULL;
 120
 121 /*
 122  * ---------- Forward declaration of static functions ----------
 123  */
 124 static mali_bool mali_executor_is_suspended(void *data);
 125 static mali_bool mali_executor_is_working(void);
 126 static void mali_executor_disable_empty_virtual(void);
 127 static mali_bool mali_executor_physical_rejoin_virtual(struct mali_group *group);
 128 static mali_bool mali_executor_has_virtual_group(void);
 129 static mali_bool mali_executor_virtual_group_is_usable(void);
 130 static void mali_executor_schedule(void);
 131 static void mali_executor_wq_schedule(void *arg);
 132 static void mali_executor_send_gp_oom_to_user(struct mali_gp_job *job, u32 added_size);
 133 static void mali_executor_complete_group(struct mali_group *group,
 134                 mali_bool success,
 135                 struct mali_gp_job **gp_job_done,
 136                 struct mali_pp_job **pp_job_done);
 137 static void mali_executor_change_state_pp_physical(struct mali_group *group,
 138                 _mali_osk_list_t *old_list,
 139                 u32 *old_count,
 140                 _mali_osk_list_t *new_list,
 141                 u32 *new_count);
 142 static mali_bool mali_executor_group_is_in_state(struct mali_group *group,
 143                 enum mali_executor_state_t state);
 144
 145 static void mali_executor_group_enable_internal(struct mali_group *group);
 146 static void mali_executor_group_disable_internal(struct mali_group *group);
 147 static void mali_executor_core_scale(unsigned int target_core_nr);
 148 static void mali_executor_core_scale_in_group_complete(struct mali_group *group);
 149 static void mali_executor_notify_core_change(u32 num_cores);
 150 static void mali_executor_wq_notify_core_change(void *arg);
 151 static void mali_executor_change_group_status_disabled(struct mali_group *group);
 152 static mali_bool mali_executor_deactivate_list_idle(mali_bool deactivate_idle_group);
 153 static void mali_executor_set_state_pp_physical(struct mali_group *group,
 154                 _mali_osk_list_t *new_list,
 155                 u32 *new_count);
 156
 157 /*
 158  * ---------- Actual implementation ----------
 159  */
 160
 161 _mali_osk_errcode_t mali_executor_initialize(void)
 162 {
 163         mali_executor_lock_obj = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_EXECUTOR);
 164         if (NULL == mali_executor_lock_obj) {
 165                 mali_executor_terminate();
 166                 return _MALI_OSK_ERR_NOMEM;
 167         }
 168
 169         executor_wq_high_pri = _mali_osk_wq_create_work_high_pri(mali_executor_wq_schedule, NULL);
 170         if (NULL == executor_wq_high_pri) {
 171                 mali_executor_terminate();
 172                 return _MALI_OSK_ERR_NOMEM;
 173         }
 174
 175         executor_working_wait_queue = _mali_osk_wait_queue_init();
 176         if (NULL == executor_working_wait_queue) {
 177                 mali_executor_terminate();
 178                 return _MALI_OSK_ERR_NOMEM;
 179         }
 180
 181         executor_wq_notify_core_change = _mali_osk_wq_create_work(mali_executor_wq_notify_core_change, NULL);
 182         if (NULL == executor_wq_notify_core_change) {
 183                 mali_executor_terminate();
 184                 return _MALI_OSK_ERR_NOMEM;
 185         }
 186
 187         executor_notify_core_change_wait_queue = _mali_osk_wait_queue_init();
 188         if (NULL == executor_notify_core_change_wait_queue) {
 189                 mali_executor_terminate();
 190                 return _MALI_OSK_ERR_NOMEM;
 191         }
 192
 193         return _MALI_OSK_ERR_OK;
 194 }
 195
 196 void mali_executor_terminate(void)
 197 {
 198         if (NULL != executor_notify_core_change_wait_queue) {
 199                 _mali_osk_wait_queue_term(executor_notify_core_change_wait_queue);
 200                 executor_notify_core_change_wait_queue = NULL;
 201         }
 202
 203         if (NULL != executor_wq_notify_core_change) {
 204                 _mali_osk_wq_delete_work(executor_wq_notify_core_change);
 205                 executor_wq_notify_core_change = NULL;
 206         }
 207
 208         if (NULL != executor_working_wait_queue) {
 209                 _mali_osk_wait_queue_term(executor_working_wait_queue);
 210                 executor_working_wait_queue = NULL;
 211         }
 212
 213         if (NULL != executor_wq_high_pri) {
 214                 _mali_osk_wq_delete_work(executor_wq_high_pri);
 215                 executor_wq_high_pri = NULL;
 216         }
 217
 218         if (NULL != mali_executor_lock_obj) {
 219                 _mali_osk_spinlock_irq_term(mali_executor_lock_obj);
 220                 mali_executor_lock_obj = NULL;
 221         }
 222 }
 223
 224 void mali_executor_populate(void)
 225 {
 226         u32 num_groups;
 227         u32 i;
 228
 229         num_groups = mali_group_get_glob_num_groups();
 230
 231         /* Do we have a virtual group? */
 232         for (i = 0; i < num_groups; i++) {
 233                 struct mali_group *group = mali_group_get_glob_group(i);
 234
 235                 if (mali_group_is_virtual(group)) {
 236                         virtual_group = group;
 237                         virtual_group_state = EXEC_STATE_INACTIVE;
 238                         break;
 239                 }
 240         }
 241
 242         /* Find all the available physical GP and PP cores */
 243         for (i = 0; i < num_groups; i++) {
 244                 struct mali_group *group = mali_group_get_glob_group(i);
 245
 246                 if (NULL != group) {
 247                         struct mali_pp_core *pp_core = mali_group_get_pp_core(group);
 248                         struct mali_gp_core *gp_core = mali_group_get_gp_core(group);
 249
 250                         if (!mali_group_is_virtual(group)) {
 251                                 if (NULL != pp_core) {
 252                                         if (0 == pp_version) {
 253                                                 /* Retrieve PP version from the first available PP core */
 254                                                 pp_version = mali_pp_core_get_version(pp_core);
 255                                         }
 256
 257                                         if (NULL != virtual_group) {
 258                                                 mali_executor_lock();
 259                                                 mali_group_add_group(virtual_group, group);
 260                                                 mali_executor_unlock();
 261                                         } else {
 262                                                 _mali_osk_list_add(&group->executor_list, &group_list_inactive);
 263                                                 group_list_inactive_count++;
 264                                         }
 265
 266                                         num_physical_pp_cores_total++;
 267                                 } else {
 268                                         MALI_DEBUG_ASSERT_POINTER(gp_core);
 269
 270                                         if (0 == gp_version) {
 271                                                 /* Retrieve GP version */
 272                                                 gp_version = mali_gp_core_get_version(gp_core);
 273                                         }
 274
 275                                         gp_group = group;
 276                                         gp_group_state = EXEC_STATE_INACTIVE;
 277                                 }
 278
 279                         }
 280                 }
 281         }
 282
 283         num_physical_pp_cores_enabled = num_physical_pp_cores_total;
 284 }
 285
 286 void mali_executor_depopulate(void)
 287 {
 288         struct mali_group *group;
 289         struct mali_group *temp;
 290
 291         MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != gp_group_state);
 292
 293         if (NULL != gp_group) {
 294                 mali_group_delete(gp_group);
 295                 gp_group = NULL;
 296         }
 297
 298         MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != virtual_group_state);
 299
 300         if (NULL != virtual_group) {
 301                 mali_group_delete(virtual_group);
 302                 virtual_group = NULL;
 303         }
 304
 305         MALI_DEBUG_ASSERT(_mali_osk_list_empty(&group_list_working));
 306
 307         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, executor_list) {
 308                 mali_group_delete(group);
 309         }
 310
 311         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive, struct mali_group, executor_list) {
 312                 mali_group_delete(group);
 313         }
 314
 315         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_disabled, struct mali_group, executor_list) {
 316                 mali_group_delete(group);
 317         }
 318 }
 319
 320 void mali_executor_suspend(void)
 321 {
 322         mali_executor_lock();
 323
 324         /* Increment the pause_count so that no more jobs will be scheduled */
 325         pause_count++;
 326
 327         mali_executor_unlock();
 328
 329         _mali_osk_wait_queue_wait_event(executor_working_wait_queue,
 330                                         mali_executor_is_suspended, NULL);
 331
 332         /*
 333          * mali_executor_complete_XX() leaves jobs in idle state.
 334          * deactivate option is used when we are going to power down
 335          * the entire GPU (OS suspend) and want a consistent SW vs HW
 336          * state.
 337          */
 338         mali_executor_lock();
 339
 340         mali_executor_deactivate_list_idle(MALI_TRUE);
 341
 342         /*
 343          * The following steps are used to deactive all of activated
 344          * (MALI_GROUP_STATE_ACTIVE) and activating (MALI_GROUP
 345          * _STAET_ACTIVATION_PENDING) groups, to make sure the variable
 346          * pd_mask_wanted is equal with 0. */
 347         if (MALI_GROUP_STATE_INACTIVE != mali_group_get_state(gp_group)) {
 348                 gp_group_state = EXEC_STATE_INACTIVE;
 349                 mali_group_deactivate(gp_group);
 350         }
 351
 352         if (mali_executor_has_virtual_group()) {
 353                 if (MALI_GROUP_STATE_INACTIVE
 354                     != mali_group_get_state(virtual_group)) {
 355                         virtual_group_state = EXEC_STATE_INACTIVE;
 356                         mali_group_deactivate(virtual_group);
 357                 }
 358         }
 359
 360         if (0 < group_list_inactive_count) {
 361                 struct mali_group *group;
 362                 struct mali_group *temp;
 363
 364                 _MALI_OSK_LIST_FOREACHENTRY(group, temp,
 365                                             &group_list_inactive,
 366                                             struct mali_group, executor_list) {
 367                         if (MALI_GROUP_STATE_ACTIVATION_PENDING
 368                             == mali_group_get_state(group)) {
 369                                 mali_group_deactivate(group);
 370                         }
 371
 372                         /*
 373                          * On mali-450 platform, we may have physical group in the group inactive
 374                          * list, and its state is MALI_GROUP_STATE_ACTIVATION_PENDING, so we only
 375                          * deactivate it is not enough, we still also need add it back to virtual group.
 376                          * And now, virtual group must be in INACTIVE state, so it's safe to add
 377                          * physical group to virtual group at this point.
 378                          */
 379                         if (NULL != virtual_group) {
 380                                 _mali_osk_list_delinit(&group->executor_list);
 381                                 group_list_inactive_count--;
 382
 383                                 mali_group_add_group(virtual_group, group);
 384                         }
 385                 }
 386         }
 387
 388         mali_executor_unlock();
 389 }
 390
 391 void mali_executor_resume(void)
 392 {
 393         mali_executor_lock();
 394
 395         /* Decrement pause_count to allow scheduling again (if it reaches 0) */
 396         pause_count--;
 397         if (0 == pause_count) {
 398                 mali_executor_schedule();
 399         }
 400
 401         mali_executor_unlock();
 402 }
 403
 404 u32 mali_executor_get_num_cores_total(void)
 405 {
 406         return num_physical_pp_cores_total;
 407 }
 408
 409 u32 mali_executor_get_num_cores_enabled(void)
 410 {
 411         return num_physical_pp_cores_enabled;
 412 }
 413
 414 struct mali_pp_core *mali_executor_get_virtual_pp(void)
 415 {
 416         MALI_DEBUG_ASSERT_POINTER(virtual_group);
 417         MALI_DEBUG_ASSERT_POINTER(virtual_group->pp_core);
 418         return virtual_group->pp_core;
 419 }
 420
 421 struct mali_group *mali_executor_get_virtual_group(void)
 422 {
 423         return virtual_group;
 424 }
 425
 426 void mali_executor_zap_all_active(struct mali_session_data *session)
 427 {
 428         struct mali_group *group;
 429         struct mali_group *temp;
 430         mali_bool ret;
 431
 432         mali_executor_lock();
 433
 434         /*
 435          * This function is a bit complicated because
 436          * mali_group_zap_session() can fail. This only happens because the
 437          * group is in an unhandled page fault status.
 438          * We need to make sure this page fault is handled before we return,
 439          * so that we know every single outstanding MMU transactions have
 440          * completed. This will allow caller to safely remove physical pages
 441          * when we have returned.
 442          */
 443
 444         MALI_DEBUG_ASSERT(NULL != gp_group);
 445         ret = mali_group_zap_session(gp_group, session);
 446         if (MALI_FALSE == ret) {
 447                 struct mali_gp_job *gp_job = NULL;
 448
 449                 mali_executor_complete_group(gp_group, MALI_FALSE, &gp_job, NULL);
 450
 451                 MALI_DEBUG_ASSERT_POINTER(gp_job);
 452
 453                 /* GP job completed, make sure it is freed */
 454                 mali_scheduler_complete_gp_job(gp_job, MALI_FALSE,
 455                                                MALI_TRUE, MALI_TRUE);
 456         }
 457
 458         if (mali_executor_has_virtual_group()) {
 459                 ret = mali_group_zap_session(virtual_group, session);
 460                 if (MALI_FALSE == ret) {
 461                         struct mali_pp_job *pp_job = NULL;
 462
 463                         mali_executor_complete_group(virtual_group, MALI_FALSE, NULL, &pp_job);
 464
 465                         if (NULL != pp_job) {
 466                                 /* PP job completed, make sure it is freed */
 467                                 mali_scheduler_complete_pp_job(pp_job, 0,
 468                                                                MALI_FALSE, MALI_TRUE);
 469                         }
 470                 }
 471         }
 472
 473         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_working,
 474                                     struct mali_group, executor_list) {
 475                 ret = mali_group_zap_session(group, session);
 476                 if (MALI_FALSE == ret) {
 477                         ret = mali_group_zap_session(group, session);
 478                         if (MALI_FALSE == ret) {
 479                                 struct mali_pp_job *pp_job = NULL;
 480
 481                                 mali_executor_complete_group(group, MALI_FALSE, NULL, &pp_job);
 482
 483                                 if (NULL != pp_job) {
 484                                         /* PP job completed, free it */
 485                                         mali_scheduler_complete_pp_job(pp_job,
 486                                                                        0, MALI_FALSE,
 487                                                                        MALI_TRUE);
 488                                 }
 489                         }
 490                 }
 491         }
 492
 493         mali_executor_unlock();
 494 }
 495
 496 void mali_executor_schedule_from_mask(mali_scheduler_mask mask, mali_bool deferred_schedule)
 497 {
 498         if (MALI_SCHEDULER_MASK_EMPTY != mask) {
 499                 if (MALI_TRUE == deferred_schedule) {
 500                         _mali_osk_wq_schedule_work_high_pri(executor_wq_high_pri);
 501                 } else {
 502                         /* Schedule from this thread*/
 503                         mali_executor_lock();
 504                         mali_executor_schedule();
 505                         mali_executor_unlock();
 506                 }
 507         }
 508 }
 509
 510 _mali_osk_errcode_t mali_executor_interrupt_gp(struct mali_group *group,
 511                 mali_bool in_upper_half)
 512 {
 513         enum mali_interrupt_result int_result;
 514         mali_bool time_out = MALI_FALSE;
 515
 516         MALI_DEBUG_PRINT(4, ("Executor: GP interrupt from %s in %s half\n",
 517                              mali_group_core_description(group),
 518                              in_upper_half ? "upper" : "bottom"));
 519
 520         mali_executor_lock();
 521         if (!mali_group_is_working(group)) {
 522                 /* Not working, so nothing to do */
 523                 mali_executor_unlock();
 524                 return _MALI_OSK_ERR_FAULT;
 525         }
 526
 527         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
 528         MALI_DEBUG_ASSERT(mali_group_is_working(group));
 529
 530         if (mali_group_has_timed_out(group)) {
 531                 int_result = MALI_INTERRUPT_RESULT_ERROR;
 532                 time_out = MALI_TRUE;
 533                 MALI_PRINT(("Executor GP: Job %d Timeout on %s\n",
 534                             mali_gp_job_get_id(group->gp_running_job),
 535                             mali_group_core_description(group)));
 536         } else {
 537                 int_result = mali_group_get_interrupt_result_gp(group);
 538                 if (MALI_INTERRUPT_RESULT_NONE == int_result) {
 539                         mali_executor_unlock();
 540                         return _MALI_OSK_ERR_FAULT;
 541                 }
 542         }
 543
 544 #if defined(CONFIG_MALI_SHARED_INTERRUPTS)
 545         if (MALI_INTERRUPT_RESULT_NONE == int_result) {
 546                 /* No interrupts signalled, so nothing to do */
 547                 mali_executor_unlock();
 548                 return _MALI_OSK_ERR_FAULT;
 549         }
 550 #else
 551         MALI_DEBUG_ASSERT(MALI_INTERRUPT_RESULT_NONE != int_result);
 552 #endif
 553
 554         mali_group_mask_all_interrupts_gp(group);
 555
 556         if (MALI_INTERRUPT_RESULT_SUCCESS_VS == int_result) {
 557                 if (mali_group_gp_is_active(group)) {
 558                         /* Only VS completed so far, while PLBU is still active */
 559
 560                         /* Enable all but the current interrupt */
 561                         mali_group_enable_interrupts_gp(group, int_result);
 562
 563                         mali_executor_unlock();
 564                         return _MALI_OSK_ERR_OK;
 565                 }
 566         } else if (MALI_INTERRUPT_RESULT_SUCCESS_PLBU == int_result) {
 567                 if (mali_group_gp_is_active(group)) {
 568                         /* Only PLBU completed so far, while VS is still active */
 569
 570                         /* Enable all but the current interrupt */
 571                         mali_group_enable_interrupts_gp(group, int_result);
 572
 573                         mali_executor_unlock();
 574                         return _MALI_OSK_ERR_OK;
 575                 }
 576         } else if (MALI_INTERRUPT_RESULT_OOM == int_result) {
 577
 578                 mali_executor_unlock();
 579
 580                 mali_group_schedule_oom_work_handler(group);
 581
 582                 return _MALI_OSK_ERR_OK;
 583         }
 584
 585         /*Add for voltage scan function*/
 586         if (MALI_INTERRUPT_RESULT_ERROR == int_result)
 587                 mali_group_error++;
 588
 589         /* We should now have a real interrupt to handle */
 590
 591         MALI_DEBUG_PRINT(4, ("Executor: Group %s completed with %s\n",
 592                              mali_group_core_description(group),
 593                              (MALI_INTERRUPT_RESULT_ERROR == int_result) ?
 594                              "ERROR" : "success"));
 595
 596         if (in_upper_half && MALI_INTERRUPT_RESULT_ERROR == int_result) {
 597                 /* Don't bother to do processing of errors in upper half */
 598                 mali_executor_unlock();
 599
 600                 if (MALI_FALSE == time_out) {
 601                         mali_group_schedule_bottom_half_gp(group);
 602                 }
 603         } else {
 604                 struct mali_gp_job *job;
 605                 mali_bool success;
 606
 607                 if (MALI_TRUE == time_out) {
 608                         mali_group_dump_status(group);
 609                 }
 610
 611                 success = (int_result != MALI_INTERRUPT_RESULT_ERROR) ?
 612                           MALI_TRUE : MALI_FALSE;
 613
 614                 mali_executor_complete_group(group, success, &job, NULL);
 615
 616                 mali_executor_unlock();
 617
 618                 /* GP jobs always fully complete */
 619                 MALI_DEBUG_ASSERT(NULL != job);
 620
 621                 /* This will notify user space and close the job object */
 622                 mali_scheduler_complete_gp_job(job, success,
 623                                                MALI_TRUE, MALI_TRUE);
 624         }
 625
 626         return _MALI_OSK_ERR_OK;
 627 }
 628
 629 _mali_osk_errcode_t mali_executor_interrupt_pp(struct mali_group *group,
 630                 mali_bool in_upper_half)
 631 {
 632         enum mali_interrupt_result int_result;
 633         mali_bool time_out = MALI_FALSE;
 634
 635         MALI_DEBUG_PRINT(4, ("Executor: PP interrupt from %s in %s half\n",
 636                              mali_group_core_description(group),
 637                              in_upper_half ? "upper" : "bottom"));
 638
 639         mali_executor_lock();
 640
 641         if (!mali_group_is_working(group)) {
 642                 /* Not working, so nothing to do */
 643                 mali_executor_unlock();
 644                 return _MALI_OSK_ERR_FAULT;
 645         }
 646
 647         if (in_upper_half) {
 648                 if (mali_group_is_in_virtual(group)) {
 649                         /* Child groups should never handle PP interrupts */
 650                         MALI_DEBUG_ASSERT(!mali_group_has_timed_out(group));
 651                         mali_executor_unlock();
 652                         return _MALI_OSK_ERR_FAULT;
 653                 }
 654         }
 655         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
 656         MALI_DEBUG_ASSERT(mali_group_is_working(group));
 657         MALI_DEBUG_ASSERT(!mali_group_is_in_virtual(group));
 658
 659         if (mali_group_has_timed_out(group)) {
 660                 int_result = MALI_INTERRUPT_RESULT_ERROR;
 661                 time_out = MALI_TRUE;
 662                 MALI_PRINT(("Executor PP: Job %d Timeout on %s\n",
 663                             mali_pp_job_get_id(group->pp_running_job),
 664                             mali_group_core_description(group)));
 665         } else {
 666                 int_result = mali_group_get_interrupt_result_pp(group);
 667                 if (MALI_INTERRUPT_RESULT_NONE == int_result) {
 668                         mali_executor_unlock();
 669                         return _MALI_OSK_ERR_FAULT;
 670                 }
 671         }
 672
 673 #if defined(CONFIG_MALI_SHARED_INTERRUPTS)
 674         if (MALI_INTERRUPT_RESULT_NONE == int_result) {
 675                 /* No interrupts signalled, so nothing to do */
 676                 mali_executor_unlock();
 677                 return _MALI_OSK_ERR_FAULT;
 678         } else if (MALI_INTERRUPT_RESULT_SUCCESS == int_result) {
 679                 if (mali_group_is_virtual(group) && mali_group_pp_is_active(group)) {
 680                         /* Some child groups are still working, so nothing to do right now */
 681                         mali_executor_unlock();
 682                         return _MALI_OSK_ERR_FAULT;
 683                 }
 684         }
 685 #else
 686         MALI_DEBUG_ASSERT(MALI_INTERRUPT_RESULT_NONE != int_result);
 687 #endif
 688
 689         /*Add voltage scan function*/
 690
 691         if (MALI_INTERRUPT_RESULT_ERROR == int_result)
 692                 mali_group_error++;
 693
 694         /* We should now have a real interrupt to handle */
 695
 696         MALI_DEBUG_PRINT(4, ("Executor: Group %s completed with %s\n",
 697                              mali_group_core_description(group),
 698                              (MALI_INTERRUPT_RESULT_ERROR == int_result) ?
 699                              "ERROR" : "success"));
 700
 701         if (in_upper_half && MALI_INTERRUPT_RESULT_ERROR == int_result) {
 702                 /* Don't bother to do processing of errors in upper half */
 703                 mali_group_mask_all_interrupts_pp(group);
 704                 mali_executor_unlock();
 705
 706                 if (MALI_FALSE == time_out) {
 707                         mali_group_schedule_bottom_half_pp(group);
 708                 }
 709         } else {
 710                 struct mali_pp_job *job = NULL;
 711                 mali_bool success;
 712
 713                 if (MALI_TRUE == time_out) {
 714                         mali_group_dump_status(group);
 715                 }
 716
 717                 success = (int_result == MALI_INTERRUPT_RESULT_SUCCESS) ?
 718                           MALI_TRUE : MALI_FALSE;
 719
 720                 mali_executor_complete_group(group, success, NULL, &job);
 721
 722                 mali_executor_unlock();
 723
 724                 if (NULL != job) {
 725                         /* Notify user space and close the job object */
 726                         mali_scheduler_complete_pp_job(job,
 727                                                        num_physical_pp_cores_total,
 728                                                        MALI_TRUE, MALI_TRUE);
 729                 }
 730         }
 731
 732         return _MALI_OSK_ERR_OK;
 733 }
 734
 735 _mali_osk_errcode_t mali_executor_interrupt_mmu(struct mali_group *group,
 736                 mali_bool in_upper_half)
 737 {
 738         enum mali_interrupt_result int_result;
 739
 740         MALI_DEBUG_PRINT(4, ("Executor: MMU interrupt from %s in %s half\n",
 741                              mali_group_core_description(group),
 742                              in_upper_half ? "upper" : "bottom"));
 743
 744         mali_executor_lock();
 745         if (!mali_group_is_working(group)) {
 746                 /* Not working, so nothing to do */
 747                 mali_executor_unlock();
 748                 return _MALI_OSK_ERR_FAULT;
 749         }
 750
 751         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
 752         MALI_DEBUG_ASSERT(mali_group_is_working(group));
 753
 754         int_result = mali_group_get_interrupt_result_mmu(group);
 755         if (MALI_INTERRUPT_RESULT_NONE == int_result) {
 756                 mali_executor_unlock();
 757                 return _MALI_OSK_ERR_FAULT;
 758         }
 759
 760 #if defined(CONFIG_MALI_SHARED_INTERRUPTS)
 761         if (MALI_INTERRUPT_RESULT_NONE == int_result) {
 762                 /* No interrupts signalled, so nothing to do */
 763                 mali_executor_unlock();
 764                 return _MALI_OSK_ERR_FAULT;
 765         }
 766 #else
 767         MALI_DEBUG_ASSERT(MALI_INTERRUPT_RESULT_ERROR == int_result);
 768 #endif
 769
 770         /* We should now have a real interrupt to handle */
 771
 772         if (in_upper_half) {
 773                 /* Don't bother to do processing of errors in upper half */
 774
 775                 struct mali_group *parent = group->parent_group;
 776
 777                 mali_mmu_mask_all_interrupts(group->mmu);
 778
 779                 mali_executor_unlock();
 780
 781                 if (NULL == parent) {
 782                         mali_group_schedule_bottom_half_mmu(group);
 783                 } else {
 784                         mali_group_schedule_bottom_half_mmu(parent);
 785                 }
 786
 787         } else {
 788                 struct mali_gp_job *gp_job = NULL;
 789                 struct mali_pp_job *pp_job = NULL;
 790
 791 #ifdef DEBUG
 792
 793                 u32 fault_address = mali_mmu_get_page_fault_addr(group->mmu);
 794                 u32 status = mali_mmu_get_status(group->mmu);
 795                 MALI_DEBUG_PRINT(2, ("Executor: Mali page fault detected at 0x%x from bus id %d of type %s on %s\n",
 796                                      (void *)(uintptr_t)fault_address,
 797                                      (status >> 6) & 0x1F,
 798                                      (status & 32) ? "write" : "read",
 799                                      group->mmu->hw_core.description));
 800                 MALI_DEBUG_PRINT(3, ("Executor: MMU rawstat = 0x%08X, MMU status = 0x%08X\n",
 801                                      mali_mmu_get_rawstat(group->mmu), status));
 802                 mali_mmu_pagedir_diag(mali_session_get_page_directory(group->session), fault_address);
 803 #endif
 804
 805                 mali_executor_complete_group(group, MALI_FALSE, &gp_job, &pp_job);
 806
 807                 mali_executor_unlock();
 808
 809                 if (NULL != gp_job) {
 810                         MALI_DEBUG_ASSERT(NULL == pp_job);
 811
 812                         /* Notify user space and close the job object */
 813                         mali_scheduler_complete_gp_job(gp_job, MALI_FALSE,
 814                                                        MALI_TRUE, MALI_TRUE);
 815                 } else if (NULL != pp_job) {
 816                         MALI_DEBUG_ASSERT(NULL == gp_job);
 817
 818                         /* Notify user space and close the job object */
 819                         mali_scheduler_complete_pp_job(pp_job,
 820                                                        num_physical_pp_cores_total,
 821                                                        MALI_TRUE, MALI_TRUE);
 822                 }
 823         }
 824
 825         return _MALI_OSK_ERR_OK;
 826 }
 827
 828 void mali_executor_group_oom(struct mali_group *group)
 829 {
 830         struct mali_gp_job *job = NULL;
 831         MALI_DEBUG_ASSERT_POINTER(group);
 832         MALI_DEBUG_ASSERT_POINTER(group->gp_core);
 833         MALI_DEBUG_ASSERT_POINTER(group->mmu);
 834
 835         mali_executor_lock();
 836
 837         job = mali_group_get_running_gp_job(group);
 838
 839         MALI_DEBUG_ASSERT_POINTER(job);
 840
 841 #if defined(CONFIG_MALI400_PROFILING)
 842         /* Give group a chance to generate a SUSPEND event */
 843         mali_group_oom(group);
 844 #endif
 845
 846         mali_gp_job_set_current_heap_addr(job, mali_gp_read_plbu_alloc_start_addr(group->gp_core));
 847
 848         mali_executor_unlock();
 849
 850         if (_MALI_OSK_ERR_OK  == mali_mem_add_mem_size(job->session, job->heap_base_addr, job->heap_grow_size)) {
 851                 _mali_osk_notification_t *new_notification = NULL;
 852
 853                 new_notification = _mali_osk_notification_create(
 854                                            _MALI_NOTIFICATION_GP_STALLED,
 855                                            sizeof(_mali_uk_gp_job_suspended_s));
 856
 857                 /* resume job with new heap,
 858                 * This will also re-enable interrupts
 859                 */
 860                 mali_executor_lock();
 861
 862                 mali_executor_send_gp_oom_to_user(job, job->heap_grow_size);
 863
 864                 if (NULL != new_notification) {
 865
 866                         mali_gp_job_set_oom_notification(job, new_notification);
 867
 868                         mali_group_resume_gp_with_new_heap(group, mali_gp_job_get_id(job),
 869                                                            job->heap_current_addr,
 870                                                            job->heap_current_addr + job->heap_grow_size);
 871                 }
 872                 mali_executor_unlock();
 873         } else {
 874                 mali_executor_lock();
 875                 mali_executor_send_gp_oom_to_user(job, 0);
 876                 mali_executor_unlock();
 877         }
 878
 879 }
 880
 881 void mali_executor_group_power_up(struct mali_group *groups[], u32 num_groups)
 882 {
 883         u32 i;
 884         mali_bool child_groups_activated = MALI_FALSE;
 885         mali_bool do_schedule = MALI_FALSE;
 886 #if defined(DEBUG)
 887         u32 num_activated = 0;
 888 #endif
 889
 890         MALI_DEBUG_ASSERT_POINTER(groups);
 891         MALI_DEBUG_ASSERT(0 < num_groups);
 892
 893         mali_executor_lock();
 894
 895         MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups\n", num_groups));
 896
 897         for (i = 0; i < num_groups; i++) {
 898                 MALI_DEBUG_PRINT(3, ("Executor: powering up group %s\n",
 899                                      mali_group_core_description(groups[i])));
 900
 901                 mali_group_power_up(groups[i]);
 902
 903                 if ((MALI_GROUP_STATE_ACTIVATION_PENDING != mali_group_get_state(groups[i]) ||
 904                      (MALI_TRUE != mali_executor_group_is_in_state(groups[i], EXEC_STATE_INACTIVE)))) {
 905                         /* nothing more to do for this group */
 906                         continue;
 907                 }
 908
 909                 MALI_DEBUG_PRINT(3, ("Executor: activating group %s\n",
 910                                      mali_group_core_description(groups[i])));
 911
 912 #if defined(DEBUG)
 913                 num_activated++;
 914 #endif
 915
 916                 if (mali_group_is_in_virtual(groups[i])) {
 917                         /*
 918                          * At least one child group of virtual group is powered on.
 919                          */
 920                         child_groups_activated = MALI_TRUE;
 921                 } else if (MALI_FALSE == mali_group_is_virtual(groups[i])) {
 922                         /* Set gp and pp not in virtual to active. */
 923                         mali_group_set_active(groups[i]);
 924                 }
 925
 926                 /* Move group from inactive to idle list */
 927                 if (groups[i] == gp_group) {
 928                         MALI_DEBUG_ASSERT(EXEC_STATE_INACTIVE ==
 929                                           gp_group_state);
 930                         gp_group_state = EXEC_STATE_IDLE;
 931                 } else if (MALI_FALSE == mali_group_is_in_virtual(groups[i])
 932                            && MALI_FALSE == mali_group_is_virtual(groups[i])) {
 933                         MALI_DEBUG_ASSERT(MALI_TRUE == mali_executor_group_is_in_state(groups[i],
 934                                           EXEC_STATE_INACTIVE));
 935
 936                         mali_executor_change_state_pp_physical(groups[i],
 937                                                                &group_list_inactive,
 938                                                                &group_list_inactive_count,
 939                                                                &group_list_idle,
 940                                                                &group_list_idle_count);
 941                 }
 942
 943                 do_schedule = MALI_TRUE;
 944         }
 945
 946         if (mali_executor_has_virtual_group() &&
 947             MALI_TRUE == child_groups_activated &&
 948             MALI_GROUP_STATE_ACTIVATION_PENDING ==
 949             mali_group_get_state(virtual_group)) {
 950                 /*
 951                  * Try to active virtual group while it may be not sucessful every time,
 952                  * because there is one situation that not all of child groups are powered on
 953                  * in one time and virtual group is in activation pending state.
 954                  */
 955                 if (mali_group_set_active(virtual_group)) {
 956                         /* Move group from inactive to idle */
 957                         MALI_DEBUG_ASSERT(EXEC_STATE_INACTIVE ==
 958                                           virtual_group_state);
 959                         virtual_group_state = EXEC_STATE_IDLE;
 960
 961                         MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups completed, %u  physical activated, 1 virtual activated.\n", num_groups, num_activated));
 962                 } else {
 963                         MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups completed, %u physical activated\n", num_groups, num_activated));
 964                 }
 965         } else {
 966                 MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups completed, %u physical activated\n", num_groups, num_activated));
 967         }
 968
 969         if (MALI_TRUE == do_schedule) {
 970                 /* Trigger a schedule */
 971                 mali_executor_schedule();
 972         }
 973
 974         mali_executor_unlock();
 975 }
 976
 977 void mali_executor_group_power_down(struct mali_group *groups[],
 978                                     u32 num_groups)
 979 {
 980         u32 i;
 981
 982         MALI_DEBUG_ASSERT_POINTER(groups);
 983         MALI_DEBUG_ASSERT(0 < num_groups);
 984
 985         mali_executor_lock();
 986
 987         MALI_DEBUG_PRINT(3, ("Executor: powering down %u groups\n", num_groups));
 988
 989         for (i = 0; i < num_groups; i++) {
 990                 /* Groups must be either disabled or inactive. while for virtual group,
 991                  * it maybe in empty state, because when we meet pm_runtime_suspend,
 992                  * virtual group could be powered off, and before we acquire mali_executor_lock,
 993                  * we must release mali_pm_state_lock, if there is a new physical job was queued,
 994                  * all of physical groups in virtual group could be pulled out, so we only can
 995                  * powered down an empty virtual group. Those physical groups will be powered
 996                  * up in following pm_runtime_resume callback function.
 997                  */
 998                 MALI_DEBUG_ASSERT(mali_executor_group_is_in_state(groups[i],
 999                                   EXEC_STATE_DISABLED) ||
1000                                   mali_executor_group_is_in_state(groups[i],
1001                                                   EXEC_STATE_INACTIVE) ||
1002                                   mali_executor_group_is_in_state(groups[i],
1003                                                   EXEC_STATE_EMPTY));
1004
1005                 MALI_DEBUG_PRINT(3, ("Executor: powering down group %s\n",
1006                                      mali_group_core_description(groups[i])));
1007
1008                 mali_group_power_down(groups[i]);
1009         }
1010
1011         MALI_DEBUG_PRINT(3, ("Executor: powering down %u groups completed\n", num_groups));
1012
1013         mali_executor_unlock();
1014 }
1015
1016 void mali_executor_abort_session(struct mali_session_data *session)
1017 {
1018         struct mali_group *group;
1019         struct mali_group *tmp_group;
1020
1021         MALI_DEBUG_ASSERT_POINTER(session);
1022         MALI_DEBUG_ASSERT(session->is_aborting);
1023
1024         MALI_DEBUG_PRINT(3,
1025                          ("Executor: Aborting all jobs from session 0x%08X.\n",
1026                           session));
1027
1028         mali_executor_lock();
1029
1030         if (mali_group_get_session(gp_group) == session) {
1031                 if (EXEC_STATE_WORKING == gp_group_state) {
1032                         struct mali_gp_job *gp_job = NULL;
1033
1034                         mali_executor_complete_group(gp_group, MALI_FALSE, &gp_job, NULL);
1035
1036                         MALI_DEBUG_ASSERT_POINTER(gp_job);
1037
1038                         /* GP job completed, make sure it is freed */
1039                         mali_scheduler_complete_gp_job(gp_job, MALI_FALSE,
1040                                                        MALI_FALSE, MALI_TRUE);
1041                 } else {
1042                         /* Same session, but not working, so just clear it */
1043                         mali_group_clear_session(gp_group);
1044                 }
1045         }
1046
1047         if (mali_executor_has_virtual_group()) {
1048                 if (EXEC_STATE_WORKING == virtual_group_state
1049                     && mali_group_get_session(virtual_group) == session) {
1050                         struct mali_pp_job *pp_job = NULL;
1051
1052                         mali_executor_complete_group(virtual_group, MALI_FALSE, NULL, &pp_job);
1053
1054                         if (NULL != pp_job) {
1055                                 /* PP job completed, make sure it is freed */
1056                                 mali_scheduler_complete_pp_job(pp_job, 0,
1057                                                                MALI_FALSE, MALI_TRUE);
1058                         }
1059                 }
1060         }
1061
1062         _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_working,
1063                                     struct mali_group, executor_list) {
1064                 if (mali_group_get_session(group) == session) {
1065                         struct mali_pp_job *pp_job = NULL;
1066
1067                         mali_executor_complete_group(group, MALI_FALSE, NULL, &pp_job);
1068
1069                         if (NULL != pp_job) {
1070                                 /* PP job completed, make sure it is freed */
1071                                 mali_scheduler_complete_pp_job(pp_job, 0,
1072                                                                MALI_FALSE, MALI_TRUE);
1073                         }
1074                 }
1075         }
1076
1077         _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_idle, struct mali_group, executor_list) {
1078                 mali_group_clear_session(group);
1079         }
1080
1081         _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_inactive, struct mali_group, executor_list) {
1082                 mali_group_clear_session(group);
1083         }
1084
1085         _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_disabled, struct mali_group, executor_list) {
1086                 mali_group_clear_session(group);
1087         }
1088
1089         mali_executor_unlock();
1090 }
1091
1092
1093 void mali_executor_core_scaling_enable(void)
1094 {
1095         /* PS: Core scaling is by default enabled */
1096         core_scaling_enabled = MALI_TRUE;
1097 }
1098
1099 void mali_executor_core_scaling_disable(void)
1100 {
1101         core_scaling_enabled = MALI_FALSE;
1102 }
1103
1104 mali_bool mali_executor_core_scaling_is_enabled(void)
1105 {
1106         return core_scaling_enabled;
1107 }
1108
1109 void mali_executor_group_enable(struct mali_group *group)
1110 {
1111         MALI_DEBUG_ASSERT_POINTER(group);
1112
1113         mali_executor_lock();
1114
1115         if ((NULL != mali_group_get_gp_core(group) || NULL != mali_group_get_pp_core(group))
1116             && (mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED))) {
1117                 mali_executor_group_enable_internal(group);
1118         }
1119
1120         mali_executor_schedule();
1121         mali_executor_unlock();
1122
1123         _mali_osk_wq_schedule_work(executor_wq_notify_core_change);
1124 }
1125
1126 /*
1127  * If a physical group is inactive or idle, we should disable it immediately,
1128  * if group is in virtual, and virtual group is idle, disable given physical group in it.
1129  */
1130 void mali_executor_group_disable(struct mali_group *group)
1131 {
1132         MALI_DEBUG_ASSERT_POINTER(group);
1133
1134         mali_executor_lock();
1135
1136         if ((NULL != mali_group_get_gp_core(group) || NULL != mali_group_get_pp_core(group))
1137             && (!mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED))) {
1138                 mali_executor_group_disable_internal(group);
1139         }
1140
1141         mali_executor_schedule();
1142         mali_executor_unlock();
1143
1144         _mali_osk_wq_schedule_work(executor_wq_notify_core_change);
1145 }
1146
1147 mali_bool mali_executor_group_is_disabled(struct mali_group *group)
1148 {
1149         /* NB: This function is not optimized for time critical usage */
1150
1151         mali_bool ret;
1152
1153         MALI_DEBUG_ASSERT_POINTER(group);
1154
1155         mali_executor_lock();
1156         ret = mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED);
1157         mali_executor_unlock();
1158
1159         return ret;
1160 }
1161
1162 int mali_executor_set_perf_level(unsigned int target_core_nr, mali_bool override)
1163 {
1164         if (target_core_nr == num_physical_pp_cores_enabled) return 0;
1165         if (MALI_FALSE == core_scaling_enabled && MALI_FALSE == override) return -EPERM;
1166         if (target_core_nr > num_physical_pp_cores_total) return -EINVAL;
1167         if (0 == target_core_nr) return -EINVAL;
1168
1169         mali_executor_core_scale(target_core_nr);
1170
1171         _mali_osk_wq_schedule_work(executor_wq_notify_core_change);
1172
1173         return 0;
1174 }
1175
1176 #if MALI_STATE_TRACKING
1177 u32 mali_executor_dump_state(char *buf, u32 size)
1178 {
1179         int n = 0;
1180         struct mali_group *group;
1181         struct mali_group *temp;
1182
1183         mali_executor_lock();
1184
1185         switch (gp_group_state) {
1186         case EXEC_STATE_INACTIVE:
1187                 n += _mali_osk_snprintf(buf + n, size - n,
1188                                         "GP group is in state INACTIVE\n");
1189                 break;
1190         case EXEC_STATE_IDLE:
1191                 n += _mali_osk_snprintf(buf + n, size - n,
1192                                         "GP group is in state IDLE\n");
1193                 break;
1194         case EXEC_STATE_WORKING:
1195                 n += _mali_osk_snprintf(buf + n, size - n,
1196                                         "GP group is in state WORKING\n");
1197                 break;
1198         default:
1199                 n += _mali_osk_snprintf(buf + n, size - n,
1200                                         "GP group is in unknown/illegal state %u\n",
1201                                         gp_group_state);
1202                 break;
1203         }
1204
1205         n += mali_group_dump_state(gp_group, buf + n, size - n);
1206
1207         n += _mali_osk_snprintf(buf + n, size - n,
1208                                 "Physical PP groups in WORKING state (count = %u):\n",
1209                                 group_list_working_count);
1210
1211         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_working, struct mali_group, executor_list) {
1212                 n += mali_group_dump_state(group, buf + n, size - n);
1213         }
1214
1215         n += _mali_osk_snprintf(buf + n, size - n,
1216                                 "Physical PP groups in IDLE state (count = %u):\n",
1217                                 group_list_idle_count);
1218
1219         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, executor_list) {
1220                 n += mali_group_dump_state(group, buf + n, size - n);
1221         }
1222
1223         n += _mali_osk_snprintf(buf + n, size - n,
1224                                 "Physical PP groups in INACTIVE state (count = %u):\n",
1225                                 group_list_inactive_count);
1226
1227         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive, struct mali_group, executor_list) {
1228                 n += mali_group_dump_state(group, buf + n, size - n);
1229         }
1230
1231         n += _mali_osk_snprintf(buf + n, size - n,
1232                                 "Physical PP groups in DISABLED state (count = %u):\n",
1233                                 group_list_disabled_count);
1234
1235         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_disabled, struct mali_group, executor_list) {
1236                 n += mali_group_dump_state(group, buf + n, size - n);
1237         }
1238
1239         if (mali_executor_has_virtual_group()) {
1240                 switch (virtual_group_state) {
1241                 case EXEC_STATE_EMPTY:
1242                         n += _mali_osk_snprintf(buf + n, size - n,
1243                                                 "Virtual PP group is in state EMPTY\n");
1244                         break;
1245                 case EXEC_STATE_INACTIVE:
1246                         n += _mali_osk_snprintf(buf + n, size - n,
1247                                                 "Virtual PP group is in state INACTIVE\n");
1248                         break;
1249                 case EXEC_STATE_IDLE:
1250                         n += _mali_osk_snprintf(buf + n, size - n,
1251                                                 "Virtual PP group is in state IDLE\n");
1252                         break;
1253                 case EXEC_STATE_WORKING:
1254                         n += _mali_osk_snprintf(buf + n, size - n,
1255                                                 "Virtual PP group is in state WORKING\n");
1256                         break;
1257                 default:
1258                         n += _mali_osk_snprintf(buf + n, size - n,
1259                                                 "Virtual PP group is in unknown/illegal state %u\n",
1260                                                 virtual_group_state);
1261                         break;
1262                 }
1263
1264                 n += mali_group_dump_state(virtual_group, buf + n, size - n);
1265         }
1266
1267         mali_executor_unlock();
1268
1269         n += _mali_osk_snprintf(buf + n, size - n, "\n");
1270
1271         return n;
1272 }
1273 #endif
1274
1275 _mali_osk_errcode_t _mali_ukk_get_pp_number_of_cores(_mali_uk_get_pp_number_of_cores_s *args)
1276 {
1277         MALI_DEBUG_ASSERT_POINTER(args);
1278         MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
1279         args->number_of_total_cores = num_physical_pp_cores_total;
1280         args->number_of_enabled_cores = num_physical_pp_cores_enabled;
1281         return _MALI_OSK_ERR_OK;
1282 }
1283
1284 _mali_osk_errcode_t _mali_ukk_get_pp_core_version(_mali_uk_get_pp_core_version_s *args)
1285 {
1286         MALI_DEBUG_ASSERT_POINTER(args);
1287         MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
1288         args->version = pp_version;
1289         return _MALI_OSK_ERR_OK;
1290 }
1291
1292 _mali_osk_errcode_t _mali_ukk_get_gp_number_of_cores(_mali_uk_get_gp_number_of_cores_s *args)
1293 {
1294         MALI_DEBUG_ASSERT_POINTER(args);
1295         MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
1296         args->number_of_cores = 1;
1297         return _MALI_OSK_ERR_OK;
1298 }
1299
1300 _mali_osk_errcode_t _mali_ukk_get_gp_core_version(_mali_uk_get_gp_core_version_s *args)
1301 {
1302         MALI_DEBUG_ASSERT_POINTER(args);
1303         MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
1304         args->version = gp_version;
1305         return _MALI_OSK_ERR_OK;
1306 }
1307
1308 _mali_osk_errcode_t _mali_ukk_gp_suspend_response(_mali_uk_gp_suspend_response_s *args)
1309 {
1310         struct mali_session_data *session;
1311         struct mali_gp_job *job;
1312
1313         MALI_DEBUG_ASSERT_POINTER(args);
1314         MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
1315
1316         session = (struct mali_session_data *)(uintptr_t)args->ctx;
1317
1318         if (_MALIGP_JOB_RESUME_WITH_NEW_HEAP == args->code) {
1319                 _mali_osk_notification_t *new_notification = NULL;
1320
1321                 new_notification = _mali_osk_notification_create(
1322                                            _MALI_NOTIFICATION_GP_STALLED,
1323                                            sizeof(_mali_uk_gp_job_suspended_s));
1324
1325                 if (NULL != new_notification) {
1326                         MALI_DEBUG_PRINT(3, ("Executor: Resuming job %u with new heap; 0x%08X - 0x%08X\n",
1327                                              args->cookie, args->arguments[0], args->arguments[1]));
1328
1329                         mali_executor_lock();
1330
1331                         /* Resume the job in question if it is still running */
1332                         job = mali_group_get_running_gp_job(gp_group);
1333                         if (NULL != job &&
1334                             args->cookie == mali_gp_job_get_id(job) &&
1335                             session == mali_gp_job_get_session(job)) {
1336                                 /*
1337                                  * Correct job is running, resume with new heap
1338                                  */
1339
1340                                 mali_gp_job_set_oom_notification(job,
1341                                                                  new_notification);
1342
1343                                 /* This will also re-enable interrupts */
1344                                 mali_group_resume_gp_with_new_heap(gp_group,
1345                                                                    args->cookie,
1346                                                                    args->arguments[0],
1347                                                                    args->arguments[1]);
1348
1349                                 job->heap_base_addr =  args->arguments[0];
1350                                 job->heap_current_addr = args->arguments[0];
1351
1352                                 mali_executor_unlock();
1353                                 return _MALI_OSK_ERR_OK;
1354                         } else {
1355                                 MALI_PRINT_ERROR(("Executor: Unable to resume, GP job no longer running.\n"));
1356
1357                                 _mali_osk_notification_delete(new_notification);
1358
1359                                 mali_executor_unlock();
1360                                 return _MALI_OSK_ERR_FAULT;
1361                         }
1362                 } else {
1363                         MALI_PRINT_ERROR(("Executor: Failed to allocate notification object. Will abort GP job.\n"));
1364                 }
1365         } else {
1366                 MALI_DEBUG_PRINT(2, ("Executor: Aborting job %u, no new heap provided\n", args->cookie));
1367         }
1368
1369         mali_executor_lock();
1370
1371         /* Abort the job in question if it is still running */
1372         job = mali_group_get_running_gp_job(gp_group);
1373         if (NULL != job &&
1374             args->cookie == mali_gp_job_get_id(job) &&
1375             session == mali_gp_job_get_session(job)) {
1376                 /* Correct job is still running */
1377                 struct mali_gp_job *job_done = NULL;
1378
1379                 mali_executor_complete_group(gp_group, MALI_FALSE, &job_done, NULL);
1380
1381                 /* The same job should have completed */
1382                 MALI_DEBUG_ASSERT(job_done == job);
1383
1384                 /* GP job completed, make sure it is freed */
1385                 mali_scheduler_complete_gp_job(job_done, MALI_FALSE,
1386                                                MALI_TRUE, MALI_TRUE);
1387         }
1388
1389         mali_executor_unlock();
1390         return _MALI_OSK_ERR_FAULT;
1391 }
1392
1393
1394 /*
1395  * ---------- Implementation of static functions ----------
1396  */
1397
1398 void mali_executor_lock(void)
1399 {
1400         _mali_osk_spinlock_irq_lock(mali_executor_lock_obj);
1401         MALI_DEBUG_PRINT(5, ("Executor: lock taken\n"));
1402 }
1403
1404 void mali_executor_unlock(void)
1405 {
1406         MALI_DEBUG_PRINT(5, ("Executor: Releasing lock\n"));
1407         _mali_osk_spinlock_irq_unlock(mali_executor_lock_obj);
1408 }
1409
1410 static mali_bool mali_executor_is_suspended(void *data)
1411 {
1412         mali_bool ret;
1413
1414         /* This callback does not use the data pointer. */
1415         MALI_IGNORE(data);
1416
1417         mali_executor_lock();
1418
1419         ret = pause_count > 0 && !mali_executor_is_working();
1420
1421         mali_executor_unlock();
1422
1423         return ret;
1424 }
1425
1426 static mali_bool mali_executor_is_working()
1427 {
1428         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1429
1430         return (0 != group_list_working_count ||
1431                 EXEC_STATE_WORKING == gp_group_state ||
1432                 EXEC_STATE_WORKING == virtual_group_state);
1433 }
1434
1435 static void mali_executor_disable_empty_virtual(void)
1436 {
1437         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1438         MALI_DEBUG_ASSERT(virtual_group_state != EXEC_STATE_EMPTY);
1439         MALI_DEBUG_ASSERT(virtual_group_state != EXEC_STATE_WORKING);
1440
1441         if (mali_group_is_empty(virtual_group)) {
1442                 virtual_group_state = EXEC_STATE_EMPTY;
1443         }
1444 }
1445
1446 static mali_bool mali_executor_physical_rejoin_virtual(struct mali_group *group)
1447 {
1448         mali_bool trigger_pm_update = MALI_FALSE;
1449
1450         MALI_DEBUG_ASSERT_POINTER(group);
1451         /* Only rejoining after job has completed (still active) */
1452         MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVE ==
1453                           mali_group_get_state(group));
1454         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1455         MALI_DEBUG_ASSERT(MALI_TRUE == mali_executor_has_virtual_group());
1456         MALI_DEBUG_ASSERT(MALI_FALSE == mali_group_is_virtual(group));
1457
1458         /* Make sure group and virtual group have same status */
1459
1460         if (MALI_GROUP_STATE_INACTIVE == mali_group_get_state(virtual_group)) {
1461                 if (mali_group_deactivate(group)) {
1462                         trigger_pm_update = MALI_TRUE;
1463                 }
1464
1465                 if (virtual_group_state == EXEC_STATE_EMPTY) {
1466                         virtual_group_state = EXEC_STATE_INACTIVE;
1467                 }
1468         } else if (MALI_GROUP_STATE_ACTIVATION_PENDING ==
1469                    mali_group_get_state(virtual_group)) {
1470                 /*
1471                  * Activation is pending for virtual group, leave
1472                  * this child group as active.
1473                  */
1474                 if (virtual_group_state == EXEC_STATE_EMPTY) {
1475                         virtual_group_state = EXEC_STATE_INACTIVE;
1476                 }
1477         } else {
1478                 MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVE ==
1479                                   mali_group_get_state(virtual_group));
1480
1481                 if (virtual_group_state == EXEC_STATE_EMPTY) {
1482                         virtual_group_state = EXEC_STATE_IDLE;
1483                 }
1484         }
1485
1486         /* Remove group from idle list */
1487         MALI_DEBUG_ASSERT(mali_executor_group_is_in_state(group,
1488                           EXEC_STATE_IDLE));
1489         _mali_osk_list_delinit(&group->executor_list);
1490         group_list_idle_count--;
1491
1492         /*
1493          * And finally rejoin the virtual group
1494          * group will start working on same job as virtual_group,
1495          * if virtual_group is working on a job
1496          */
1497         mali_group_add_group(virtual_group, group);
1498
1499         return trigger_pm_update;
1500 }
1501
1502 static mali_bool mali_executor_has_virtual_group(void)
1503 {
1504 #if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470))
1505         return (NULL != virtual_group) ? MALI_TRUE : MALI_FALSE;
1506 #else
1507         return MALI_FALSE;
1508 #endif /* (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) */
1509 }
1510
1511 static mali_bool mali_executor_virtual_group_is_usable(void)
1512 {
1513 #if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470))
1514         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1515         return ((EXEC_STATE_INACTIVE == virtual_group_state ||
1516                  EXEC_STATE_IDLE == virtual_group_state) && (virtual_group->state != MALI_GROUP_STATE_ACTIVATION_PENDING)) ?
1517                MALI_TRUE : MALI_FALSE;
1518 #else
1519         return MALI_FALSE;
1520 #endif /* (defined(CONFIG_MALI450) || defined(CONFIG_MALI470)) */
1521 }
1522
1523 static mali_bool mali_executor_tackle_gp_bound(void)
1524 {
1525         struct mali_pp_job *job;
1526
1527         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1528
1529         job = mali_scheduler_job_pp_physical_peek();
1530
1531         if (NULL != job && MALI_TRUE == mali_is_mali400()) {
1532                 if (0 < group_list_working_count &&
1533                     mali_pp_job_is_large_and_unstarted(job)) {
1534                         return MALI_TRUE;
1535                 }
1536         }
1537
1538         return MALI_FALSE;
1539 }
1540
1541 /*
1542  * This is where jobs are actually started.
1543  */
1544 static void mali_executor_schedule(void)
1545 {
1546         u32 i;
1547         u32 num_physical_needed = 0;
1548         u32 num_physical_to_process = 0;
1549         mali_bool trigger_pm_update = MALI_FALSE;
1550         mali_bool deactivate_idle_group = MALI_TRUE;
1551
1552         /* Physical groups + jobs to start in this function */
1553         struct mali_group *groups_to_start[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS];
1554         struct mali_pp_job *jobs_to_start[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS];
1555         u32 sub_jobs_to_start[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS];
1556         int num_jobs_to_start = 0;
1557
1558         /* Virtual job to start in this function */
1559         struct mali_pp_job *virtual_job_to_start = NULL;
1560
1561         /* GP job to start in this function */
1562         struct mali_gp_job *gp_job_to_start = NULL;
1563
1564         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1565
1566         if (pause_count > 0) {
1567                 /* Execution is suspended, don't schedule any jobs. */
1568                 return;
1569         }
1570
1571         /* Lock needed in order to safely handle the job queues */
1572         mali_scheduler_lock();
1573
1574         /* 1. Activate gp firstly if have gp job queued. */
1575         if (EXEC_STATE_INACTIVE == gp_group_state &&
1576             0 < mali_scheduler_job_gp_count()) {
1577
1578                 enum mali_group_state state =
1579                         mali_group_activate(gp_group);
1580                 if (MALI_GROUP_STATE_ACTIVE == state) {
1581                         /* Set GP group state to idle */
1582                         gp_group_state = EXEC_STATE_IDLE;
1583                 } else {
1584                         trigger_pm_update = MALI_TRUE;
1585                 }
1586         }
1587
1588         /* 2. Prepare as many physical groups as needed/possible */
1589
1590         num_physical_needed = mali_scheduler_job_physical_head_count();
1591
1592         /* On mali-450 platform, we don't need to enter in this block frequently. */
1593         if (0 < num_physical_needed) {
1594
1595                 if (num_physical_needed <= group_list_idle_count) {
1596                         /* We have enough groups on idle list already */
1597                         num_physical_to_process = num_physical_needed;
1598                         num_physical_needed = 0;
1599                 } else {
1600                         /* We need to get a hold of some more groups */
1601                         num_physical_to_process = group_list_idle_count;
1602                         num_physical_needed -= group_list_idle_count;
1603                 }
1604
1605                 if (0 < num_physical_needed) {
1606
1607                         /* 2.1. Activate groups which are inactive */
1608
1609                         struct mali_group *group;
1610                         struct mali_group *temp;
1611
1612                         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive,
1613                                                     struct mali_group, executor_list) {
1614                                 enum mali_group_state state =
1615                                         mali_group_activate(group);
1616                                 if (MALI_GROUP_STATE_ACTIVE == state) {
1617                                         /* Move from inactive to idle */
1618                                         mali_executor_change_state_pp_physical(group,
1619                                                                                &group_list_inactive,
1620                                                                                &group_list_inactive_count,
1621                                                                                &group_list_idle,
1622                                                                                &group_list_idle_count);
1623                                         num_physical_to_process++;
1624                                 } else {
1625                                         trigger_pm_update = MALI_TRUE;
1626                                 }
1627
1628                                 num_physical_needed--;
1629                                 if (0 == num_physical_needed) {
1630                                         /* We have activated all the groups we need */
1631                                         break;
1632                                 }
1633                         }
1634                 }
1635
1636                 if (mali_executor_virtual_group_is_usable()) {
1637
1638                         /*
1639                          * 2.2. And finally, steal and activate groups
1640                          * from virtual group if we need even more
1641                          */
1642                         while (0 < num_physical_needed) {
1643                                 struct mali_group *group;
1644
1645                                 group = mali_group_acquire_group(virtual_group);
1646                                 if (NULL != group) {
1647                                         enum mali_group_state state;
1648
1649                                         mali_executor_disable_empty_virtual();
1650
1651                                         state = mali_group_activate(group);
1652                                         if (MALI_GROUP_STATE_ACTIVE == state) {
1653                                                 /* Group is ready, add to idle list */
1654                                                 _mali_osk_list_add(
1655                                                         &group->executor_list,
1656                                                         &group_list_idle);
1657                                                 group_list_idle_count++;
1658                                                 num_physical_to_process++;
1659                                         } else {
1660                                                 /*
1661                                                  * Group is not ready yet,
1662                                                  * add to inactive list
1663                                                  */
1664                                                 _mali_osk_list_add(
1665                                                         &group->executor_list,
1666                                                         &group_list_inactive);
1667                                                 group_list_inactive_count++;
1668
1669                                                 trigger_pm_update = MALI_TRUE;
1670                                         }
1671                                         num_physical_needed--;
1672                                 } else {
1673                                         /*
1674                                          * We could not get enough groups
1675                                          * from the virtual group.
1676                                          */
1677                                         break;
1678                                 }
1679                         }
1680                 }
1681
1682                 /* 2.3. Assign physical jobs to groups */
1683
1684                 if (0 < num_physical_to_process) {
1685                         struct mali_group *group;
1686                         struct mali_group *temp;
1687
1688                         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle,
1689                                                     struct mali_group, executor_list) {
1690                                 struct mali_pp_job *job = NULL;
1691                                 u32 sub_job = MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS;
1692
1693                                 MALI_DEBUG_ASSERT(num_jobs_to_start <
1694                                                   MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS);
1695
1696                                 MALI_DEBUG_ASSERT(0 <
1697                                                   mali_scheduler_job_physical_head_count());
1698
1699                                 if (mali_executor_hint_is_enabled(
1700                                             MALI_EXECUTOR_HINT_GP_BOUND)) {
1701                                         if (MALI_TRUE == mali_executor_tackle_gp_bound()) {
1702                                                 /*
1703                                                 * We're gp bound,
1704                                                 * don't start this right now.
1705                                                 */
1706                                                 deactivate_idle_group = MALI_FALSE;
1707                                                 num_physical_to_process = 0;
1708                                                 break;
1709                                         }
1710                                 }
1711
1712                                 job = mali_scheduler_job_pp_physical_get(
1713                                               &sub_job);
1714
1715                                 MALI_DEBUG_ASSERT_POINTER(job);
1716                                 MALI_DEBUG_ASSERT(sub_job <= MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS);
1717
1718                                 /* Put job + group on list of jobs to start later on */
1719
1720                                 groups_to_start[num_jobs_to_start] = group;
1721                                 jobs_to_start[num_jobs_to_start] = job;
1722                                 sub_jobs_to_start[num_jobs_to_start] = sub_job;
1723                                 num_jobs_to_start++;
1724
1725                                 /* Move group from idle to working */
1726                                 mali_executor_change_state_pp_physical(group,
1727                                                                        &group_list_idle,
1728                                                                        &group_list_idle_count,
1729                                                                        &group_list_working,
1730                                                                        &group_list_working_count);
1731
1732                                 num_physical_to_process--;
1733                                 if (0 == num_physical_to_process) {
1734                                         /* Got all we needed */
1735                                         break;
1736                                 }
1737                         }
1738                 }
1739         }
1740
1741
1742         /* 3. Deactivate idle pp group , must put deactive here before active vitual group
1743          *    for cover case first only has physical job in normal queue but group inactive,
1744          *    so delay the job start go to active group, when group activated,
1745          *    call scheduler again, but now if we get high queue virtual job,
1746          *    we will do nothing in schedule cause executor schedule stop
1747          */
1748
1749         if (MALI_TRUE == mali_executor_deactivate_list_idle(deactivate_idle_group
1750                         && (!mali_timeline_has_physical_pp_job()))) {
1751                 trigger_pm_update = MALI_TRUE;
1752         }
1753
1754         /* 4. Activate virtual group, if needed */
1755
1756         if (EXEC_STATE_INACTIVE == virtual_group_state &&
1757             0 < mali_scheduler_job_next_is_virtual()) {
1758                 enum mali_group_state state =
1759                         mali_group_activate(virtual_group);
1760                 if (MALI_GROUP_STATE_ACTIVE == state) {
1761                         /* Set virtual group state to idle */
1762                         virtual_group_state = EXEC_STATE_IDLE;
1763                 } else {
1764                         trigger_pm_update = MALI_TRUE;
1765                 }
1766         }
1767
1768         /* 5. To power up group asap, we trigger pm update here. */
1769
1770         if (MALI_TRUE == trigger_pm_update) {
1771                 trigger_pm_update = MALI_FALSE;
1772                 mali_pm_update_async();
1773         }
1774
1775         /* 6. Assign jobs to idle virtual group (or deactivate if no job) */
1776
1777         if (EXEC_STATE_IDLE == virtual_group_state) {
1778                 if (0 < mali_scheduler_job_next_is_virtual()) {
1779                         virtual_job_to_start =
1780                                 mali_scheduler_job_pp_virtual_get();
1781                         virtual_group_state = EXEC_STATE_WORKING;
1782                 } else if (!mali_timeline_has_virtual_pp_job()) {
1783                         virtual_group_state = EXEC_STATE_INACTIVE;
1784
1785                         if (mali_group_deactivate(virtual_group)) {
1786                                 trigger_pm_update = MALI_TRUE;
1787                         }
1788                 }
1789         }
1790
1791         /* 7. Assign job to idle GP group (or deactivate if no job) */
1792
1793         if (EXEC_STATE_IDLE == gp_group_state) {
1794                 if (0 < mali_scheduler_job_gp_count()) {
1795                         gp_job_to_start = mali_scheduler_job_gp_get();
1796                         gp_group_state = EXEC_STATE_WORKING;
1797                 } else if (!mali_timeline_has_gp_job()) {
1798                         gp_group_state = EXEC_STATE_INACTIVE;
1799                         if (mali_group_deactivate(gp_group)) {
1800                                 trigger_pm_update = MALI_TRUE;
1801                         }
1802                 }
1803         }
1804
1805         /* 8. We no longer need the schedule/queue lock */
1806
1807         mali_scheduler_unlock();
1808
1809         /* 9. start jobs */
1810
1811         if (NULL != virtual_job_to_start) {
1812                 MALI_DEBUG_ASSERT(!mali_group_pp_is_active(virtual_group));
1813                 mali_group_start_pp_job(virtual_group,
1814                                         virtual_job_to_start, 0);
1815         }
1816
1817         for (i = 0; i < num_jobs_to_start; i++) {
1818                 MALI_DEBUG_ASSERT(!mali_group_pp_is_active(
1819                                           groups_to_start[i]));
1820                 mali_group_start_pp_job(groups_to_start[i],
1821                                         jobs_to_start[i],
1822                                         sub_jobs_to_start[i]);
1823         }
1824
1825         MALI_DEBUG_ASSERT_POINTER(gp_group);
1826
1827         if (NULL != gp_job_to_start) {
1828                 MALI_DEBUG_ASSERT(!mali_group_gp_is_active(gp_group));
1829                 mali_group_start_gp_job(gp_group, gp_job_to_start);
1830         }
1831
1832         /* 10. Trigger any pending PM updates */
1833         if (MALI_TRUE == trigger_pm_update) {
1834                 mali_pm_update_async();
1835         }
1836 }
1837
1838 /* Handler for deferred schedule requests */
1839 static void mali_executor_wq_schedule(void *arg)
1840 {
1841         MALI_IGNORE(arg);
1842         mali_executor_lock();
1843         mali_executor_schedule();
1844         mali_executor_unlock();
1845 }
1846
1847 static void mali_executor_send_gp_oom_to_user(struct mali_gp_job *job, u32 added_size)
1848 {
1849         _mali_uk_gp_job_suspended_s *jobres;
1850         _mali_osk_notification_t *notification;
1851
1852         notification = mali_gp_job_get_oom_notification(job);
1853
1854         /*
1855          * Remember the id we send to user space, so we have something to
1856          * verify when we get a response
1857          */
1858         gp_returned_cookie = mali_gp_job_get_id(job);
1859
1860         jobres = (_mali_uk_gp_job_suspended_s *)notification->result_buffer;
1861         jobres->user_job_ptr = mali_gp_job_get_user_id(job);
1862         jobres->cookie = gp_returned_cookie;
1863         jobres->heap_added_size = added_size;
1864         mali_session_send_notification(mali_gp_job_get_session(job),
1865                                        notification);
1866 }
1867 static struct mali_gp_job *mali_executor_complete_gp(struct mali_group *group,
1868                 mali_bool success)
1869 {
1870         struct mali_gp_job *job;
1871
1872         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1873
1874         /* Extracts the needed HW status from core and reset */
1875         job = mali_group_complete_gp(group, success);
1876
1877         MALI_DEBUG_ASSERT_POINTER(job);
1878
1879         /* Core is now ready to go into idle list */
1880         gp_group_state = EXEC_STATE_IDLE;
1881
1882         /* This will potentially queue more GP and PP jobs */
1883         mali_timeline_tracker_release(&job->tracker);
1884
1885         /* Signal PP job */
1886         mali_gp_job_signal_pp_tracker(job, success);
1887
1888         return job;
1889 }
1890
1891 static struct mali_pp_job *mali_executor_complete_pp(struct mali_group *group,
1892                 mali_bool success)
1893 {
1894         struct mali_pp_job *job;
1895         u32 sub_job;
1896         mali_bool job_is_done;
1897
1898         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1899
1900         /* Extracts the needed HW status from core and reset */
1901         job = mali_group_complete_pp(group, success, &sub_job);
1902
1903         MALI_DEBUG_ASSERT_POINTER(job);
1904
1905         /* Core is now ready to go into idle list */
1906         if (mali_group_is_virtual(group)) {
1907                 virtual_group_state = EXEC_STATE_IDLE;
1908         } else {
1909                 /* Move from working to idle state */
1910                 mali_executor_change_state_pp_physical(group,
1911                                                        &group_list_working,
1912                                                        &group_list_working_count,
1913                                                        &group_list_idle,
1914                                                        &group_list_idle_count);
1915         }
1916
1917         /* It is the executor module which owns the jobs themselves by now */
1918         mali_pp_job_mark_sub_job_completed(job, success);
1919         job_is_done = mali_pp_job_is_complete(job);
1920
1921         if (job_is_done) {
1922                 /* This will potentially queue more GP and PP jobs */
1923                 mali_timeline_tracker_release(&job->tracker);
1924         }
1925
1926         return job;
1927 }
1928
1929 static void mali_executor_complete_group(struct mali_group *group,
1930                 mali_bool success,
1931                 struct mali_gp_job **gp_job_done,
1932                 struct mali_pp_job **pp_job_done)
1933 {
1934         struct mali_gp_core *gp_core = mali_group_get_gp_core(group);
1935         struct mali_pp_core *pp_core = mali_group_get_pp_core(group);
1936         struct mali_gp_job *gp_job = NULL;
1937         struct mali_pp_job *pp_job = NULL;
1938         mali_bool pp_job_is_done = MALI_TRUE;
1939
1940         if (NULL != gp_core) {
1941                 gp_job = mali_executor_complete_gp(group, success);
1942         } else {
1943                 MALI_DEBUG_ASSERT_POINTER(pp_core);
1944                 MALI_IGNORE(pp_core);
1945                 pp_job = mali_executor_complete_pp(group, success);
1946
1947                 pp_job_is_done = mali_pp_job_is_complete(pp_job);
1948         }
1949
1950         if (pause_count > 0) {
1951                 /* Execution has been suspended */
1952
1953                 if (!mali_executor_is_working()) {
1954                         /* Last job completed, wake up sleepers */
1955                         _mali_osk_wait_queue_wake_up(
1956                                 executor_working_wait_queue);
1957                 }
1958         } else if (MALI_TRUE == mali_group_disable_requested(group)) {
1959                 mali_executor_core_scale_in_group_complete(group);
1960
1961                 mali_executor_schedule();
1962         } else {
1963                 /* try to schedule new jobs */
1964                 mali_executor_schedule();
1965         }
1966
1967         if (NULL != gp_job) {
1968                 MALI_DEBUG_ASSERT_POINTER(gp_job_done);
1969                 *gp_job_done = gp_job;
1970         } else if (pp_job_is_done) {
1971                 MALI_DEBUG_ASSERT_POINTER(pp_job);
1972                 MALI_DEBUG_ASSERT_POINTER(pp_job_done);
1973                 *pp_job_done = pp_job;
1974         }
1975 }
1976
1977 static void mali_executor_change_state_pp_physical(struct mali_group *group,
1978                 _mali_osk_list_t *old_list,
1979                 u32 *old_count,
1980                 _mali_osk_list_t *new_list,
1981                 u32 *new_count)
1982 {
1983         /*
1984          * It's a bit more complicated to change the state for the physical PP
1985          * groups since their state is determined by the list they are on.
1986          */
1987 #if defined(DEBUG)
1988         mali_bool found = MALI_FALSE;
1989         struct mali_group *group_iter;
1990         struct mali_group *temp;
1991         u32 old_counted = 0;
1992         u32 new_counted = 0;
1993
1994         MALI_DEBUG_ASSERT_POINTER(group);
1995         MALI_DEBUG_ASSERT_POINTER(old_list);
1996         MALI_DEBUG_ASSERT_POINTER(old_count);
1997         MALI_DEBUG_ASSERT_POINTER(new_list);
1998         MALI_DEBUG_ASSERT_POINTER(new_count);
1999
2000         /*
2001          * Verify that group is present on old list,
2002          * and that the count is correct
2003          */
2004
2005         _MALI_OSK_LIST_FOREACHENTRY(group_iter, temp, old_list,
2006                                     struct mali_group, executor_list) {
2007                 old_counted++;
2008                 if (group == group_iter) {
2009                         found = MALI_TRUE;
2010                 }
2011         }
2012
2013         _MALI_OSK_LIST_FOREACHENTRY(group_iter, temp, new_list,
2014                                     struct mali_group, executor_list) {
2015                 new_counted++;
2016         }
2017
2018         if (MALI_FALSE == found) {
2019                 if (old_list == &group_list_idle) {
2020                         MALI_DEBUG_PRINT(1, (" old Group list is idle,"));
2021                 } else if (old_list == &group_list_inactive) {
2022                         MALI_DEBUG_PRINT(1, (" old Group list is inactive,"));
2023                 } else if (old_list == &group_list_working) {
2024                         MALI_DEBUG_PRINT(1, (" old Group list is working,"));
2025                 } else if (old_list == &group_list_disabled) {
2026                         MALI_DEBUG_PRINT(1, (" old Group list is disable,"));
2027                 }
2028
2029                 if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_WORKING)) {
2030                         MALI_DEBUG_PRINT(1, (" group in working \n"));
2031                 } else if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_INACTIVE)) {
2032                         MALI_DEBUG_PRINT(1, (" group in inactive \n"));
2033                 } else if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_IDLE)) {
2034                         MALI_DEBUG_PRINT(1, (" group in idle \n"));
2035                 } else if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED)) {
2036                         MALI_DEBUG_PRINT(1, (" but group in disabled \n"));
2037                 }
2038         }
2039
2040         MALI_DEBUG_ASSERT(MALI_TRUE == found);
2041         MALI_DEBUG_ASSERT(0 < (*old_count));
2042         MALI_DEBUG_ASSERT((*old_count) == old_counted);
2043         MALI_DEBUG_ASSERT((*new_count) == new_counted);
2044 #endif
2045
2046         _mali_osk_list_move(&group->executor_list, new_list);
2047         (*old_count)--;
2048         (*new_count)++;
2049 }
2050
2051 static void mali_executor_set_state_pp_physical(struct mali_group *group,
2052                 _mali_osk_list_t *new_list,
2053                 u32 *new_count)
2054 {
2055         _mali_osk_list_add(&group->executor_list, new_list);
2056         (*new_count)++;
2057 }
2058
2059 static mali_bool mali_executor_group_is_in_state(struct mali_group *group,
2060                 enum mali_executor_state_t state)
2061 {
2062         MALI_DEBUG_ASSERT_POINTER(group);
2063         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
2064
2065         if (gp_group == group) {
2066                 if (gp_group_state == state) {
2067                         return MALI_TRUE;
2068                 }
2069         } else if (virtual_group == group || mali_group_is_in_virtual(group)) {
2070                 if (virtual_group_state == state) {
2071                         return MALI_TRUE;
2072                 }
2073         } else {
2074                 /* Physical PP group */
2075                 struct mali_group *group_iter;
2076                 struct mali_group *temp;
2077                 _mali_osk_list_t *list;
2078
2079                 if (EXEC_STATE_DISABLED == state) {
2080                         list = &group_list_disabled;
2081                 } else if (EXEC_STATE_INACTIVE == state) {
2082                         list = &group_list_inactive;
2083                 } else if (EXEC_STATE_IDLE == state) {
2084                         list = &group_list_idle;
2085                 } else {
2086                         MALI_DEBUG_ASSERT(EXEC_STATE_WORKING == state);
2087                         list = &group_list_working;
2088                 }
2089
2090                 _MALI_OSK_LIST_FOREACHENTRY(group_iter, temp, list,
2091                                             struct mali_group, executor_list) {
2092                         if (group_iter == group) {
2093                                 return MALI_TRUE;
2094                         }
2095                 }
2096         }
2097
2098         /* group not in correct state */
2099         return MALI_FALSE;
2100 }
2101
2102 static void mali_executor_group_enable_internal(struct mali_group *group)
2103 {
2104         MALI_DEBUG_ASSERT(group);
2105         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
2106         MALI_DEBUG_ASSERT(mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED));
2107
2108         /* Put into inactive state (== "lowest" enabled state) */
2109         if (group == gp_group) {
2110                 MALI_DEBUG_ASSERT(EXEC_STATE_DISABLED == gp_group_state);
2111                 gp_group_state = EXEC_STATE_INACTIVE;
2112         } else {
2113                 mali_executor_change_state_pp_physical(group,
2114                                                        &group_list_disabled,
2115                                                        &group_list_disabled_count,
2116                                                        &group_list_inactive,
2117                                                        &group_list_inactive_count);
2118
2119                 ++num_physical_pp_cores_enabled;
2120                 MALI_DEBUG_PRINT(4, ("Enabling group id %d \n", group->pp_core->core_id));
2121         }
2122
2123         if (MALI_GROUP_STATE_ACTIVE == mali_group_activate(group)) {
2124                 MALI_DEBUG_ASSERT(MALI_TRUE == mali_group_power_is_on(group));
2125
2126                 /* Move from inactive to idle */
2127                 if (group == gp_group) {
2128                         gp_group_state = EXEC_STATE_IDLE;
2129                 } else {
2130                         mali_executor_change_state_pp_physical(group,
2131                                                                &group_list_inactive,
2132                                                                &group_list_inactive_count,
2133                                                                &group_list_idle,
2134                                                                &group_list_idle_count);
2135
2136                         if (mali_executor_has_virtual_group()) {
2137                                 if (mali_executor_physical_rejoin_virtual(group)) {
2138                                         mali_pm_update_async();
2139                                 }
2140                         }
2141                 }
2142         } else {
2143                 mali_pm_update_async();
2144         }
2145 }
2146
2147 static void mali_executor_group_disable_internal(struct mali_group *group)
2148 {
2149         mali_bool working;
2150
2151         MALI_DEBUG_ASSERT_POINTER(group);
2152         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
2153         MALI_DEBUG_ASSERT(!mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED));
2154
2155         working = mali_executor_group_is_in_state(group, EXEC_STATE_WORKING);
2156         if (MALI_TRUE == working) {
2157                 /** Group to be disabled once it completes current work,
2158                  * when virtual group completes, also check child groups for this flag */
2159                 mali_group_set_disable_request(group, MALI_TRUE);
2160                 return;
2161         }
2162
2163         /* Put into disabled state */
2164         if (group == gp_group) {
2165                 /* GP group */
2166                 MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != gp_group_state);
2167                 gp_group_state = EXEC_STATE_DISABLED;
2168         } else {
2169                 if (mali_group_is_in_virtual(group)) {
2170                         /* A child group of virtual group. move the specific group from virtual group */
2171                         MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != virtual_group_state);
2172
2173                         mali_executor_set_state_pp_physical(group,
2174                                                             &group_list_disabled,
2175                                                             &group_list_disabled_count);
2176
2177                         mali_group_remove_group(virtual_group, group);
2178                         mali_executor_disable_empty_virtual();
2179                 } else {
2180                         mali_executor_change_group_status_disabled(group);
2181                 }
2182
2183                 --num_physical_pp_cores_enabled;
2184                 MALI_DEBUG_PRINT(4, ("Disabling group id %d \n", group->pp_core->core_id));
2185         }
2186
2187         if (MALI_GROUP_STATE_INACTIVE != group->state) {
2188                 if (MALI_TRUE == mali_group_deactivate(group)) {
2189                         mali_pm_update_async();
2190                 }
2191         }
2192 }
2193
2194 static void mali_executor_notify_core_change(u32 num_cores)
2195 {
2196         mali_bool done = MALI_FALSE;
2197
2198         if (mali_is_mali450() || mali_is_mali470()) {
2199                 return;
2200         }
2201
2202         /*
2203          * This function gets a bit complicated because we can't hold the session lock while
2204          * allocating notification objects.
2205          */
2206         while (!done) {
2207                 u32 i;
2208                 u32 num_sessions_alloc;
2209                 u32 num_sessions_with_lock;
2210                 u32 used_notification_objects = 0;
2211                 _mali_osk_notification_t **notobjs;
2212
2213                 /* Pre allocate the number of notifications objects we need right now (might change after lock has been taken) */
2214                 num_sessions_alloc = mali_session_get_count();
2215                 if (0 == num_sessions_alloc) {
2216                         /* No sessions to report to */
2217                         return;
2218                 }
2219
2220                 notobjs = (_mali_osk_notification_t **)_mali_osk_malloc(sizeof(_mali_osk_notification_t *) * num_sessions_alloc);
2221                 if (NULL == notobjs) {
2222                         MALI_PRINT_ERROR(("Failed to notify user space session about num PP core change (alloc failure)\n"));
2223                         /* there is probably no point in trying again, system must be really low on memory and probably unusable now anyway */
2224                         return;
2225                 }
2226
2227                 for (i = 0; i < num_sessions_alloc; i++) {
2228                         notobjs[i] = _mali_osk_notification_create(_MALI_NOTIFICATION_PP_NUM_CORE_CHANGE, sizeof(_mali_uk_pp_num_cores_changed_s));
2229                         if (NULL != notobjs[i]) {
2230                                 _mali_uk_pp_num_cores_changed_s *data = notobjs[i]->result_buffer;
2231                                 data->number_of_enabled_cores = num_cores;
2232                         } else {
2233                                 MALI_PRINT_ERROR(("Failed to notify user space session about num PP core change (alloc failure %u)\n", i));
2234                         }
2235                 }
2236
2237                 mali_session_lock();
2238
2239                 /* number of sessions will not change while we hold the lock */
2240                 num_sessions_with_lock = mali_session_get_count();
2241
2242                 if (num_sessions_alloc >= num_sessions_with_lock) {
2243                         /* We have allocated enough notification objects for all the sessions atm */
2244                         struct mali_session_data *session, *tmp;
2245                         MALI_SESSION_FOREACH(session, tmp, link) {
2246                                 MALI_DEBUG_ASSERT(used_notification_objects < num_sessions_alloc);
2247                                 if (NULL != notobjs[used_notification_objects]) {
2248                                         mali_session_send_notification(session, notobjs[used_notification_objects]);
2249                                         notobjs[used_notification_objects] = NULL; /* Don't track this notification object any more */
2250                                 }
2251                                 used_notification_objects++;
2252                         }
2253                         done = MALI_TRUE;
2254                 }
2255
2256                 mali_session_unlock();
2257
2258                 /* Delete any remaining/unused notification objects */
2259                 for (; used_notification_objects < num_sessions_alloc; used_notification_objects++) {
2260                         if (NULL != notobjs[used_notification_objects]) {
2261                                 _mali_osk_notification_delete(notobjs[used_notification_objects]);
2262                         }
2263                 }
2264
2265                 _mali_osk_free(notobjs);
2266         }
2267 }
2268
2269 static mali_bool mali_executor_core_scaling_is_done(void *data)
2270 {
2271         u32 i;
2272         u32 num_groups;
2273         mali_bool ret = MALI_TRUE;
2274
2275         MALI_IGNORE(data);
2276
2277         mali_executor_lock();
2278
2279         num_groups = mali_group_get_glob_num_groups();
2280
2281         for (i = 0; i < num_groups; i++) {
2282                 struct mali_group *group = mali_group_get_glob_group(i);
2283
2284                 if (NULL != group) {
2285                         if (MALI_TRUE == group->disable_requested && NULL != mali_group_get_pp_core(group)) {
2286                                 ret = MALI_FALSE;
2287                                 break;
2288                         }
2289                 }
2290         }
2291         mali_executor_unlock();
2292
2293         return ret;
2294 }
2295
2296 static void mali_executor_wq_notify_core_change(void *arg)
2297 {
2298         MALI_IGNORE(arg);
2299
2300         if (mali_is_mali450() || mali_is_mali470()) {
2301                 return;
2302         }
2303
2304         _mali_osk_wait_queue_wait_event(executor_notify_core_change_wait_queue,
2305                                         mali_executor_core_scaling_is_done, NULL);
2306
2307         mali_executor_notify_core_change(num_physical_pp_cores_enabled);
2308 }
2309
2310 /**
2311  * Clear all disable request from the _last_ core scaling behavior.
2312  */
2313 static void mali_executor_core_scaling_reset(void)
2314 {
2315         u32 i;
2316         u32 num_groups;
2317
2318         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
2319
2320         num_groups = mali_group_get_glob_num_groups();
2321
2322         for (i = 0; i < num_groups; i++) {
2323                 struct mali_group *group = mali_group_get_glob_group(i);
2324
2325                 if (NULL != group) {
2326                         group->disable_requested = MALI_FALSE;
2327                 }
2328         }
2329
2330         for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) {
2331                 core_scaling_delay_up_mask[i] = 0;
2332         }
2333 }
2334
2335 static void mali_executor_core_scale(unsigned int target_core_nr)
2336 {
2337         int current_core_scaling_mask[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 };
2338         int target_core_scaling_mask[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 };
2339         int i;
2340
2341         MALI_DEBUG_ASSERT(0 < target_core_nr);
2342         MALI_DEBUG_ASSERT(num_physical_pp_cores_total >= target_core_nr);
2343
2344         mali_executor_lock();
2345
2346         if (target_core_nr < num_physical_pp_cores_enabled) {
2347                 MALI_DEBUG_PRINT(2, ("Requesting %d cores: disabling %d cores\n", target_core_nr, num_physical_pp_cores_enabled - target_core_nr));
2348         } else {
2349                 MALI_DEBUG_PRINT(2, ("Requesting %d cores: enabling %d cores\n", target_core_nr, target_core_nr - num_physical_pp_cores_enabled));
2350         }
2351
2352         /* When a new core scaling request is comming,  we should remove the un-doing
2353          * part of the last core scaling request.  It's safe because we have only one
2354          * lock(executor lock) protection. */
2355         mali_executor_core_scaling_reset();
2356
2357         mali_pm_get_best_power_cost_mask(num_physical_pp_cores_enabled, current_core_scaling_mask);
2358         mali_pm_get_best_power_cost_mask(target_core_nr, target_core_scaling_mask);
2359
2360         for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) {
2361                 target_core_scaling_mask[i] = target_core_scaling_mask[i] - current_core_scaling_mask[i];
2362                 MALI_DEBUG_PRINT(5, ("target_core_scaling_mask[%d] = %d\n", i, target_core_scaling_mask[i]));
2363         }
2364
2365         for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) {
2366                 if (0 > target_core_scaling_mask[i]) {
2367                         struct mali_pm_domain *domain;
2368
2369                         domain = mali_pm_domain_get_from_index(i);
2370
2371                         /* Domain is valid and has pp cores */
2372                         if ((NULL != domain) && !(_mali_osk_list_empty(&domain->group_list))) {
2373                                 struct mali_group *group;
2374                                 struct mali_group *temp;
2375
2376                                 _MALI_OSK_LIST_FOREACHENTRY(group, temp, &domain->group_list, struct mali_group, pm_domain_list) {
2377                                         if (NULL != mali_group_get_pp_core(group) && (!mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED))
2378                                             && (!mali_group_is_virtual(group))) {
2379                                                 mali_executor_group_disable_internal(group);
2380                                                 target_core_scaling_mask[i]++;
2381                                                 if ((0 == target_core_scaling_mask[i])) {
2382                                                         break;
2383                                                 }
2384
2385                                         }
2386                                 }
2387                         }
2388                 }
2389         }
2390
2391         for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) {
2392                 /**
2393                  * Target_core_scaling_mask[i] is bigger than 0,
2394                  * means we need to enable some pp cores in
2395                  * this domain whose domain index is i.
2396                  */
2397                 if (0 < target_core_scaling_mask[i]) {
2398                         struct mali_pm_domain *domain;
2399
2400                         if (num_physical_pp_cores_enabled >= target_core_nr) {
2401                                 break;
2402                         }
2403
2404                         domain = mali_pm_domain_get_from_index(i);
2405
2406                         /* Domain is valid and has pp cores */
2407                         if ((NULL != domain) && !(_mali_osk_list_empty(&domain->group_list))) {
2408                                 struct mali_group *group;
2409                                 struct mali_group *temp;
2410
2411                                 _MALI_OSK_LIST_FOREACHENTRY(group, temp, &domain->group_list, struct mali_group, pm_domain_list) {
2412                                         if (NULL != mali_group_get_pp_core(group) && mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED)
2413                                             && (!mali_group_is_virtual(group))) {
2414                                                 mali_executor_group_enable_internal(group);
2415                                                 target_core_scaling_mask[i]--;
2416
2417                                                 if ((0 == target_core_scaling_mask[i]) || num_physical_pp_cores_enabled == target_core_nr) {
2418                                                         break;
2419                                                 }
2420                                         }
2421                                 }
2422                         }
2423                 }
2424         }
2425
2426         /**
2427          * Here, we may still have some pp cores not been enabled because of some
2428          * pp cores need to be disabled are still in working state.
2429          */
2430         for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) {
2431                 if (0 < target_core_scaling_mask[i]) {
2432                         core_scaling_delay_up_mask[i] = target_core_scaling_mask[i];
2433                 }
2434         }
2435
2436         mali_executor_schedule();
2437         mali_executor_unlock();
2438 }
2439
2440 static void mali_executor_core_scale_in_group_complete(struct mali_group *group)
2441 {
2442         int num_pp_cores_disabled = 0;
2443         int num_pp_cores_to_enable = 0;
2444         int i;
2445
2446         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
2447         MALI_DEBUG_ASSERT(MALI_TRUE == mali_group_disable_requested(group));
2448
2449         /* Disable child group of virtual group */
2450         if (mali_group_is_virtual(group)) {
2451                 struct mali_group *child;
2452                 struct mali_group *temp;
2453
2454                 _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) {
2455                         if (MALI_TRUE == mali_group_disable_requested(child)) {
2456                                 mali_group_set_disable_request(child, MALI_FALSE);
2457                                 mali_executor_group_disable_internal(child);
2458                                 num_pp_cores_disabled++;
2459                         }
2460                 }
2461                 mali_group_set_disable_request(group, MALI_FALSE);
2462         } else {
2463                 mali_executor_group_disable_internal(group);
2464                 mali_group_set_disable_request(group, MALI_FALSE);
2465                 if (NULL != mali_group_get_pp_core(group)) {
2466                         num_pp_cores_disabled++;
2467                 }
2468         }
2469
2470         num_pp_cores_to_enable = num_pp_cores_disabled;
2471
2472         for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) {
2473                 if (0 < core_scaling_delay_up_mask[i]) {
2474                         struct mali_pm_domain *domain;
2475
2476                         if (0 == num_pp_cores_to_enable) {
2477                                 break;
2478                         }
2479
2480                         domain = mali_pm_domain_get_from_index(i);
2481
2482                         /* Domain is valid and has pp cores */
2483                         if ((NULL != domain) && !(_mali_osk_list_empty(&domain->group_list))) {
2484                                 struct mali_group *disabled_group;
2485                                 struct mali_group *temp;
2486
2487                                 _MALI_OSK_LIST_FOREACHENTRY(disabled_group, temp, &domain->group_list, struct mali_group, pm_domain_list) {
2488                                         if (NULL != mali_group_get_pp_core(disabled_group) && mali_executor_group_is_in_state(disabled_group, EXEC_STATE_DISABLED)) {
2489                                                 mali_executor_group_enable_internal(disabled_group);
2490                                                 core_scaling_delay_up_mask[i]--;
2491                                                 num_pp_cores_to_enable--;
2492
2493                                                 if ((0 == core_scaling_delay_up_mask[i]) || 0 == num_pp_cores_to_enable) {
2494                                                         break;
2495                                                 }
2496                                         }
2497                                 }
2498                         }
2499                 }
2500         }
2501
2502         _mali_osk_wait_queue_wake_up(executor_notify_core_change_wait_queue);
2503 }
2504
2505 static void mali_executor_change_group_status_disabled(struct mali_group *group)
2506 {
2507         /* Physical PP group */
2508         mali_bool idle;
2509
2510         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
2511
2512         idle = mali_executor_group_is_in_state(group, EXEC_STATE_IDLE);
2513         if (MALI_TRUE == idle) {
2514                 mali_executor_change_state_pp_physical(group,
2515                                                        &group_list_idle,
2516                                                        &group_list_idle_count,
2517                                                        &group_list_disabled,
2518                                                        &group_list_disabled_count);
2519         } else {
2520                 mali_executor_change_state_pp_physical(group,
2521                                                        &group_list_inactive,
2522                                                        &group_list_inactive_count,
2523                                                        &group_list_disabled,
2524                                                        &group_list_disabled_count);
2525         }
2526 }
2527
2528 static mali_bool mali_executor_deactivate_list_idle(mali_bool deactivate_idle_group)
2529 {
2530         mali_bool trigger_pm_update = MALI_FALSE;
2531
2532         if (group_list_idle_count > 0) {
2533                 if (mali_executor_has_virtual_group()) {
2534
2535                         /* Rejoin virtual group on Mali-450 */
2536
2537                         struct mali_group *group;
2538                         struct mali_group *temp;
2539
2540                         _MALI_OSK_LIST_FOREACHENTRY(group, temp,
2541                                                     &group_list_idle,
2542                                                     struct mali_group, executor_list) {
2543                                 if (mali_executor_physical_rejoin_virtual(
2544                                             group)) {
2545                                         trigger_pm_update = MALI_TRUE;
2546                                 }
2547                         }
2548                 } else if (deactivate_idle_group) {
2549                         struct mali_group *group;
2550                         struct mali_group *temp;
2551
2552                         /* Deactivate group on Mali-300/400 */
2553
2554                         _MALI_OSK_LIST_FOREACHENTRY(group, temp,
2555                                                     &group_list_idle,
2556                                                     struct mali_group, executor_list) {
2557                                 if (mali_group_deactivate(group)) {
2558                                         trigger_pm_update = MALI_TRUE;
2559                                 }
2560
2561                                 /* Move from idle to inactive */
2562                                 mali_executor_change_state_pp_physical(group,
2563                                                                        &group_list_idle,
2564                                                                        &group_list_idle_count,
2565                                                                        &group_list_inactive,
2566                                                                        &group_list_inactive_count);
2567                         }
2568                 }
2569         }
2570
2571         return trigger_pm_update;
2572 }
2573
2574 void mali_executor_running_status_print(void)
2575 {
2576         struct mali_group *group = NULL;
2577         struct mali_group *temp = NULL;
2578
2579         MALI_PRINT(("GP running job: %p\n", gp_group->gp_running_job));
2580         if ((gp_group->gp_core) && (gp_group->is_working)) {
2581                 mali_group_dump_status(gp_group);
2582         }
2583         MALI_PRINT(("Physical PP groups in WORKING state (count = %u):\n", group_list_working_count));
2584         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_working, struct mali_group, executor_list) {
2585                 MALI_PRINT(("PP running job: %p, subjob %d \n", group->pp_running_job, group->pp_running_sub_job));
2586                 mali_group_dump_status(group);
2587         }
2588         MALI_PRINT(("Physical PP groups in INACTIVE state (count = %u):\n", group_list_inactive_count));
2589         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive, struct mali_group, executor_list) {
2590                 MALI_PRINT(("\tPP status %d, SW power: %s\n", group->state, group->power_is_on ? "On" : "Off"));
2591                 MALI_PRINT(("\tPP #%d: %s\n", group->pp_core->core_id, group->pp_core->hw_core.description));
2592         }
2593         MALI_PRINT(("Physical PP groups in IDLE state (count = %u):\n", group_list_idle_count));
2594         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, executor_list) {
2595                 MALI_PRINT(("\tPP status %d, SW power: %s\n", group->state, group->power_is_on ? "On" : "Off"));
2596                 MALI_PRINT(("\tPP #%d: %s\n", group->pp_core->core_id, group->pp_core->hw_core.description));
2597         }
2598         MALI_PRINT(("Physical PP groups in DISABLED state (count = %u):\n", group_list_disabled_count));
2599         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_disabled, struct mali_group, executor_list) {
2600                 MALI_PRINT(("\tPP status %d, SW power: %s\n", group->state, group->power_is_on ? "On" : "Off"));
2601                 MALI_PRINT(("\tPP #%d: %s\n", group->pp_core->core_id, group->pp_core->hw_core.description));
2602         }
2603
2604         if (mali_executor_has_virtual_group()) {
2605                 MALI_PRINT(("Virtual group running job: %p\n", virtual_group->pp_running_job));
2606                 MALI_PRINT(("Virtual group status: %d\n", virtual_group_state));
2607                 MALI_PRINT(("Virtual group->status: %d\n", virtual_group->state));
2608                 MALI_PRINT(("\tSW power: %s\n", virtual_group->power_is_on ? "On" : "Off"));
2609                 _MALI_OSK_LIST_FOREACHENTRY(group, temp, &virtual_group->group_list,
2610                                             struct mali_group, group_list) {
2611                         int i = 0;
2612                         MALI_PRINT(("\tchild group(%s) running job: %p\n", group->pp_core->hw_core.description, group->pp_running_job));
2613                         MALI_PRINT(("\tchild group(%s)->status: %d\n", group->pp_core->hw_core.description, group->state));
2614                         MALI_PRINT(("\tchild group(%s) SW power: %s\n", group->pp_core->hw_core.description, group->power_is_on ? "On" : "Off"));
2615                         if (group->pm_domain) {
2616                                 MALI_PRINT(("\tPower domain: id %u\n", mali_pm_domain_get_id(group->pm_domain)));
2617                                 MALI_PRINT(("\tMask:0x%04x \n", mali_pm_domain_get_mask(group->pm_domain)));
2618                                 MALI_PRINT(("\tUse-count:%u \n", mali_pm_domain_get_use_count(group->pm_domain)));
2619                                 MALI_PRINT(("\tCurrent power status:%s \n", (mali_pm_domain_get_mask(group->pm_domain)& mali_pm_get_current_mask()) ? "On" : "Off"));
2620                                 MALI_PRINT(("\tWanted  power status:%s \n", (mali_pm_domain_get_mask(group->pm_domain)& mali_pm_get_wanted_mask()) ? "On" : "Off"));
2621                         }
2622
2623                         for (i = 0; i < 2; i++) {
2624                                 if (NULL != group->l2_cache_core[i]) {
2625                                         struct mali_pm_domain *domain;
2626                                         domain = mali_l2_cache_get_pm_domain(group->l2_cache_core[i]);
2627                                         MALI_PRINT(("\t L2(index %d) group SW power: %s\n", i, group->l2_cache_core[i]->power_is_on ? "On" : "Off"));
2628                                         if (domain) {
2629                                                 MALI_PRINT(("\tL2 Power domain: id %u\n", mali_pm_domain_get_id(domain)));
2630                                                 MALI_PRINT(("\tL2 Mask:0x%04x \n", mali_pm_domain_get_mask(domain)));
2631                                                 MALI_PRINT(("\tL2 Use-count:%u \n", mali_pm_domain_get_use_count(domain)));
2632                                                 MALI_PRINT(("\tL2 Current power status:%s \n", (mali_pm_domain_get_mask(domain) & mali_pm_get_current_mask()) ? "On" : "Off"));
2633                                                 MALI_PRINT(("\tL2 Wanted  power status:%s \n", (mali_pm_domain_get_mask(domain) & mali_pm_get_wanted_mask()) ? "On" : "Off"));
2634                                         }
2635                                 }
2636                         }
2637                 }
2638                 if (EXEC_STATE_WORKING == virtual_group_state) {
2639                         mali_group_dump_status(virtual_group);
2640                 }
2641         }
2642 }
2643
2644 void mali_executor_status_dump(void)
2645 {
2646         mali_executor_lock();
2647         mali_scheduler_lock();
2648
2649         /* print schedule queue status */
2650         mali_scheduler_gp_pp_job_queue_print();
2651
2652         mali_scheduler_unlock();
2653         mali_executor_unlock();
2654 }