drivers/gpu/arm/mali400/mali/common/mali_executor.c

   1 /*
   2  * Copyright (C) 2012-2014 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_executor.h"
  12 #include "mali_scheduler.h"
  13 #include "mali_kernel_common.h"
  14 #include "mali_kernel_core.h"
  15 #include "mali_osk.h"
  16 #include "mali_osk_list.h"
  17 #include "mali_pp.h"
  18 #include "mali_pp_job.h"
  19 #include "mali_group.h"
  20 #include "mali_pm.h"
  21 #include "mali_timeline.h"
  22 #include "mali_osk_profiling.h"
  23 #include "mali_session.h"
  24
  25 /*Add for voltage scan function*/
  26 extern u32 mali_group_error;
  27
  28 /*
  29  * If dma_buf with map on demand is used, we defer job deletion and job queue
  30  * if in atomic context, since both might sleep.
  31  */
  32 #if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
  33 #define MALI_EXECUTOR_USE_DEFERRED_PP_JOB_DELETE 1
  34 #define MALI_EXECUTOR_USE_DEFERRED_PP_JOB_QUEUE 1
  35 #endif /* !defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH) */
  36
  37 /*
  38  * ---------- static type definitions (structs, enums, etc) ----------
  39  */
  40
  41 enum mali_executor_state_t {
  42         EXEC_STATE_NOT_PRESENT, /* Virtual group on Mali-300/400 (do not use) */
  43         EXEC_STATE_DISABLED,    /* Disabled by core scaling (do not use) */
  44         EXEC_STATE_EMPTY,       /* No child groups for virtual group (do not use) */
  45         EXEC_STATE_INACTIVE,    /* Can be used, but must be activate first */
  46         EXEC_STATE_IDLE,        /* Active and ready to be used */
  47         EXEC_STATE_WORKING,     /* Executing a job */
  48 };
  49
  50 /*
  51  * ---------- global variables (exported due to inline functions) ----------
  52  */
  53
  54 /* Lock for this module (protecting all HW access except L2 caches) */
  55 _mali_osk_spinlock_irq_t *mali_executor_lock_obj = NULL;
  56
  57 mali_bool mali_executor_hints[MALI_EXECUTOR_HINT_MAX];
  58
  59 /*
  60  * ---------- static variables ----------
  61  */
  62
  63 /* Used to defer job scheduling */
  64 static _mali_osk_wq_work_t *executor_wq_high_pri = NULL;
  65
  66 /* Store version from GP and PP (user space wants to know this) */
  67 static u32 pp_version = 0;
  68 static u32 gp_version = 0;
  69
  70 /* List of physical PP groups which are disabled by some external source */
  71 static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_disabled);
  72 static u32 group_list_disabled_count = 0;
  73
  74 /* List of groups which can be used, but activate first */
  75 static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_inactive);
  76 static u32 group_list_inactive_count = 0;
  77
  78 /* List of groups which are active and ready to be used */
  79 static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_idle);
  80 static u32 group_list_idle_count = 0;
  81
  82 /* List of groups which are executing a job */
  83 static _MALI_OSK_LIST_HEAD_STATIC_INIT(group_list_working);
  84 static u32 group_list_working_count = 0;
  85
  86 /* Virtual group (if any) */
  87 static struct mali_group *virtual_group = NULL;
  88
  89 /* Virtual group state is tracked with a state variable instead of 4 lists */
  90 static enum mali_executor_state_t virtual_group_state = EXEC_STATE_NOT_PRESENT;
  91
  92 /* GP group */
  93 static struct mali_group *gp_group = NULL;
  94
  95 /* GP group state is tracked with a state variable instead of 4 lists */
  96 static enum mali_executor_state_t gp_group_state = EXEC_STATE_NOT_PRESENT;
  97
  98 static u32 gp_returned_cookie = 0;
  99
 100 /* Total number of physical PP cores present */
 101 static u32 num_physical_pp_cores_total = 0;
 102
 103 /* Number of physical cores which are enabled */
 104 static u32 num_physical_pp_cores_enabled = 0;
 105
 106 /* Enable or disable core scaling */
 107 static mali_bool core_scaling_enabled = MALI_TRUE;
 108
 109 /* Variables to allow safe pausing of the scheduler */
 110 static _mali_osk_wait_queue_t *executor_working_wait_queue = NULL;
 111 static u32 pause_count = 0;
 112
 113 /* PP cores haven't been enabled because of some pp cores haven't been disabled. */
 114 static int core_scaling_delay_up_mask[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 };
 115
 116 /* Variables used to implement notify pp core changes to userspace when core scaling
 117  * is finished in mali_executor_complete_group() function. */
 118 static _mali_osk_wq_work_t *executor_wq_notify_core_change = NULL;
 119 static _mali_osk_wait_queue_t *executor_notify_core_change_wait_queue = NULL;
 120
 121 /*
 122  * ---------- Forward declaration of static functions ----------
 123  */
 124 static void mali_executor_lock(void);
 125 static void mali_executor_unlock(void);
 126 static mali_bool mali_executor_is_suspended(void *data);
 127 static mali_bool mali_executor_is_working(void);
 128 static void mali_executor_disable_empty_virtual(void);
 129 static mali_bool mali_executor_physical_rejoin_virtual(struct mali_group *group);
 130 static mali_bool mali_executor_has_virtual_group(void);
 131 static mali_bool mali_executor_virtual_group_is_usable(void);
 132 static void mali_executor_schedule(void);
 133 static void mali_executor_wq_schedule(void *arg);
 134 static void mali_executor_send_gp_oom_to_user(struct mali_gp_job *job);
 135 static void mali_executor_complete_group(struct mali_group *group,
 136                 mali_bool success,
 137                 mali_bool release_jobs,
 138                 struct mali_gp_job **gp_job_done,
 139                 struct mali_pp_job **pp_job_done);
 140 static void mali_executor_change_state_pp_physical(struct mali_group *group,
 141                 _mali_osk_list_t *old_list,
 142                 u32 *old_count,
 143                 _mali_osk_list_t *new_list,
 144                 u32 *new_count);
 145 static mali_bool mali_executor_group_is_in_state(struct mali_group *group,
 146                 enum mali_executor_state_t state);
 147
 148 static void mali_executor_group_enable_internal(struct mali_group *group);
 149 static void mali_executor_group_disable_internal(struct mali_group *group);
 150 static void mali_executor_core_scale(unsigned int target_core_nr);
 151 static void mali_executor_core_scale_in_group_complete(struct mali_group *group);
 152 static void mali_executor_notify_core_change(u32 num_cores);
 153 static void mali_executor_wq_notify_core_change(void *arg);
 154 static void mali_executor_change_group_status_disabled(struct mali_group *group);
 155 static mali_bool mali_executor_deactivate_list_idle(mali_bool deactivate_idle_group);
 156 static void mali_executor_set_state_pp_physical(struct mali_group *group,
 157                 _mali_osk_list_t *new_list,
 158                 u32 *new_count);
 159
 160 /*
 161  * ---------- Actual implementation ----------
 162  */
 163
 164 _mali_osk_errcode_t mali_executor_initialize(void)
 165 {
 166         mali_executor_lock_obj = _mali_osk_spinlock_irq_init(_MALI_OSK_LOCKFLAG_ORDERED, _MALI_OSK_LOCK_ORDER_EXECUTOR);
 167         if (NULL == mali_executor_lock_obj) {
 168                 mali_executor_terminate();
 169                 return _MALI_OSK_ERR_NOMEM;
 170         }
 171
 172         executor_wq_high_pri = _mali_osk_wq_create_work_high_pri(mali_executor_wq_schedule, NULL);
 173         if (NULL == executor_wq_high_pri) {
 174                 mali_executor_terminate();
 175                 return _MALI_OSK_ERR_NOMEM;
 176         }
 177
 178         executor_working_wait_queue = _mali_osk_wait_queue_init();
 179         if (NULL == executor_working_wait_queue) {
 180                 mali_executor_terminate();
 181                 return _MALI_OSK_ERR_NOMEM;
 182         }
 183
 184         executor_wq_notify_core_change = _mali_osk_wq_create_work(mali_executor_wq_notify_core_change, NULL);
 185         if (NULL == executor_wq_notify_core_change) {
 186                 mali_executor_terminate();
 187                 return _MALI_OSK_ERR_NOMEM;
 188         }
 189
 190         executor_notify_core_change_wait_queue = _mali_osk_wait_queue_init();
 191         if (NULL == executor_notify_core_change_wait_queue) {
 192                 mali_executor_terminate();
 193                 return _MALI_OSK_ERR_NOMEM;
 194         }
 195
 196         return _MALI_OSK_ERR_OK;
 197 }
 198
 199 void mali_executor_terminate(void)
 200 {
 201         if (NULL != executor_notify_core_change_wait_queue) {
 202                 _mali_osk_wait_queue_term(executor_notify_core_change_wait_queue);
 203                 executor_notify_core_change_wait_queue = NULL;
 204         }
 205
 206         if (NULL != executor_wq_notify_core_change) {
 207                 _mali_osk_wq_delete_work(executor_wq_notify_core_change);
 208                 executor_wq_notify_core_change = NULL;
 209         }
 210
 211         if (NULL != executor_working_wait_queue) {
 212                 _mali_osk_wait_queue_term(executor_working_wait_queue);
 213                 executor_working_wait_queue = NULL;
 214         }
 215
 216         if (NULL != executor_wq_high_pri) {
 217                 _mali_osk_wq_delete_work(executor_wq_high_pri);
 218                 executor_wq_high_pri = NULL;
 219         }
 220
 221         if (NULL != mali_executor_lock_obj) {
 222                 _mali_osk_spinlock_irq_term(mali_executor_lock_obj);
 223                 mali_executor_lock_obj = NULL;
 224         }
 225 }
 226
 227 void mali_executor_populate(void)
 228 {
 229         u32 num_groups;
 230         u32 i;
 231
 232         num_groups = mali_group_get_glob_num_groups();
 233
 234         /* Do we have a virtual group? */
 235         for (i = 0; i < num_groups; i++) {
 236                 struct mali_group *group = mali_group_get_glob_group(i);
 237
 238                 if (mali_group_is_virtual(group)) {
 239                         virtual_group = group;
 240                         virtual_group_state = EXEC_STATE_INACTIVE;
 241                         break;
 242                 }
 243         }
 244
 245         /* Find all the available physical GP and PP cores */
 246         for (i = 0; i < num_groups; i++) {
 247                 struct mali_group *group = mali_group_get_glob_group(i);
 248
 249                 if (NULL != group) {
 250                         struct mali_pp_core *pp_core = mali_group_get_pp_core(group);
 251                         struct mali_gp_core *gp_core = mali_group_get_gp_core(group);
 252
 253                         if (!mali_group_is_virtual(group)) {
 254                                 if (NULL != pp_core) {
 255                                         if (0 == pp_version) {
 256                                                 /* Retrieve PP version from the first available PP core */
 257                                                 pp_version = mali_pp_core_get_version(pp_core);
 258                                         }
 259
 260                                         if (NULL != virtual_group) {
 261                                                 mali_executor_lock();
 262                                                 mali_group_add_group(virtual_group, group);
 263                                                 mali_executor_unlock();
 264                                         } else {
 265                                                 _mali_osk_list_add(&group->executor_list, &group_list_inactive);
 266                                                 group_list_inactive_count++;
 267                                         }
 268
 269                                         num_physical_pp_cores_total++;
 270                                 } else {
 271                                         MALI_DEBUG_ASSERT_POINTER(gp_core);
 272
 273                                         if (0 == gp_version) {
 274                                                 /* Retrieve GP version */
 275                                                 gp_version = mali_gp_core_get_version(gp_core);
 276                                         }
 277
 278                                         gp_group = group;
 279                                         gp_group_state = EXEC_STATE_INACTIVE;
 280                                 }
 281
 282                         }
 283                 }
 284         }
 285
 286         num_physical_pp_cores_enabled = num_physical_pp_cores_total;
 287 }
 288
 289 void mali_executor_depopulate(void)
 290 {
 291         struct mali_group *group;
 292         struct mali_group *temp;
 293
 294         MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != gp_group_state);
 295
 296         if (NULL != gp_group) {
 297                 mali_group_delete(gp_group);
 298                 gp_group = NULL;
 299         }
 300
 301         MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != virtual_group_state);
 302
 303         if (NULL != virtual_group) {
 304                 mali_group_delete(virtual_group);
 305                 virtual_group = NULL;
 306         }
 307
 308         MALI_DEBUG_ASSERT(_mali_osk_list_empty(&group_list_working));
 309
 310         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, executor_list) {
 311                 mali_group_delete(group);
 312         }
 313
 314         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive, struct mali_group, executor_list) {
 315                 mali_group_delete(group);
 316         }
 317
 318         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_disabled, struct mali_group, executor_list) {
 319                 mali_group_delete(group);
 320         }
 321 }
 322
 323 void mali_executor_suspend(void)
 324 {
 325         mali_executor_lock();
 326
 327         /* Increment the pause_count so that no more jobs will be scheduled */
 328         pause_count++;
 329
 330         mali_executor_unlock();
 331
 332         _mali_osk_wait_queue_wait_event(executor_working_wait_queue,
 333                                         mali_executor_is_suspended, NULL);
 334
 335         /*
 336          * mali_executor_complete_XX() leaves jobs in idle state.
 337          * deactivate option is used when we are going to power down
 338          * the entire GPU (OS suspend) and want a consistent SW vs HW
 339          * state.
 340          */
 341         mali_executor_lock();
 342
 343         mali_executor_deactivate_list_idle(MALI_TRUE);
 344
 345         /*
 346          * The following steps are used to deactive all of activated
 347          * (MALI_GROUP_STATE_ACTIVE) and activating (MALI_GROUP
 348          * _STAET_ACTIVATION_PENDING) groups, to make sure the variable
 349          * pd_mask_wanted is equal with 0. */
 350         if (MALI_GROUP_STATE_INACTIVE != mali_group_get_state(gp_group)) {
 351                 gp_group_state = EXEC_STATE_INACTIVE;
 352                 mali_group_deactivate(gp_group);
 353         }
 354
 355         if (mali_executor_has_virtual_group()) {
 356                 if (MALI_GROUP_STATE_INACTIVE
 357                     != mali_group_get_state(virtual_group)) {
 358                         virtual_group_state = EXEC_STATE_INACTIVE;
 359                         mali_group_deactivate(virtual_group);
 360                 }
 361         }
 362
 363         if (0 < group_list_inactive_count) {
 364                 struct mali_group *group;
 365                 struct mali_group *temp;
 366
 367                 _MALI_OSK_LIST_FOREACHENTRY(group, temp,
 368                                             &group_list_inactive,
 369                                             struct mali_group, executor_list) {
 370                         if (MALI_GROUP_STATE_ACTIVATION_PENDING
 371                             == mali_group_get_state(group)) {
 372                                 mali_group_deactivate(group);
 373                         }
 374
 375                         /*
 376                          * On mali-450 platform, we may have physical group in the group inactive
 377                          * list, and its state is MALI_GROUP_STATE_ACTIVATION_PENDING, so we only
 378                          * deactivate it is not enough, we still also need add it back to virtual group.
 379                          * And now, virtual group must be in INACTIVE state, so it's safe to add
 380                          * physical group to virtual group at this point.
 381                          */
 382                         if (NULL != virtual_group) {
 383                                 _mali_osk_list_delinit(&group->executor_list);
 384                                 group_list_inactive_count--;
 385
 386                                 mali_group_add_group(virtual_group, group);
 387                         }
 388                 }
 389         }
 390
 391         mali_executor_unlock();
 392 }
 393
 394 void mali_executor_resume(void)
 395 {
 396         mali_executor_lock();
 397
 398         /* Decrement pause_count to allow scheduling again (if it reaches 0) */
 399         pause_count--;
 400         if (0 == pause_count) {
 401                 mali_executor_schedule();
 402         }
 403
 404         mali_executor_unlock();
 405 }
 406
 407 u32 mali_executor_get_num_cores_total(void)
 408 {
 409         return num_physical_pp_cores_total;
 410 }
 411
 412 u32 mali_executor_get_num_cores_enabled(void)
 413 {
 414         return num_physical_pp_cores_enabled;
 415 }
 416
 417 struct mali_pp_core *mali_executor_get_virtual_pp(void)
 418 {
 419         MALI_DEBUG_ASSERT_POINTER(virtual_group);
 420         MALI_DEBUG_ASSERT_POINTER(virtual_group->pp_core);
 421         return virtual_group->pp_core;
 422 }
 423
 424 struct mali_group *mali_executor_get_virtual_group(void)
 425 {
 426         return virtual_group;
 427 }
 428
 429 void mali_executor_zap_all_active(struct mali_session_data *session)
 430 {
 431         struct mali_group *group;
 432         struct mali_group *temp;
 433         mali_bool ret;
 434
 435         mali_executor_lock();
 436
 437         /*
 438          * This function is a bit complicated because
 439          * mali_group_zap_session() can fail. This only happens because the
 440          * group is in an unhandled page fault status.
 441          * We need to make sure this page fault is handled before we return,
 442          * so that we know every single outstanding MMU transactions have
 443          * completed. This will allow caller to safely remove physical pages
 444          * when we have returned.
 445          */
 446
 447         MALI_DEBUG_ASSERT(NULL != gp_group);
 448         ret = mali_group_zap_session(gp_group, session);
 449         if (MALI_FALSE == ret) {
 450                 struct mali_gp_job *gp_job = NULL;
 451
 452                 mali_executor_complete_group(gp_group, MALI_FALSE,
 453                                              MALI_TRUE, &gp_job, NULL);
 454
 455                 MALI_DEBUG_ASSERT_POINTER(gp_job);
 456
 457                 /* GP job completed, make sure it is freed */
 458                 mali_scheduler_complete_gp_job(gp_job, MALI_FALSE,
 459                                                MALI_TRUE, MALI_TRUE);
 460         }
 461
 462         if (mali_executor_has_virtual_group()) {
 463                 ret = mali_group_zap_session(virtual_group, session);
 464                 if (MALI_FALSE == ret) {
 465                         struct mali_pp_job *pp_job = NULL;
 466
 467                         mali_executor_complete_group(virtual_group, MALI_FALSE,
 468                                                      MALI_TRUE, NULL, &pp_job);
 469
 470                         if (NULL != pp_job) {
 471                                 /* PP job completed, make sure it is freed */
 472                                 mali_scheduler_complete_pp_job(pp_job, 0,
 473                                                                MALI_FALSE, MALI_TRUE);
 474                         }
 475                 }
 476         }
 477
 478         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_working,
 479                                     struct mali_group, executor_list) {
 480                 ret = mali_group_zap_session(group, session);
 481                 if (MALI_FALSE == ret) {
 482                         ret = mali_group_zap_session(group, session);
 483                         if (MALI_FALSE == ret) {
 484                                 struct mali_pp_job *pp_job = NULL;
 485
 486                                 mali_executor_complete_group(group, MALI_FALSE,
 487                                                              MALI_TRUE, NULL, &pp_job);
 488
 489                                 if (NULL != pp_job) {
 490                                         /* PP job completed, free it */
 491                                         mali_scheduler_complete_pp_job(pp_job,
 492                                                                        0, MALI_FALSE,
 493                                                                        MALI_TRUE);
 494                                 }
 495                         }
 496                 }
 497         }
 498
 499         mali_executor_unlock();
 500 }
 501
 502 void mali_executor_schedule_from_mask(mali_scheduler_mask mask, mali_bool deferred_schedule)
 503 {
 504         if (MALI_SCHEDULER_MASK_EMPTY != mask) {
 505                 if (MALI_TRUE == deferred_schedule) {
 506                         _mali_osk_wq_schedule_work_high_pri(executor_wq_high_pri);
 507                 } else {
 508                         /* Schedule from this thread*/
 509                         mali_executor_lock();
 510                         mali_executor_schedule();
 511                         mali_executor_unlock();
 512                 }
 513         }
 514 }
 515
 516 _mali_osk_errcode_t mali_executor_interrupt_gp(struct mali_group *group,
 517                 mali_bool in_upper_half)
 518 {
 519         enum mali_interrupt_result int_result;
 520         mali_bool time_out = MALI_FALSE;
 521
 522         MALI_DEBUG_PRINT(4, ("Executor: GP interrupt from %s in %s half\n",
 523                              mali_group_core_description(group),
 524                              in_upper_half ? "upper" : "bottom"));
 525
 526         mali_executor_lock();
 527         if (!mali_group_is_working(group)) {
 528                 /* Not working, so nothing to do */
 529                 mali_executor_unlock();
 530                 return _MALI_OSK_ERR_FAULT;
 531         }
 532
 533         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
 534         MALI_DEBUG_ASSERT(mali_group_is_working(group));
 535
 536         if (mali_group_has_timed_out(group)) {
 537                 int_result = MALI_INTERRUPT_RESULT_ERROR;
 538                 time_out = MALI_TRUE;
 539                 MALI_PRINT(("Executor GP: Job %d Timeout on %s\n",
 540                             mali_gp_job_get_id(group->gp_running_job),
 541                             mali_group_core_description(group)));
 542         } else {
 543                 int_result = mali_group_get_interrupt_result_gp(group);
 544                 if (MALI_INTERRUPT_RESULT_NONE == int_result) {
 545                         mali_executor_unlock();
 546                         return _MALI_OSK_ERR_FAULT;
 547                 }
 548         }
 549
 550 #if defined(CONFIG_MALI_SHARED_INTERRUPTS)
 551         if (MALI_INTERRUPT_RESULT_NONE == int_result) {
 552                 /* No interrupts signalled, so nothing to do */
 553                 mali_executor_unlock();
 554                 return _MALI_OSK_ERR_FAULT;
 555         }
 556 #else
 557         MALI_DEBUG_ASSERT(MALI_INTERRUPT_RESULT_NONE != int_result);
 558 #endif
 559
 560         mali_group_mask_all_interrupts_gp(group);
 561
 562         if (MALI_INTERRUPT_RESULT_SUCCESS_VS == int_result) {
 563                 if (mali_group_gp_is_active(group)) {
 564                         /* Only VS completed so far, while PLBU is still active */
 565
 566                         /* Enable all but the current interrupt */
 567                         mali_group_enable_interrupts_gp(group, int_result);
 568
 569                         mali_executor_unlock();
 570                         return _MALI_OSK_ERR_OK;
 571                 }
 572         } else if (MALI_INTERRUPT_RESULT_SUCCESS_PLBU == int_result) {
 573                 if (mali_group_gp_is_active(group)) {
 574                         /* Only PLBU completed so far, while VS is still active */
 575
 576                         /* Enable all but the current interrupt */
 577                         mali_group_enable_interrupts_gp(group, int_result);
 578
 579                         mali_executor_unlock();
 580                         return _MALI_OSK_ERR_OK;
 581                 }
 582         } else if (MALI_INTERRUPT_RESULT_OOM == int_result) {
 583                 struct mali_gp_job *job = mali_group_get_running_gp_job(group);
 584
 585                 /* PLBU out of mem */
 586                 MALI_DEBUG_PRINT(3, ("Executor: PLBU needs more heap memory\n"));
 587
 588 #if defined(CONFIG_MALI400_PROFILING)
 589                 /* Give group a chance to generate a SUSPEND event */
 590                 mali_group_oom(group);
 591 #endif
 592
 593                 /*
 594                  * no need to hold interrupt raised while
 595                  * waiting for more memory.
 596                  */
 597                 mali_executor_send_gp_oom_to_user(job);
 598
 599                 mali_executor_unlock();
 600
 601                 return _MALI_OSK_ERR_OK;
 602         }
 603
 604         /*Add for voltage scan function*/
 605         if (MALI_INTERRUPT_RESULT_ERROR == int_result)
 606                 mali_group_error++;
 607
 608         /* We should now have a real interrupt to handle */
 609
 610         MALI_DEBUG_PRINT(4, ("Executor: Group %s completed with %s\n",
 611                              mali_group_core_description(group),
 612                              (MALI_INTERRUPT_RESULT_ERROR == int_result) ?
 613                              "ERROR" : "success"));
 614
 615         if (in_upper_half && MALI_INTERRUPT_RESULT_ERROR == int_result) {
 616                 /* Don't bother to do processing of errors in upper half */
 617                 mali_executor_unlock();
 618
 619                 if (MALI_FALSE == time_out) {
 620                         mali_group_schedule_bottom_half_gp(group);
 621                 }
 622         } else {
 623                 struct mali_gp_job *job;
 624                 mali_bool success;
 625
 626                 success = (int_result != MALI_INTERRUPT_RESULT_ERROR) ?
 627                           MALI_TRUE : MALI_FALSE;
 628
 629                 mali_executor_complete_group(group, success,
 630                                              MALI_TRUE, &job, NULL);
 631
 632                 mali_executor_unlock();
 633
 634                 /* GP jobs always fully complete */
 635                 MALI_DEBUG_ASSERT(NULL != job);
 636
 637                 /* This will notify user space and close the job object */
 638                 mali_scheduler_complete_gp_job(job, success,
 639                                                MALI_TRUE, MALI_TRUE);
 640         }
 641
 642         return _MALI_OSK_ERR_OK;
 643 }
 644
 645 _mali_osk_errcode_t mali_executor_interrupt_pp(struct mali_group *group,
 646                 mali_bool in_upper_half)
 647 {
 648         enum mali_interrupt_result int_result;
 649         mali_bool time_out = MALI_FALSE;
 650
 651         MALI_DEBUG_PRINT(4, ("Executor: PP interrupt from %s in %s half\n",
 652                              mali_group_core_description(group),
 653                              in_upper_half ? "upper" : "bottom"));
 654
 655         mali_executor_lock();
 656
 657         if (!mali_group_is_working(group)) {
 658                 /* Not working, so nothing to do */
 659                 mali_executor_unlock();
 660                 return _MALI_OSK_ERR_FAULT;
 661         }
 662
 663         if (in_upper_half) {
 664                 if (mali_group_is_in_virtual(group)) {
 665                         /* Child groups should never handle PP interrupts */
 666                         MALI_DEBUG_ASSERT(!mali_group_has_timed_out(group));
 667                         mali_executor_unlock();
 668                         return _MALI_OSK_ERR_FAULT;
 669                 }
 670         }
 671         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
 672         MALI_DEBUG_ASSERT(mali_group_is_working(group));
 673         MALI_DEBUG_ASSERT(!mali_group_is_in_virtual(group));
 674
 675         if (mali_group_has_timed_out(group)) {
 676                 int_result = MALI_INTERRUPT_RESULT_ERROR;
 677                 time_out = MALI_TRUE;
 678                 MALI_PRINT(("Executor PP: Job %d Timeout on %s\n",
 679                             mali_pp_job_get_id(group->pp_running_job),
 680                             mali_group_core_description(group)));
 681         } else {
 682                 int_result = mali_group_get_interrupt_result_pp(group);
 683                 if (MALI_INTERRUPT_RESULT_NONE == int_result) {
 684                         mali_executor_unlock();
 685                         return _MALI_OSK_ERR_FAULT;
 686                 }
 687         }
 688
 689 #if defined(CONFIG_MALI_SHARED_INTERRUPTS)
 690         if (MALI_INTERRUPT_RESULT_NONE == int_result) {
 691                 /* No interrupts signalled, so nothing to do */
 692                 mali_executor_unlock();
 693                 return _MALI_OSK_ERR_FAULT;
 694         } else if (MALI_INTERRUPT_RESULT_SUCCESS == int_result) {
 695                 if (mali_group_is_virtual(group) && mali_group_pp_is_active(group)) {
 696                         /* Some child groups are still working, so nothing to do right now */
 697                         mali_executor_unlock();
 698                         return _MALI_OSK_ERR_FAULT;
 699                 }
 700         }
 701 #else
 702         MALI_DEBUG_ASSERT(MALI_INTERRUPT_RESULT_NONE != int_result);
 703         if (!mali_group_has_timed_out(group)) {
 704                 MALI_DEBUG_ASSERT(!mali_group_pp_is_active(group));
 705         }
 706 #endif
 707
 708         /*Add voltage scan function*/
 709
 710         if (MALI_INTERRUPT_RESULT_ERROR == int_result)
 711                 mali_group_error++;
 712
 713         /* We should now have a real interrupt to handle */
 714
 715         MALI_DEBUG_PRINT(4, ("Executor: Group %s completed with %s\n",
 716                              mali_group_core_description(group),
 717                              (MALI_INTERRUPT_RESULT_ERROR == int_result) ?
 718                              "ERROR" : "success"));
 719
 720         if (in_upper_half && MALI_INTERRUPT_RESULT_ERROR == int_result) {
 721                 /* Don't bother to do processing of errors in upper half */
 722                 mali_group_mask_all_interrupts_pp(group);
 723                 mali_executor_unlock();
 724
 725                 if (MALI_FALSE == time_out) {
 726                         mali_group_schedule_bottom_half_pp(group);
 727                 }
 728         } else {
 729                 struct mali_pp_job *job = NULL;
 730                 mali_bool success;
 731
 732                 success = (int_result == MALI_INTERRUPT_RESULT_SUCCESS) ?
 733                           MALI_TRUE : MALI_FALSE;
 734
 735                 mali_executor_complete_group(group, success,
 736                                              MALI_TRUE, NULL, &job);
 737
 738                 mali_executor_unlock();
 739
 740                 if (NULL != job) {
 741                         /* Notify user space and close the job object */
 742                         mali_scheduler_complete_pp_job(job,
 743                                                        num_physical_pp_cores_total,
 744                                                        MALI_TRUE, MALI_TRUE);
 745                 }
 746         }
 747
 748         return _MALI_OSK_ERR_OK;
 749 }
 750
 751 _mali_osk_errcode_t mali_executor_interrupt_mmu(struct mali_group *group,
 752                 mali_bool in_upper_half)
 753 {
 754         enum mali_interrupt_result int_result;
 755
 756         MALI_DEBUG_PRINT(4, ("Executor: MMU interrupt from %s in %s half\n",
 757                              mali_group_core_description(group),
 758                              in_upper_half ? "upper" : "bottom"));
 759
 760         mali_executor_lock();
 761         if (!mali_group_is_working(group)) {
 762                 /* Not working, so nothing to do */
 763                 mali_executor_unlock();
 764                 return _MALI_OSK_ERR_FAULT;
 765         }
 766
 767         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
 768         MALI_DEBUG_ASSERT(mali_group_is_working(group));
 769
 770         int_result = mali_group_get_interrupt_result_mmu(group);
 771         if (MALI_INTERRUPT_RESULT_NONE == int_result) {
 772                 mali_executor_unlock();
 773                 return _MALI_OSK_ERR_FAULT;
 774         }
 775
 776 #if defined(CONFIG_MALI_SHARED_INTERRUPTS)
 777         if (MALI_INTERRUPT_RESULT_NONE == int_result) {
 778                 /* No interrupts signalled, so nothing to do */
 779                 mali_executor_unlock();
 780                 return _MALI_OSK_ERR_FAULT;
 781         }
 782 #else
 783         MALI_DEBUG_ASSERT(MALI_INTERRUPT_RESULT_ERROR == int_result);
 784 #endif
 785
 786         /* We should now have a real interrupt to handle */
 787
 788         if (in_upper_half) {
 789                 /* Don't bother to do processing of errors in upper half */
 790
 791                 struct mali_group *parent = group->parent_group;
 792
 793                 mali_mmu_mask_all_interrupts(group->mmu);
 794
 795                 mali_executor_unlock();
 796
 797                 if (NULL == parent) {
 798                         mali_group_schedule_bottom_half_mmu(group);
 799                 } else {
 800                         mali_group_schedule_bottom_half_mmu(parent);
 801                 }
 802
 803         } else {
 804                 struct mali_gp_job *gp_job = NULL;
 805                 struct mali_pp_job *pp_job = NULL;
 806
 807 #ifdef DEBUG
 808
 809                 u32 fault_address = mali_mmu_get_page_fault_addr(group->mmu);
 810                 u32 status = mali_mmu_get_status(group->mmu);
 811                 MALI_DEBUG_PRINT(2, ("Executor: Mali page fault detected at 0x%x from bus id %d of type %s on %s\n",
 812                                      (void *)(uintptr_t)fault_address,
 813                                      (status >> 6) & 0x1F,
 814                                      (status & 32) ? "write" : "read",
 815                                      group->mmu->hw_core.description));
 816                 MALI_DEBUG_PRINT(3, ("Executor: MMU rawstat = 0x%08X, MMU status = 0x%08X\n",
 817                                      mali_mmu_get_rawstat(group->mmu), status));
 818 #endif
 819
 820                 mali_executor_complete_group(group, MALI_FALSE,
 821                                              MALI_TRUE, &gp_job, &pp_job);
 822
 823                 mali_executor_unlock();
 824
 825                 if (NULL != gp_job) {
 826                         MALI_DEBUG_ASSERT(NULL == pp_job);
 827
 828                         /* Notify user space and close the job object */
 829                         mali_scheduler_complete_gp_job(gp_job, MALI_FALSE,
 830                                                        MALI_TRUE, MALI_TRUE);
 831                 } else if (NULL != pp_job) {
 832                         MALI_DEBUG_ASSERT(NULL == gp_job);
 833
 834                         /* Notify user space and close the job object */
 835                         mali_scheduler_complete_pp_job(pp_job,
 836                                                        num_physical_pp_cores_total,
 837                                                        MALI_TRUE, MALI_TRUE);
 838                 }
 839         }
 840
 841         return _MALI_OSK_ERR_OK;
 842 }
 843
 844 void mali_executor_group_power_up(struct mali_group *groups[], u32 num_groups)
 845 {
 846         u32 i;
 847         mali_bool child_groups_activated = MALI_FALSE;
 848         mali_bool do_schedule = MALI_FALSE;
 849 #if defined(DEBUG)
 850         u32 num_activated = 0;
 851 #endif
 852
 853         MALI_DEBUG_ASSERT_POINTER(groups);
 854         MALI_DEBUG_ASSERT(0 < num_groups);
 855
 856         mali_executor_lock();
 857
 858         MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups\n", num_groups));
 859
 860         for (i = 0; i < num_groups; i++) {
 861                 MALI_DEBUG_PRINT(3, ("Executor: powering up group %s\n",
 862                                      mali_group_core_description(groups[i])));
 863
 864                 mali_group_power_up(groups[i]);
 865
 866                 if ((MALI_GROUP_STATE_ACTIVATION_PENDING != mali_group_get_state(groups[i]) ||
 867                      (MALI_TRUE != mali_executor_group_is_in_state(groups[i], EXEC_STATE_INACTIVE)))) {
 868                         /* nothing more to do for this group */
 869                         continue;
 870                 }
 871
 872                 MALI_DEBUG_PRINT(3, ("Executor: activating group %s\n",
 873                                      mali_group_core_description(groups[i])));
 874
 875 #if defined(DEBUG)
 876                 num_activated++;
 877 #endif
 878
 879                 if (mali_group_is_in_virtual(groups[i])) {
 880                         /*
 881                          * At least one child group of virtual group is powered on.
 882                          */
 883                         child_groups_activated = MALI_TRUE;
 884                 } else if (MALI_FALSE == mali_group_is_virtual(groups[i])) {
 885                         /* Set gp and pp not in virtual to active. */
 886                         mali_group_set_active(groups[i]);
 887                 }
 888
 889                 /* Move group from inactive to idle list */
 890                 if (groups[i] == gp_group) {
 891                         MALI_DEBUG_ASSERT(EXEC_STATE_INACTIVE ==
 892                                           gp_group_state);
 893                         gp_group_state = EXEC_STATE_IDLE;
 894                 } else if (MALI_FALSE == mali_group_is_in_virtual(groups[i])
 895                            && MALI_FALSE == mali_group_is_virtual(groups[i])) {
 896                         MALI_DEBUG_ASSERT(MALI_TRUE == mali_executor_group_is_in_state(groups[i],
 897                                           EXEC_STATE_INACTIVE));
 898
 899                         mali_executor_change_state_pp_physical(groups[i],
 900                                                                &group_list_inactive,
 901                                                                &group_list_inactive_count,
 902                                                                &group_list_idle,
 903                                                                &group_list_idle_count);
 904                 }
 905
 906                 do_schedule = MALI_TRUE;
 907         }
 908
 909         if (mali_executor_has_virtual_group() &&
 910             MALI_TRUE == child_groups_activated &&
 911             MALI_GROUP_STATE_ACTIVATION_PENDING ==
 912             mali_group_get_state(virtual_group)) {
 913                 /*
 914                  * Try to active virtual group while it may be not sucessful every time,
 915                  * because there is one situation that not all of child groups are powered on
 916                  * in one time and virtual group is in activation pending state.
 917                  */
 918                 if (mali_group_set_active(virtual_group)) {
 919                         /* Move group from inactive to idle */
 920                         MALI_DEBUG_ASSERT(EXEC_STATE_INACTIVE ==
 921                                           virtual_group_state);
 922                         virtual_group_state = EXEC_STATE_IDLE;
 923
 924                         MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups completed, %u  physical activated, 1 virtual activated.\n", num_groups, num_activated));
 925                 } else {
 926                         MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups completed, %u physical activated\n", num_groups, num_activated));
 927                 }
 928         } else {
 929                 MALI_DEBUG_PRINT(3, ("Executor: powering up %u groups completed, %u physical activated\n", num_groups, num_activated));
 930         }
 931
 932         if (MALI_TRUE == do_schedule) {
 933                 /* Trigger a schedule */
 934                 mali_executor_schedule();
 935         }
 936
 937         mali_executor_unlock();
 938 }
 939
 940 void mali_executor_group_power_down(struct mali_group *groups[],
 941                                     u32 num_groups)
 942 {
 943         u32 i;
 944
 945         MALI_DEBUG_ASSERT_POINTER(groups);
 946         MALI_DEBUG_ASSERT(0 < num_groups);
 947
 948         mali_executor_lock();
 949
 950         MALI_DEBUG_PRINT(3, ("Executor: powering down %u groups\n", num_groups));
 951
 952         for (i = 0; i < num_groups; i++) {
 953                 /* Groups must be either disabled or inactive */
 954                 MALI_DEBUG_ASSERT(mali_executor_group_is_in_state(groups[i],
 955                                   EXEC_STATE_DISABLED) ||
 956                                   mali_executor_group_is_in_state(groups[i],
 957                                                   EXEC_STATE_INACTIVE));
 958
 959                 MALI_DEBUG_PRINT(3, ("Executor: powering down group %s\n",
 960                                      mali_group_core_description(groups[i])));
 961
 962                 mali_group_power_down(groups[i]);
 963         }
 964
 965         MALI_DEBUG_PRINT(3, ("Executor: powering down %u groups completed\n", num_groups));
 966
 967         mali_executor_unlock();
 968 }
 969
 970 void mali_executor_abort_session(struct mali_session_data *session)
 971 {
 972         struct mali_group *group;
 973         struct mali_group *tmp_group;
 974
 975         MALI_DEBUG_ASSERT_POINTER(session);
 976         MALI_DEBUG_ASSERT(session->is_aborting);
 977
 978         MALI_DEBUG_PRINT(3,
 979                          ("Executor: Aborting all jobs from session 0x%08X.\n",
 980                           session));
 981
 982         mali_executor_lock();
 983
 984         if (mali_group_get_session(gp_group) == session) {
 985                 if (EXEC_STATE_WORKING == gp_group_state) {
 986                         struct mali_gp_job *gp_job = NULL;
 987
 988                         mali_executor_complete_group(gp_group, MALI_FALSE,
 989                                                      MALI_TRUE, &gp_job, NULL);
 990
 991                         MALI_DEBUG_ASSERT_POINTER(gp_job);
 992
 993                         /* GP job completed, make sure it is freed */
 994                         mali_scheduler_complete_gp_job(gp_job, MALI_FALSE,
 995                                                        MALI_FALSE, MALI_TRUE);
 996                 } else {
 997                         /* Same session, but not working, so just clear it */
 998                         mali_group_clear_session(gp_group);
 999                 }
1000         }
1001
1002         if (mali_executor_has_virtual_group()) {
1003                 if (EXEC_STATE_WORKING == virtual_group_state
1004                     && mali_group_get_session(virtual_group) == session) {
1005                         struct mali_pp_job *pp_job = NULL;
1006
1007                         mali_executor_complete_group(virtual_group, MALI_FALSE,
1008                                                      MALI_FALSE, NULL, &pp_job);
1009
1010                         if (NULL != pp_job) {
1011                                 /* PP job completed, make sure it is freed */
1012                                 mali_scheduler_complete_pp_job(pp_job, 0,
1013                                                                MALI_FALSE, MALI_TRUE);
1014                         }
1015                 }
1016         }
1017
1018         _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_working,
1019                                     struct mali_group, executor_list) {
1020                 if (mali_group_get_session(group) == session) {
1021                         struct mali_pp_job *pp_job = NULL;
1022
1023                         mali_executor_complete_group(group, MALI_FALSE,
1024                                                      MALI_FALSE, NULL, &pp_job);
1025
1026                         if (NULL != pp_job) {
1027                                 /* PP job completed, make sure it is freed */
1028                                 mali_scheduler_complete_pp_job(pp_job, 0,
1029                                                                MALI_FALSE, MALI_TRUE);
1030                         }
1031                 }
1032         }
1033
1034         _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_idle, struct mali_group, executor_list) {
1035                 mali_group_clear_session(group);
1036         }
1037
1038         _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_inactive, struct mali_group, executor_list) {
1039                 mali_group_clear_session(group);
1040         }
1041
1042         _MALI_OSK_LIST_FOREACHENTRY(group, tmp_group, &group_list_disabled, struct mali_group, executor_list) {
1043                 mali_group_clear_session(group);
1044         }
1045
1046         mali_executor_unlock();
1047 }
1048
1049
1050 void mali_executor_core_scaling_enable(void)
1051 {
1052         /* PS: Core scaling is by default enabled */
1053         core_scaling_enabled = MALI_TRUE;
1054 }
1055
1056 void mali_executor_core_scaling_disable(void)
1057 {
1058         core_scaling_enabled = MALI_FALSE;
1059 }
1060
1061 mali_bool mali_executor_core_scaling_is_enabled(void)
1062 {
1063         return core_scaling_enabled;
1064 }
1065
1066 void mali_executor_group_enable(struct mali_group *group)
1067 {
1068         MALI_DEBUG_ASSERT_POINTER(group);
1069
1070         mali_executor_lock();
1071
1072         if ((NULL != mali_group_get_gp_core(group) || NULL != mali_group_get_pp_core(group))
1073             && (mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED))) {
1074                 mali_executor_group_enable_internal(group);
1075         }
1076
1077         mali_executor_schedule();
1078         mali_executor_unlock();
1079
1080         _mali_osk_wq_schedule_work(executor_wq_notify_core_change);
1081 }
1082
1083 /*
1084  * If a physical group is inactive or idle, we should disable it immediately,
1085  * if group is in virtual, and virtual group is idle, disable given physical group in it.
1086  */
1087 void mali_executor_group_disable(struct mali_group *group)
1088 {
1089         MALI_DEBUG_ASSERT_POINTER(group);
1090
1091         mali_executor_lock();
1092
1093         if ((NULL != mali_group_get_gp_core(group) || NULL != mali_group_get_pp_core(group))
1094             && (!mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED))) {
1095                 mali_executor_group_disable_internal(group);
1096         }
1097
1098         mali_executor_schedule();
1099         mali_executor_unlock();
1100
1101         _mali_osk_wq_schedule_work(executor_wq_notify_core_change);
1102 }
1103
1104 mali_bool mali_executor_group_is_disabled(struct mali_group *group)
1105 {
1106         /* NB: This function is not optimized for time critical usage */
1107
1108         mali_bool ret;
1109
1110         MALI_DEBUG_ASSERT_POINTER(group);
1111
1112         mali_executor_lock();
1113         ret = mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED);
1114         mali_executor_unlock();
1115
1116         return ret;
1117 }
1118
1119 int mali_executor_set_perf_level(unsigned int target_core_nr, mali_bool override)
1120 {
1121         if (target_core_nr == num_physical_pp_cores_enabled) return 0;
1122         if (MALI_FALSE == core_scaling_enabled && MALI_FALSE == override) return -EPERM;
1123         if (target_core_nr > num_physical_pp_cores_total) return -EINVAL;
1124         if (0 == target_core_nr) return -EINVAL;
1125
1126         mali_executor_core_scale(target_core_nr);
1127
1128         _mali_osk_wq_schedule_work(executor_wq_notify_core_change);
1129
1130         return 0;
1131 }
1132
1133 #if MALI_STATE_TRACKING
1134 u32 mali_executor_dump_state(char *buf, u32 size)
1135 {
1136         int n = 0;
1137         struct mali_group *group;
1138         struct mali_group *temp;
1139
1140         mali_executor_lock();
1141
1142         switch (gp_group_state) {
1143         case EXEC_STATE_INACTIVE:
1144                 n += _mali_osk_snprintf(buf + n, size - n,
1145                                         "GP group is in state INACTIVE\n");
1146                 break;
1147         case EXEC_STATE_IDLE:
1148                 n += _mali_osk_snprintf(buf + n, size - n,
1149                                         "GP group is in state IDLE\n");
1150                 break;
1151         case EXEC_STATE_WORKING:
1152                 n += _mali_osk_snprintf(buf + n, size - n,
1153                                         "GP group is in state WORKING\n");
1154                 break;
1155         default:
1156                 n += _mali_osk_snprintf(buf + n, size - n,
1157                                         "GP group is in unknown/illegal state %u\n",
1158                                         gp_group_state);
1159                 break;
1160         }
1161
1162         n += mali_group_dump_state(gp_group, buf + n, size - n);
1163
1164         n += _mali_osk_snprintf(buf + n, size - n,
1165                                 "Physical PP groups in WORKING state (count = %u):\n",
1166                                 group_list_working_count);
1167
1168         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_working, struct mali_group, executor_list) {
1169                 n += mali_group_dump_state(group, buf + n, size - n);
1170         }
1171
1172         n += _mali_osk_snprintf(buf + n, size - n,
1173                                 "Physical PP groups in IDLE state (count = %u):\n",
1174                                 group_list_idle_count);
1175
1176         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle, struct mali_group, executor_list) {
1177                 n += mali_group_dump_state(group, buf + n, size - n);
1178         }
1179
1180         n += _mali_osk_snprintf(buf + n, size - n,
1181                                 "Physical PP groups in INACTIVE state (count = %u):\n",
1182                                 group_list_inactive_count);
1183
1184         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive, struct mali_group, executor_list) {
1185                 n += mali_group_dump_state(group, buf + n, size - n);
1186         }
1187
1188         n += _mali_osk_snprintf(buf + n, size - n,
1189                                 "Physical PP groups in DISABLED state (count = %u):\n",
1190                                 group_list_disabled_count);
1191
1192         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_disabled, struct mali_group, executor_list) {
1193                 n += mali_group_dump_state(group, buf + n, size - n);
1194         }
1195
1196         if (mali_executor_has_virtual_group()) {
1197                 switch (virtual_group_state) {
1198                 case EXEC_STATE_EMPTY:
1199                         n += _mali_osk_snprintf(buf + n, size - n,
1200                                                 "Virtual PP group is in state EMPTY\n");
1201                         break;
1202                 case EXEC_STATE_INACTIVE:
1203                         n += _mali_osk_snprintf(buf + n, size - n,
1204                                                 "Virtual PP group is in state INACTIVE\n");
1205                         break;
1206                 case EXEC_STATE_IDLE:
1207                         n += _mali_osk_snprintf(buf + n, size - n,
1208                                                 "Virtual PP group is in state IDLE\n");
1209                         break;
1210                 case EXEC_STATE_WORKING:
1211                         n += _mali_osk_snprintf(buf + n, size - n,
1212                                                 "Virtual PP group is in state WORKING\n");
1213                         break;
1214                 default:
1215                         n += _mali_osk_snprintf(buf + n, size - n,
1216                                                 "Virtual PP group is in unknown/illegal state %u\n",
1217                                                 virtual_group_state);
1218                         break;
1219                 }
1220
1221                 n += mali_group_dump_state(virtual_group, buf + n, size - n);
1222         }
1223
1224         mali_executor_unlock();
1225
1226         n += _mali_osk_snprintf(buf + n, size - n, "\n");
1227
1228         return n;
1229 }
1230 #endif
1231
1232 _mali_osk_errcode_t _mali_ukk_get_pp_number_of_cores(_mali_uk_get_pp_number_of_cores_s *args)
1233 {
1234         MALI_DEBUG_ASSERT_POINTER(args);
1235         MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
1236         args->number_of_total_cores = num_physical_pp_cores_total;
1237         args->number_of_enabled_cores = num_physical_pp_cores_enabled;
1238         return _MALI_OSK_ERR_OK;
1239 }
1240
1241 _mali_osk_errcode_t _mali_ukk_get_pp_core_version(_mali_uk_get_pp_core_version_s *args)
1242 {
1243         MALI_DEBUG_ASSERT_POINTER(args);
1244         MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
1245         args->version = pp_version;
1246         return _MALI_OSK_ERR_OK;
1247 }
1248
1249 _mali_osk_errcode_t _mali_ukk_get_gp_number_of_cores(_mali_uk_get_gp_number_of_cores_s *args)
1250 {
1251         MALI_DEBUG_ASSERT_POINTER(args);
1252         MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
1253         args->number_of_cores = 1;
1254         return _MALI_OSK_ERR_OK;
1255 }
1256
1257 _mali_osk_errcode_t _mali_ukk_get_gp_core_version(_mali_uk_get_gp_core_version_s *args)
1258 {
1259         MALI_DEBUG_ASSERT_POINTER(args);
1260         MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
1261         args->version = gp_version;
1262         return _MALI_OSK_ERR_OK;
1263 }
1264
1265 _mali_osk_errcode_t _mali_ukk_gp_suspend_response(_mali_uk_gp_suspend_response_s *args)
1266 {
1267         struct mali_session_data *session;
1268         struct mali_gp_job *job;
1269
1270         MALI_DEBUG_ASSERT_POINTER(args);
1271         MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
1272
1273         session = (struct mali_session_data *)(uintptr_t)args->ctx;
1274
1275         if (_MALIGP_JOB_RESUME_WITH_NEW_HEAP == args->code) {
1276                 _mali_osk_notification_t *new_notification = NULL;
1277
1278                 new_notification = _mali_osk_notification_create(
1279                                            _MALI_NOTIFICATION_GP_STALLED,
1280                                            sizeof(_mali_uk_gp_job_suspended_s));
1281
1282                 if (NULL != new_notification) {
1283                         MALI_DEBUG_PRINT(3, ("Executor: Resuming job %u with new heap; 0x%08X - 0x%08X\n",
1284                                              args->cookie, args->arguments[0], args->arguments[1]));
1285
1286                         mali_executor_lock();
1287
1288                         /* Resume the job in question if it is still running */
1289                         job = mali_group_get_running_gp_job(gp_group);
1290                         if (NULL != job &&
1291                             args->cookie == mali_gp_job_get_id(job) &&
1292                             session == mali_gp_job_get_session(job)) {
1293                                 /*
1294                                  * Correct job is running, resume with new heap
1295                                  */
1296
1297                                 mali_gp_job_set_oom_notification(job,
1298                                                                  new_notification);
1299
1300                                 /* This will also re-enable interrupts */
1301                                 mali_group_resume_gp_with_new_heap(gp_group,
1302                                                                    args->cookie,
1303                                                                    args->arguments[0],
1304                                                                    args->arguments[1]);
1305
1306                                 mali_executor_unlock();
1307                                 return _MALI_OSK_ERR_OK;
1308                         } else {
1309                                 MALI_PRINT_ERROR(("Executor: Unable to resume, GP job no longer running.\n"));
1310
1311                                 _mali_osk_notification_delete(new_notification);
1312
1313                                 mali_executor_unlock();
1314                                 return _MALI_OSK_ERR_FAULT;
1315                         }
1316                 } else {
1317                         MALI_PRINT_ERROR(("Executor: Failed to allocate notification object. Will abort GP job.\n"));
1318                 }
1319         } else {
1320                 MALI_DEBUG_PRINT(2, ("Executor: Aborting job %u, no new heap provided\n", args->cookie));
1321         }
1322
1323         mali_executor_lock();
1324
1325         /* Abort the job in question if it is still running */
1326         job = mali_group_get_running_gp_job(gp_group);
1327         if (NULL != job &&
1328             args->cookie == mali_gp_job_get_id(job) &&
1329             session == mali_gp_job_get_session(job)) {
1330                 /* Correct job is still running */
1331                 struct mali_gp_job *job_done = NULL;
1332
1333                 mali_executor_complete_group(gp_group, MALI_FALSE,
1334                                              MALI_TRUE, &job_done, NULL);
1335
1336                 /* The same job should have completed */
1337                 MALI_DEBUG_ASSERT(job_done == job);
1338
1339                 /* GP job completed, make sure it is freed */
1340                 mali_scheduler_complete_gp_job(job_done, MALI_FALSE,
1341                                                MALI_TRUE, MALI_TRUE);
1342         }
1343
1344         mali_executor_unlock();
1345         return _MALI_OSK_ERR_FAULT;
1346 }
1347
1348
1349 /*
1350  * ---------- Implementation of static functions ----------
1351  */
1352
1353 static void mali_executor_lock(void)
1354 {
1355         _mali_osk_spinlock_irq_lock(mali_executor_lock_obj);
1356         MALI_DEBUG_PRINT(5, ("Executor: lock taken\n"));
1357 }
1358
1359 static void mali_executor_unlock(void)
1360 {
1361         MALI_DEBUG_PRINT(5, ("Executor: Releasing lock\n"));
1362         _mali_osk_spinlock_irq_unlock(mali_executor_lock_obj);
1363 }
1364
1365 static mali_bool mali_executor_is_suspended(void *data)
1366 {
1367         mali_bool ret;
1368
1369         /* This callback does not use the data pointer. */
1370         MALI_IGNORE(data);
1371
1372         mali_executor_lock();
1373
1374         ret = pause_count > 0 && !mali_executor_is_working();
1375
1376         mali_executor_unlock();
1377
1378         return ret;
1379 }
1380
1381 static mali_bool mali_executor_is_working()
1382 {
1383         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1384
1385         return (0 != group_list_working_count ||
1386                 EXEC_STATE_WORKING == gp_group_state ||
1387                 EXEC_STATE_WORKING == virtual_group_state);
1388 }
1389
1390 static void mali_executor_disable_empty_virtual(void)
1391 {
1392         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1393         MALI_DEBUG_ASSERT(virtual_group_state != EXEC_STATE_EMPTY);
1394         MALI_DEBUG_ASSERT(virtual_group_state != EXEC_STATE_WORKING);
1395
1396         if (mali_group_is_empty(virtual_group)) {
1397                 virtual_group_state = EXEC_STATE_EMPTY;
1398         }
1399 }
1400
1401 static mali_bool mali_executor_physical_rejoin_virtual(struct mali_group *group)
1402 {
1403         mali_bool trigger_pm_update = MALI_FALSE;
1404
1405         MALI_DEBUG_ASSERT_POINTER(group);
1406         /* Only rejoining after job has completed (still active) */
1407         MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVE ==
1408                           mali_group_get_state(group));
1409         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1410         MALI_DEBUG_ASSERT(MALI_TRUE == mali_executor_has_virtual_group());
1411         MALI_DEBUG_ASSERT(MALI_FALSE == mali_group_is_virtual(group));
1412
1413         /* Make sure group and virtual group have same status */
1414
1415         if (MALI_GROUP_STATE_INACTIVE == mali_group_get_state(virtual_group)) {
1416                 if (mali_group_deactivate(group)) {
1417                         trigger_pm_update = MALI_TRUE;
1418                 }
1419
1420                 if (virtual_group_state == EXEC_STATE_EMPTY) {
1421                         virtual_group_state = EXEC_STATE_INACTIVE;
1422                 }
1423         } else if (MALI_GROUP_STATE_ACTIVATION_PENDING ==
1424                    mali_group_get_state(virtual_group)) {
1425                 /*
1426                  * Activation is pending for virtual group, leave
1427                  * this child group as active.
1428                  */
1429                 if (virtual_group_state == EXEC_STATE_EMPTY) {
1430                         virtual_group_state = EXEC_STATE_INACTIVE;
1431                 }
1432         } else {
1433                 MALI_DEBUG_ASSERT(MALI_GROUP_STATE_ACTIVE ==
1434                                   mali_group_get_state(virtual_group));
1435
1436                 if (virtual_group_state == EXEC_STATE_EMPTY) {
1437                         virtual_group_state = EXEC_STATE_IDLE;
1438                 }
1439         }
1440
1441         /* Remove group from idle list */
1442         MALI_DEBUG_ASSERT(mali_executor_group_is_in_state(group,
1443                           EXEC_STATE_IDLE));
1444         _mali_osk_list_delinit(&group->executor_list);
1445         group_list_idle_count--;
1446
1447         /*
1448          * And finally rejoin the virtual group
1449          * group will start working on same job as virtual_group,
1450          * if virtual_group is working on a job
1451          */
1452         mali_group_add_group(virtual_group, group);
1453
1454         return trigger_pm_update;
1455 }
1456
1457 static mali_bool mali_executor_has_virtual_group(void)
1458 {
1459 #if defined(CONFIG_MALI450)
1460         return (NULL != virtual_group) ? MALI_TRUE : MALI_FALSE;
1461 #else
1462         return MALI_FALSE;
1463 #endif /* defined(CONFIG_MALI450) */
1464 }
1465
1466 static mali_bool mali_executor_virtual_group_is_usable(void)
1467 {
1468 #if defined(CONFIG_MALI450)
1469         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1470         return (EXEC_STATE_INACTIVE == virtual_group_state ||
1471                 EXEC_STATE_IDLE == virtual_group_state) ?
1472                MALI_TRUE : MALI_FALSE;
1473 #else
1474         return MALI_FALSE;
1475 #endif /* defined(CONFIG_MALI450) */
1476 }
1477
1478 static mali_bool mali_executor_tackle_gp_bound(void)
1479 {
1480         struct mali_pp_job *job;
1481
1482         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1483
1484         job = mali_scheduler_job_pp_physical_peek();
1485
1486         if (NULL != job && MALI_TRUE == mali_is_mali400()) {
1487                 if (0 < group_list_working_count &&
1488                     mali_pp_job_is_large_and_unstarted(job)) {
1489                         return MALI_TRUE;
1490                 }
1491         }
1492
1493         return MALI_FALSE;
1494 }
1495
1496 /*
1497  * This is where jobs are actually started.
1498  */
1499 static void mali_executor_schedule(void)
1500 {
1501         u32 i;
1502         u32 num_physical_needed = 0;
1503         u32 num_physical_to_process = 0;
1504         mali_bool trigger_pm_update = MALI_FALSE;
1505         mali_bool deactivate_idle_group = MALI_TRUE;
1506
1507         /* Physical groups + jobs to start in this function */
1508         struct mali_group *groups_to_start[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS];
1509         struct mali_pp_job *jobs_to_start[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS];
1510         u32 sub_jobs_to_start[MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS];
1511         int num_jobs_to_start = 0;
1512
1513         /* Virtual job to start in this function */
1514         struct mali_pp_job *virtual_job_to_start = NULL;
1515
1516         /* GP job to start in this function */
1517         struct mali_gp_job *gp_job_to_start = NULL;
1518
1519         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1520
1521         if (pause_count > 0) {
1522                 /* Execution is suspended, don't schedule any jobs. */
1523                 return;
1524         }
1525
1526         /* Lock needed in order to safely handle the job queues */
1527         mali_scheduler_lock();
1528
1529         /* 1. Activate gp firstly if have gp job queued. */
1530         if (EXEC_STATE_INACTIVE == gp_group_state &&
1531             0 < mali_scheduler_job_gp_count()) {
1532
1533                 enum mali_group_state state =
1534                         mali_group_activate(gp_group);
1535                 if (MALI_GROUP_STATE_ACTIVE == state) {
1536                         /* Set GP group state to idle */
1537                         gp_group_state = EXEC_STATE_IDLE;
1538                 } else {
1539                         trigger_pm_update = MALI_TRUE;
1540                 }
1541         }
1542
1543         /* 2. Prepare as many physical groups as needed/possible */
1544
1545         num_physical_needed = mali_scheduler_job_physical_head_count();
1546
1547         /* On mali-450 platform, we don't need to enter in this block frequently. */
1548         if (0 < num_physical_needed) {
1549
1550                 if (num_physical_needed <= group_list_idle_count) {
1551                         /* We have enough groups on idle list already */
1552                         num_physical_to_process = num_physical_needed;
1553                         num_physical_needed = 0;
1554                 } else {
1555                         /* We need to get a hold of some more groups */
1556                         num_physical_to_process = group_list_idle_count;
1557                         num_physical_needed -= group_list_idle_count;
1558                 }
1559
1560                 if (0 < num_physical_needed) {
1561
1562                         /* 2.1. Activate groups which are inactive */
1563
1564                         struct mali_group *group;
1565                         struct mali_group *temp;
1566
1567                         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_inactive,
1568                                                     struct mali_group, executor_list) {
1569                                 enum mali_group_state state =
1570                                         mali_group_activate(group);
1571                                 if (MALI_GROUP_STATE_ACTIVE == state) {
1572                                         /* Move from inactive to idle */
1573                                         mali_executor_change_state_pp_physical(group,
1574                                                                                &group_list_inactive,
1575                                                                                &group_list_inactive_count,
1576                                                                                &group_list_idle,
1577                                                                                &group_list_idle_count);
1578                                         num_physical_to_process++;
1579                                 } else {
1580                                         trigger_pm_update = MALI_TRUE;
1581                                 }
1582
1583                                 num_physical_needed--;
1584                                 if (0 == num_physical_needed) {
1585                                         /* We have activated all the groups we need */
1586                                         break;
1587                                 }
1588                         }
1589                 }
1590
1591                 if (mali_executor_virtual_group_is_usable()) {
1592
1593                         /*
1594                          * 2.2. And finally, steal and activate groups
1595                          * from virtual group if we need even more
1596                          */
1597                         while (0 < num_physical_needed) {
1598                                 struct mali_group *group;
1599
1600                                 group = mali_group_acquire_group(virtual_group);
1601                                 if (NULL != group) {
1602                                         enum mali_group_state state;
1603
1604                                         mali_executor_disable_empty_virtual();
1605
1606                                         state = mali_group_activate(group);
1607                                         if (MALI_GROUP_STATE_ACTIVE == state) {
1608                                                 /* Group is ready, add to idle list */
1609                                                 _mali_osk_list_add(
1610                                                         &group->executor_list,
1611                                                         &group_list_idle);
1612                                                 group_list_idle_count++;
1613                                                 num_physical_to_process++;
1614                                         } else {
1615                                                 /*
1616                                                  * Group is not ready yet,
1617                                                  * add to inactive list
1618                                                  */
1619                                                 _mali_osk_list_add(
1620                                                         &group->executor_list,
1621                                                         &group_list_inactive);
1622                                                 group_list_inactive_count++;
1623
1624                                                 trigger_pm_update = MALI_TRUE;
1625                                         }
1626                                         num_physical_needed--;
1627                                 } else {
1628                                         /*
1629                                          * We could not get enough groups
1630                                          * from the virtual group.
1631                                          */
1632                                         break;
1633                                 }
1634                         }
1635                 }
1636
1637                 /* 2.3. Assign physical jobs to groups */
1638
1639                 if (0 < num_physical_to_process) {
1640                         struct mali_group *group;
1641                         struct mali_group *temp;
1642
1643                         _MALI_OSK_LIST_FOREACHENTRY(group, temp, &group_list_idle,
1644                                                     struct mali_group, executor_list) {
1645                                 struct mali_pp_job *job = NULL;
1646                                 u32 sub_job = MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS;
1647
1648                                 MALI_DEBUG_ASSERT(num_jobs_to_start <
1649                                                   MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS);
1650
1651                                 MALI_DEBUG_ASSERT(0 <
1652                                                   mali_scheduler_job_physical_head_count());
1653
1654                                 if (mali_executor_hint_is_enabled(
1655                                             MALI_EXECUTOR_HINT_GP_BOUND)) {
1656                                         if (MALI_TRUE == mali_executor_tackle_gp_bound()) {
1657                                                 /*
1658                                                 * We're gp bound,
1659                                                 * don't start this right now.
1660                                                 */
1661                                                 deactivate_idle_group = MALI_FALSE;
1662                                                 num_physical_to_process = 0;
1663                                                 break;
1664                                         }
1665                                 }
1666
1667                                 job = mali_scheduler_job_pp_physical_get(
1668                                               &sub_job);
1669
1670                                 MALI_DEBUG_ASSERT_POINTER(job);
1671                                 MALI_DEBUG_ASSERT(sub_job <= MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS);
1672
1673                                 /* Put job + group on list of jobs to start later on */
1674
1675                                 groups_to_start[num_jobs_to_start] = group;
1676                                 jobs_to_start[num_jobs_to_start] = job;
1677                                 sub_jobs_to_start[num_jobs_to_start] = sub_job;
1678                                 num_jobs_to_start++;
1679
1680                                 /* Move group from idle to working */
1681                                 mali_executor_change_state_pp_physical(group,
1682                                                                        &group_list_idle,
1683                                                                        &group_list_idle_count,
1684                                                                        &group_list_working,
1685                                                                        &group_list_working_count);
1686
1687                                 num_physical_to_process--;
1688                                 if (0 == num_physical_to_process) {
1689                                         /* Got all we needed */
1690                                         break;
1691                                 }
1692                         }
1693                 }
1694         }
1695
1696         /* 3. Activate virtual group, if needed */
1697
1698         if (EXEC_STATE_INACTIVE == virtual_group_state &&
1699             0 < mali_scheduler_job_next_is_virtual()) {
1700                 enum mali_group_state state =
1701                         mali_group_activate(virtual_group);
1702                 if (MALI_GROUP_STATE_ACTIVE == state) {
1703                         /* Set virtual group state to idle */
1704                         virtual_group_state = EXEC_STATE_IDLE;
1705                 } else {
1706                         trigger_pm_update = MALI_TRUE;
1707                 }
1708         }
1709
1710         /* 4. To power up group asap, we trigger pm update here. */
1711
1712         if (MALI_TRUE == trigger_pm_update) {
1713                 trigger_pm_update = MALI_FALSE;
1714                 mali_pm_update_async();
1715         }
1716
1717         /* 5. Deactivate idle pp group */
1718
1719         if (MALI_TRUE == mali_executor_deactivate_list_idle(deactivate_idle_group
1720                         && (!mali_timeline_has_physical_pp_job()))) {
1721                 trigger_pm_update = MALI_TRUE;
1722         }
1723
1724         /* 6. Assign jobs to idle virtual group (or deactivate if no job) */
1725
1726         if (EXEC_STATE_IDLE == virtual_group_state) {
1727                 if (0 < mali_scheduler_job_next_is_virtual()) {
1728                         virtual_job_to_start =
1729                                 mali_scheduler_job_pp_virtual_get();
1730                         virtual_group_state = EXEC_STATE_WORKING;
1731                 } else if (!mali_timeline_has_virtual_pp_job()) {
1732                         virtual_group_state = EXEC_STATE_INACTIVE;
1733
1734                         if (mali_group_deactivate(virtual_group)) {
1735                                 trigger_pm_update = MALI_TRUE;
1736                         }
1737                 }
1738         }
1739
1740         /* 7. Assign job to idle GP group (or deactivate if no job) */
1741
1742         if (EXEC_STATE_IDLE == gp_group_state) {
1743                 if (0 < mali_scheduler_job_gp_count()) {
1744                         gp_job_to_start = mali_scheduler_job_gp_get();
1745                         gp_group_state = EXEC_STATE_WORKING;
1746                 } else if (!mali_timeline_has_gp_job()) {
1747                         gp_group_state = EXEC_STATE_INACTIVE;
1748                         if (mali_group_deactivate(gp_group)) {
1749                                 trigger_pm_update = MALI_TRUE;
1750                         }
1751                 }
1752         }
1753
1754         /* 8. We no longer need the schedule/queue lock */
1755
1756         mali_scheduler_unlock();
1757
1758         /* 9. start jobs */
1759
1760         if (NULL != virtual_job_to_start) {
1761                 MALI_DEBUG_ASSERT(!mali_group_pp_is_active(virtual_group));
1762                 mali_group_start_pp_job(virtual_group,
1763                                         virtual_job_to_start, 0);
1764         }
1765
1766         for (i = 0; i < num_jobs_to_start; i++) {
1767                 MALI_DEBUG_ASSERT(!mali_group_pp_is_active(
1768                                           groups_to_start[i]));
1769                 mali_group_start_pp_job(groups_to_start[i],
1770                                         jobs_to_start[i],
1771                                         sub_jobs_to_start[i]);
1772         }
1773
1774         MALI_DEBUG_ASSERT_POINTER(gp_group);
1775
1776         if (NULL != gp_job_to_start) {
1777                 MALI_DEBUG_ASSERT(!mali_group_gp_is_active(gp_group));
1778                 mali_group_start_gp_job(gp_group, gp_job_to_start);
1779         }
1780
1781         /* 10. Trigger any pending PM updates */
1782         if (MALI_TRUE == trigger_pm_update) {
1783                 mali_pm_update_async();
1784         }
1785 }
1786
1787 /* Handler for deferred schedule requests */
1788 static void mali_executor_wq_schedule(void *arg)
1789 {
1790         MALI_IGNORE(arg);
1791         mali_executor_lock();
1792         mali_executor_schedule();
1793         mali_executor_unlock();
1794 }
1795
1796 static void mali_executor_send_gp_oom_to_user(struct mali_gp_job *job)
1797 {
1798         _mali_uk_gp_job_suspended_s *jobres;
1799         _mali_osk_notification_t *notification;
1800
1801         notification = mali_gp_job_get_oom_notification(job);
1802
1803         /*
1804          * Remember the id we send to user space, so we have something to
1805          * verify when we get a response
1806          */
1807         gp_returned_cookie = mali_gp_job_get_id(job);
1808
1809         jobres = (_mali_uk_gp_job_suspended_s *)notification->result_buffer;
1810         jobres->user_job_ptr = mali_gp_job_get_user_id(job);
1811         jobres->cookie = gp_returned_cookie;
1812
1813         mali_session_send_notification(mali_gp_job_get_session(job),
1814                                        notification);
1815 }
1816 static struct mali_gp_job *mali_executor_complete_gp(struct mali_group *group,
1817                 mali_bool success,
1818                 mali_bool release_jobs)
1819 {
1820         struct mali_gp_job *job;
1821
1822         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1823
1824         /* Extracts the needed HW status from core and reset */
1825         job = mali_group_complete_gp(group, success);
1826
1827         MALI_DEBUG_ASSERT_POINTER(job);
1828
1829         /* Core is now ready to go into idle list */
1830         gp_group_state = EXEC_STATE_IDLE;
1831
1832         if (release_jobs) {
1833                 /* This will potentially queue more GP and PP jobs */
1834                 mali_timeline_tracker_release(&job->tracker);
1835
1836                 /* Signal PP job */
1837                 mali_gp_job_signal_pp_tracker(job, success);
1838         }
1839
1840         return job;
1841 }
1842
1843 static struct mali_pp_job *mali_executor_complete_pp(struct mali_group *group,
1844                 mali_bool success,
1845                 mali_bool release_jobs)
1846 {
1847         struct mali_pp_job *job;
1848         u32 sub_job;
1849         mali_bool job_is_done;
1850
1851         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
1852
1853         /* Extracts the needed HW status from core and reset */
1854         job = mali_group_complete_pp(group, success, &sub_job);
1855
1856         MALI_DEBUG_ASSERT_POINTER(job);
1857
1858         /* Core is now ready to go into idle list */
1859         if (mali_group_is_virtual(group)) {
1860                 virtual_group_state = EXEC_STATE_IDLE;
1861         } else {
1862                 /* Move from working to idle state */
1863                 mali_executor_change_state_pp_physical(group,
1864                                                        &group_list_working,
1865                                                        &group_list_working_count,
1866                                                        &group_list_idle,
1867                                                        &group_list_idle_count);
1868         }
1869
1870         /* It is the executor module which owns the jobs themselves by now */
1871         mali_pp_job_mark_sub_job_completed(job, success);
1872         job_is_done = mali_pp_job_is_complete(job);
1873
1874         if (job_is_done && release_jobs) {
1875                 /* This will potentially queue more GP and PP jobs */
1876                 mali_timeline_tracker_release(&job->tracker);
1877         }
1878
1879         return job;
1880 }
1881
1882 static void mali_executor_complete_group(struct mali_group *group,
1883                 mali_bool success,
1884                 mali_bool release_jobs,
1885                 struct mali_gp_job **gp_job_done,
1886                 struct mali_pp_job **pp_job_done)
1887 {
1888         struct mali_gp_core *gp_core = mali_group_get_gp_core(group);
1889         struct mali_pp_core *pp_core = mali_group_get_pp_core(group);
1890         struct mali_gp_job *gp_job = NULL;
1891         struct mali_pp_job *pp_job = NULL;
1892         mali_bool pp_job_is_done = MALI_TRUE;
1893
1894         if (NULL != gp_core) {
1895                 gp_job = mali_executor_complete_gp(group,
1896                                                    success, release_jobs);
1897         } else {
1898                 MALI_DEBUG_ASSERT_POINTER(pp_core);
1899                 MALI_IGNORE(pp_core);
1900                 pp_job = mali_executor_complete_pp(group,
1901                                                    success, release_jobs);
1902
1903                 pp_job_is_done = mali_pp_job_is_complete(pp_job);
1904         }
1905
1906         if (pause_count > 0) {
1907                 /* Execution has been suspended */
1908
1909                 if (!mali_executor_is_working()) {
1910                         /* Last job completed, wake up sleepers */
1911                         _mali_osk_wait_queue_wake_up(
1912                                 executor_working_wait_queue);
1913                 }
1914         } else if (MALI_TRUE == mali_group_disable_requested(group)) {
1915                 mali_executor_core_scale_in_group_complete(group);
1916
1917                 mali_executor_schedule();
1918         } else {
1919                 /* try to schedule new jobs */
1920                 mali_executor_schedule();
1921         }
1922
1923         if (NULL != gp_job) {
1924                 MALI_DEBUG_ASSERT_POINTER(gp_job_done);
1925                 *gp_job_done = gp_job;
1926         } else if (pp_job_is_done) {
1927                 MALI_DEBUG_ASSERT_POINTER(pp_job);
1928                 MALI_DEBUG_ASSERT_POINTER(pp_job_done);
1929                 *pp_job_done = pp_job;
1930         }
1931 }
1932
1933 static void mali_executor_change_state_pp_physical(struct mali_group *group,
1934                 _mali_osk_list_t *old_list,
1935                 u32 *old_count,
1936                 _mali_osk_list_t *new_list,
1937                 u32 *new_count)
1938 {
1939         /*
1940          * It's a bit more complicated to change the state for the physical PP
1941          * groups since their state is determined by the list they are on.
1942          */
1943 #if defined(DEBUG)
1944         mali_bool found = MALI_FALSE;
1945         struct mali_group *group_iter;
1946         struct mali_group *temp;
1947         u32 old_counted = 0;
1948         u32 new_counted = 0;
1949
1950         MALI_DEBUG_ASSERT_POINTER(group);
1951         MALI_DEBUG_ASSERT_POINTER(old_list);
1952         MALI_DEBUG_ASSERT_POINTER(old_count);
1953         MALI_DEBUG_ASSERT_POINTER(new_list);
1954         MALI_DEBUG_ASSERT_POINTER(new_count);
1955
1956         /*
1957          * Verify that group is present on old list,
1958          * and that the count is correct
1959          */
1960
1961         _MALI_OSK_LIST_FOREACHENTRY(group_iter, temp, old_list,
1962                                     struct mali_group, executor_list) {
1963                 old_counted++;
1964                 if (group == group_iter) {
1965                         found = MALI_TRUE;
1966                 }
1967         }
1968
1969         _MALI_OSK_LIST_FOREACHENTRY(group_iter, temp, new_list,
1970                                     struct mali_group, executor_list) {
1971                 new_counted++;
1972         }
1973
1974         if (MALI_FALSE == found) {
1975                 if (old_list == &group_list_idle) {
1976                         MALI_DEBUG_PRINT(1, (" old Group list is idle,"));
1977                 } else if (old_list == &group_list_inactive) {
1978                         MALI_DEBUG_PRINT(1, (" old Group list is inactive,"));
1979                 } else if (old_list == &group_list_working) {
1980                         MALI_DEBUG_PRINT(1, (" old Group list is working,"));
1981                 } else if (old_list == &group_list_disabled) {
1982                         MALI_DEBUG_PRINT(1, (" old Group list is disable,"));
1983                 }
1984
1985                 if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_WORKING)) {
1986                         MALI_DEBUG_PRINT(1, (" group in working \n"));
1987                 } else if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_INACTIVE)) {
1988                         MALI_DEBUG_PRINT(1, (" group in inactive \n"));
1989                 } else if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_IDLE)) {
1990                         MALI_DEBUG_PRINT(1, (" group in idle \n"));
1991                 } else if (MALI_TRUE == mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED)) {
1992                         MALI_DEBUG_PRINT(1, (" but group in disabled \n"));
1993                 }
1994         }
1995
1996         MALI_DEBUG_ASSERT(MALI_TRUE == found);
1997         MALI_DEBUG_ASSERT(0 < (*old_count));
1998         MALI_DEBUG_ASSERT((*old_count) == old_counted);
1999         MALI_DEBUG_ASSERT((*new_count) == new_counted);
2000 #endif
2001
2002         _mali_osk_list_move(&group->executor_list, new_list);
2003         (*old_count)--;
2004         (*new_count)++;
2005 }
2006
2007 static void mali_executor_set_state_pp_physical(struct mali_group *group,
2008                 _mali_osk_list_t *new_list,
2009                 u32 *new_count)
2010 {
2011         _mali_osk_list_add(&group->executor_list, new_list);
2012         (*new_count)++;
2013 }
2014
2015 static mali_bool mali_executor_group_is_in_state(struct mali_group *group,
2016                 enum mali_executor_state_t state)
2017 {
2018         MALI_DEBUG_ASSERT_POINTER(group);
2019         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
2020
2021         if (gp_group == group) {
2022                 if (gp_group_state == state) {
2023                         return MALI_TRUE;
2024                 }
2025         } else if (virtual_group == group || mali_group_is_in_virtual(group)) {
2026                 if (virtual_group_state == state) {
2027                         return MALI_TRUE;
2028                 }
2029         } else {
2030                 /* Physical PP group */
2031                 struct mali_group *group_iter;
2032                 struct mali_group *temp;
2033                 _mali_osk_list_t *list;
2034
2035                 if (EXEC_STATE_DISABLED == state) {
2036                         list = &group_list_disabled;
2037                 } else if (EXEC_STATE_INACTIVE == state) {
2038                         list = &group_list_inactive;
2039                 } else if (EXEC_STATE_IDLE == state) {
2040                         list = &group_list_idle;
2041                 } else {
2042                         MALI_DEBUG_ASSERT(EXEC_STATE_WORKING == state);
2043                         list = &group_list_working;
2044                 }
2045
2046                 _MALI_OSK_LIST_FOREACHENTRY(group_iter, temp, list,
2047                                             struct mali_group, executor_list) {
2048                         if (group_iter == group) {
2049                                 return MALI_TRUE;
2050                         }
2051                 }
2052         }
2053
2054         /* group not in correct state */
2055         return MALI_FALSE;
2056 }
2057
2058 static void mali_executor_group_enable_internal(struct mali_group *group)
2059 {
2060         MALI_DEBUG_ASSERT(group);
2061         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
2062         MALI_DEBUG_ASSERT(mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED));
2063
2064         /* Put into inactive state (== "lowest" enabled state) */
2065         if (group == gp_group) {
2066                 MALI_DEBUG_ASSERT(EXEC_STATE_DISABLED == gp_group_state);
2067                 gp_group_state = EXEC_STATE_INACTIVE;
2068         } else {
2069                 mali_executor_change_state_pp_physical(group,
2070                                                        &group_list_disabled,
2071                                                        &group_list_disabled_count,
2072                                                        &group_list_inactive,
2073                                                        &group_list_inactive_count);
2074
2075                 ++num_physical_pp_cores_enabled;
2076                 MALI_DEBUG_PRINT(4, ("Enabling group id %d \n", group->pp_core->core_id));
2077         }
2078
2079         if (MALI_GROUP_STATE_ACTIVE == mali_group_activate(group)) {
2080                 MALI_DEBUG_ASSERT(MALI_TRUE == mali_group_power_is_on(group));
2081
2082                 /* Move from inactive to idle */
2083                 if (group == gp_group) {
2084                         gp_group_state = EXEC_STATE_IDLE;
2085                 } else {
2086                         mali_executor_change_state_pp_physical(group,
2087                                                                &group_list_inactive,
2088                                                                &group_list_inactive_count,
2089                                                                &group_list_idle,
2090                                                                &group_list_idle_count);
2091
2092                         if (mali_executor_has_virtual_group()) {
2093                                 if (mali_executor_physical_rejoin_virtual(group)) {
2094                                         mali_pm_update_async();
2095                                 }
2096                         }
2097                 }
2098         } else {
2099                 mali_pm_update_async();
2100         }
2101 }
2102
2103 static void mali_executor_group_disable_internal(struct mali_group *group)
2104 {
2105         mali_bool working;
2106
2107         MALI_DEBUG_ASSERT_POINTER(group);
2108         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
2109         MALI_DEBUG_ASSERT(!mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED));
2110
2111         working = mali_executor_group_is_in_state(group, EXEC_STATE_WORKING);
2112         if (MALI_TRUE == working) {
2113                 /** Group to be disabled once it completes current work,
2114                  * when virtual group completes, also check child groups for this flag */
2115                 mali_group_set_disable_request(group, MALI_TRUE);
2116                 return;
2117         }
2118
2119         /* Put into disabled state */
2120         if (group == gp_group) {
2121                 /* GP group */
2122                 MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != gp_group_state);
2123                 gp_group_state = EXEC_STATE_DISABLED;
2124         } else {
2125                 if (mali_group_is_in_virtual(group)) {
2126                         /* A child group of virtual group. move the specific group from virtual group */
2127                         MALI_DEBUG_ASSERT(EXEC_STATE_WORKING != virtual_group_state);
2128
2129                         mali_executor_set_state_pp_physical(group,
2130                                                             &group_list_disabled,
2131                                                             &group_list_disabled_count);
2132
2133                         mali_group_remove_group(virtual_group, group);
2134                         mali_executor_disable_empty_virtual();
2135                 } else {
2136                         mali_executor_change_group_status_disabled(group);
2137                 }
2138
2139                 --num_physical_pp_cores_enabled;
2140                 MALI_DEBUG_PRINT(4, ("Disabling group id %d \n", group->pp_core->core_id));
2141         }
2142
2143         if (MALI_GROUP_STATE_INACTIVE != group->state) {
2144                 if (MALI_TRUE == mali_group_deactivate(group)) {
2145                         mali_pm_update_async();
2146                 }
2147         }
2148 }
2149
2150 static void mali_executor_notify_core_change(u32 num_cores)
2151 {
2152         mali_bool done = MALI_FALSE;
2153
2154         if (mali_is_mali450()) {
2155                 return;
2156         }
2157
2158         /*
2159          * This function gets a bit complicated because we can't hold the session lock while
2160          * allocating notification objects.
2161          */
2162         while (!done) {
2163                 u32 i;
2164                 u32 num_sessions_alloc;
2165                 u32 num_sessions_with_lock;
2166                 u32 used_notification_objects = 0;
2167                 _mali_osk_notification_t **notobjs;
2168
2169                 /* Pre allocate the number of notifications objects we need right now (might change after lock has been taken) */
2170                 num_sessions_alloc = mali_session_get_count();
2171                 if (0 == num_sessions_alloc) {
2172                         /* No sessions to report to */
2173                         return;
2174                 }
2175
2176                 notobjs = (_mali_osk_notification_t **)_mali_osk_malloc(sizeof(_mali_osk_notification_t *) * num_sessions_alloc);
2177                 if (NULL == notobjs) {
2178                         MALI_PRINT_ERROR(("Failed to notify user space session about num PP core change (alloc failure)\n"));
2179                         /* there is probably no point in trying again, system must be really low on memory and probably unusable now anyway */
2180                         return;
2181                 }
2182
2183                 for (i = 0; i < num_sessions_alloc; i++) {
2184                         notobjs[i] = _mali_osk_notification_create(_MALI_NOTIFICATION_PP_NUM_CORE_CHANGE, sizeof(_mali_uk_pp_num_cores_changed_s));
2185                         if (NULL != notobjs[i]) {
2186                                 _mali_uk_pp_num_cores_changed_s *data = notobjs[i]->result_buffer;
2187                                 data->number_of_enabled_cores = num_cores;
2188                         } else {
2189                                 MALI_PRINT_ERROR(("Failed to notify user space session about num PP core change (alloc failure %u)\n", i));
2190                         }
2191                 }
2192
2193                 mali_session_lock();
2194
2195                 /* number of sessions will not change while we hold the lock */
2196                 num_sessions_with_lock = mali_session_get_count();
2197
2198                 if (num_sessions_alloc >= num_sessions_with_lock) {
2199                         /* We have allocated enough notification objects for all the sessions atm */
2200                         struct mali_session_data *session, *tmp;
2201                         MALI_SESSION_FOREACH(session, tmp, link) {
2202                                 MALI_DEBUG_ASSERT(used_notification_objects < num_sessions_alloc);
2203                                 if (NULL != notobjs[used_notification_objects]) {
2204                                         mali_session_send_notification(session, notobjs[used_notification_objects]);
2205                                         notobjs[used_notification_objects] = NULL; /* Don't track this notification object any more */
2206                                 }
2207                                 used_notification_objects++;
2208                         }
2209                         done = MALI_TRUE;
2210                 }
2211
2212                 mali_session_unlock();
2213
2214                 /* Delete any remaining/unused notification objects */
2215                 for (; used_notification_objects < num_sessions_alloc; used_notification_objects++) {
2216                         if (NULL != notobjs[used_notification_objects]) {
2217                                 _mali_osk_notification_delete(notobjs[used_notification_objects]);
2218                         }
2219                 }
2220
2221                 _mali_osk_free(notobjs);
2222         }
2223 }
2224
2225 static mali_bool mali_executor_core_scaling_is_done(void *data)
2226 {
2227         u32 i;
2228         u32 num_groups;
2229         mali_bool ret = MALI_TRUE;
2230
2231         MALI_IGNORE(data);
2232
2233         mali_executor_lock();
2234
2235         num_groups = mali_group_get_glob_num_groups();
2236
2237         for (i = 0; i < num_groups; i++) {
2238                 struct mali_group *group = mali_group_get_glob_group(i);
2239
2240                 if (NULL != group) {
2241                         if (MALI_TRUE == group->disable_requested && NULL != mali_group_get_pp_core(group)) {
2242                                 ret = MALI_FALSE;
2243                                 break;
2244                         }
2245                 }
2246         }
2247         mali_executor_unlock();
2248
2249         return ret;
2250 }
2251
2252 static void mali_executor_wq_notify_core_change(void *arg)
2253 {
2254         MALI_IGNORE(arg);
2255
2256         if (mali_is_mali450()) {
2257                 return;
2258         }
2259
2260         _mali_osk_wait_queue_wait_event(executor_notify_core_change_wait_queue,
2261                                         mali_executor_core_scaling_is_done, NULL);
2262
2263         mali_executor_notify_core_change(num_physical_pp_cores_enabled);
2264 }
2265
2266 /**
2267  * Clear all disable request from the _last_ core scaling behavior.
2268  */
2269 static void mali_executor_core_scaling_reset(void)
2270 {
2271         u32 i;
2272         u32 num_groups;
2273
2274         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
2275
2276         num_groups = mali_group_get_glob_num_groups();
2277
2278         for (i = 0; i < num_groups; i++) {
2279                 struct mali_group *group = mali_group_get_glob_group(i);
2280
2281                 if (NULL != group) {
2282                         group->disable_requested = MALI_FALSE;
2283                 }
2284         }
2285
2286         for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) {
2287                 core_scaling_delay_up_mask[i] = 0;
2288         }
2289 }
2290
2291 static void mali_executor_core_scale(unsigned int target_core_nr)
2292 {
2293         int current_core_scaling_mask[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 };
2294         int target_core_scaling_mask[MALI_MAX_NUMBER_OF_DOMAINS] = { 0 };
2295         mali_bool update_global_core_scaling_mask = MALI_FALSE;
2296         int i;
2297
2298         MALI_DEBUG_ASSERT(0 < target_core_nr);
2299         MALI_DEBUG_ASSERT(num_physical_pp_cores_total >= target_core_nr);
2300
2301         mali_executor_lock();
2302
2303         if (target_core_nr < num_physical_pp_cores_enabled) {
2304                 MALI_DEBUG_PRINT(2, ("Requesting %d cores: disabling %d cores\n", target_core_nr, num_physical_pp_cores_enabled - target_core_nr));
2305         } else {
2306                 MALI_DEBUG_PRINT(2, ("Requesting %d cores: enabling %d cores\n", target_core_nr, target_core_nr - num_physical_pp_cores_enabled));
2307         }
2308
2309         /* When a new core scaling request is comming,  we should remove the un-doing
2310          * part of the last core scaling request.  It's safe because we have only one
2311          * lock(executor lock) protection. */
2312         mali_executor_core_scaling_reset();
2313
2314         mali_pm_get_best_power_cost_mask(num_physical_pp_cores_enabled, current_core_scaling_mask);
2315         mali_pm_get_best_power_cost_mask(target_core_nr, target_core_scaling_mask);
2316
2317         for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) {
2318                 target_core_scaling_mask[i] = target_core_scaling_mask[i] - current_core_scaling_mask[i];
2319                 MALI_DEBUG_PRINT(5, ("target_core_scaling_mask[%d] = %d\n", i, target_core_scaling_mask[i]));
2320         }
2321
2322         for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) {
2323                 if (0 > target_core_scaling_mask[i]) {
2324                         struct mali_pm_domain *domain;
2325
2326                         domain = mali_pm_domain_get_from_index(i);
2327
2328                         /* Domain is valid and has pp cores */
2329                         if ((NULL != domain) && !(_mali_osk_list_empty(&domain->group_list))) {
2330                                 struct mali_group *group;
2331                                 struct mali_group *temp;
2332
2333                                 _MALI_OSK_LIST_FOREACHENTRY(group, temp, &domain->group_list, struct mali_group, pm_domain_list) {
2334                                         if (NULL != mali_group_get_pp_core(group) && (!mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED))
2335                                             && (!mali_group_is_virtual(group))) {
2336                                                 mali_executor_group_disable_internal(group);
2337                                                 target_core_scaling_mask[i]++;
2338                                                 if ((0 == target_core_scaling_mask[i])) {
2339                                                         break;
2340                                                 }
2341
2342                                         }
2343                                 }
2344                         }
2345                 }
2346         }
2347
2348         for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) {
2349                 /**
2350                  * Target_core_scaling_mask[i] is bigger than 0,
2351                  * means we need to enable some pp cores in
2352                  * this domain whose domain index is i.
2353                  */
2354                 if (0 < target_core_scaling_mask[i]) {
2355                         struct mali_pm_domain *domain;
2356
2357                         if (num_physical_pp_cores_enabled >= target_core_nr) {
2358                                 update_global_core_scaling_mask = MALI_TRUE;
2359                                 break;
2360                         }
2361
2362                         domain = mali_pm_domain_get_from_index(i);
2363
2364                         /* Domain is valid and has pp cores */
2365                         if ((NULL != domain) && !(_mali_osk_list_empty(&domain->group_list))) {
2366                                 struct mali_group *group;
2367                                 struct mali_group *temp;
2368
2369                                 _MALI_OSK_LIST_FOREACHENTRY(group, temp, &domain->group_list, struct mali_group, pm_domain_list) {
2370                                         if (NULL != mali_group_get_pp_core(group) && mali_executor_group_is_in_state(group, EXEC_STATE_DISABLED)
2371                                             && (!mali_group_is_virtual(group))) {
2372                                                 mali_executor_group_enable_internal(group);
2373                                                 target_core_scaling_mask[i]--;
2374
2375                                                 if ((0 == target_core_scaling_mask[i]) || num_physical_pp_cores_enabled == target_core_nr) {
2376                                                         break;
2377                                                 }
2378                                         }
2379                                 }
2380                         }
2381                 }
2382         }
2383
2384         /**
2385          * Here, we may still have some pp cores not been enabled because of some
2386          * pp cores need to be disabled are still in working state.
2387          */
2388         if (update_global_core_scaling_mask) {
2389                 for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) {
2390                         if (0 < target_core_scaling_mask[i]) {
2391                                 core_scaling_delay_up_mask[i] = target_core_scaling_mask[i];
2392                         }
2393                 }
2394         }
2395
2396         mali_executor_schedule();
2397         mali_executor_unlock();
2398 }
2399
2400 static void mali_executor_core_scale_in_group_complete(struct mali_group *group)
2401 {
2402         int num_pp_cores_disabled = 0;
2403         int num_pp_cores_to_enable = 0;
2404         int i;
2405
2406         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
2407         MALI_DEBUG_ASSERT(MALI_TRUE == mali_group_disable_requested(group));
2408
2409         /* Disable child group of virtual group */
2410         if (mali_group_is_virtual(group)) {
2411                 struct mali_group *child;
2412                 struct mali_group *temp;
2413
2414                 _MALI_OSK_LIST_FOREACHENTRY(child, temp, &group->group_list, struct mali_group, group_list) {
2415                         if (MALI_TRUE == mali_group_disable_requested(child)) {
2416                                 mali_group_set_disable_request(child, MALI_FALSE);
2417                                 mali_executor_group_disable_internal(child);
2418                                 num_pp_cores_disabled++;
2419                         }
2420                 }
2421                 mali_group_set_disable_request(group, MALI_FALSE);
2422         } else {
2423                 mali_executor_group_disable_internal(group);
2424                 mali_group_set_disable_request(group, MALI_FALSE);
2425                 if (NULL != mali_group_get_pp_core(group)) {
2426                         num_pp_cores_disabled++;
2427                 }
2428         }
2429
2430         num_pp_cores_to_enable = num_pp_cores_disabled;
2431
2432         for (i = 0; i < MALI_MAX_NUMBER_OF_DOMAINS; i++) {
2433                 if (0 < core_scaling_delay_up_mask[i]) {
2434                         struct mali_pm_domain *domain;
2435
2436                         if (0 == num_pp_cores_to_enable) {
2437                                 break;
2438                         }
2439
2440                         domain = mali_pm_domain_get_from_index(i);
2441
2442                         /* Domain is valid and has pp cores */
2443                         if ((NULL != domain) && !(_mali_osk_list_empty(&domain->group_list))) {
2444                                 struct mali_group *disabled_group;
2445                                 struct mali_group *temp;
2446
2447                                 _MALI_OSK_LIST_FOREACHENTRY(disabled_group, temp, &domain->group_list, struct mali_group, pm_domain_list) {
2448                                         if (NULL != mali_group_get_pp_core(disabled_group) && mali_executor_group_is_in_state(disabled_group, EXEC_STATE_DISABLED)) {
2449                                                 mali_executor_group_enable_internal(disabled_group);
2450                                                 core_scaling_delay_up_mask[i]--;
2451                                                 num_pp_cores_to_enable--;
2452
2453                                                 if ((0 == core_scaling_delay_up_mask[i]) || 0 == num_pp_cores_to_enable) {
2454                                                         break;
2455                                                 }
2456                                         }
2457                                 }
2458                         }
2459                 }
2460         }
2461
2462         _mali_osk_wait_queue_wake_up(executor_notify_core_change_wait_queue);
2463 }
2464
2465 static void mali_executor_change_group_status_disabled(struct mali_group *group)
2466 {
2467         /* Physical PP group */
2468         mali_bool idle;
2469
2470         MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
2471
2472         idle = mali_executor_group_is_in_state(group, EXEC_STATE_IDLE);
2473         if (MALI_TRUE == idle) {
2474                 mali_executor_change_state_pp_physical(group,
2475                                                        &group_list_idle,
2476                                                        &group_list_idle_count,
2477                                                        &group_list_disabled,
2478                                                        &group_list_disabled_count);
2479         } else {
2480                 mali_executor_change_state_pp_physical(group,
2481                                                        &group_list_inactive,
2482                                                        &group_list_inactive_count,
2483                                                        &group_list_disabled,
2484                                                        &group_list_disabled_count);
2485         }
2486 }
2487
2488 static mali_bool mali_executor_deactivate_list_idle(mali_bool deactivate_idle_group)
2489 {
2490         mali_bool trigger_pm_update = MALI_FALSE;
2491
2492         if (group_list_idle_count > 0) {
2493                 if (mali_executor_has_virtual_group()) {
2494
2495                         /* Rejoin virtual group on Mali-450 */
2496
2497                         struct mali_group *group;
2498                         struct mali_group *temp;
2499
2500                         _MALI_OSK_LIST_FOREACHENTRY(group, temp,
2501                                                     &group_list_idle,
2502                                                     struct mali_group, executor_list) {
2503                                 if (mali_executor_physical_rejoin_virtual(
2504                                             group)) {
2505                                         trigger_pm_update = MALI_TRUE;
2506                                 }
2507                         }
2508                 } else if (deactivate_idle_group) {
2509                         struct mali_group *group;
2510                         struct mali_group *temp;
2511
2512                         /* Deactivate group on Mali-300/400 */
2513
2514                         _MALI_OSK_LIST_FOREACHENTRY(group, temp,
2515                                                     &group_list_idle,
2516                                                     struct mali_group, executor_list) {
2517                                 if (mali_group_deactivate(group)) {
2518                                         trigger_pm_update = MALI_TRUE;
2519                                 }
2520
2521                                 /* Move from idle to inactive */
2522                                 mali_executor_change_state_pp_physical(group,
2523                                                                        &group_list_idle,
2524                                                                        &group_list_idle_count,
2525                                                                        &group_list_inactive,
2526                                                                        &group_list_inactive_count);
2527                         }
2528                 }
2529         }
2530
2531         return trigger_pm_update;
2532 }