drivers/gpu/arm/mali400/mali/common/mali_scheduler.c

   1 /*
   2  * Copyright (C) 2012-2016 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_scheduler.h"
  12 #include "mali_kernel_common.h"
  13 #include "mali_osk.h"
  14 #include "mali_osk_profiling.h"
  15 #include "mali_kernel_utilization.h"
  16 #include "mali_timeline.h"
  17 #include "mali_gp_job.h"
  18 #include "mali_pp_job.h"
  19 #include "mali_executor.h"
  20 #include "mali_group.h"
  21 #include <linux/wait.h>
  22 #include <linux/sched.h>
  23 #include "mali_pm_metrics.h"
  24
  25 #if defined(CONFIG_DMA_SHARED_BUFFER)
  26 #include "mali_memory_dma_buf.h"
  27 #endif
  28
  29 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
  30 #include <linux/sched.h>
  31 #include <trace/events/gpu.h>
  32 #endif
  33 /*
  34  * ---------- static defines/constants ----------
  35  */
  36
  37 /*
  38  * If dma_buf with map on demand is used, we defer job queue
  39  * if in atomic context, since both might sleep.
  40  */
  41 #if defined(CONFIG_DMA_SHARED_BUFFER)
  42 #if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
  43 #define MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE 1
  44 #endif
  45 #endif
  46
  47
  48 /*
  49  * ---------- global variables (exported due to inline functions) ----------
  50  */
  51
  52 /* Lock protecting this module */
  53 _mali_osk_spinlock_irq_t *mali_scheduler_lock_obj = NULL;
  54
  55 /* Queue of jobs to be executed on the GP group */
  56 struct mali_scheduler_job_queue job_queue_gp;
  57
  58 /* Queue of PP jobs */
  59 struct mali_scheduler_job_queue job_queue_pp;
  60
  61 _mali_osk_atomic_t mali_job_id_autonumber;
  62 _mali_osk_atomic_t mali_job_cache_order_autonumber;
  63 /*
  64  * ---------- static variables ----------
  65  */
  66
  67 _mali_osk_wq_work_t *scheduler_wq_pp_job_delete = NULL;
  68 _mali_osk_spinlock_irq_t *scheduler_pp_job_delete_lock = NULL;
  69 static _MALI_OSK_LIST_HEAD_STATIC_INIT(scheduler_pp_job_deletion_queue);
  70
  71 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
  72 static _mali_osk_wq_work_t *scheduler_wq_pp_job_queue = NULL;
  73 static _mali_osk_spinlock_irq_t *scheduler_pp_job_queue_lock = NULL;
  74 static _MALI_OSK_LIST_HEAD_STATIC_INIT(scheduler_pp_job_queue_list);
  75 #endif
  76
  77 /*
  78  * ---------- Forward declaration of static functions ----------
  79  */
  80
  81 static mali_timeline_point mali_scheduler_submit_gp_job(
  82         struct mali_session_data *session, struct mali_gp_job *job);
  83 static mali_timeline_point mali_scheduler_submit_pp_job(
  84         struct mali_session_data *session, struct mali_pp_job *job);
  85
  86 static mali_bool mali_scheduler_queue_gp_job(struct mali_gp_job *job);
  87 static mali_bool mali_scheduler_queue_pp_job(struct mali_pp_job *job);
  88
  89 static void mali_scheduler_return_gp_job_to_user(struct mali_gp_job *job,
  90                 mali_bool success);
  91
  92 static void mali_scheduler_deferred_pp_job_delete(struct mali_pp_job *job);
  93 void mali_scheduler_do_pp_job_delete(void *arg);
  94
  95 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
  96 static void mali_scheduler_deferred_pp_job_queue(struct mali_pp_job *job);
  97 static void mali_scheduler_do_pp_job_queue(void *arg);
  98 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
  99
 100 /*
 101  * ---------- Actual implementation ----------
 102  */
 103
 104 _mali_osk_errcode_t mali_scheduler_initialize(void)
 105 {
 106         _mali_osk_atomic_init(&mali_job_id_autonumber, 0);
 107         _mali_osk_atomic_init(&mali_job_cache_order_autonumber, 0);
 108
 109         _MALI_OSK_INIT_LIST_HEAD(&job_queue_gp.normal_pri);
 110         _MALI_OSK_INIT_LIST_HEAD(&job_queue_gp.high_pri);
 111         job_queue_gp.depth = 0;
 112         job_queue_gp.big_job_num = 0;
 113
 114         _MALI_OSK_INIT_LIST_HEAD(&job_queue_pp.normal_pri);
 115         _MALI_OSK_INIT_LIST_HEAD(&job_queue_pp.high_pri);
 116         job_queue_pp.depth = 0;
 117         job_queue_pp.big_job_num = 0;
 118
 119         mali_scheduler_lock_obj = _mali_osk_spinlock_irq_init(
 120                                           _MALI_OSK_LOCKFLAG_ORDERED,
 121                                           _MALI_OSK_LOCK_ORDER_SCHEDULER);
 122         if (NULL == mali_scheduler_lock_obj) {
 123                 mali_scheduler_terminate();
 124         }
 125
 126         scheduler_wq_pp_job_delete = _mali_osk_wq_create_work(
 127                                              mali_scheduler_do_pp_job_delete, NULL);
 128         if (NULL == scheduler_wq_pp_job_delete) {
 129                 mali_scheduler_terminate();
 130                 return _MALI_OSK_ERR_FAULT;
 131         }
 132
 133         scheduler_pp_job_delete_lock = _mali_osk_spinlock_irq_init(
 134                                                _MALI_OSK_LOCKFLAG_ORDERED,
 135                                                _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED);
 136         if (NULL == scheduler_pp_job_delete_lock) {
 137                 mali_scheduler_terminate();
 138                 return _MALI_OSK_ERR_FAULT;
 139         }
 140
 141 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
 142         scheduler_wq_pp_job_queue = _mali_osk_wq_create_work(
 143                                             mali_scheduler_do_pp_job_queue, NULL);
 144         if (NULL == scheduler_wq_pp_job_queue) {
 145                 mali_scheduler_terminate();
 146                 return _MALI_OSK_ERR_FAULT;
 147         }
 148
 149         scheduler_pp_job_queue_lock = _mali_osk_spinlock_irq_init(
 150                                               _MALI_OSK_LOCKFLAG_ORDERED,
 151                                               _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED);
 152         if (NULL == scheduler_pp_job_queue_lock) {
 153                 mali_scheduler_terminate();
 154                 return _MALI_OSK_ERR_FAULT;
 155         }
 156 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
 157
 158         return _MALI_OSK_ERR_OK;
 159 }
 160
 161 void mali_scheduler_terminate(void)
 162 {
 163 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
 164         if (NULL != scheduler_pp_job_queue_lock) {
 165                 _mali_osk_spinlock_irq_term(scheduler_pp_job_queue_lock);
 166                 scheduler_pp_job_queue_lock = NULL;
 167         }
 168
 169         if (NULL != scheduler_wq_pp_job_queue) {
 170                 _mali_osk_wq_delete_work(scheduler_wq_pp_job_queue);
 171                 scheduler_wq_pp_job_queue = NULL;
 172         }
 173 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
 174
 175         if (NULL != scheduler_pp_job_delete_lock) {
 176                 _mali_osk_spinlock_irq_term(scheduler_pp_job_delete_lock);
 177                 scheduler_pp_job_delete_lock = NULL;
 178         }
 179
 180         if (NULL != scheduler_wq_pp_job_delete) {
 181                 _mali_osk_wq_delete_work(scheduler_wq_pp_job_delete);
 182                 scheduler_wq_pp_job_delete = NULL;
 183         }
 184
 185         if (NULL != mali_scheduler_lock_obj) {
 186                 _mali_osk_spinlock_irq_term(mali_scheduler_lock_obj);
 187                 mali_scheduler_lock_obj = NULL;
 188         }
 189
 190         _mali_osk_atomic_term(&mali_job_cache_order_autonumber);
 191         _mali_osk_atomic_term(&mali_job_id_autonumber);
 192 }
 193
 194 u32 mali_scheduler_job_physical_head_count(mali_bool gpu_mode_is_secure)
 195 {
 196         /*
 197          * Count how many physical sub jobs are present from the head of queue
 198          * until the first virtual job is present.
 199          * Early out when we have reached maximum number of PP cores (8)
 200          */
 201         u32 count = 0;
 202         struct mali_pp_job *job;
 203         struct mali_pp_job *temp;
 204
 205         /* Check for partially started normal pri jobs */
 206         if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
 207                 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
 208
 209                 job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next,
 210                                            struct mali_pp_job, list);
 211
 212                 MALI_DEBUG_ASSERT_POINTER(job);
 213
 214                 if (MALI_TRUE == mali_pp_job_has_started_sub_jobs(job)) {
 215                         /*
 216                          * Remember; virtual jobs can't be queued and started
 217                          * at the same time, so this must be a physical job
 218                          */
 219                         if ((MALI_FALSE  == gpu_mode_is_secure && MALI_FALSE == mali_pp_job_is_protected_job(job))
 220                             || (MALI_TRUE  == gpu_mode_is_secure && MALI_TRUE == mali_pp_job_is_protected_job(job))) {
 221
 222                                 count += mali_pp_job_unstarted_sub_job_count(job);
 223                                 if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) {
 224                                         return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS;
 225                                 }
 226                         }
 227                 }
 228         }
 229
 230         _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.high_pri,
 231                                     struct mali_pp_job, list) {
 232                 if ((MALI_FALSE == mali_pp_job_is_virtual(job))
 233                     && ((MALI_FALSE  == gpu_mode_is_secure && MALI_FALSE == mali_pp_job_is_protected_job(job))
 234                         || (MALI_TRUE  == gpu_mode_is_secure && MALI_TRUE == mali_pp_job_is_protected_job(job)))) {
 235
 236                         count += mali_pp_job_unstarted_sub_job_count(job);
 237                         if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) {
 238                                 return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS;
 239                         }
 240                 } else {
 241                         /* Came across a virtual job, so stop counting */
 242                         return count;
 243                 }
 244         }
 245
 246         _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.normal_pri,
 247                                     struct mali_pp_job, list) {
 248                 if ((MALI_FALSE == mali_pp_job_is_virtual(job))
 249                     && (MALI_FALSE == mali_pp_job_has_started_sub_jobs(job))
 250                     && ((MALI_FALSE  == gpu_mode_is_secure && MALI_FALSE == mali_pp_job_is_protected_job(job))
 251                         || (MALI_TRUE  == gpu_mode_is_secure && MALI_TRUE == mali_pp_job_is_protected_job(job)))) {
 252
 253                         count += mali_pp_job_unstarted_sub_job_count(job);
 254                         if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) {
 255                                 return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS;
 256                         }
 257                 } else {
 258                         /* Came across a virtual job, so stop counting */
 259                         return count;
 260                 }
 261         }
 262         return count;
 263 }
 264
 265 struct mali_pp_job *mali_scheduler_job_pp_next(void)
 266 {
 267         struct mali_pp_job *job;
 268         struct mali_pp_job *temp;
 269
 270         MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
 271
 272         /* Check for partially started normal pri jobs */
 273         if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
 274                 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
 275
 276                 job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next,
 277                                            struct mali_pp_job, list);
 278
 279                 MALI_DEBUG_ASSERT_POINTER(job);
 280
 281                 if (MALI_TRUE == mali_pp_job_has_started_sub_jobs(job)) {
 282                         return job;
 283                 }
 284         }
 285
 286         _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.high_pri,
 287                                     struct mali_pp_job, list) {
 288                 return job;
 289         }
 290
 291         _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.normal_pri,
 292                                     struct mali_pp_job, list) {
 293                 return job;
 294         }
 295
 296         return NULL;
 297 }
 298
 299 mali_bool mali_scheduler_job_next_is_virtual(void)
 300 {
 301         struct mali_pp_job *job;
 302
 303         job = mali_scheduler_job_pp_virtual_peek();
 304         if (NULL != job) {
 305                 MALI_DEBUG_ASSERT(mali_pp_job_is_virtual(job));
 306
 307                 return MALI_TRUE;
 308         }
 309
 310         return MALI_FALSE;
 311 }
 312
 313 struct mali_gp_job *mali_scheduler_job_gp_get(void)
 314 {
 315         _mali_osk_list_t *queue;
 316         struct mali_gp_job *job = NULL;
 317
 318         MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
 319         MALI_DEBUG_ASSERT(0 < job_queue_gp.depth);
 320         MALI_DEBUG_ASSERT(job_queue_gp.big_job_num <= job_queue_gp.depth);
 321
 322         if (!_mali_osk_list_empty(&job_queue_gp.high_pri)) {
 323                 queue = &job_queue_gp.high_pri;
 324         } else {
 325                 queue = &job_queue_gp.normal_pri;
 326                 MALI_DEBUG_ASSERT(!_mali_osk_list_empty(queue));
 327         }
 328
 329         job = _MALI_OSK_LIST_ENTRY(queue->next, struct mali_gp_job, list);
 330
 331         MALI_DEBUG_ASSERT_POINTER(job);
 332
 333         mali_gp_job_list_remove(job);
 334         job_queue_gp.depth--;
 335         if (job->big_job) {
 336                 job_queue_gp.big_job_num --;
 337                 if (job_queue_gp.big_job_num < MALI_MAX_PENDING_BIG_JOB) {
 338                         /* wake up process */
 339                         wait_queue_head_t *queue = mali_session_get_wait_queue();
 340                         wake_up(queue);
 341                 }
 342         }
 343         return job;
 344 }
 345
 346 struct mali_pp_job *mali_scheduler_job_pp_physical_peek(void)
 347 {
 348         struct mali_pp_job *job = NULL;
 349         struct mali_pp_job *tmp_job = NULL;
 350
 351         MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
 352
 353         /*
 354          * For PP jobs we favour partially started jobs in normal
 355          * priority queue over unstarted jobs in high priority queue
 356          */
 357
 358         if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
 359                 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
 360
 361                 tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next,
 362                                                struct mali_pp_job, list);
 363                 MALI_DEBUG_ASSERT(NULL != tmp_job);
 364
 365                 if (MALI_FALSE == mali_pp_job_is_virtual(tmp_job)) {
 366                         job = tmp_job;
 367                 }
 368         }
 369
 370         if (NULL == job ||
 371             MALI_FALSE == mali_pp_job_has_started_sub_jobs(job)) {
 372                 /*
 373                  * There isn't a partially started job in normal queue, so
 374                  * look in high priority queue.
 375                  */
 376                 if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) {
 377                         MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
 378
 379                         tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.high_pri.next,
 380                                                        struct mali_pp_job, list);
 381                         MALI_DEBUG_ASSERT(NULL != tmp_job);
 382
 383                         if (MALI_FALSE == mali_pp_job_is_virtual(tmp_job)) {
 384                                 job = tmp_job;
 385                         }
 386                 }
 387         }
 388
 389         return job;
 390 }
 391
 392 struct mali_pp_job *mali_scheduler_job_pp_virtual_peek(void)
 393 {
 394         struct mali_pp_job *job = NULL;
 395         struct mali_pp_job *tmp_job = NULL;
 396
 397         MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
 398
 399         if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) {
 400                 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
 401
 402                 tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.high_pri.next,
 403                                                struct mali_pp_job, list);
 404
 405                 if (MALI_TRUE == mali_pp_job_is_virtual(tmp_job)) {
 406                         job = tmp_job;
 407                 }
 408         }
 409
 410         if (NULL == job) {
 411                 if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
 412                         MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
 413
 414                         tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next,
 415                                                        struct mali_pp_job, list);
 416
 417                         if (MALI_TRUE == mali_pp_job_is_virtual(tmp_job)) {
 418                                 job = tmp_job;
 419                         }
 420                 }
 421         }
 422
 423         return job;
 424 }
 425
 426 struct mali_pp_job *mali_scheduler_job_pp_physical_get(u32 *sub_job)
 427 {
 428         struct mali_pp_job *job = mali_scheduler_job_pp_physical_peek();
 429
 430         MALI_DEBUG_ASSERT(MALI_FALSE == mali_pp_job_is_virtual(job));
 431
 432         if (NULL != job) {
 433                 *sub_job = mali_pp_job_get_first_unstarted_sub_job(job);
 434
 435                 mali_pp_job_mark_sub_job_started(job, *sub_job);
 436                 if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(job)) {
 437                         /* Remove from queue when last sub job has been retrieved */
 438                         mali_pp_job_list_remove(job);
 439                 }
 440
 441                 job_queue_pp.depth--;
 442
 443                 /*
 444                  * Job about to start so it is no longer be
 445                  * possible to discard WB
 446                  */
 447                 mali_pp_job_fb_lookup_remove(job);
 448         }
 449
 450         return job;
 451 }
 452
 453 struct mali_pp_job *mali_scheduler_job_pp_virtual_get(void)
 454 {
 455         struct mali_pp_job *job = mali_scheduler_job_pp_virtual_peek();
 456
 457         MALI_DEBUG_ASSERT(MALI_TRUE == mali_pp_job_is_virtual(job));
 458
 459         if (NULL != job) {
 460                 MALI_DEBUG_ASSERT(0 ==
 461                                   mali_pp_job_get_first_unstarted_sub_job(job));
 462                 MALI_DEBUG_ASSERT(1 ==
 463                                   mali_pp_job_get_sub_job_count(job));
 464
 465                 mali_pp_job_mark_sub_job_started(job, 0);
 466
 467                 mali_pp_job_list_remove(job);
 468
 469                 job_queue_pp.depth--;
 470
 471                 /*
 472                  * Job about to start so it is no longer be
 473                  * possible to discard WB
 474                  */
 475                 mali_pp_job_fb_lookup_remove(job);
 476         }
 477
 478         return job;
 479 }
 480
 481 mali_scheduler_mask mali_scheduler_activate_gp_job(struct mali_gp_job *job)
 482 {
 483         MALI_DEBUG_ASSERT_POINTER(job);
 484
 485         MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Timeline activation for job %u (0x%08X).\n",
 486                              mali_gp_job_get_id(job), job));
 487
 488         mali_scheduler_lock();
 489
 490         if (!mali_scheduler_queue_gp_job(job)) {
 491                 /* Failed to enqueue job, release job (with error) */
 492
 493                 mali_scheduler_unlock();
 494
 495                 mali_timeline_tracker_release(mali_gp_job_get_tracker(job));
 496                 mali_gp_job_signal_pp_tracker(job, MALI_FALSE);
 497
 498                 /* This will notify user space and close the job object */
 499                 mali_scheduler_complete_gp_job(job, MALI_FALSE,
 500                                                MALI_TRUE, MALI_FALSE);
 501
 502                 return MALI_SCHEDULER_MASK_EMPTY;
 503         }
 504
 505         mali_scheduler_unlock();
 506
 507         return MALI_SCHEDULER_MASK_GP;
 508 }
 509
 510 mali_scheduler_mask mali_scheduler_activate_pp_job(struct mali_pp_job *job)
 511 {
 512         MALI_DEBUG_ASSERT_POINTER(job);
 513
 514         MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Timeline activation for job %u (0x%08X).\n",
 515                              mali_pp_job_get_id(job), job));
 516
 517         if (MALI_TRUE == mali_timeline_tracker_activation_error(
 518                     mali_pp_job_get_tracker(job))) {
 519                 MALI_DEBUG_PRINT(3, ("Mali PP scheduler: Job %u (0x%08X) activated with error, aborting.\n",
 520                                      mali_pp_job_get_id(job), job));
 521
 522                 mali_scheduler_lock();
 523                 mali_pp_job_fb_lookup_remove(job);
 524                 mali_pp_job_mark_unstarted_failed(job);
 525                 mali_scheduler_unlock();
 526
 527                 mali_timeline_tracker_release(mali_pp_job_get_tracker(job));
 528
 529                 /* This will notify user space and close the job object */
 530                 mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, MALI_FALSE);
 531
 532                 return MALI_SCHEDULER_MASK_EMPTY;
 533         }
 534
 535 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
 536         if (mali_pp_job_needs_dma_buf_mapping(job)) {
 537                 mali_scheduler_deferred_pp_job_queue(job);
 538                 return MALI_SCHEDULER_MASK_EMPTY;
 539         }
 540 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
 541
 542         mali_scheduler_lock();
 543
 544         if (!mali_scheduler_queue_pp_job(job)) {
 545                 /* Failed to enqueue job, release job (with error) */
 546                 mali_pp_job_fb_lookup_remove(job);
 547                 mali_pp_job_mark_unstarted_failed(job);
 548                 mali_scheduler_unlock();
 549
 550                 mali_timeline_tracker_release(mali_pp_job_get_tracker(job));
 551
 552                 /* This will notify user space and close the job object */
 553                 mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, MALI_FALSE);
 554
 555                 return MALI_SCHEDULER_MASK_EMPTY;
 556         }
 557
 558         mali_scheduler_unlock();
 559         return MALI_SCHEDULER_MASK_PP;
 560 }
 561
 562 void mali_scheduler_complete_gp_job(struct mali_gp_job *job,
 563                                     mali_bool success,
 564                                     mali_bool user_notification,
 565                                     mali_bool dequeued)
 566 {
 567         if (user_notification) {
 568                 mali_scheduler_return_gp_job_to_user(job, success);
 569         }
 570
 571         if (dequeued) {
 572                 _mali_osk_pm_dev_ref_put();
 573
 574                 if (mali_utilization_enabled()) {
 575                         mali_utilization_gp_end();
 576                 }
 577                 mali_pm_record_gpu_idle(MALI_TRUE);
 578         }
 579
 580         mali_gp_job_delete(job);
 581 }
 582
 583 void mali_scheduler_complete_pp_job(struct mali_pp_job *job,
 584                                     u32 num_cores_in_virtual,
 585                                     mali_bool user_notification,
 586                                     mali_bool dequeued)
 587 {
 588         job->user_notification = user_notification;
 589         job->num_pp_cores_in_virtual = num_cores_in_virtual;
 590
 591         if (dequeued) {
 592 #if defined(CONFIG_MALI_DVFS)
 593                 if (mali_pp_job_is_window_surface(job)) {
 594                         struct mali_session_data *session;
 595                         session = mali_pp_job_get_session(job);
 596                         mali_session_inc_num_window_jobs(session);
 597                 }
 598 #endif
 599
 600                 _mali_osk_pm_dev_ref_put();
 601
 602                 if (mali_utilization_enabled()) {
 603                         mali_utilization_pp_end();
 604                 }
 605                 mali_pm_record_gpu_idle(MALI_FALSE);
 606         }
 607
 608         /* With ZRAM feature enabled, all pp jobs will be force to use deferred delete. */
 609         mali_scheduler_deferred_pp_job_delete(job);
 610 }
 611
 612 void mali_scheduler_abort_session(struct mali_session_data *session)
 613 {
 614         struct mali_gp_job *gp_job;
 615         struct mali_gp_job *gp_tmp;
 616         struct mali_pp_job *pp_job;
 617         struct mali_pp_job *pp_tmp;
 618         _MALI_OSK_LIST_HEAD_STATIC_INIT(removed_jobs_gp);
 619         _MALI_OSK_LIST_HEAD_STATIC_INIT(removed_jobs_pp);
 620
 621         MALI_DEBUG_ASSERT_POINTER(session);
 622         MALI_DEBUG_ASSERT(session->is_aborting);
 623
 624         MALI_DEBUG_PRINT(3, ("Mali scheduler: Aborting all queued jobs from session 0x%08X.\n",
 625                              session));
 626
 627         mali_scheduler_lock();
 628
 629         /* Remove from GP normal priority queue */
 630         _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &job_queue_gp.normal_pri,
 631                                     struct mali_gp_job, list) {
 632                 if (mali_gp_job_get_session(gp_job) == session) {
 633                         mali_gp_job_list_move(gp_job, &removed_jobs_gp);
 634                         job_queue_gp.depth--;
 635                         job_queue_gp.big_job_num -= gp_job->big_job ? 1 : 0;
 636                 }
 637         }
 638
 639         /* Remove from GP high priority queue */
 640         _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &job_queue_gp.high_pri,
 641                                     struct mali_gp_job, list) {
 642                 if (mali_gp_job_get_session(gp_job) == session) {
 643                         mali_gp_job_list_move(gp_job, &removed_jobs_gp);
 644                         job_queue_gp.depth--;
 645                         job_queue_gp.big_job_num -= gp_job->big_job ? 1 : 0;
 646                 }
 647         }
 648
 649         /* Remove from PP normal priority queue */
 650         _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp,
 651                                     &job_queue_pp.normal_pri,
 652                                     struct mali_pp_job, list) {
 653                 if (mali_pp_job_get_session(pp_job) == session) {
 654                         mali_pp_job_fb_lookup_remove(pp_job);
 655
 656                         job_queue_pp.depth -=
 657                                 mali_pp_job_unstarted_sub_job_count(
 658                                         pp_job);
 659                         mali_pp_job_mark_unstarted_failed(pp_job);
 660
 661                         if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(pp_job)) {
 662                                 if (mali_pp_job_is_complete(pp_job)) {
 663                                         mali_pp_job_list_move(pp_job,
 664                                                               &removed_jobs_pp);
 665                                 } else {
 666                                         mali_pp_job_list_remove(pp_job);
 667                                 }
 668                         }
 669                 }
 670         }
 671
 672         /* Remove from PP high priority queue */
 673         _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp,
 674                                     &job_queue_pp.high_pri,
 675                                     struct mali_pp_job, list) {
 676                 if (mali_pp_job_get_session(pp_job) == session) {
 677                         mali_pp_job_fb_lookup_remove(pp_job);
 678
 679                         job_queue_pp.depth -=
 680                                 mali_pp_job_unstarted_sub_job_count(
 681                                         pp_job);
 682                         mali_pp_job_mark_unstarted_failed(pp_job);
 683
 684                         if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(pp_job)) {
 685                                 if (mali_pp_job_is_complete(pp_job)) {
 686                                         mali_pp_job_list_move(pp_job,
 687                                                               &removed_jobs_pp);
 688                                 } else {
 689                                         mali_pp_job_list_remove(pp_job);
 690                                 }
 691                         }
 692                 }
 693         }
 694
 695         /*
 696          * Release scheduler lock so we can release trackers
 697          * (which will potentially queue new jobs)
 698          */
 699         mali_scheduler_unlock();
 700
 701         /* Release and complete all (non-running) found GP jobs  */
 702         _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &removed_jobs_gp,
 703                                     struct mali_gp_job, list) {
 704                 mali_timeline_tracker_release(mali_gp_job_get_tracker(gp_job));
 705                 mali_gp_job_signal_pp_tracker(gp_job, MALI_FALSE);
 706                 _mali_osk_list_delinit(&gp_job->list);
 707                 mali_scheduler_complete_gp_job(gp_job,
 708                                                MALI_FALSE, MALI_FALSE, MALI_TRUE);
 709         }
 710
 711         /* Release and complete non-running PP jobs */
 712         _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp, &removed_jobs_pp,
 713                                     struct mali_pp_job, list) {
 714                 mali_timeline_tracker_release(mali_pp_job_get_tracker(pp_job));
 715                 _mali_osk_list_delinit(&pp_job->list);
 716                 mali_scheduler_complete_pp_job(pp_job, 0,
 717                                                MALI_FALSE, MALI_TRUE);
 718         }
 719 }
 720
 721 _mali_osk_errcode_t _mali_ukk_gp_start_job(void *ctx,
 722                 _mali_uk_gp_start_job_s *uargs)
 723 {
 724         struct mali_session_data *session;
 725         struct mali_gp_job *job;
 726         mali_timeline_point point;
 727         u32 __user *point_ptr = NULL;
 728
 729         MALI_DEBUG_ASSERT_POINTER(uargs);
 730         MALI_DEBUG_ASSERT_POINTER(ctx);
 731
 732         session = (struct mali_session_data *)(uintptr_t)ctx;
 733
 734         job = mali_gp_job_create(session, uargs, mali_scheduler_get_new_id(),
 735                                  NULL);
 736         if (NULL == job) {
 737                 MALI_PRINT_ERROR(("Failed to create GP job.\n"));
 738                 return _MALI_OSK_ERR_NOMEM;
 739         }
 740
 741         point_ptr = (u32 __user *)(uintptr_t)mali_gp_job_get_timeline_point_ptr(job);
 742
 743         point = mali_scheduler_submit_gp_job(session, job);
 744
 745         if (0 != _mali_osk_put_user(((u32) point), point_ptr)) {
 746                 /*
 747                  * Let user space know that something failed
 748                  * after the job was started.
 749                  */
 750                 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
 751         }
 752
 753         return _MALI_OSK_ERR_OK;
 754 }
 755
 756 _mali_osk_errcode_t _mali_ukk_pp_start_job(void *ctx,
 757                 _mali_uk_pp_start_job_s *uargs)
 758 {
 759         struct mali_session_data *session;
 760         struct mali_pp_job *job;
 761         mali_timeline_point point;
 762         u32 __user *point_ptr = NULL;
 763
 764         MALI_DEBUG_ASSERT_POINTER(uargs);
 765         MALI_DEBUG_ASSERT_POINTER(ctx);
 766
 767         session = (struct mali_session_data *)(uintptr_t)ctx;
 768
 769         job = mali_pp_job_create(session, uargs, mali_scheduler_get_new_id());
 770         if (NULL == job) {
 771                 MALI_PRINT_ERROR(("Failed to create PP job.\n"));
 772                 return _MALI_OSK_ERR_NOMEM;
 773         }
 774
 775         point_ptr = (u32 __user *)(uintptr_t)mali_pp_job_get_timeline_point_ptr(job);
 776
 777         point = mali_scheduler_submit_pp_job(session, job);
 778         job = NULL;
 779
 780         if (0 != _mali_osk_put_user(((u32) point), point_ptr)) {
 781                 /*
 782                  * Let user space know that something failed
 783                  * after the job was started.
 784                  */
 785                 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
 786         }
 787
 788         return _MALI_OSK_ERR_OK;
 789 }
 790
 791 _mali_osk_errcode_t _mali_ukk_pp_and_gp_start_job(void *ctx,
 792                 _mali_uk_pp_and_gp_start_job_s *uargs)
 793 {
 794         struct mali_session_data *session;
 795         _mali_uk_pp_and_gp_start_job_s kargs;
 796         struct mali_pp_job *pp_job;
 797         struct mali_gp_job *gp_job;
 798         u32 __user *point_ptr = NULL;
 799         mali_timeline_point point;
 800         _mali_uk_pp_start_job_s __user *pp_args;
 801         _mali_uk_gp_start_job_s __user *gp_args;
 802
 803         MALI_DEBUG_ASSERT_POINTER(ctx);
 804         MALI_DEBUG_ASSERT_POINTER(uargs);
 805
 806         session = (struct mali_session_data *) ctx;
 807
 808         if (0 != _mali_osk_copy_from_user(&kargs, uargs,
 809                                           sizeof(_mali_uk_pp_and_gp_start_job_s))) {
 810                 return _MALI_OSK_ERR_NOMEM;
 811         }
 812
 813         pp_args = (_mali_uk_pp_start_job_s __user *)(uintptr_t)kargs.pp_args;
 814         gp_args = (_mali_uk_gp_start_job_s __user *)(uintptr_t)kargs.gp_args;
 815
 816         pp_job = mali_pp_job_create(session, pp_args,
 817                                     mali_scheduler_get_new_id());
 818         if (NULL == pp_job) {
 819                 MALI_PRINT_ERROR(("Failed to create PP job.\n"));
 820                 return _MALI_OSK_ERR_NOMEM;
 821         }
 822
 823         gp_job = mali_gp_job_create(session, gp_args,
 824                                     mali_scheduler_get_new_id(),
 825                                     mali_pp_job_get_tracker(pp_job));
 826         if (NULL == gp_job) {
 827                 MALI_PRINT_ERROR(("Failed to create GP job.\n"));
 828                 mali_pp_job_delete(pp_job);
 829                 return _MALI_OSK_ERR_NOMEM;
 830         }
 831
 832         point_ptr = (u32 __user *)(uintptr_t)mali_pp_job_get_timeline_point_ptr(pp_job);
 833
 834         /* Submit GP job. */
 835         mali_scheduler_submit_gp_job(session, gp_job);
 836         gp_job = NULL;
 837
 838         /* Submit PP job. */
 839         point = mali_scheduler_submit_pp_job(session, pp_job);
 840         pp_job = NULL;
 841
 842         if (0 != _mali_osk_put_user(((u32) point), point_ptr)) {
 843                 /*
 844                  * Let user space know that something failed
 845                  * after the jobs were started.
 846                  */
 847                 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
 848         }
 849
 850         return _MALI_OSK_ERR_OK;
 851 }
 852
 853 void _mali_ukk_pp_job_disable_wb(_mali_uk_pp_disable_wb_s *args)
 854 {
 855         struct mali_session_data *session;
 856         struct mali_pp_job *job;
 857         struct mali_pp_job *tmp;
 858         u32 fb_lookup_id;
 859
 860         MALI_DEBUG_ASSERT_POINTER(args);
 861         MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
 862
 863         session = (struct mali_session_data *)(uintptr_t)args->ctx;
 864
 865         fb_lookup_id = args->fb_id & MALI_PP_JOB_FB_LOOKUP_LIST_MASK;
 866
 867         mali_scheduler_lock();
 868
 869         /* Iterate over all jobs for given frame builder_id. */
 870         _MALI_OSK_LIST_FOREACHENTRY(job, tmp,
 871                                     &session->pp_job_fb_lookup_list[fb_lookup_id],
 872                                     struct mali_pp_job, session_fb_lookup_list) {
 873                 MALI_DEBUG_CODE(u32 disable_mask = 0);
 874
 875                 if (mali_pp_job_get_frame_builder_id(job) !=
 876                     (u32) args->fb_id) {
 877                         MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Disable WB mismatching FB.\n"));
 878                         continue;
 879                 }
 880
 881                 MALI_DEBUG_CODE(disable_mask |= 0xD << (4 * 3));
 882
 883                 if (mali_pp_job_get_wb0_source_addr(job) == args->wb0_memory) {
 884                         MALI_DEBUG_CODE(disable_mask |= 0x1 << (4 * 1));
 885                         mali_pp_job_disable_wb0(job);
 886                 }
 887
 888                 if (mali_pp_job_get_wb1_source_addr(job) == args->wb1_memory) {
 889                         MALI_DEBUG_CODE(disable_mask |= 0x2 << (4 * 2));
 890                         mali_pp_job_disable_wb1(job);
 891                 }
 892
 893                 if (mali_pp_job_get_wb2_source_addr(job) == args->wb2_memory) {
 894                         MALI_DEBUG_CODE(disable_mask |= 0x3 << (4 * 3));
 895                         mali_pp_job_disable_wb2(job);
 896                 }
 897                 MALI_DEBUG_PRINT(3, ("Mali PP scheduler: Disable WB: 0x%X.\n",
 898                                      disable_mask));
 899         }
 900
 901         mali_scheduler_unlock();
 902 }
 903
 904 #if MALI_STATE_TRACKING
 905 u32 mali_scheduler_dump_state(char *buf, u32 size)
 906 {
 907         int n = 0;
 908
 909         n += _mali_osk_snprintf(buf + n, size - n, "GP queues\n");
 910         n += _mali_osk_snprintf(buf + n, size - n,
 911                                 "\tQueue depth: %u\n", job_queue_gp.depth);
 912         n += _mali_osk_snprintf(buf + n, size - n,
 913                                 "\tNormal priority queue is %s\n",
 914                                 _mali_osk_list_empty(&job_queue_gp.normal_pri) ?
 915                                 "empty" : "not empty");
 916         n += _mali_osk_snprintf(buf + n, size - n,
 917                                 "\tHigh priority queue is %s\n",
 918                                 _mali_osk_list_empty(&job_queue_gp.high_pri) ?
 919                                 "empty" : "not empty");
 920
 921         n += _mali_osk_snprintf(buf + n, size - n,
 922                                 "PP queues\n");
 923         n += _mali_osk_snprintf(buf + n, size - n,
 924                                 "\tQueue depth: %u\n", job_queue_pp.depth);
 925         n += _mali_osk_snprintf(buf + n, size - n,
 926                                 "\tNormal priority queue is %s\n",
 927                                 _mali_osk_list_empty(&job_queue_pp.normal_pri)
 928                                 ? "empty" : "not empty");
 929         n += _mali_osk_snprintf(buf + n, size - n,
 930                                 "\tHigh priority queue is %s\n",
 931                                 _mali_osk_list_empty(&job_queue_pp.high_pri)
 932                                 ? "empty" : "not empty");
 933
 934         n += _mali_osk_snprintf(buf + n, size - n, "\n");
 935
 936         return n;
 937 }
 938 #endif
 939
 940 /*
 941  * ---------- Implementation of static functions ----------
 942  */
 943
 944 static mali_timeline_point mali_scheduler_submit_gp_job(
 945         struct mali_session_data *session, struct mali_gp_job *job)
 946 {
 947         mali_timeline_point point;
 948
 949         MALI_DEBUG_ASSERT_POINTER(session);
 950         MALI_DEBUG_ASSERT_POINTER(job);
 951
 952         /* Add job to Timeline system. */
 953         point = mali_timeline_system_add_tracker(session->timeline_system,
 954                         mali_gp_job_get_tracker(job), MALI_TIMELINE_GP);
 955
 956         return point;
 957 }
 958
 959 static mali_timeline_point mali_scheduler_submit_pp_job(
 960         struct mali_session_data *session, struct mali_pp_job *job)
 961 {
 962         mali_timeline_point point;
 963
 964         MALI_DEBUG_ASSERT_POINTER(session);
 965         MALI_DEBUG_ASSERT_POINTER(job);
 966
 967         mali_scheduler_lock();
 968         /*
 969          * Adding job to the lookup list used to quickly discard
 970          * writeback units of queued jobs.
 971          */
 972         mali_pp_job_fb_lookup_add(job);
 973         mali_scheduler_unlock();
 974
 975         /* Add job to Timeline system. */
 976         point = mali_timeline_system_add_tracker(session->timeline_system,
 977                         mali_pp_job_get_tracker(job), MALI_TIMELINE_PP);
 978
 979         return point;
 980 }
 981
 982 static mali_bool mali_scheduler_queue_gp_job(struct mali_gp_job *job)
 983 {
 984         struct mali_session_data *session;
 985         _mali_osk_list_t *queue;
 986
 987         MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
 988         MALI_DEBUG_ASSERT_POINTER(job);
 989
 990         session = mali_gp_job_get_session(job);
 991         MALI_DEBUG_ASSERT_POINTER(session);
 992
 993         if (unlikely(session->is_aborting)) {
 994                 MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Job %u (0x%08X) queued while session is aborting.\n",
 995                                      mali_gp_job_get_id(job), job));
 996                 return MALI_FALSE; /* job not queued */
 997         }
 998
 999         mali_gp_job_set_cache_order(job, mali_scheduler_get_new_cache_order());
1000
1001         /* Determine which queue the job should be added to. */
1002         if (session->use_high_priority_job_queue) {
1003                 queue = &job_queue_gp.high_pri;
1004         } else {
1005                 queue = &job_queue_gp.normal_pri;
1006         }
1007
1008         job_queue_gp.depth += 1;
1009         job_queue_gp.big_job_num += (job->big_job) ? 1 : 0;
1010
1011         /* Add job to queue (mali_gp_job_queue_add find correct place). */
1012         mali_gp_job_list_add(job, queue);
1013
1014         /*
1015          * We hold a PM reference for every job we hold queued (and running)
1016          * It is important that we take this reference after job has been
1017          * added the the queue so that any runtime resume could schedule this
1018          * job right there and then.
1019          */
1020         _mali_osk_pm_dev_ref_get_async();
1021
1022         if (mali_utilization_enabled()) {
1023                 /*
1024                  * We cheat a little bit by counting the GP as busy from the
1025                  * time a GP job is queued. This will be fine because we only
1026                  * loose the tiny idle gap between jobs, but we will instead
1027                  * get less utilization work to do (less locks taken)
1028                  */
1029                 mali_utilization_gp_start();
1030         }
1031
1032         mali_pm_record_gpu_active(MALI_TRUE);
1033
1034         /* Add profiling events for job enqueued */
1035         _mali_osk_profiling_add_event(
1036                 MALI_PROFILING_EVENT_TYPE_SINGLE |
1037                 MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1038                 MALI_PROFILING_EVENT_REASON_SINGLE_SW_GP_ENQUEUE,
1039                 mali_gp_job_get_pid(job),
1040                 mali_gp_job_get_tid(job),
1041                 mali_gp_job_get_frame_builder_id(job),
1042                 mali_gp_job_get_flush_id(job),
1043                 0);
1044
1045 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
1046         trace_gpu_job_enqueue(mali_gp_job_get_tid(job),
1047                               mali_gp_job_get_id(job), "GP");
1048 #endif
1049
1050         MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Job %u (0x%08X) queued\n",
1051                              mali_gp_job_get_id(job), job));
1052
1053         return MALI_TRUE; /* job queued */
1054 }
1055
1056 static mali_bool mali_scheduler_queue_pp_job(struct mali_pp_job *job)
1057 {
1058         struct mali_session_data *session;
1059         _mali_osk_list_t *queue = NULL;
1060
1061         MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
1062         MALI_DEBUG_ASSERT_POINTER(job);
1063
1064         session = mali_pp_job_get_session(job);
1065         MALI_DEBUG_ASSERT_POINTER(session);
1066
1067         if (unlikely(session->is_aborting)) {
1068                 MALI_DEBUG_PRINT(2, ("Mali PP scheduler: Job %u (0x%08X) queued while session is aborting.\n",
1069                                      mali_pp_job_get_id(job), job));
1070                 return MALI_FALSE; /* job not queued */
1071         } else if (unlikely(MALI_SWAP_IN_FAIL == job->swap_status)) {
1072                 MALI_DEBUG_PRINT(2, ("Mali PP scheduler: Job %u (0x%08X) queued while swap in failed.\n",
1073                                      mali_pp_job_get_id(job), job));
1074                 return MALI_FALSE;
1075         }
1076
1077         mali_pp_job_set_cache_order(job, mali_scheduler_get_new_cache_order());
1078
1079         if (session->use_high_priority_job_queue) {
1080                 queue = &job_queue_pp.high_pri;
1081         } else {
1082                 queue = &job_queue_pp.normal_pri;
1083         }
1084
1085         job_queue_pp.depth +=
1086                 mali_pp_job_get_sub_job_count(job);
1087
1088         /* Add job to queue (mali_gp_job_queue_add find correct place). */
1089         mali_pp_job_list_add(job, queue);
1090
1091         /*
1092          * We hold a PM reference for every job we hold queued (and running)
1093          * It is important that we take this reference after job has been
1094          * added the the queue so that any runtime resume could schedule this
1095          * job right there and then.
1096          */
1097         _mali_osk_pm_dev_ref_get_async();
1098
1099         if (mali_utilization_enabled()) {
1100                 /*
1101                  * We cheat a little bit by counting the PP as busy from the
1102                  * time a PP job is queued. This will be fine because we only
1103                  * loose the tiny idle gap between jobs, but we will instead
1104                  * get less utilization work to do (less locks taken)
1105                  */
1106                 mali_utilization_pp_start();
1107         }
1108
1109         mali_pm_record_gpu_active(MALI_FALSE);
1110
1111         /* Add profiling events for job enqueued */
1112         _mali_osk_profiling_add_event(
1113                 MALI_PROFILING_EVENT_TYPE_SINGLE |
1114                 MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1115                 MALI_PROFILING_EVENT_REASON_SINGLE_SW_PP_ENQUEUE,
1116                 mali_pp_job_get_pid(job),
1117                 mali_pp_job_get_tid(job),
1118                 mali_pp_job_get_frame_builder_id(job),
1119                 mali_pp_job_get_flush_id(job),
1120                 0);
1121
1122 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
1123         trace_gpu_job_enqueue(mali_pp_job_get_tid(job),
1124                               mali_pp_job_get_id(job), "PP");
1125 #endif
1126
1127         MALI_DEBUG_PRINT(3, ("Mali PP scheduler: %s job %u (0x%08X) with %u parts queued.\n",
1128                              mali_pp_job_is_virtual(job)
1129                              ? "Virtual" : "Physical",
1130                              mali_pp_job_get_id(job), job,
1131                              mali_pp_job_get_sub_job_count(job)));
1132
1133         return MALI_TRUE; /* job queued */
1134 }
1135
1136 static void mali_scheduler_return_gp_job_to_user(struct mali_gp_job *job,
1137                 mali_bool success)
1138 {
1139         _mali_uk_gp_job_finished_s *jobres;
1140         struct mali_session_data *session;
1141         _mali_osk_notification_t *notification;
1142
1143         MALI_DEBUG_ASSERT_POINTER(job);
1144
1145         session = mali_gp_job_get_session(job);
1146         MALI_DEBUG_ASSERT_POINTER(session);
1147
1148         notification = mali_gp_job_get_finished_notification(job);
1149         MALI_DEBUG_ASSERT_POINTER(notification);
1150
1151         jobres = notification->result_buffer;
1152         MALI_DEBUG_ASSERT_POINTER(jobres);
1153
1154         jobres->pending_big_job_num = mali_scheduler_job_gp_big_job_count();
1155
1156         jobres->user_job_ptr = mali_gp_job_get_user_id(job);
1157         if (MALI_TRUE == success) {
1158                 jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS;
1159         } else {
1160                 jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR;
1161         }
1162         jobres->heap_current_addr = mali_gp_job_get_current_heap_addr(job);
1163         jobres->perf_counter0 = mali_gp_job_get_perf_counter_value0(job);
1164         jobres->perf_counter1 = mali_gp_job_get_perf_counter_value1(job);
1165
1166         mali_session_send_notification(session, notification);
1167 }
1168
1169 void mali_scheduler_return_pp_job_to_user(struct mali_pp_job *job,
1170                 u32 num_cores_in_virtual)
1171 {
1172         u32 i;
1173         u32 num_counters_to_copy;
1174         _mali_uk_pp_job_finished_s *jobres;
1175         struct mali_session_data *session;
1176         _mali_osk_notification_t *notification;
1177
1178         if (MALI_TRUE == mali_pp_job_use_no_notification(job)) {
1179                 return;
1180         }
1181
1182         MALI_DEBUG_ASSERT_POINTER(job);
1183
1184         session = mali_pp_job_get_session(job);
1185         MALI_DEBUG_ASSERT_POINTER(session);
1186
1187         notification = mali_pp_job_get_finished_notification(job);
1188         MALI_DEBUG_ASSERT_POINTER(notification);
1189
1190         jobres = notification->result_buffer;
1191         MALI_DEBUG_ASSERT_POINTER(jobres);
1192
1193         jobres->user_job_ptr = mali_pp_job_get_user_id(job);
1194         if (MALI_TRUE == mali_pp_job_was_success(job)) {
1195                 jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS;
1196         } else {
1197                 jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR;
1198         }
1199
1200         if (mali_pp_job_is_virtual(job)) {
1201                 num_counters_to_copy = num_cores_in_virtual;
1202         } else {
1203                 num_counters_to_copy = mali_pp_job_get_sub_job_count(job);
1204         }
1205
1206         for (i = 0; i < num_counters_to_copy; i++) {
1207                 jobres->perf_counter0[i] =
1208                         mali_pp_job_get_perf_counter_value0(job, i);
1209                 jobres->perf_counter1[i] =
1210                         mali_pp_job_get_perf_counter_value1(job, i);
1211                 jobres->perf_counter_src0 =
1212                         mali_pp_job_get_pp_counter_global_src0();
1213                 jobres->perf_counter_src1 =
1214                         mali_pp_job_get_pp_counter_global_src1();
1215         }
1216
1217         mali_session_send_notification(session, notification);
1218 }
1219
1220 static void mali_scheduler_deferred_pp_job_delete(struct mali_pp_job *job)
1221 {
1222         MALI_DEBUG_ASSERT_POINTER(job);
1223
1224         _mali_osk_spinlock_irq_lock(scheduler_pp_job_delete_lock);
1225         mali_pp_job_list_addtail(job, &scheduler_pp_job_deletion_queue);
1226         _mali_osk_spinlock_irq_unlock(scheduler_pp_job_delete_lock);
1227
1228         _mali_osk_wq_schedule_work(scheduler_wq_pp_job_delete);
1229 }
1230
1231 void mali_scheduler_do_pp_job_delete(void *arg)
1232 {
1233         _MALI_OSK_LIST_HEAD_STATIC_INIT(list);
1234         struct mali_pp_job *job;
1235         struct mali_pp_job *tmp;
1236
1237         MALI_IGNORE(arg);
1238
1239         /*
1240          * Quickly "unhook" the jobs pending to be deleted, so we can release
1241          * the lock before we start deleting the job objects
1242          * (without any locks held)
1243          */
1244         _mali_osk_spinlock_irq_lock(scheduler_pp_job_delete_lock);
1245         _mali_osk_list_move_list(&scheduler_pp_job_deletion_queue, &list);
1246         _mali_osk_spinlock_irq_unlock(scheduler_pp_job_delete_lock);
1247
1248         _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list,
1249                                     struct mali_pp_job, list) {
1250                 _mali_osk_list_delinit(&job->list);
1251
1252                 mali_pp_job_delete(job); /* delete the job object itself */
1253         }
1254 }
1255
1256 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
1257
1258 static void mali_scheduler_deferred_pp_job_queue(struct mali_pp_job *job)
1259 {
1260         MALI_DEBUG_ASSERT_POINTER(job);
1261
1262         _mali_osk_spinlock_irq_lock(scheduler_pp_job_queue_lock);
1263         mali_pp_job_list_addtail(job, &scheduler_pp_job_queue_list);
1264         _mali_osk_spinlock_irq_unlock(scheduler_pp_job_queue_lock);
1265
1266         _mali_osk_wq_schedule_work(scheduler_wq_pp_job_queue);
1267 }
1268
1269 static void mali_scheduler_do_pp_job_queue(void *arg)
1270 {
1271         _MALI_OSK_LIST_HEAD_STATIC_INIT(list);
1272         struct mali_pp_job *job;
1273         struct mali_pp_job *tmp;
1274         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1275
1276         MALI_IGNORE(arg);
1277
1278         /*
1279          * Quickly "unhook" the jobs pending to be queued, so we can release
1280          * the lock before we start queueing the job objects
1281          * (without any locks held)
1282          */
1283         _mali_osk_spinlock_irq_lock(scheduler_pp_job_queue_lock);
1284         _mali_osk_list_move_list(&scheduler_pp_job_queue_list, &list);
1285         _mali_osk_spinlock_irq_unlock(scheduler_pp_job_queue_lock);
1286
1287         /* First loop through all jobs and do the pre-work (no locks needed) */
1288         _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list,
1289                                     struct mali_pp_job, list) {
1290                 if (mali_pp_job_needs_dma_buf_mapping(job)) {
1291                         /*
1292                          * This operation could fail, but we continue anyway,
1293                          * because the worst that could happen is that this
1294                          * job will fail due to a Mali page fault.
1295                          */
1296                         mali_dma_buf_map_job(job);
1297                 }
1298         }
1299
1300         mali_scheduler_lock();
1301
1302         /* Then loop through all jobs again to queue them (lock needed) */
1303         _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list,
1304                                     struct mali_pp_job, list) {
1305
1306                 /* Remove from scheduler_pp_job_queue_list before queueing */
1307                 mali_pp_job_list_remove(job);
1308
1309                 if (mali_scheduler_queue_pp_job(job)) {
1310                         /* Job queued successfully */
1311                         schedule_mask |= MALI_SCHEDULER_MASK_PP;
1312                 } else {
1313                         /* Failed to enqueue job, release job (with error) */
1314                         mali_pp_job_fb_lookup_remove(job);
1315                         mali_pp_job_mark_unstarted_failed(job);
1316
1317                         /* unlock scheduler in this uncommon case */
1318                         mali_scheduler_unlock();
1319
1320                         schedule_mask |= mali_timeline_tracker_release(
1321                                                  mali_pp_job_get_tracker(job));
1322
1323                         /* Notify user space and close the job object */
1324                         mali_scheduler_complete_pp_job(job, 0, MALI_TRUE,
1325                                                        MALI_FALSE);
1326
1327                         mali_scheduler_lock();
1328                 }
1329         }
1330
1331         mali_scheduler_unlock();
1332
1333         /* Trigger scheduling of jobs */
1334         mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE);
1335 }
1336
1337 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
1338
1339 void mali_scheduler_gp_pp_job_queue_print(void)
1340 {
1341         struct mali_gp_job *gp_job = NULL;
1342         struct mali_gp_job *tmp_gp_job = NULL;
1343         struct mali_pp_job *pp_job = NULL;
1344         struct mali_pp_job *tmp_pp_job = NULL;
1345
1346         MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
1347         MALI_DEBUG_ASSERT_LOCK_HELD(mali_executor_lock_obj);
1348
1349         /* dump job queup status */
1350         if ((0 == job_queue_gp.depth) && (0 == job_queue_pp.depth)) {
1351                 MALI_PRINT(("No GP&PP job in the job queue.\n"));
1352                 return;
1353         }
1354
1355         MALI_PRINT(("Total (%d) GP job in the job queue.\n", job_queue_gp.depth));
1356         if (job_queue_gp.depth > 0) {
1357                 if (!_mali_osk_list_empty(&job_queue_gp.high_pri)) {
1358                         _MALI_OSK_LIST_FOREACHENTRY(gp_job, tmp_gp_job, &job_queue_gp.high_pri,
1359                                                     struct mali_gp_job, list) {
1360                                 MALI_PRINT(("GP job(%p) id = %d tid = %d pid = %d in the gp job high_pri queue\n", gp_job, gp_job->id, gp_job->tid, gp_job->pid));
1361                         }
1362                 }
1363
1364                 if (!_mali_osk_list_empty(&job_queue_gp.normal_pri)) {
1365                         _MALI_OSK_LIST_FOREACHENTRY(gp_job, tmp_gp_job, &job_queue_gp.normal_pri,
1366                                                     struct mali_gp_job, list) {
1367                                 MALI_PRINT(("GP job(%p) id = %d tid = %d pid = %d in the gp job normal_pri queue\n", gp_job, gp_job->id, gp_job->tid, gp_job->pid));
1368                         }
1369                 }
1370         }
1371
1372         MALI_PRINT(("Total (%d) PP job in the job queue.\n", job_queue_pp.depth));
1373         if (job_queue_pp.depth > 0) {
1374                 if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) {
1375                         _MALI_OSK_LIST_FOREACHENTRY(pp_job, tmp_pp_job, &job_queue_pp.high_pri,
1376                                                     struct mali_pp_job, list) {
1377                                 if (mali_pp_job_is_virtual(pp_job)) {
1378                                         MALI_PRINT(("PP Virtual job(%p) id = %d tid = %d pid = %d in the pp job high_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid));
1379                                 } else {
1380                                         MALI_PRINT(("PP Physical job(%p) id = %d tid = %d pid = %d in the pp job high_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid));
1381                                 }
1382                         }
1383                 }
1384
1385                 if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
1386                         _MALI_OSK_LIST_FOREACHENTRY(pp_job, tmp_pp_job, &job_queue_pp.normal_pri,
1387                                                     struct mali_pp_job, list) {
1388                                 if (mali_pp_job_is_virtual(pp_job)) {
1389                                         MALI_PRINT(("PP Virtual job(%p) id = %d tid = %d pid = %d in the pp job normal_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid));
1390                                 } else {
1391                                         MALI_PRINT(("PP Physical job(%p) id = %d tid = %d pid = %d in the pp job normal_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid));
1392                                 }
1393                         }
1394                 }
1395         }
1396
1397         /* dump group running job status */
1398         mali_executor_running_status_print();
1399 }