drivers/gpu/arm/mali400/mali/common/mali_scheduler.c

   1 /*
   2  * Copyright (C) 2012-2015 ARM Limited. All rights reserved.
   3  *
   4  * This program is free software and is provided to you under the terms of the GNU General Public License version 2
   5  * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
   6  *
   7  * A copy of the licence is included with the program, and can also be obtained from Free Software
   8  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
   9  */
  10
  11 #include "mali_scheduler.h"
  12 #include "mali_kernel_common.h"
  13 #include "mali_osk.h"
  14 #include "mali_osk_profiling.h"
  15 #include "mali_kernel_utilization.h"
  16 #include "mali_timeline.h"
  17 #include "mali_gp_job.h"
  18 #include "mali_pp_job.h"
  19 #include "mali_executor.h"
  20 #include "mali_group.h"
  21 #include <linux/wait.h>
  22 #include <linux/sched.h>
  23
  24
  25 #if defined(CONFIG_DMA_SHARED_BUFFER)
  26 #include "mali_memory_dma_buf.h"
  27 #endif
  28
  29 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
  30 #include <linux/sched.h>
  31 #include <trace/events/gpu.h>
  32 #endif
  33 /*
  34  * ---------- static defines/constants ----------
  35  */
  36
  37 /*
  38  * If dma_buf with map on demand is used, we defer job queue
  39  * if in atomic context, since both might sleep.
  40  */
  41 #if defined(CONFIG_DMA_SHARED_BUFFER)
  42 #if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
  43 #define MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE 1
  44 #endif
  45 #endif
  46
  47
  48 /*
  49  * ---------- global variables (exported due to inline functions) ----------
  50  */
  51
  52 /* Lock protecting this module */
  53 _mali_osk_spinlock_irq_t *mali_scheduler_lock_obj = NULL;
  54
  55 /* Queue of jobs to be executed on the GP group */
  56 struct mali_scheduler_job_queue job_queue_gp;
  57
  58 /* Queue of PP jobs */
  59 struct mali_scheduler_job_queue job_queue_pp;
  60
  61 _mali_osk_atomic_t mali_job_id_autonumber;
  62 _mali_osk_atomic_t mali_job_cache_order_autonumber;
  63 /*
  64  * ---------- static variables ----------
  65  */
  66
  67 _mali_osk_wq_work_t *scheduler_wq_pp_job_delete = NULL;
  68 _mali_osk_spinlock_irq_t *scheduler_pp_job_delete_lock = NULL;
  69 static _MALI_OSK_LIST_HEAD_STATIC_INIT(scheduler_pp_job_deletion_queue);
  70
  71 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
  72 static _mali_osk_wq_work_t *scheduler_wq_pp_job_queue = NULL;
  73 static _mali_osk_spinlock_irq_t *scheduler_pp_job_queue_lock = NULL;
  74 static _MALI_OSK_LIST_HEAD_STATIC_INIT(scheduler_pp_job_queue_list);
  75 #endif
  76
  77 /*
  78  * ---------- Forward declaration of static functions ----------
  79  */
  80
  81 static mali_timeline_point mali_scheduler_submit_gp_job(
  82         struct mali_session_data *session, struct mali_gp_job *job);
  83 static mali_timeline_point mali_scheduler_submit_pp_job(
  84         struct mali_session_data *session, struct mali_pp_job *job);
  85
  86 static mali_bool mali_scheduler_queue_gp_job(struct mali_gp_job *job);
  87 static mali_bool mali_scheduler_queue_pp_job(struct mali_pp_job *job);
  88
  89 static void mali_scheduler_return_gp_job_to_user(struct mali_gp_job *job,
  90                 mali_bool success);
  91
  92 static void mali_scheduler_deferred_pp_job_delete(struct mali_pp_job *job);
  93 void mali_scheduler_do_pp_job_delete(void *arg);
  94
  95 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
  96 static void mali_scheduler_deferred_pp_job_queue(struct mali_pp_job *job);
  97 static void mali_scheduler_do_pp_job_queue(void *arg);
  98 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
  99
 100 /*
 101  * ---------- Actual implementation ----------
 102  */
 103
 104 _mali_osk_errcode_t mali_scheduler_initialize(void)
 105 {
 106         _mali_osk_atomic_init(&mali_job_id_autonumber, 0);
 107         _mali_osk_atomic_init(&mali_job_cache_order_autonumber, 0);
 108
 109         _MALI_OSK_INIT_LIST_HEAD(&job_queue_gp.normal_pri);
 110         _MALI_OSK_INIT_LIST_HEAD(&job_queue_gp.high_pri);
 111         job_queue_gp.depth = 0;
 112         job_queue_gp.big_job_num = 0;
 113
 114         _MALI_OSK_INIT_LIST_HEAD(&job_queue_pp.normal_pri);
 115         _MALI_OSK_INIT_LIST_HEAD(&job_queue_pp.high_pri);
 116         job_queue_pp.depth = 0;
 117         job_queue_pp.big_job_num = 0;
 118
 119         mali_scheduler_lock_obj = _mali_osk_spinlock_irq_init(
 120                                           _MALI_OSK_LOCKFLAG_ORDERED,
 121                                           _MALI_OSK_LOCK_ORDER_SCHEDULER);
 122         if (NULL == mali_scheduler_lock_obj) {
 123                 mali_scheduler_terminate();
 124         }
 125
 126         scheduler_wq_pp_job_delete = _mali_osk_wq_create_work(
 127                                              mali_scheduler_do_pp_job_delete, NULL);
 128         if (NULL == scheduler_wq_pp_job_delete) {
 129                 mali_scheduler_terminate();
 130                 return _MALI_OSK_ERR_FAULT;
 131         }
 132
 133         scheduler_pp_job_delete_lock = _mali_osk_spinlock_irq_init(
 134                                                _MALI_OSK_LOCKFLAG_ORDERED,
 135                                                _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED);
 136         if (NULL == scheduler_pp_job_delete_lock) {
 137                 mali_scheduler_terminate();
 138                 return _MALI_OSK_ERR_FAULT;
 139         }
 140
 141 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
 142         scheduler_wq_pp_job_queue = _mali_osk_wq_create_work(
 143                                             mali_scheduler_do_pp_job_queue, NULL);
 144         if (NULL == scheduler_wq_pp_job_queue) {
 145                 mali_scheduler_terminate();
 146                 return _MALI_OSK_ERR_FAULT;
 147         }
 148
 149         scheduler_pp_job_queue_lock = _mali_osk_spinlock_irq_init(
 150                                               _MALI_OSK_LOCKFLAG_ORDERED,
 151                                               _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED);
 152         if (NULL == scheduler_pp_job_queue_lock) {
 153                 mali_scheduler_terminate();
 154                 return _MALI_OSK_ERR_FAULT;
 155         }
 156 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
 157
 158         return _MALI_OSK_ERR_OK;
 159 }
 160
 161 void mali_scheduler_terminate(void)
 162 {
 163 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
 164         if (NULL != scheduler_pp_job_queue_lock) {
 165                 _mali_osk_spinlock_irq_term(scheduler_pp_job_queue_lock);
 166                 scheduler_pp_job_queue_lock = NULL;
 167         }
 168
 169         if (NULL != scheduler_wq_pp_job_queue) {
 170                 _mali_osk_wq_delete_work(scheduler_wq_pp_job_queue);
 171                 scheduler_wq_pp_job_queue = NULL;
 172         }
 173 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
 174
 175         if (NULL != scheduler_pp_job_delete_lock) {
 176                 _mali_osk_spinlock_irq_term(scheduler_pp_job_delete_lock);
 177                 scheduler_pp_job_delete_lock = NULL;
 178         }
 179
 180         if (NULL != scheduler_wq_pp_job_delete) {
 181                 _mali_osk_wq_delete_work(scheduler_wq_pp_job_delete);
 182                 scheduler_wq_pp_job_delete = NULL;
 183         }
 184
 185         if (NULL != mali_scheduler_lock_obj) {
 186                 _mali_osk_spinlock_irq_term(mali_scheduler_lock_obj);
 187                 mali_scheduler_lock_obj = NULL;
 188         }
 189
 190         _mali_osk_atomic_term(&mali_job_cache_order_autonumber);
 191         _mali_osk_atomic_term(&mali_job_id_autonumber);
 192 }
 193
 194 u32 mali_scheduler_job_physical_head_count(void)
 195 {
 196         /*
 197          * Count how many physical sub jobs are present from the head of queue
 198          * until the first virtual job is present.
 199          * Early out when we have reached maximum number of PP cores (8)
 200          */
 201         u32 count = 0;
 202         struct mali_pp_job *job;
 203         struct mali_pp_job *temp;
 204
 205         /* Check for partially started normal pri jobs */
 206         if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
 207                 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
 208
 209                 job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next,
 210                                            struct mali_pp_job, list);
 211
 212                 MALI_DEBUG_ASSERT_POINTER(job);
 213
 214                 if (MALI_TRUE == mali_pp_job_has_started_sub_jobs(job)) {
 215                         /*
 216                          * Remember; virtual jobs can't be queued and started
 217                          * at the same time, so this must be a physical job
 218                          */
 219                         count += mali_pp_job_unstarted_sub_job_count(job);
 220                         if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) {
 221                                 return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS;
 222                         }
 223                 }
 224         }
 225
 226         _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.high_pri,
 227                                     struct mali_pp_job, list) {
 228                 if (MALI_FALSE == mali_pp_job_is_virtual(job)) {
 229                         count += mali_pp_job_unstarted_sub_job_count(job);
 230                         if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) {
 231                                 return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS;
 232                         }
 233                 } else {
 234                         /* Came across a virtual job, so stop counting */
 235                         return count;
 236                 }
 237         }
 238
 239         _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.normal_pri,
 240                                     struct mali_pp_job, list) {
 241                 if (MALI_FALSE == mali_pp_job_is_virtual(job)) {
 242                         /* any partially started is already counted */
 243                         if (MALI_FALSE == mali_pp_job_has_started_sub_jobs(job)) {
 244                                 count += mali_pp_job_unstarted_sub_job_count(job);
 245                                 if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <=
 246                                     count) {
 247                                         return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS;
 248                                 }
 249                         }
 250                 } else {
 251                         /* Came across a virtual job, so stop counting */
 252                         return count;
 253                 }
 254         }
 255
 256         return count;
 257 }
 258
 259 mali_bool mali_scheduler_job_next_is_virtual(void)
 260 {
 261         struct mali_pp_job *job;
 262
 263         job = mali_scheduler_job_pp_virtual_peek();
 264         if (NULL != job) {
 265                 MALI_DEBUG_ASSERT(mali_pp_job_is_virtual(job));
 266
 267                 return MALI_TRUE;
 268         }
 269
 270         return MALI_FALSE;
 271 }
 272
 273 struct mali_gp_job *mali_scheduler_job_gp_get(void)
 274 {
 275         _mali_osk_list_t *queue;
 276         struct mali_gp_job *job = NULL;
 277
 278         MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
 279         MALI_DEBUG_ASSERT(0 < job_queue_gp.depth);
 280         MALI_DEBUG_ASSERT(job_queue_gp.big_job_num <= job_queue_gp.depth);
 281
 282         if (!_mali_osk_list_empty(&job_queue_gp.high_pri)) {
 283                 queue = &job_queue_gp.high_pri;
 284         } else {
 285                 queue = &job_queue_gp.normal_pri;
 286                 MALI_DEBUG_ASSERT(!_mali_osk_list_empty(queue));
 287         }
 288
 289         job = _MALI_OSK_LIST_ENTRY(queue->next, struct mali_gp_job, list);
 290
 291         MALI_DEBUG_ASSERT_POINTER(job);
 292
 293         mali_gp_job_list_remove(job);
 294         job_queue_gp.depth--;
 295         if (job->big_job) {
 296                 job_queue_gp.big_job_num --;
 297                 if (job_queue_gp.big_job_num < MALI_MAX_PENDING_BIG_JOB) {
 298                         /* wake up process */
 299                         wait_queue_head_t *queue = mali_session_get_wait_queue();
 300                         wake_up(queue);
 301                 }
 302         }
 303         return job;
 304 }
 305
 306 struct mali_pp_job *mali_scheduler_job_pp_physical_peek(void)
 307 {
 308         struct mali_pp_job *job = NULL;
 309         struct mali_pp_job *tmp_job = NULL;
 310
 311         MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
 312
 313         /*
 314          * For PP jobs we favour partially started jobs in normal
 315          * priority queue over unstarted jobs in high priority queue
 316          */
 317
 318         if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
 319                 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
 320
 321                 tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next,
 322                                                struct mali_pp_job, list);
 323                 MALI_DEBUG_ASSERT(NULL != tmp_job);
 324
 325                 if (MALI_FALSE == mali_pp_job_is_virtual(tmp_job)) {
 326                         job = tmp_job;
 327                 }
 328         }
 329
 330         if (NULL == job ||
 331             MALI_FALSE == mali_pp_job_has_started_sub_jobs(job)) {
 332                 /*
 333                  * There isn't a partially started job in normal queue, so
 334                  * look in high priority queue.
 335                  */
 336                 if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) {
 337                         MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
 338
 339                         tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.high_pri.next,
 340                                                        struct mali_pp_job, list);
 341                         MALI_DEBUG_ASSERT(NULL != tmp_job);
 342
 343                         if (MALI_FALSE == mali_pp_job_is_virtual(tmp_job)) {
 344                                 job = tmp_job;
 345                         }
 346                 }
 347         }
 348
 349         return job;
 350 }
 351
 352 struct mali_pp_job *mali_scheduler_job_pp_virtual_peek(void)
 353 {
 354         struct mali_pp_job *job = NULL;
 355         struct mali_pp_job *tmp_job = NULL;
 356
 357         MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
 358
 359         if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) {
 360                 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
 361
 362                 tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.high_pri.next,
 363                                                struct mali_pp_job, list);
 364
 365                 if (MALI_TRUE == mali_pp_job_is_virtual(tmp_job)) {
 366                         job = tmp_job;
 367                 }
 368         }
 369
 370         if (NULL == job) {
 371                 if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
 372                         MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
 373
 374                         tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next,
 375                                                        struct mali_pp_job, list);
 376
 377                         if (MALI_TRUE == mali_pp_job_is_virtual(tmp_job)) {
 378                                 job = tmp_job;
 379                         }
 380                 }
 381         }
 382
 383         return job;
 384 }
 385
 386 struct mali_pp_job *mali_scheduler_job_pp_physical_get(u32 *sub_job)
 387 {
 388         struct mali_pp_job *job = mali_scheduler_job_pp_physical_peek();
 389
 390         MALI_DEBUG_ASSERT(MALI_FALSE == mali_pp_job_is_virtual(job));
 391
 392         if (NULL != job) {
 393                 *sub_job = mali_pp_job_get_first_unstarted_sub_job(job);
 394
 395                 mali_pp_job_mark_sub_job_started(job, *sub_job);
 396                 if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(job)) {
 397                         /* Remove from queue when last sub job has been retrieved */
 398                         mali_pp_job_list_remove(job);
 399                 }
 400
 401                 job_queue_pp.depth--;
 402
 403                 /*
 404                  * Job about to start so it is no longer be
 405                  * possible to discard WB
 406                  */
 407                 mali_pp_job_fb_lookup_remove(job);
 408         }
 409
 410         return job;
 411 }
 412
 413 struct mali_pp_job *mali_scheduler_job_pp_virtual_get(void)
 414 {
 415         struct mali_pp_job *job = mali_scheduler_job_pp_virtual_peek();
 416
 417         MALI_DEBUG_ASSERT(MALI_TRUE == mali_pp_job_is_virtual(job));
 418
 419         if (NULL != job) {
 420                 MALI_DEBUG_ASSERT(0 ==
 421                                   mali_pp_job_get_first_unstarted_sub_job(job));
 422                 MALI_DEBUG_ASSERT(1 ==
 423                                   mali_pp_job_get_sub_job_count(job));
 424
 425                 mali_pp_job_mark_sub_job_started(job, 0);
 426
 427                 mali_pp_job_list_remove(job);
 428
 429                 job_queue_pp.depth--;
 430
 431                 /*
 432                  * Job about to start so it is no longer be
 433                  * possible to discard WB
 434                  */
 435                 mali_pp_job_fb_lookup_remove(job);
 436         }
 437
 438         return job;
 439 }
 440
 441 mali_scheduler_mask mali_scheduler_activate_gp_job(struct mali_gp_job *job)
 442 {
 443         MALI_DEBUG_ASSERT_POINTER(job);
 444
 445         MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Timeline activation for job %u (0x%08X).\n",
 446                              mali_gp_job_get_id(job), job));
 447
 448         mali_scheduler_lock();
 449
 450         if (!mali_scheduler_queue_gp_job(job)) {
 451                 /* Failed to enqueue job, release job (with error) */
 452
 453                 mali_scheduler_unlock();
 454
 455                 mali_timeline_tracker_release(mali_gp_job_get_tracker(job));
 456                 mali_gp_job_signal_pp_tracker(job, MALI_FALSE);
 457
 458                 /* This will notify user space and close the job object */
 459                 mali_scheduler_complete_gp_job(job, MALI_FALSE,
 460                                                MALI_TRUE, MALI_FALSE);
 461
 462                 return MALI_SCHEDULER_MASK_EMPTY;
 463         }
 464
 465         mali_scheduler_unlock();
 466
 467         return MALI_SCHEDULER_MASK_GP;
 468 }
 469
 470 mali_scheduler_mask mali_scheduler_activate_pp_job(struct mali_pp_job *job)
 471 {
 472         MALI_DEBUG_ASSERT_POINTER(job);
 473
 474         MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Timeline activation for job %u (0x%08X).\n",
 475                              mali_pp_job_get_id(job), job));
 476
 477         if (MALI_TRUE == mali_timeline_tracker_activation_error(
 478                     mali_pp_job_get_tracker(job))) {
 479                 MALI_DEBUG_PRINT(3, ("Mali PP scheduler: Job %u (0x%08X) activated with error, aborting.\n",
 480                                      mali_pp_job_get_id(job), job));
 481
 482                 mali_scheduler_lock();
 483                 mali_pp_job_fb_lookup_remove(job);
 484                 mali_pp_job_mark_unstarted_failed(job);
 485                 mali_scheduler_unlock();
 486
 487                 mali_timeline_tracker_release(mali_pp_job_get_tracker(job));
 488
 489                 /* This will notify user space and close the job object */
 490                 mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, MALI_FALSE);
 491
 492                 return MALI_SCHEDULER_MASK_EMPTY;
 493         }
 494
 495 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
 496         if (mali_pp_job_needs_dma_buf_mapping(job)) {
 497                 mali_scheduler_deferred_pp_job_queue(job);
 498                 return MALI_SCHEDULER_MASK_EMPTY;
 499         }
 500 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
 501
 502         mali_scheduler_lock();
 503
 504         if (!mali_scheduler_queue_pp_job(job)) {
 505                 /* Failed to enqueue job, release job (with error) */
 506                 mali_pp_job_fb_lookup_remove(job);
 507                 mali_pp_job_mark_unstarted_failed(job);
 508                 mali_scheduler_unlock();
 509
 510                 mali_timeline_tracker_release(mali_pp_job_get_tracker(job));
 511
 512                 /* This will notify user space and close the job object */
 513                 mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, MALI_FALSE);
 514
 515                 return MALI_SCHEDULER_MASK_EMPTY;
 516         }
 517
 518         mali_scheduler_unlock();
 519         return MALI_SCHEDULER_MASK_PP;
 520 }
 521
 522 void mali_scheduler_complete_gp_job(struct mali_gp_job *job,
 523                                     mali_bool success,
 524                                     mali_bool user_notification,
 525                                     mali_bool dequeued)
 526 {
 527         if (user_notification) {
 528                 mali_scheduler_return_gp_job_to_user(job, success);
 529         }
 530
 531         if (dequeued) {
 532                 _mali_osk_pm_dev_ref_put();
 533
 534                 if (mali_utilization_enabled()) {
 535                         mali_utilization_gp_end();
 536                 }
 537         }
 538
 539         mali_gp_job_delete(job);
 540 }
 541
 542 void mali_scheduler_complete_pp_job(struct mali_pp_job *job,
 543                                     u32 num_cores_in_virtual,
 544                                     mali_bool user_notification,
 545                                     mali_bool dequeued)
 546 {
 547         job->user_notification = user_notification;
 548         job->num_pp_cores_in_virtual = num_cores_in_virtual;
 549
 550         if (dequeued) {
 551 #if defined(CONFIG_MALI_DVFS)
 552                 if (mali_pp_job_is_window_surface(job)) {
 553                         struct mali_session_data *session;
 554                         session = mali_pp_job_get_session(job);
 555                         mali_session_inc_num_window_jobs(session);
 556                 }
 557 #endif
 558
 559                 _mali_osk_pm_dev_ref_put();
 560
 561                 if (mali_utilization_enabled()) {
 562                         mali_utilization_pp_end();
 563                 }
 564         }
 565
 566         /* With ZRAM feature enabled, all pp jobs will be force to use deferred delete. */
 567         mali_scheduler_deferred_pp_job_delete(job);
 568 }
 569
 570 void mali_scheduler_abort_session(struct mali_session_data *session)
 571 {
 572         struct mali_gp_job *gp_job;
 573         struct mali_gp_job *gp_tmp;
 574         struct mali_pp_job *pp_job;
 575         struct mali_pp_job *pp_tmp;
 576         _MALI_OSK_LIST_HEAD_STATIC_INIT(removed_jobs_gp);
 577         _MALI_OSK_LIST_HEAD_STATIC_INIT(removed_jobs_pp);
 578
 579         MALI_DEBUG_ASSERT_POINTER(session);
 580         MALI_DEBUG_ASSERT(session->is_aborting);
 581
 582         MALI_DEBUG_PRINT(3, ("Mali scheduler: Aborting all queued jobs from session 0x%08X.\n",
 583                              session));
 584
 585         mali_scheduler_lock();
 586
 587         /* Remove from GP normal priority queue */
 588         _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &job_queue_gp.normal_pri,
 589                                     struct mali_gp_job, list) {
 590                 if (mali_gp_job_get_session(gp_job) == session) {
 591                         mali_gp_job_list_move(gp_job, &removed_jobs_gp);
 592                         job_queue_gp.depth--;
 593                         job_queue_gp.big_job_num -= gp_job->big_job ? 1 : 0;
 594                 }
 595         }
 596
 597         /* Remove from GP high priority queue */
 598         _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &job_queue_gp.high_pri,
 599                                     struct mali_gp_job, list) {
 600                 if (mali_gp_job_get_session(gp_job) == session) {
 601                         mali_gp_job_list_move(gp_job, &removed_jobs_gp);
 602                         job_queue_gp.depth--;
 603                         job_queue_gp.big_job_num -= gp_job->big_job ? 1 : 0;
 604                 }
 605         }
 606
 607         /* Remove from PP normal priority queue */
 608         _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp,
 609                                     &job_queue_pp.normal_pri,
 610                                     struct mali_pp_job, list) {
 611                 if (mali_pp_job_get_session(pp_job) == session) {
 612                         mali_pp_job_fb_lookup_remove(pp_job);
 613
 614                         job_queue_pp.depth -=
 615                                 mali_pp_job_unstarted_sub_job_count(
 616                                         pp_job);
 617                         mali_pp_job_mark_unstarted_failed(pp_job);
 618
 619                         if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(pp_job)) {
 620                                 if (mali_pp_job_is_complete(pp_job)) {
 621                                         mali_pp_job_list_move(pp_job,
 622                                                               &removed_jobs_pp);
 623                                 } else {
 624                                         mali_pp_job_list_remove(pp_job);
 625                                 }
 626                         }
 627                 }
 628         }
 629
 630         /* Remove from PP high priority queue */
 631         _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp,
 632                                     &job_queue_pp.high_pri,
 633                                     struct mali_pp_job, list) {
 634                 if (mali_pp_job_get_session(pp_job) == session) {
 635                         mali_pp_job_fb_lookup_remove(pp_job);
 636
 637                         job_queue_pp.depth -=
 638                                 mali_pp_job_unstarted_sub_job_count(
 639                                         pp_job);
 640                         mali_pp_job_mark_unstarted_failed(pp_job);
 641
 642                         if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(pp_job)) {
 643                                 if (mali_pp_job_is_complete(pp_job)) {
 644                                         mali_pp_job_list_move(pp_job,
 645                                                               &removed_jobs_pp);
 646                                 } else {
 647                                         mali_pp_job_list_remove(pp_job);
 648                                 }
 649                         }
 650                 }
 651         }
 652
 653         /*
 654          * Release scheduler lock so we can release trackers
 655          * (which will potentially queue new jobs)
 656          */
 657         mali_scheduler_unlock();
 658
 659         /* Release and complete all (non-running) found GP jobs  */
 660         _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &removed_jobs_gp,
 661                                     struct mali_gp_job, list) {
 662                 mali_timeline_tracker_release(mali_gp_job_get_tracker(gp_job));
 663                 mali_gp_job_signal_pp_tracker(gp_job, MALI_FALSE);
 664                 _mali_osk_list_delinit(&gp_job->list);
 665                 mali_scheduler_complete_gp_job(gp_job,
 666                                                MALI_FALSE, MALI_FALSE, MALI_TRUE);
 667         }
 668
 669         /* Release and complete non-running PP jobs */
 670         _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp, &removed_jobs_pp,
 671                                     struct mali_pp_job, list) {
 672                 mali_timeline_tracker_release(mali_pp_job_get_tracker(pp_job));
 673                 _mali_osk_list_delinit(&pp_job->list);
 674                 mali_scheduler_complete_pp_job(pp_job, 0,
 675                                                MALI_FALSE, MALI_TRUE);
 676         }
 677 }
 678
 679 _mali_osk_errcode_t _mali_ukk_gp_start_job(void *ctx,
 680                 _mali_uk_gp_start_job_s *uargs)
 681 {
 682         struct mali_session_data *session;
 683         struct mali_gp_job *job;
 684         mali_timeline_point point;
 685         u32 __user *point_ptr = NULL;
 686
 687         MALI_DEBUG_ASSERT_POINTER(uargs);
 688         MALI_DEBUG_ASSERT_POINTER(ctx);
 689
 690         session = (struct mali_session_data *)(uintptr_t)ctx;
 691
 692         job = mali_gp_job_create(session, uargs, mali_scheduler_get_new_id(),
 693                                  NULL);
 694         if (NULL == job) {
 695                 MALI_PRINT_ERROR(("Failed to create GP job.\n"));
 696                 return _MALI_OSK_ERR_NOMEM;
 697         }
 698
 699         point_ptr = (u32 __user *)(uintptr_t)mali_gp_job_get_timeline_point_ptr(job);
 700
 701         point = mali_scheduler_submit_gp_job(session, job);
 702
 703         if (0 != _mali_osk_put_user(((u32) point), point_ptr)) {
 704                 /*
 705                  * Let user space know that something failed
 706                  * after the job was started.
 707                  */
 708                 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
 709         }
 710
 711         return _MALI_OSK_ERR_OK;
 712 }
 713
 714 _mali_osk_errcode_t _mali_ukk_pp_start_job(void *ctx,
 715                 _mali_uk_pp_start_job_s *uargs)
 716 {
 717         struct mali_session_data *session;
 718         struct mali_pp_job *job;
 719         mali_timeline_point point;
 720         u32 __user *point_ptr = NULL;
 721
 722         MALI_DEBUG_ASSERT_POINTER(uargs);
 723         MALI_DEBUG_ASSERT_POINTER(ctx);
 724
 725         session = (struct mali_session_data *)(uintptr_t)ctx;
 726
 727         job = mali_pp_job_create(session, uargs, mali_scheduler_get_new_id());
 728         if (NULL == job) {
 729                 MALI_PRINT_ERROR(("Failed to create PP job.\n"));
 730                 return _MALI_OSK_ERR_NOMEM;
 731         }
 732
 733         point_ptr = (u32 __user *)(uintptr_t)mali_pp_job_get_timeline_point_ptr(job);
 734
 735         point = mali_scheduler_submit_pp_job(session, job);
 736         job = NULL;
 737
 738         if (0 != _mali_osk_put_user(((u32) point), point_ptr)) {
 739                 /*
 740                  * Let user space know that something failed
 741                  * after the job was started.
 742                  */
 743                 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
 744         }
 745
 746         return _MALI_OSK_ERR_OK;
 747 }
 748
 749 _mali_osk_errcode_t _mali_ukk_pp_and_gp_start_job(void *ctx,
 750                 _mali_uk_pp_and_gp_start_job_s *uargs)
 751 {
 752         struct mali_session_data *session;
 753         _mali_uk_pp_and_gp_start_job_s kargs;
 754         struct mali_pp_job *pp_job;
 755         struct mali_gp_job *gp_job;
 756         u32 __user *point_ptr = NULL;
 757         mali_timeline_point point;
 758         _mali_uk_pp_start_job_s __user *pp_args;
 759         _mali_uk_gp_start_job_s __user *gp_args;
 760
 761         MALI_DEBUG_ASSERT_POINTER(ctx);
 762         MALI_DEBUG_ASSERT_POINTER(uargs);
 763
 764         session = (struct mali_session_data *) ctx;
 765
 766         if (0 != _mali_osk_copy_from_user(&kargs, uargs,
 767                                           sizeof(_mali_uk_pp_and_gp_start_job_s))) {
 768                 return _MALI_OSK_ERR_NOMEM;
 769         }
 770
 771         pp_args = (_mali_uk_pp_start_job_s __user *)(uintptr_t)kargs.pp_args;
 772         gp_args = (_mali_uk_gp_start_job_s __user *)(uintptr_t)kargs.gp_args;
 773
 774         pp_job = mali_pp_job_create(session, pp_args,
 775                                     mali_scheduler_get_new_id());
 776         if (NULL == pp_job) {
 777                 MALI_PRINT_ERROR(("Failed to create PP job.\n"));
 778                 return _MALI_OSK_ERR_NOMEM;
 779         }
 780
 781         gp_job = mali_gp_job_create(session, gp_args,
 782                                     mali_scheduler_get_new_id(),
 783                                     mali_pp_job_get_tracker(pp_job));
 784         if (NULL == gp_job) {
 785                 MALI_PRINT_ERROR(("Failed to create GP job.\n"));
 786                 mali_pp_job_delete(pp_job);
 787                 return _MALI_OSK_ERR_NOMEM;
 788         }
 789
 790         point_ptr = (u32 __user *)(uintptr_t)mali_pp_job_get_timeline_point_ptr(pp_job);
 791
 792         /* Submit GP job. */
 793         mali_scheduler_submit_gp_job(session, gp_job);
 794         gp_job = NULL;
 795
 796         /* Submit PP job. */
 797         point = mali_scheduler_submit_pp_job(session, pp_job);
 798         pp_job = NULL;
 799
 800         if (0 != _mali_osk_put_user(((u32) point), point_ptr)) {
 801                 /*
 802                  * Let user space know that something failed
 803                  * after the jobs were started.
 804                  */
 805                 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
 806         }
 807
 808         return _MALI_OSK_ERR_OK;
 809 }
 810
 811 void _mali_ukk_pp_job_disable_wb(_mali_uk_pp_disable_wb_s *args)
 812 {
 813         struct mali_session_data *session;
 814         struct mali_pp_job *job;
 815         struct mali_pp_job *tmp;
 816         u32 fb_lookup_id;
 817
 818         MALI_DEBUG_ASSERT_POINTER(args);
 819         MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
 820
 821         session = (struct mali_session_data *)(uintptr_t)args->ctx;
 822
 823         fb_lookup_id = args->fb_id & MALI_PP_JOB_FB_LOOKUP_LIST_MASK;
 824
 825         mali_scheduler_lock();
 826
 827         /* Iterate over all jobs for given frame builder_id. */
 828         _MALI_OSK_LIST_FOREACHENTRY(job, tmp,
 829                                     &session->pp_job_fb_lookup_list[fb_lookup_id],
 830                                     struct mali_pp_job, session_fb_lookup_list) {
 831                 MALI_DEBUG_CODE(u32 disable_mask = 0);
 832
 833                 if (mali_pp_job_get_frame_builder_id(job) !=
 834                     (u32) args->fb_id) {
 835                         MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Disable WB mismatching FB.\n"));
 836                         continue;
 837                 }
 838
 839                 MALI_DEBUG_CODE(disable_mask |= 0xD << (4 * 3));
 840
 841                 if (mali_pp_job_get_wb0_source_addr(job) == args->wb0_memory) {
 842                         MALI_DEBUG_CODE(disable_mask |= 0x1 << (4 * 1));
 843                         mali_pp_job_disable_wb0(job);
 844                 }
 845
 846                 if (mali_pp_job_get_wb1_source_addr(job) == args->wb1_memory) {
 847                         MALI_DEBUG_CODE(disable_mask |= 0x2 << (4 * 2));
 848                         mali_pp_job_disable_wb1(job);
 849                 }
 850
 851                 if (mali_pp_job_get_wb2_source_addr(job) == args->wb2_memory) {
 852                         MALI_DEBUG_CODE(disable_mask |= 0x3 << (4 * 3));
 853                         mali_pp_job_disable_wb2(job);
 854                 }
 855                 MALI_DEBUG_PRINT(3, ("Mali PP scheduler: Disable WB: 0x%X.\n",
 856                                      disable_mask));
 857         }
 858
 859         mali_scheduler_unlock();
 860 }
 861
 862 #if MALI_STATE_TRACKING
 863 u32 mali_scheduler_dump_state(char *buf, u32 size)
 864 {
 865         int n = 0;
 866
 867         n += _mali_osk_snprintf(buf + n, size - n, "GP queues\n");
 868         n += _mali_osk_snprintf(buf + n, size - n,
 869                                 "\tQueue depth: %u\n", job_queue_gp.depth);
 870         n += _mali_osk_snprintf(buf + n, size - n,
 871                                 "\tNormal priority queue is %s\n",
 872                                 _mali_osk_list_empty(&job_queue_gp.normal_pri) ?
 873                                 "empty" : "not empty");
 874         n += _mali_osk_snprintf(buf + n, size - n,
 875                                 "\tHigh priority queue is %s\n",
 876                                 _mali_osk_list_empty(&job_queue_gp.high_pri) ?
 877                                 "empty" : "not empty");
 878
 879         n += _mali_osk_snprintf(buf + n, size - n,
 880                                 "PP queues\n");
 881         n += _mali_osk_snprintf(buf + n, size - n,
 882                                 "\tQueue depth: %u\n", job_queue_pp.depth);
 883         n += _mali_osk_snprintf(buf + n, size - n,
 884                                 "\tNormal priority queue is %s\n",
 885                                 _mali_osk_list_empty(&job_queue_pp.normal_pri)
 886                                 ? "empty" : "not empty");
 887         n += _mali_osk_snprintf(buf + n, size - n,
 888                                 "\tHigh priority queue is %s\n",
 889                                 _mali_osk_list_empty(&job_queue_pp.high_pri)
 890                                 ? "empty" : "not empty");
 891
 892         n += _mali_osk_snprintf(buf + n, size - n, "\n");
 893
 894         return n;
 895 }
 896 #endif
 897
 898 /*
 899  * ---------- Implementation of static functions ----------
 900  */
 901
 902 static mali_timeline_point mali_scheduler_submit_gp_job(
 903         struct mali_session_data *session, struct mali_gp_job *job)
 904 {
 905         mali_timeline_point point;
 906
 907         MALI_DEBUG_ASSERT_POINTER(session);
 908         MALI_DEBUG_ASSERT_POINTER(job);
 909
 910         /* Add job to Timeline system. */
 911         point = mali_timeline_system_add_tracker(session->timeline_system,
 912                         mali_gp_job_get_tracker(job), MALI_TIMELINE_GP);
 913
 914         return point;
 915 }
 916
 917 static mali_timeline_point mali_scheduler_submit_pp_job(
 918         struct mali_session_data *session, struct mali_pp_job *job)
 919 {
 920         mali_timeline_point point;
 921
 922         MALI_DEBUG_ASSERT_POINTER(session);
 923         MALI_DEBUG_ASSERT_POINTER(job);
 924
 925         mali_scheduler_lock();
 926         /*
 927          * Adding job to the lookup list used to quickly discard
 928          * writeback units of queued jobs.
 929          */
 930         mali_pp_job_fb_lookup_add(job);
 931         mali_scheduler_unlock();
 932
 933         /* Add job to Timeline system. */
 934         point = mali_timeline_system_add_tracker(session->timeline_system,
 935                         mali_pp_job_get_tracker(job), MALI_TIMELINE_PP);
 936
 937         return point;
 938 }
 939
 940 static mali_bool mali_scheduler_queue_gp_job(struct mali_gp_job *job)
 941 {
 942         struct mali_session_data *session;
 943         _mali_osk_list_t *queue;
 944
 945         MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
 946         MALI_DEBUG_ASSERT_POINTER(job);
 947
 948         session = mali_gp_job_get_session(job);
 949         MALI_DEBUG_ASSERT_POINTER(session);
 950
 951         if (unlikely(session->is_aborting)) {
 952                 MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Job %u (0x%08X) queued while session is aborting.\n",
 953                                      mali_gp_job_get_id(job), job));
 954                 return MALI_FALSE; /* job not queued */
 955         }
 956
 957         mali_gp_job_set_cache_order(job, mali_scheduler_get_new_cache_order());
 958
 959         /* Determine which queue the job should be added to. */
 960         if (session->use_high_priority_job_queue) {
 961                 queue = &job_queue_gp.high_pri;
 962         } else {
 963                 queue = &job_queue_gp.normal_pri;
 964         }
 965
 966         job_queue_gp.depth += 1;
 967         job_queue_gp.big_job_num += (job->big_job) ? 1 : 0;
 968
 969         /* Add job to queue (mali_gp_job_queue_add find correct place). */
 970         mali_gp_job_list_add(job, queue);
 971
 972         /*
 973          * We hold a PM reference for every job we hold queued (and running)
 974          * It is important that we take this reference after job has been
 975          * added the the queue so that any runtime resume could schedule this
 976          * job right there and then.
 977          */
 978         _mali_osk_pm_dev_ref_get_async();
 979
 980         if (mali_utilization_enabled()) {
 981                 /*
 982                  * We cheat a little bit by counting the GP as busy from the
 983                  * time a GP job is queued. This will be fine because we only
 984                  * loose the tiny idle gap between jobs, but we will instead
 985                  * get less utilization work to do (less locks taken)
 986                  */
 987                 mali_utilization_gp_start();
 988         }
 989
 990         /* Add profiling events for job enqueued */
 991         _mali_osk_profiling_add_event(
 992                 MALI_PROFILING_EVENT_TYPE_SINGLE |
 993                 MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
 994                 MALI_PROFILING_EVENT_REASON_SINGLE_SW_GP_ENQUEUE,
 995                 mali_gp_job_get_pid(job),
 996                 mali_gp_job_get_tid(job),
 997                 mali_gp_job_get_frame_builder_id(job),
 998                 mali_gp_job_get_flush_id(job),
 999                 0);
1000
1001 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
1002         trace_gpu_job_enqueue(mali_gp_job_get_tid(job),
1003                               mali_gp_job_get_id(job), "GP");
1004 #endif
1005
1006         MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Job %u (0x%08X) queued\n",
1007                              mali_gp_job_get_id(job), job));
1008
1009         return MALI_TRUE; /* job queued */
1010 }
1011
1012 static mali_bool mali_scheduler_queue_pp_job(struct mali_pp_job *job)
1013 {
1014         struct mali_session_data *session;
1015         _mali_osk_list_t *queue = NULL;
1016
1017         MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
1018         MALI_DEBUG_ASSERT_POINTER(job);
1019
1020         session = mali_pp_job_get_session(job);
1021         MALI_DEBUG_ASSERT_POINTER(session);
1022
1023         if (unlikely(session->is_aborting)) {
1024                 MALI_DEBUG_PRINT(2, ("Mali PP scheduler: Job %u (0x%08X) queued while session is aborting.\n",
1025                                      mali_pp_job_get_id(job), job));
1026                 return MALI_FALSE; /* job not queued */
1027         } else if (unlikely(MALI_SWAP_IN_FAIL == job->swap_status)) {
1028                 MALI_DEBUG_PRINT(2, ("Mali PP scheduler: Job %u (0x%08X) queued while swap in failed.\n",
1029                                      mali_pp_job_get_id(job), job));
1030                 return MALI_FALSE;
1031         }
1032
1033         mali_pp_job_set_cache_order(job, mali_scheduler_get_new_cache_order());
1034
1035         if (session->use_high_priority_job_queue) {
1036                 queue = &job_queue_pp.high_pri;
1037         } else {
1038                 queue = &job_queue_pp.normal_pri;
1039         }
1040
1041         job_queue_pp.depth +=
1042                 mali_pp_job_get_sub_job_count(job);
1043
1044         /* Add job to queue (mali_gp_job_queue_add find correct place). */
1045         mali_pp_job_list_add(job, queue);
1046
1047         /*
1048          * We hold a PM reference for every job we hold queued (and running)
1049          * It is important that we take this reference after job has been
1050          * added the the queue so that any runtime resume could schedule this
1051          * job right there and then.
1052          */
1053         _mali_osk_pm_dev_ref_get_async();
1054
1055         if (mali_utilization_enabled()) {
1056                 /*
1057                  * We cheat a little bit by counting the PP as busy from the
1058                  * time a PP job is queued. This will be fine because we only
1059                  * loose the tiny idle gap between jobs, but we will instead
1060                  * get less utilization work to do (less locks taken)
1061                  */
1062                 mali_utilization_pp_start();
1063         }
1064
1065         /* Add profiling events for job enqueued */
1066
1067         _mali_osk_profiling_add_event(
1068                 MALI_PROFILING_EVENT_TYPE_SINGLE |
1069                 MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1070                 MALI_PROFILING_EVENT_REASON_SINGLE_SW_PP_ENQUEUE,
1071                 mali_pp_job_get_pid(job),
1072                 mali_pp_job_get_tid(job),
1073                 mali_pp_job_get_frame_builder_id(job),
1074                 mali_pp_job_get_flush_id(job),
1075                 0);
1076
1077 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
1078         trace_gpu_job_enqueue(mali_pp_job_get_tid(job),
1079                               mali_pp_job_get_id(job), "PP");
1080 #endif
1081
1082         MALI_DEBUG_PRINT(3, ("Mali PP scheduler: %s job %u (0x%08X) with %u parts queued.\n",
1083                              mali_pp_job_is_virtual(job)
1084                              ? "Virtual" : "Physical",
1085                              mali_pp_job_get_id(job), job,
1086                              mali_pp_job_get_sub_job_count(job)));
1087
1088         return MALI_TRUE; /* job queued */
1089 }
1090
1091 static void mali_scheduler_return_gp_job_to_user(struct mali_gp_job *job,
1092                 mali_bool success)
1093 {
1094         _mali_uk_gp_job_finished_s *jobres;
1095         struct mali_session_data *session;
1096         _mali_osk_notification_t *notification;
1097
1098         MALI_DEBUG_ASSERT_POINTER(job);
1099
1100         session = mali_gp_job_get_session(job);
1101         MALI_DEBUG_ASSERT_POINTER(session);
1102
1103         notification = mali_gp_job_get_finished_notification(job);
1104         MALI_DEBUG_ASSERT_POINTER(notification);
1105
1106         jobres = notification->result_buffer;
1107         MALI_DEBUG_ASSERT_POINTER(jobres);
1108
1109         jobres->pending_big_job_num = mali_scheduler_job_gp_big_job_count();
1110
1111         jobres->user_job_ptr = mali_gp_job_get_user_id(job);
1112         if (MALI_TRUE == success) {
1113                 jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS;
1114         } else {
1115                 jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR;
1116         }
1117         jobres->heap_current_addr = mali_gp_job_get_current_heap_addr(job);
1118         jobres->perf_counter0 = mali_gp_job_get_perf_counter_value0(job);
1119         jobres->perf_counter1 = mali_gp_job_get_perf_counter_value1(job);
1120
1121         mali_session_send_notification(session, notification);
1122 }
1123
1124 void mali_scheduler_return_pp_job_to_user(struct mali_pp_job *job,
1125                 u32 num_cores_in_virtual)
1126 {
1127         u32 i;
1128         u32 num_counters_to_copy;
1129         _mali_uk_pp_job_finished_s *jobres;
1130         struct mali_session_data *session;
1131         _mali_osk_notification_t *notification;
1132
1133         if (MALI_TRUE == mali_pp_job_use_no_notification(job)) {
1134                 return;
1135         }
1136
1137         MALI_DEBUG_ASSERT_POINTER(job);
1138
1139         session = mali_pp_job_get_session(job);
1140         MALI_DEBUG_ASSERT_POINTER(session);
1141
1142         notification = mali_pp_job_get_finished_notification(job);
1143         MALI_DEBUG_ASSERT_POINTER(notification);
1144
1145         jobres = notification->result_buffer;
1146         MALI_DEBUG_ASSERT_POINTER(jobres);
1147
1148         jobres->user_job_ptr = mali_pp_job_get_user_id(job);
1149         if (MALI_TRUE == mali_pp_job_was_success(job)) {
1150                 jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS;
1151         } else {
1152                 jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR;
1153         }
1154
1155         if (mali_pp_job_is_virtual(job)) {
1156                 num_counters_to_copy = num_cores_in_virtual;
1157         } else {
1158                 num_counters_to_copy = mali_pp_job_get_sub_job_count(job);
1159         }
1160
1161         for (i = 0; i < num_counters_to_copy; i++) {
1162                 jobres->perf_counter0[i] =
1163                         mali_pp_job_get_perf_counter_value0(job, i);
1164                 jobres->perf_counter1[i] =
1165                         mali_pp_job_get_perf_counter_value1(job, i);
1166                 jobres->perf_counter_src0 =
1167                         mali_pp_job_get_pp_counter_global_src0();
1168                 jobres->perf_counter_src1 =
1169                         mali_pp_job_get_pp_counter_global_src1();
1170         }
1171
1172         mali_session_send_notification(session, notification);
1173 }
1174
1175 static void mali_scheduler_deferred_pp_job_delete(struct mali_pp_job *job)
1176 {
1177         MALI_DEBUG_ASSERT_POINTER(job);
1178
1179         _mali_osk_spinlock_irq_lock(scheduler_pp_job_delete_lock);
1180         mali_pp_job_list_addtail(job, &scheduler_pp_job_deletion_queue);
1181         _mali_osk_spinlock_irq_unlock(scheduler_pp_job_delete_lock);
1182
1183         _mali_osk_wq_schedule_work(scheduler_wq_pp_job_delete);
1184 }
1185
1186 void mali_scheduler_do_pp_job_delete(void *arg)
1187 {
1188         _MALI_OSK_LIST_HEAD_STATIC_INIT(list);
1189         struct mali_pp_job *job;
1190         struct mali_pp_job *tmp;
1191
1192         MALI_IGNORE(arg);
1193
1194         /*
1195          * Quickly "unhook" the jobs pending to be deleted, so we can release
1196          * the lock before we start deleting the job objects
1197          * (without any locks held)
1198          */
1199         _mali_osk_spinlock_irq_lock(scheduler_pp_job_delete_lock);
1200         _mali_osk_list_move_list(&scheduler_pp_job_deletion_queue, &list);
1201         _mali_osk_spinlock_irq_unlock(scheduler_pp_job_delete_lock);
1202
1203         _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list,
1204                                     struct mali_pp_job, list) {
1205                 _mali_osk_list_delinit(&job->list);
1206
1207                 mali_pp_job_delete(job); /* delete the job object itself */
1208         }
1209 }
1210
1211 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
1212
1213 static void mali_scheduler_deferred_pp_job_queue(struct mali_pp_job *job)
1214 {
1215         MALI_DEBUG_ASSERT_POINTER(job);
1216
1217         _mali_osk_spinlock_irq_lock(scheduler_pp_job_queue_lock);
1218         mali_pp_job_list_addtail(job, &scheduler_pp_job_queue_list);
1219         _mali_osk_spinlock_irq_unlock(scheduler_pp_job_queue_lock);
1220
1221         _mali_osk_wq_schedule_work(scheduler_wq_pp_job_queue);
1222 }
1223
1224 static void mali_scheduler_do_pp_job_queue(void *arg)
1225 {
1226         _MALI_OSK_LIST_HEAD_STATIC_INIT(list);
1227         struct mali_pp_job *job;
1228         struct mali_pp_job *tmp;
1229         mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1230
1231         MALI_IGNORE(arg);
1232
1233         /*
1234          * Quickly "unhook" the jobs pending to be queued, so we can release
1235          * the lock before we start queueing the job objects
1236          * (without any locks held)
1237          */
1238         _mali_osk_spinlock_irq_lock(scheduler_pp_job_queue_lock);
1239         _mali_osk_list_move_list(&scheduler_pp_job_queue_list, &list);
1240         _mali_osk_spinlock_irq_unlock(scheduler_pp_job_queue_lock);
1241
1242         /* First loop through all jobs and do the pre-work (no locks needed) */
1243         _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list,
1244                                     struct mali_pp_job, list) {
1245                 if (mali_pp_job_needs_dma_buf_mapping(job)) {
1246                         /*
1247                          * This operation could fail, but we continue anyway,
1248                          * because the worst that could happen is that this
1249                          * job will fail due to a Mali page fault.
1250                          */
1251                         mali_dma_buf_map_job(job);
1252                 }
1253         }
1254
1255         mali_scheduler_lock();
1256
1257         /* Then loop through all jobs again to queue them (lock needed) */
1258         _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list,
1259                                     struct mali_pp_job, list) {
1260
1261                 /* Remove from scheduler_pp_job_queue_list before queueing */
1262                 mali_pp_job_list_remove(job);
1263
1264                 if (mali_scheduler_queue_pp_job(job)) {
1265                         /* Job queued successfully */
1266                         schedule_mask |= MALI_SCHEDULER_MASK_PP;
1267                 } else {
1268                         /* Failed to enqueue job, release job (with error) */
1269                         mali_pp_job_fb_lookup_remove(job);
1270                         mali_pp_job_mark_unstarted_failed(job);
1271
1272                         /* unlock scheduler in this uncommon case */
1273                         mali_scheduler_unlock();
1274
1275                         schedule_mask |= mali_timeline_tracker_release(
1276                                                  mali_pp_job_get_tracker(job));
1277
1278                         /* Notify user space and close the job object */
1279                         mali_scheduler_complete_pp_job(job, 0, MALI_TRUE,
1280                                                        MALI_FALSE);
1281
1282                         mali_scheduler_lock();
1283                 }
1284         }
1285
1286         mali_scheduler_unlock();
1287
1288         /* Trigger scheduling of jobs */
1289         mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE);
1290 }
1291
1292 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
1293
1294 void mali_scheduler_gp_pp_job_queue_print(void)
1295 {
1296         struct mali_gp_job *gp_job = NULL;
1297         struct mali_gp_job *tmp_gp_job = NULL;
1298         struct mali_pp_job *pp_job = NULL;
1299         struct mali_pp_job *tmp_pp_job = NULL;
1300
1301         MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
1302         MALI_DEBUG_ASSERT_LOCK_HELD(mali_executor_lock_obj);
1303
1304         /* dump job queup status */
1305         if ((0 == job_queue_gp.depth) && (0 == job_queue_pp.depth)) {
1306                 MALI_PRINT(("No GP&PP job in the job queue.\n"));
1307                 return;
1308         }
1309
1310         MALI_PRINT(("Total (%d) GP job in the job queue.\n", job_queue_gp.depth));
1311         if (job_queue_gp.depth > 0) {
1312                 if (!_mali_osk_list_empty(&job_queue_gp.high_pri)) {
1313                         _MALI_OSK_LIST_FOREACHENTRY(gp_job, tmp_gp_job, &job_queue_gp.high_pri,
1314                                                     struct mali_gp_job, list) {
1315                                 MALI_PRINT(("GP job(%p) id = %d tid = %d pid = %d in the gp job high_pri queue\n", gp_job, gp_job->id, gp_job->tid, gp_job->pid));
1316                         }
1317                 }
1318
1319                 if (!_mali_osk_list_empty(&job_queue_gp.normal_pri)) {
1320                         _MALI_OSK_LIST_FOREACHENTRY(gp_job, tmp_gp_job, &job_queue_gp.normal_pri,
1321                                                     struct mali_gp_job, list) {
1322                                 MALI_PRINT(("GP job(%p) id = %d tid = %d pid = %d in the gp job normal_pri queue\n", gp_job, gp_job->id, gp_job->tid, gp_job->pid));
1323                         }
1324                 }
1325         }
1326
1327         MALI_PRINT(("Total (%d) PP job in the job queue.\n", job_queue_pp.depth));
1328         if (job_queue_pp.depth > 0) {
1329                 if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) {
1330                         _MALI_OSK_LIST_FOREACHENTRY(pp_job, tmp_pp_job, &job_queue_pp.high_pri,
1331                                                     struct mali_pp_job, list) {
1332                                 if (mali_pp_job_is_virtual(pp_job)) {
1333                                         MALI_PRINT(("PP Virtual job(%p) id = %d tid = %d pid = %d in the pp job high_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid));
1334                                 } else {
1335                                         MALI_PRINT(("PP Physical job(%p) id = %d tid = %d pid = %d in the pp job high_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid));
1336                                 }
1337                         }
1338                 }
1339
1340                 if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
1341                         _MALI_OSK_LIST_FOREACHENTRY(pp_job, tmp_pp_job, &job_queue_pp.normal_pri,
1342                                                     struct mali_pp_job, list) {
1343                                 if (mali_pp_job_is_virtual(pp_job)) {
1344                                         MALI_PRINT(("PP Virtual job(%p) id = %d tid = %d pid = %d in the pp job normal_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid));
1345                                 } else {
1346                                         MALI_PRINT(("PP Physical job(%p) id = %d tid = %d pid = %d in the pp job normal_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid));
1347                                 }
1348                         }
1349                 }
1350         }
1351
1352         /* dump group running job status */
1353         mali_executor_running_status_print();
1354 }