2 * Copyright (C) 2012-2016 ARM Limited. All rights reserved.
4 * This program is free software and is provided to you under the terms of the GNU General Public License version 2
5 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
7 * A copy of the licence is included with the program, and can also be obtained from Free Software
8 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
11 #include "mali_scheduler.h"
12 #include "mali_kernel_common.h"
14 #include "mali_osk_profiling.h"
15 #include "mali_kernel_utilization.h"
16 #include "mali_timeline.h"
17 #include "mali_gp_job.h"
18 #include "mali_pp_job.h"
19 #include "mali_executor.h"
20 #include "mali_group.h"
21 #include <linux/wait.h>
22 #include <linux/sched.h>
23 #include "mali_pm_metrics.h"
25 #if defined(CONFIG_DMA_SHARED_BUFFER)
26 #include "mali_memory_dma_buf.h"
29 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
30 #include <linux/sched.h>
31 #include <trace/events/gpu.h>
34 * ---------- static defines/constants ----------
38 * If dma_buf with map on demand is used, we defer job queue
39 * if in atomic context, since both might sleep.
41 #if defined(CONFIG_DMA_SHARED_BUFFER)
42 #if !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
43 #define MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE 1
49 * ---------- global variables (exported due to inline functions) ----------
52 /* Lock protecting this module */
53 _mali_osk_spinlock_irq_t *mali_scheduler_lock_obj = NULL;
55 /* Queue of jobs to be executed on the GP group */
56 struct mali_scheduler_job_queue job_queue_gp;
58 /* Queue of PP jobs */
59 struct mali_scheduler_job_queue job_queue_pp;
61 _mali_osk_atomic_t mali_job_id_autonumber;
62 _mali_osk_atomic_t mali_job_cache_order_autonumber;
64 * ---------- static variables ----------
67 _mali_osk_wq_work_t *scheduler_wq_pp_job_delete = NULL;
68 _mali_osk_spinlock_irq_t *scheduler_pp_job_delete_lock = NULL;
69 static _MALI_OSK_LIST_HEAD_STATIC_INIT(scheduler_pp_job_deletion_queue);
71 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
72 static _mali_osk_wq_work_t *scheduler_wq_pp_job_queue = NULL;
73 static _mali_osk_spinlock_irq_t *scheduler_pp_job_queue_lock = NULL;
74 static _MALI_OSK_LIST_HEAD_STATIC_INIT(scheduler_pp_job_queue_list);
78 * ---------- Forward declaration of static functions ----------
81 static mali_timeline_point mali_scheduler_submit_gp_job(
82 struct mali_session_data *session, struct mali_gp_job *job);
83 static mali_timeline_point mali_scheduler_submit_pp_job(
84 struct mali_session_data *session, struct mali_pp_job *job);
86 static mali_bool mali_scheduler_queue_gp_job(struct mali_gp_job *job);
87 static mali_bool mali_scheduler_queue_pp_job(struct mali_pp_job *job);
89 static void mali_scheduler_return_gp_job_to_user(struct mali_gp_job *job,
92 static void mali_scheduler_deferred_pp_job_delete(struct mali_pp_job *job);
93 void mali_scheduler_do_pp_job_delete(void *arg);
95 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
96 static void mali_scheduler_deferred_pp_job_queue(struct mali_pp_job *job);
97 static void mali_scheduler_do_pp_job_queue(void *arg);
98 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
101 * ---------- Actual implementation ----------
104 _mali_osk_errcode_t mali_scheduler_initialize(void)
106 _mali_osk_atomic_init(&mali_job_id_autonumber, 0);
107 _mali_osk_atomic_init(&mali_job_cache_order_autonumber, 0);
109 _MALI_OSK_INIT_LIST_HEAD(&job_queue_gp.normal_pri);
110 _MALI_OSK_INIT_LIST_HEAD(&job_queue_gp.high_pri);
111 job_queue_gp.depth = 0;
112 job_queue_gp.big_job_num = 0;
114 _MALI_OSK_INIT_LIST_HEAD(&job_queue_pp.normal_pri);
115 _MALI_OSK_INIT_LIST_HEAD(&job_queue_pp.high_pri);
116 job_queue_pp.depth = 0;
117 job_queue_pp.big_job_num = 0;
119 mali_scheduler_lock_obj = _mali_osk_spinlock_irq_init(
120 _MALI_OSK_LOCKFLAG_ORDERED,
121 _MALI_OSK_LOCK_ORDER_SCHEDULER);
122 if (NULL == mali_scheduler_lock_obj) {
123 mali_scheduler_terminate();
126 scheduler_wq_pp_job_delete = _mali_osk_wq_create_work(
127 mali_scheduler_do_pp_job_delete, NULL);
128 if (NULL == scheduler_wq_pp_job_delete) {
129 mali_scheduler_terminate();
130 return _MALI_OSK_ERR_FAULT;
133 scheduler_pp_job_delete_lock = _mali_osk_spinlock_irq_init(
134 _MALI_OSK_LOCKFLAG_ORDERED,
135 _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED);
136 if (NULL == scheduler_pp_job_delete_lock) {
137 mali_scheduler_terminate();
138 return _MALI_OSK_ERR_FAULT;
141 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
142 scheduler_wq_pp_job_queue = _mali_osk_wq_create_work(
143 mali_scheduler_do_pp_job_queue, NULL);
144 if (NULL == scheduler_wq_pp_job_queue) {
145 mali_scheduler_terminate();
146 return _MALI_OSK_ERR_FAULT;
149 scheduler_pp_job_queue_lock = _mali_osk_spinlock_irq_init(
150 _MALI_OSK_LOCKFLAG_ORDERED,
151 _MALI_OSK_LOCK_ORDER_SCHEDULER_DEFERRED);
152 if (NULL == scheduler_pp_job_queue_lock) {
153 mali_scheduler_terminate();
154 return _MALI_OSK_ERR_FAULT;
156 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
158 return _MALI_OSK_ERR_OK;
161 void mali_scheduler_terminate(void)
163 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
164 if (NULL != scheduler_pp_job_queue_lock) {
165 _mali_osk_spinlock_irq_term(scheduler_pp_job_queue_lock);
166 scheduler_pp_job_queue_lock = NULL;
169 if (NULL != scheduler_wq_pp_job_queue) {
170 _mali_osk_wq_delete_work(scheduler_wq_pp_job_queue);
171 scheduler_wq_pp_job_queue = NULL;
173 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
175 if (NULL != scheduler_pp_job_delete_lock) {
176 _mali_osk_spinlock_irq_term(scheduler_pp_job_delete_lock);
177 scheduler_pp_job_delete_lock = NULL;
180 if (NULL != scheduler_wq_pp_job_delete) {
181 _mali_osk_wq_delete_work(scheduler_wq_pp_job_delete);
182 scheduler_wq_pp_job_delete = NULL;
185 if (NULL != mali_scheduler_lock_obj) {
186 _mali_osk_spinlock_irq_term(mali_scheduler_lock_obj);
187 mali_scheduler_lock_obj = NULL;
190 _mali_osk_atomic_term(&mali_job_cache_order_autonumber);
191 _mali_osk_atomic_term(&mali_job_id_autonumber);
194 u32 mali_scheduler_job_physical_head_count(mali_bool gpu_mode_is_secure)
197 * Count how many physical sub jobs are present from the head of queue
198 * until the first virtual job is present.
199 * Early out when we have reached maximum number of PP cores (8)
202 struct mali_pp_job *job;
203 struct mali_pp_job *temp;
205 /* Check for partially started normal pri jobs */
206 if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
207 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
209 job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next,
210 struct mali_pp_job, list);
212 MALI_DEBUG_ASSERT_POINTER(job);
214 if (MALI_TRUE == mali_pp_job_has_started_sub_jobs(job)) {
216 * Remember; virtual jobs can't be queued and started
217 * at the same time, so this must be a physical job
219 if ((MALI_FALSE == gpu_mode_is_secure && MALI_FALSE == mali_pp_job_is_protected_job(job))
220 || (MALI_TRUE == gpu_mode_is_secure && MALI_TRUE == mali_pp_job_is_protected_job(job))) {
222 count += mali_pp_job_unstarted_sub_job_count(job);
223 if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) {
224 return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS;
230 _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.high_pri,
231 struct mali_pp_job, list) {
232 if ((MALI_FALSE == mali_pp_job_is_virtual(job))
233 && ((MALI_FALSE == gpu_mode_is_secure && MALI_FALSE == mali_pp_job_is_protected_job(job))
234 || (MALI_TRUE == gpu_mode_is_secure && MALI_TRUE == mali_pp_job_is_protected_job(job)))) {
236 count += mali_pp_job_unstarted_sub_job_count(job);
237 if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) {
238 return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS;
241 /* Came across a virtual job, so stop counting */
246 _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.normal_pri,
247 struct mali_pp_job, list) {
248 if ((MALI_FALSE == mali_pp_job_is_virtual(job))
249 && (MALI_FALSE == mali_pp_job_has_started_sub_jobs(job))
250 && ((MALI_FALSE == gpu_mode_is_secure && MALI_FALSE == mali_pp_job_is_protected_job(job))
251 || (MALI_TRUE == gpu_mode_is_secure && MALI_TRUE == mali_pp_job_is_protected_job(job)))) {
253 count += mali_pp_job_unstarted_sub_job_count(job);
254 if (MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS <= count) {
255 return MALI_MAX_NUMBER_OF_PHYSICAL_PP_GROUPS;
258 /* Came across a virtual job, so stop counting */
265 struct mali_pp_job *mali_scheduler_job_pp_next(void)
267 struct mali_pp_job *job;
268 struct mali_pp_job *temp;
270 MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
272 /* Check for partially started normal pri jobs */
273 if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
274 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
276 job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next,
277 struct mali_pp_job, list);
279 MALI_DEBUG_ASSERT_POINTER(job);
281 if (MALI_TRUE == mali_pp_job_has_started_sub_jobs(job)) {
286 _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.high_pri,
287 struct mali_pp_job, list) {
291 _MALI_OSK_LIST_FOREACHENTRY(job, temp, &job_queue_pp.normal_pri,
292 struct mali_pp_job, list) {
299 mali_bool mali_scheduler_job_next_is_virtual(void)
301 struct mali_pp_job *job;
303 job = mali_scheduler_job_pp_virtual_peek();
305 MALI_DEBUG_ASSERT(mali_pp_job_is_virtual(job));
313 struct mali_gp_job *mali_scheduler_job_gp_get(void)
315 _mali_osk_list_t *queue;
316 struct mali_gp_job *job = NULL;
318 MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
319 MALI_DEBUG_ASSERT(0 < job_queue_gp.depth);
320 MALI_DEBUG_ASSERT(job_queue_gp.big_job_num <= job_queue_gp.depth);
322 if (!_mali_osk_list_empty(&job_queue_gp.high_pri)) {
323 queue = &job_queue_gp.high_pri;
325 queue = &job_queue_gp.normal_pri;
326 MALI_DEBUG_ASSERT(!_mali_osk_list_empty(queue));
329 job = _MALI_OSK_LIST_ENTRY(queue->next, struct mali_gp_job, list);
331 MALI_DEBUG_ASSERT_POINTER(job);
333 mali_gp_job_list_remove(job);
334 job_queue_gp.depth--;
336 job_queue_gp.big_job_num --;
337 if (job_queue_gp.big_job_num < MALI_MAX_PENDING_BIG_JOB) {
338 /* wake up process */
339 wait_queue_head_t *queue = mali_session_get_wait_queue();
346 struct mali_pp_job *mali_scheduler_job_pp_physical_peek(void)
348 struct mali_pp_job *job = NULL;
349 struct mali_pp_job *tmp_job = NULL;
351 MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
354 * For PP jobs we favour partially started jobs in normal
355 * priority queue over unstarted jobs in high priority queue
358 if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
359 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
361 tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next,
362 struct mali_pp_job, list);
363 MALI_DEBUG_ASSERT(NULL != tmp_job);
365 if (MALI_FALSE == mali_pp_job_is_virtual(tmp_job)) {
371 MALI_FALSE == mali_pp_job_has_started_sub_jobs(job)) {
373 * There isn't a partially started job in normal queue, so
374 * look in high priority queue.
376 if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) {
377 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
379 tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.high_pri.next,
380 struct mali_pp_job, list);
381 MALI_DEBUG_ASSERT(NULL != tmp_job);
383 if (MALI_FALSE == mali_pp_job_is_virtual(tmp_job)) {
392 struct mali_pp_job *mali_scheduler_job_pp_virtual_peek(void)
394 struct mali_pp_job *job = NULL;
395 struct mali_pp_job *tmp_job = NULL;
397 MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
399 if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) {
400 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
402 tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.high_pri.next,
403 struct mali_pp_job, list);
405 if (MALI_TRUE == mali_pp_job_is_virtual(tmp_job)) {
411 if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
412 MALI_DEBUG_ASSERT(0 < job_queue_pp.depth);
414 tmp_job = _MALI_OSK_LIST_ENTRY(job_queue_pp.normal_pri.next,
415 struct mali_pp_job, list);
417 if (MALI_TRUE == mali_pp_job_is_virtual(tmp_job)) {
426 struct mali_pp_job *mali_scheduler_job_pp_physical_get(u32 *sub_job)
428 struct mali_pp_job *job = mali_scheduler_job_pp_physical_peek();
430 MALI_DEBUG_ASSERT(MALI_FALSE == mali_pp_job_is_virtual(job));
433 *sub_job = mali_pp_job_get_first_unstarted_sub_job(job);
435 mali_pp_job_mark_sub_job_started(job, *sub_job);
436 if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(job)) {
437 /* Remove from queue when last sub job has been retrieved */
438 mali_pp_job_list_remove(job);
441 job_queue_pp.depth--;
444 * Job about to start so it is no longer be
445 * possible to discard WB
447 mali_pp_job_fb_lookup_remove(job);
453 struct mali_pp_job *mali_scheduler_job_pp_virtual_get(void)
455 struct mali_pp_job *job = mali_scheduler_job_pp_virtual_peek();
457 MALI_DEBUG_ASSERT(MALI_TRUE == mali_pp_job_is_virtual(job));
460 MALI_DEBUG_ASSERT(0 ==
461 mali_pp_job_get_first_unstarted_sub_job(job));
462 MALI_DEBUG_ASSERT(1 ==
463 mali_pp_job_get_sub_job_count(job));
465 mali_pp_job_mark_sub_job_started(job, 0);
467 mali_pp_job_list_remove(job);
469 job_queue_pp.depth--;
472 * Job about to start so it is no longer be
473 * possible to discard WB
475 mali_pp_job_fb_lookup_remove(job);
481 mali_scheduler_mask mali_scheduler_activate_gp_job(struct mali_gp_job *job)
483 MALI_DEBUG_ASSERT_POINTER(job);
485 MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Timeline activation for job %u (0x%08X).\n",
486 mali_gp_job_get_id(job), job));
488 mali_scheduler_lock();
490 if (!mali_scheduler_queue_gp_job(job)) {
491 /* Failed to enqueue job, release job (with error) */
493 mali_scheduler_unlock();
495 mali_timeline_tracker_release(mali_gp_job_get_tracker(job));
496 mali_gp_job_signal_pp_tracker(job, MALI_FALSE);
498 /* This will notify user space and close the job object */
499 mali_scheduler_complete_gp_job(job, MALI_FALSE,
500 MALI_TRUE, MALI_FALSE);
502 return MALI_SCHEDULER_MASK_EMPTY;
505 mali_scheduler_unlock();
507 return MALI_SCHEDULER_MASK_GP;
510 mali_scheduler_mask mali_scheduler_activate_pp_job(struct mali_pp_job *job)
512 MALI_DEBUG_ASSERT_POINTER(job);
514 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Timeline activation for job %u (0x%08X).\n",
515 mali_pp_job_get_id(job), job));
517 if (MALI_TRUE == mali_timeline_tracker_activation_error(
518 mali_pp_job_get_tracker(job))) {
519 MALI_DEBUG_PRINT(3, ("Mali PP scheduler: Job %u (0x%08X) activated with error, aborting.\n",
520 mali_pp_job_get_id(job), job));
522 mali_scheduler_lock();
523 mali_pp_job_fb_lookup_remove(job);
524 mali_pp_job_mark_unstarted_failed(job);
525 mali_scheduler_unlock();
527 mali_timeline_tracker_release(mali_pp_job_get_tracker(job));
529 /* This will notify user space and close the job object */
530 mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, MALI_FALSE);
532 return MALI_SCHEDULER_MASK_EMPTY;
535 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
536 if (mali_pp_job_needs_dma_buf_mapping(job)) {
537 mali_scheduler_deferred_pp_job_queue(job);
538 return MALI_SCHEDULER_MASK_EMPTY;
540 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
542 mali_scheduler_lock();
544 if (!mali_scheduler_queue_pp_job(job)) {
545 /* Failed to enqueue job, release job (with error) */
546 mali_pp_job_fb_lookup_remove(job);
547 mali_pp_job_mark_unstarted_failed(job);
548 mali_scheduler_unlock();
550 mali_timeline_tracker_release(mali_pp_job_get_tracker(job));
552 /* This will notify user space and close the job object */
553 mali_scheduler_complete_pp_job(job, 0, MALI_TRUE, MALI_FALSE);
555 return MALI_SCHEDULER_MASK_EMPTY;
558 mali_scheduler_unlock();
559 return MALI_SCHEDULER_MASK_PP;
562 void mali_scheduler_complete_gp_job(struct mali_gp_job *job,
564 mali_bool user_notification,
567 if (user_notification) {
568 mali_scheduler_return_gp_job_to_user(job, success);
572 _mali_osk_pm_dev_ref_put();
574 if (mali_utilization_enabled()) {
575 mali_utilization_gp_end();
577 mali_pm_record_gpu_idle(MALI_TRUE);
580 mali_gp_job_delete(job);
583 void mali_scheduler_complete_pp_job(struct mali_pp_job *job,
584 u32 num_cores_in_virtual,
585 mali_bool user_notification,
588 job->user_notification = user_notification;
589 job->num_pp_cores_in_virtual = num_cores_in_virtual;
592 #if defined(CONFIG_MALI_DVFS)
593 if (mali_pp_job_is_window_surface(job)) {
594 struct mali_session_data *session;
595 session = mali_pp_job_get_session(job);
596 mali_session_inc_num_window_jobs(session);
600 _mali_osk_pm_dev_ref_put();
602 if (mali_utilization_enabled()) {
603 mali_utilization_pp_end();
605 mali_pm_record_gpu_idle(MALI_FALSE);
608 /* With ZRAM feature enabled, all pp jobs will be force to use deferred delete. */
609 mali_scheduler_deferred_pp_job_delete(job);
612 void mali_scheduler_abort_session(struct mali_session_data *session)
614 struct mali_gp_job *gp_job;
615 struct mali_gp_job *gp_tmp;
616 struct mali_pp_job *pp_job;
617 struct mali_pp_job *pp_tmp;
618 _MALI_OSK_LIST_HEAD_STATIC_INIT(removed_jobs_gp);
619 _MALI_OSK_LIST_HEAD_STATIC_INIT(removed_jobs_pp);
621 MALI_DEBUG_ASSERT_POINTER(session);
622 MALI_DEBUG_ASSERT(session->is_aborting);
624 MALI_DEBUG_PRINT(3, ("Mali scheduler: Aborting all queued jobs from session 0x%08X.\n",
627 mali_scheduler_lock();
629 /* Remove from GP normal priority queue */
630 _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &job_queue_gp.normal_pri,
631 struct mali_gp_job, list) {
632 if (mali_gp_job_get_session(gp_job) == session) {
633 mali_gp_job_list_move(gp_job, &removed_jobs_gp);
634 job_queue_gp.depth--;
635 job_queue_gp.big_job_num -= gp_job->big_job ? 1 : 0;
639 /* Remove from GP high priority queue */
640 _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &job_queue_gp.high_pri,
641 struct mali_gp_job, list) {
642 if (mali_gp_job_get_session(gp_job) == session) {
643 mali_gp_job_list_move(gp_job, &removed_jobs_gp);
644 job_queue_gp.depth--;
645 job_queue_gp.big_job_num -= gp_job->big_job ? 1 : 0;
649 /* Remove from PP normal priority queue */
650 _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp,
651 &job_queue_pp.normal_pri,
652 struct mali_pp_job, list) {
653 if (mali_pp_job_get_session(pp_job) == session) {
654 mali_pp_job_fb_lookup_remove(pp_job);
656 job_queue_pp.depth -=
657 mali_pp_job_unstarted_sub_job_count(
659 mali_pp_job_mark_unstarted_failed(pp_job);
661 if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(pp_job)) {
662 if (mali_pp_job_is_complete(pp_job)) {
663 mali_pp_job_list_move(pp_job,
666 mali_pp_job_list_remove(pp_job);
672 /* Remove from PP high priority queue */
673 _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp,
674 &job_queue_pp.high_pri,
675 struct mali_pp_job, list) {
676 if (mali_pp_job_get_session(pp_job) == session) {
677 mali_pp_job_fb_lookup_remove(pp_job);
679 job_queue_pp.depth -=
680 mali_pp_job_unstarted_sub_job_count(
682 mali_pp_job_mark_unstarted_failed(pp_job);
684 if (MALI_FALSE == mali_pp_job_has_unstarted_sub_jobs(pp_job)) {
685 if (mali_pp_job_is_complete(pp_job)) {
686 mali_pp_job_list_move(pp_job,
689 mali_pp_job_list_remove(pp_job);
696 * Release scheduler lock so we can release trackers
697 * (which will potentially queue new jobs)
699 mali_scheduler_unlock();
701 /* Release and complete all (non-running) found GP jobs */
702 _MALI_OSK_LIST_FOREACHENTRY(gp_job, gp_tmp, &removed_jobs_gp,
703 struct mali_gp_job, list) {
704 mali_timeline_tracker_release(mali_gp_job_get_tracker(gp_job));
705 mali_gp_job_signal_pp_tracker(gp_job, MALI_FALSE);
706 _mali_osk_list_delinit(&gp_job->list);
707 mali_scheduler_complete_gp_job(gp_job,
708 MALI_FALSE, MALI_FALSE, MALI_TRUE);
711 /* Release and complete non-running PP jobs */
712 _MALI_OSK_LIST_FOREACHENTRY(pp_job, pp_tmp, &removed_jobs_pp,
713 struct mali_pp_job, list) {
714 mali_timeline_tracker_release(mali_pp_job_get_tracker(pp_job));
715 _mali_osk_list_delinit(&pp_job->list);
716 mali_scheduler_complete_pp_job(pp_job, 0,
717 MALI_FALSE, MALI_TRUE);
721 _mali_osk_errcode_t _mali_ukk_gp_start_job(void *ctx,
722 _mali_uk_gp_start_job_s *uargs)
724 struct mali_session_data *session;
725 struct mali_gp_job *job;
726 mali_timeline_point point;
727 u32 __user *point_ptr = NULL;
729 MALI_DEBUG_ASSERT_POINTER(uargs);
730 MALI_DEBUG_ASSERT_POINTER(ctx);
732 session = (struct mali_session_data *)(uintptr_t)ctx;
734 job = mali_gp_job_create(session, uargs, mali_scheduler_get_new_id(),
737 MALI_PRINT_ERROR(("Failed to create GP job.\n"));
738 return _MALI_OSK_ERR_NOMEM;
741 point_ptr = (u32 __user *)(uintptr_t)mali_gp_job_get_timeline_point_ptr(job);
743 point = mali_scheduler_submit_gp_job(session, job);
745 if (0 != _mali_osk_put_user(((u32) point), point_ptr)) {
747 * Let user space know that something failed
748 * after the job was started.
750 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
753 return _MALI_OSK_ERR_OK;
756 _mali_osk_errcode_t _mali_ukk_pp_start_job(void *ctx,
757 _mali_uk_pp_start_job_s *uargs)
759 struct mali_session_data *session;
760 struct mali_pp_job *job;
761 mali_timeline_point point;
762 u32 __user *point_ptr = NULL;
764 MALI_DEBUG_ASSERT_POINTER(uargs);
765 MALI_DEBUG_ASSERT_POINTER(ctx);
767 session = (struct mali_session_data *)(uintptr_t)ctx;
769 job = mali_pp_job_create(session, uargs, mali_scheduler_get_new_id());
771 MALI_PRINT_ERROR(("Failed to create PP job.\n"));
772 return _MALI_OSK_ERR_NOMEM;
775 point_ptr = (u32 __user *)(uintptr_t)mali_pp_job_get_timeline_point_ptr(job);
777 point = mali_scheduler_submit_pp_job(session, job);
780 if (0 != _mali_osk_put_user(((u32) point), point_ptr)) {
782 * Let user space know that something failed
783 * after the job was started.
785 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
788 return _MALI_OSK_ERR_OK;
791 _mali_osk_errcode_t _mali_ukk_pp_and_gp_start_job(void *ctx,
792 _mali_uk_pp_and_gp_start_job_s *uargs)
794 struct mali_session_data *session;
795 _mali_uk_pp_and_gp_start_job_s kargs;
796 struct mali_pp_job *pp_job;
797 struct mali_gp_job *gp_job;
798 u32 __user *point_ptr = NULL;
799 mali_timeline_point point;
800 _mali_uk_pp_start_job_s __user *pp_args;
801 _mali_uk_gp_start_job_s __user *gp_args;
803 MALI_DEBUG_ASSERT_POINTER(ctx);
804 MALI_DEBUG_ASSERT_POINTER(uargs);
806 session = (struct mali_session_data *) ctx;
808 if (0 != _mali_osk_copy_from_user(&kargs, uargs,
809 sizeof(_mali_uk_pp_and_gp_start_job_s))) {
810 return _MALI_OSK_ERR_NOMEM;
813 pp_args = (_mali_uk_pp_start_job_s __user *)(uintptr_t)kargs.pp_args;
814 gp_args = (_mali_uk_gp_start_job_s __user *)(uintptr_t)kargs.gp_args;
816 pp_job = mali_pp_job_create(session, pp_args,
817 mali_scheduler_get_new_id());
818 if (NULL == pp_job) {
819 MALI_PRINT_ERROR(("Failed to create PP job.\n"));
820 return _MALI_OSK_ERR_NOMEM;
823 gp_job = mali_gp_job_create(session, gp_args,
824 mali_scheduler_get_new_id(),
825 mali_pp_job_get_tracker(pp_job));
826 if (NULL == gp_job) {
827 MALI_PRINT_ERROR(("Failed to create GP job.\n"));
828 mali_pp_job_delete(pp_job);
829 return _MALI_OSK_ERR_NOMEM;
832 point_ptr = (u32 __user *)(uintptr_t)mali_pp_job_get_timeline_point_ptr(pp_job);
835 mali_scheduler_submit_gp_job(session, gp_job);
839 point = mali_scheduler_submit_pp_job(session, pp_job);
842 if (0 != _mali_osk_put_user(((u32) point), point_ptr)) {
844 * Let user space know that something failed
845 * after the jobs were started.
847 return _MALI_OSK_ERR_ITEM_NOT_FOUND;
850 return _MALI_OSK_ERR_OK;
853 void _mali_ukk_pp_job_disable_wb(_mali_uk_pp_disable_wb_s *args)
855 struct mali_session_data *session;
856 struct mali_pp_job *job;
857 struct mali_pp_job *tmp;
860 MALI_DEBUG_ASSERT_POINTER(args);
861 MALI_DEBUG_ASSERT(NULL != (void *)(uintptr_t)args->ctx);
863 session = (struct mali_session_data *)(uintptr_t)args->ctx;
865 fb_lookup_id = args->fb_id & MALI_PP_JOB_FB_LOOKUP_LIST_MASK;
867 mali_scheduler_lock();
869 /* Iterate over all jobs for given frame builder_id. */
870 _MALI_OSK_LIST_FOREACHENTRY(job, tmp,
871 &session->pp_job_fb_lookup_list[fb_lookup_id],
872 struct mali_pp_job, session_fb_lookup_list) {
873 MALI_DEBUG_CODE(u32 disable_mask = 0);
875 if (mali_pp_job_get_frame_builder_id(job) !=
877 MALI_DEBUG_PRINT(4, ("Mali PP scheduler: Disable WB mismatching FB.\n"));
881 MALI_DEBUG_CODE(disable_mask |= 0xD << (4 * 3));
883 if (mali_pp_job_get_wb0_source_addr(job) == args->wb0_memory) {
884 MALI_DEBUG_CODE(disable_mask |= 0x1 << (4 * 1));
885 mali_pp_job_disable_wb0(job);
888 if (mali_pp_job_get_wb1_source_addr(job) == args->wb1_memory) {
889 MALI_DEBUG_CODE(disable_mask |= 0x2 << (4 * 2));
890 mali_pp_job_disable_wb1(job);
893 if (mali_pp_job_get_wb2_source_addr(job) == args->wb2_memory) {
894 MALI_DEBUG_CODE(disable_mask |= 0x3 << (4 * 3));
895 mali_pp_job_disable_wb2(job);
897 MALI_DEBUG_PRINT(3, ("Mali PP scheduler: Disable WB: 0x%X.\n",
901 mali_scheduler_unlock();
904 #if MALI_STATE_TRACKING
905 u32 mali_scheduler_dump_state(char *buf, u32 size)
909 n += _mali_osk_snprintf(buf + n, size - n, "GP queues\n");
910 n += _mali_osk_snprintf(buf + n, size - n,
911 "\tQueue depth: %u\n", job_queue_gp.depth);
912 n += _mali_osk_snprintf(buf + n, size - n,
913 "\tNormal priority queue is %s\n",
914 _mali_osk_list_empty(&job_queue_gp.normal_pri) ?
915 "empty" : "not empty");
916 n += _mali_osk_snprintf(buf + n, size - n,
917 "\tHigh priority queue is %s\n",
918 _mali_osk_list_empty(&job_queue_gp.high_pri) ?
919 "empty" : "not empty");
921 n += _mali_osk_snprintf(buf + n, size - n,
923 n += _mali_osk_snprintf(buf + n, size - n,
924 "\tQueue depth: %u\n", job_queue_pp.depth);
925 n += _mali_osk_snprintf(buf + n, size - n,
926 "\tNormal priority queue is %s\n",
927 _mali_osk_list_empty(&job_queue_pp.normal_pri)
928 ? "empty" : "not empty");
929 n += _mali_osk_snprintf(buf + n, size - n,
930 "\tHigh priority queue is %s\n",
931 _mali_osk_list_empty(&job_queue_pp.high_pri)
932 ? "empty" : "not empty");
934 n += _mali_osk_snprintf(buf + n, size - n, "\n");
941 * ---------- Implementation of static functions ----------
944 static mali_timeline_point mali_scheduler_submit_gp_job(
945 struct mali_session_data *session, struct mali_gp_job *job)
947 mali_timeline_point point;
949 MALI_DEBUG_ASSERT_POINTER(session);
950 MALI_DEBUG_ASSERT_POINTER(job);
952 /* Add job to Timeline system. */
953 point = mali_timeline_system_add_tracker(session->timeline_system,
954 mali_gp_job_get_tracker(job), MALI_TIMELINE_GP);
959 static mali_timeline_point mali_scheduler_submit_pp_job(
960 struct mali_session_data *session, struct mali_pp_job *job)
962 mali_timeline_point point;
964 MALI_DEBUG_ASSERT_POINTER(session);
965 MALI_DEBUG_ASSERT_POINTER(job);
967 mali_scheduler_lock();
969 * Adding job to the lookup list used to quickly discard
970 * writeback units of queued jobs.
972 mali_pp_job_fb_lookup_add(job);
973 mali_scheduler_unlock();
975 /* Add job to Timeline system. */
976 point = mali_timeline_system_add_tracker(session->timeline_system,
977 mali_pp_job_get_tracker(job), MALI_TIMELINE_PP);
982 static mali_bool mali_scheduler_queue_gp_job(struct mali_gp_job *job)
984 struct mali_session_data *session;
985 _mali_osk_list_t *queue;
987 MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
988 MALI_DEBUG_ASSERT_POINTER(job);
990 session = mali_gp_job_get_session(job);
991 MALI_DEBUG_ASSERT_POINTER(session);
993 if (unlikely(session->is_aborting)) {
994 MALI_DEBUG_PRINT(4, ("Mali GP scheduler: Job %u (0x%08X) queued while session is aborting.\n",
995 mali_gp_job_get_id(job), job));
996 return MALI_FALSE; /* job not queued */
999 mali_gp_job_set_cache_order(job, mali_scheduler_get_new_cache_order());
1001 /* Determine which queue the job should be added to. */
1002 if (session->use_high_priority_job_queue) {
1003 queue = &job_queue_gp.high_pri;
1005 queue = &job_queue_gp.normal_pri;
1008 job_queue_gp.depth += 1;
1009 job_queue_gp.big_job_num += (job->big_job) ? 1 : 0;
1011 /* Add job to queue (mali_gp_job_queue_add find correct place). */
1012 mali_gp_job_list_add(job, queue);
1015 * We hold a PM reference for every job we hold queued (and running)
1016 * It is important that we take this reference after job has been
1017 * added the the queue so that any runtime resume could schedule this
1018 * job right there and then.
1020 _mali_osk_pm_dev_ref_get_async();
1022 if (mali_utilization_enabled()) {
1024 * We cheat a little bit by counting the GP as busy from the
1025 * time a GP job is queued. This will be fine because we only
1026 * loose the tiny idle gap between jobs, but we will instead
1027 * get less utilization work to do (less locks taken)
1029 mali_utilization_gp_start();
1032 mali_pm_record_gpu_active(MALI_TRUE);
1034 /* Add profiling events for job enqueued */
1035 _mali_osk_profiling_add_event(
1036 MALI_PROFILING_EVENT_TYPE_SINGLE |
1037 MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1038 MALI_PROFILING_EVENT_REASON_SINGLE_SW_GP_ENQUEUE,
1039 mali_gp_job_get_pid(job),
1040 mali_gp_job_get_tid(job),
1041 mali_gp_job_get_frame_builder_id(job),
1042 mali_gp_job_get_flush_id(job),
1045 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
1046 trace_gpu_job_enqueue(mali_gp_job_get_tid(job),
1047 mali_gp_job_get_id(job), "GP");
1050 MALI_DEBUG_PRINT(3, ("Mali GP scheduler: Job %u (0x%08X) queued\n",
1051 mali_gp_job_get_id(job), job));
1053 return MALI_TRUE; /* job queued */
1056 static mali_bool mali_scheduler_queue_pp_job(struct mali_pp_job *job)
1058 struct mali_session_data *session;
1059 _mali_osk_list_t *queue = NULL;
1061 MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
1062 MALI_DEBUG_ASSERT_POINTER(job);
1064 session = mali_pp_job_get_session(job);
1065 MALI_DEBUG_ASSERT_POINTER(session);
1067 if (unlikely(session->is_aborting)) {
1068 MALI_DEBUG_PRINT(2, ("Mali PP scheduler: Job %u (0x%08X) queued while session is aborting.\n",
1069 mali_pp_job_get_id(job), job));
1070 return MALI_FALSE; /* job not queued */
1071 } else if (unlikely(MALI_SWAP_IN_FAIL == job->swap_status)) {
1072 MALI_DEBUG_PRINT(2, ("Mali PP scheduler: Job %u (0x%08X) queued while swap in failed.\n",
1073 mali_pp_job_get_id(job), job));
1077 mali_pp_job_set_cache_order(job, mali_scheduler_get_new_cache_order());
1079 if (session->use_high_priority_job_queue) {
1080 queue = &job_queue_pp.high_pri;
1082 queue = &job_queue_pp.normal_pri;
1085 job_queue_pp.depth +=
1086 mali_pp_job_get_sub_job_count(job);
1088 /* Add job to queue (mali_gp_job_queue_add find correct place). */
1089 mali_pp_job_list_add(job, queue);
1092 * We hold a PM reference for every job we hold queued (and running)
1093 * It is important that we take this reference after job has been
1094 * added the the queue so that any runtime resume could schedule this
1095 * job right there and then.
1097 _mali_osk_pm_dev_ref_get_async();
1099 if (mali_utilization_enabled()) {
1101 * We cheat a little bit by counting the PP as busy from the
1102 * time a PP job is queued. This will be fine because we only
1103 * loose the tiny idle gap between jobs, but we will instead
1104 * get less utilization work to do (less locks taken)
1106 mali_utilization_pp_start();
1109 mali_pm_record_gpu_active(MALI_FALSE);
1111 /* Add profiling events for job enqueued */
1112 _mali_osk_profiling_add_event(
1113 MALI_PROFILING_EVENT_TYPE_SINGLE |
1114 MALI_PROFILING_EVENT_CHANNEL_SOFTWARE |
1115 MALI_PROFILING_EVENT_REASON_SINGLE_SW_PP_ENQUEUE,
1116 mali_pp_job_get_pid(job),
1117 mali_pp_job_get_tid(job),
1118 mali_pp_job_get_frame_builder_id(job),
1119 mali_pp_job_get_flush_id(job),
1122 #if defined(CONFIG_GPU_TRACEPOINTS) && defined(CONFIG_TRACEPOINTS)
1123 trace_gpu_job_enqueue(mali_pp_job_get_tid(job),
1124 mali_pp_job_get_id(job), "PP");
1127 MALI_DEBUG_PRINT(3, ("Mali PP scheduler: %s job %u (0x%08X) with %u parts queued.\n",
1128 mali_pp_job_is_virtual(job)
1129 ? "Virtual" : "Physical",
1130 mali_pp_job_get_id(job), job,
1131 mali_pp_job_get_sub_job_count(job)));
1133 return MALI_TRUE; /* job queued */
1136 static void mali_scheduler_return_gp_job_to_user(struct mali_gp_job *job,
1139 _mali_uk_gp_job_finished_s *jobres;
1140 struct mali_session_data *session;
1141 _mali_osk_notification_t *notification;
1143 MALI_DEBUG_ASSERT_POINTER(job);
1145 session = mali_gp_job_get_session(job);
1146 MALI_DEBUG_ASSERT_POINTER(session);
1148 notification = mali_gp_job_get_finished_notification(job);
1149 MALI_DEBUG_ASSERT_POINTER(notification);
1151 jobres = notification->result_buffer;
1152 MALI_DEBUG_ASSERT_POINTER(jobres);
1154 jobres->pending_big_job_num = mali_scheduler_job_gp_big_job_count();
1156 jobres->user_job_ptr = mali_gp_job_get_user_id(job);
1157 if (MALI_TRUE == success) {
1158 jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS;
1160 jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR;
1162 jobres->heap_current_addr = mali_gp_job_get_current_heap_addr(job);
1163 jobres->perf_counter0 = mali_gp_job_get_perf_counter_value0(job);
1164 jobres->perf_counter1 = mali_gp_job_get_perf_counter_value1(job);
1166 mali_session_send_notification(session, notification);
1169 void mali_scheduler_return_pp_job_to_user(struct mali_pp_job *job,
1170 u32 num_cores_in_virtual)
1173 u32 num_counters_to_copy;
1174 _mali_uk_pp_job_finished_s *jobres;
1175 struct mali_session_data *session;
1176 _mali_osk_notification_t *notification;
1178 if (MALI_TRUE == mali_pp_job_use_no_notification(job)) {
1182 MALI_DEBUG_ASSERT_POINTER(job);
1184 session = mali_pp_job_get_session(job);
1185 MALI_DEBUG_ASSERT_POINTER(session);
1187 notification = mali_pp_job_get_finished_notification(job);
1188 MALI_DEBUG_ASSERT_POINTER(notification);
1190 jobres = notification->result_buffer;
1191 MALI_DEBUG_ASSERT_POINTER(jobres);
1193 jobres->user_job_ptr = mali_pp_job_get_user_id(job);
1194 if (MALI_TRUE == mali_pp_job_was_success(job)) {
1195 jobres->status = _MALI_UK_JOB_STATUS_END_SUCCESS;
1197 jobres->status = _MALI_UK_JOB_STATUS_END_UNKNOWN_ERR;
1200 if (mali_pp_job_is_virtual(job)) {
1201 num_counters_to_copy = num_cores_in_virtual;
1203 num_counters_to_copy = mali_pp_job_get_sub_job_count(job);
1206 for (i = 0; i < num_counters_to_copy; i++) {
1207 jobres->perf_counter0[i] =
1208 mali_pp_job_get_perf_counter_value0(job, i);
1209 jobres->perf_counter1[i] =
1210 mali_pp_job_get_perf_counter_value1(job, i);
1211 jobres->perf_counter_src0 =
1212 mali_pp_job_get_pp_counter_global_src0();
1213 jobres->perf_counter_src1 =
1214 mali_pp_job_get_pp_counter_global_src1();
1217 mali_session_send_notification(session, notification);
1220 static void mali_scheduler_deferred_pp_job_delete(struct mali_pp_job *job)
1222 MALI_DEBUG_ASSERT_POINTER(job);
1224 _mali_osk_spinlock_irq_lock(scheduler_pp_job_delete_lock);
1225 mali_pp_job_list_addtail(job, &scheduler_pp_job_deletion_queue);
1226 _mali_osk_spinlock_irq_unlock(scheduler_pp_job_delete_lock);
1228 _mali_osk_wq_schedule_work(scheduler_wq_pp_job_delete);
1231 void mali_scheduler_do_pp_job_delete(void *arg)
1233 _MALI_OSK_LIST_HEAD_STATIC_INIT(list);
1234 struct mali_pp_job *job;
1235 struct mali_pp_job *tmp;
1240 * Quickly "unhook" the jobs pending to be deleted, so we can release
1241 * the lock before we start deleting the job objects
1242 * (without any locks held)
1244 _mali_osk_spinlock_irq_lock(scheduler_pp_job_delete_lock);
1245 _mali_osk_list_move_list(&scheduler_pp_job_deletion_queue, &list);
1246 _mali_osk_spinlock_irq_unlock(scheduler_pp_job_delete_lock);
1248 _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list,
1249 struct mali_pp_job, list) {
1250 _mali_osk_list_delinit(&job->list);
1252 mali_pp_job_delete(job); /* delete the job object itself */
1256 #if defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE)
1258 static void mali_scheduler_deferred_pp_job_queue(struct mali_pp_job *job)
1260 MALI_DEBUG_ASSERT_POINTER(job);
1262 _mali_osk_spinlock_irq_lock(scheduler_pp_job_queue_lock);
1263 mali_pp_job_list_addtail(job, &scheduler_pp_job_queue_list);
1264 _mali_osk_spinlock_irq_unlock(scheduler_pp_job_queue_lock);
1266 _mali_osk_wq_schedule_work(scheduler_wq_pp_job_queue);
1269 static void mali_scheduler_do_pp_job_queue(void *arg)
1271 _MALI_OSK_LIST_HEAD_STATIC_INIT(list);
1272 struct mali_pp_job *job;
1273 struct mali_pp_job *tmp;
1274 mali_scheduler_mask schedule_mask = MALI_SCHEDULER_MASK_EMPTY;
1279 * Quickly "unhook" the jobs pending to be queued, so we can release
1280 * the lock before we start queueing the job objects
1281 * (without any locks held)
1283 _mali_osk_spinlock_irq_lock(scheduler_pp_job_queue_lock);
1284 _mali_osk_list_move_list(&scheduler_pp_job_queue_list, &list);
1285 _mali_osk_spinlock_irq_unlock(scheduler_pp_job_queue_lock);
1287 /* First loop through all jobs and do the pre-work (no locks needed) */
1288 _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list,
1289 struct mali_pp_job, list) {
1290 if (mali_pp_job_needs_dma_buf_mapping(job)) {
1292 * This operation could fail, but we continue anyway,
1293 * because the worst that could happen is that this
1294 * job will fail due to a Mali page fault.
1296 mali_dma_buf_map_job(job);
1300 mali_scheduler_lock();
1302 /* Then loop through all jobs again to queue them (lock needed) */
1303 _MALI_OSK_LIST_FOREACHENTRY(job, tmp, &list,
1304 struct mali_pp_job, list) {
1306 /* Remove from scheduler_pp_job_queue_list before queueing */
1307 mali_pp_job_list_remove(job);
1309 if (mali_scheduler_queue_pp_job(job)) {
1310 /* Job queued successfully */
1311 schedule_mask |= MALI_SCHEDULER_MASK_PP;
1313 /* Failed to enqueue job, release job (with error) */
1314 mali_pp_job_fb_lookup_remove(job);
1315 mali_pp_job_mark_unstarted_failed(job);
1317 /* unlock scheduler in this uncommon case */
1318 mali_scheduler_unlock();
1320 schedule_mask |= mali_timeline_tracker_release(
1321 mali_pp_job_get_tracker(job));
1323 /* Notify user space and close the job object */
1324 mali_scheduler_complete_pp_job(job, 0, MALI_TRUE,
1327 mali_scheduler_lock();
1331 mali_scheduler_unlock();
1333 /* Trigger scheduling of jobs */
1334 mali_executor_schedule_from_mask(schedule_mask, MALI_FALSE);
1337 #endif /* defined(MALI_SCHEDULER_USE_DEFERRED_PP_JOB_QUEUE) */
1339 void mali_scheduler_gp_pp_job_queue_print(void)
1341 struct mali_gp_job *gp_job = NULL;
1342 struct mali_gp_job *tmp_gp_job = NULL;
1343 struct mali_pp_job *pp_job = NULL;
1344 struct mali_pp_job *tmp_pp_job = NULL;
1346 MALI_DEBUG_ASSERT_LOCK_HELD(mali_scheduler_lock_obj);
1347 MALI_DEBUG_ASSERT_LOCK_HELD(mali_executor_lock_obj);
1349 /* dump job queup status */
1350 if ((0 == job_queue_gp.depth) && (0 == job_queue_pp.depth)) {
1351 MALI_PRINT(("No GP&PP job in the job queue.\n"));
1355 MALI_PRINT(("Total (%d) GP job in the job queue.\n", job_queue_gp.depth));
1356 if (job_queue_gp.depth > 0) {
1357 if (!_mali_osk_list_empty(&job_queue_gp.high_pri)) {
1358 _MALI_OSK_LIST_FOREACHENTRY(gp_job, tmp_gp_job, &job_queue_gp.high_pri,
1359 struct mali_gp_job, list) {
1360 MALI_PRINT(("GP job(%p) id = %d tid = %d pid = %d in the gp job high_pri queue\n", gp_job, gp_job->id, gp_job->tid, gp_job->pid));
1364 if (!_mali_osk_list_empty(&job_queue_gp.normal_pri)) {
1365 _MALI_OSK_LIST_FOREACHENTRY(gp_job, tmp_gp_job, &job_queue_gp.normal_pri,
1366 struct mali_gp_job, list) {
1367 MALI_PRINT(("GP job(%p) id = %d tid = %d pid = %d in the gp job normal_pri queue\n", gp_job, gp_job->id, gp_job->tid, gp_job->pid));
1372 MALI_PRINT(("Total (%d) PP job in the job queue.\n", job_queue_pp.depth));
1373 if (job_queue_pp.depth > 0) {
1374 if (!_mali_osk_list_empty(&job_queue_pp.high_pri)) {
1375 _MALI_OSK_LIST_FOREACHENTRY(pp_job, tmp_pp_job, &job_queue_pp.high_pri,
1376 struct mali_pp_job, list) {
1377 if (mali_pp_job_is_virtual(pp_job)) {
1378 MALI_PRINT(("PP Virtual job(%p) id = %d tid = %d pid = %d in the pp job high_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid));
1380 MALI_PRINT(("PP Physical job(%p) id = %d tid = %d pid = %d in the pp job high_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid));
1385 if (!_mali_osk_list_empty(&job_queue_pp.normal_pri)) {
1386 _MALI_OSK_LIST_FOREACHENTRY(pp_job, tmp_pp_job, &job_queue_pp.normal_pri,
1387 struct mali_pp_job, list) {
1388 if (mali_pp_job_is_virtual(pp_job)) {
1389 MALI_PRINT(("PP Virtual job(%p) id = %d tid = %d pid = %d in the pp job normal_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid));
1391 MALI_PRINT(("PP Physical job(%p) id = %d tid = %d pid = %d in the pp job normal_pri queue\n", pp_job, pp_job->id, pp_job->tid, pp_job->pid));
1397 /* dump group running job status */
1398 mali_executor_running_status_print();