2 * Copyright (C) 2011-2016 ARM Limited. All rights reserved.
4 * This program is free software and is provided to you under the terms of the GNU General Public License version 2
5 * as published by the Free Software Foundation, and any use by you of this program is subject to the terms of such GNU licence.
7 * A copy of the licence is included with the program, and can also be obtained from Free Software
8 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
11 #ifndef __MALI_PP_JOB_H__
12 #define __MALI_PP_JOB_H__
15 #include "mali_osk_list.h"
16 #include "mali_uk_types.h"
17 #include "mali_session.h"
18 #include "mali_kernel_common.h"
19 #include "regs/mali_200_regs.h"
20 #include "mali_kernel_core.h"
21 #include "mali_dlbu.h"
22 #include "mali_timeline.h"
23 #include "mali_scheduler.h"
24 #include "mali_executor.h"
25 #if defined(CONFIG_DMA_SHARED_BUFFER) && !defined(CONFIG_MALI_DMA_BUF_MAP_ON_ATTACH)
26 #include "linux/mali_memory_dma_buf.h"
28 #if defined(CONFIG_MALI_DMA_BUF_FENCE)
29 #include "linux/mali_dma_fence.h"
30 #include <linux/fence.h>
33 typedef enum pp_job_status {
40 * This structure represents a PP job, including all sub jobs.
42 * The PP job object itself is not protected by any single lock,
43 * but relies on other locks instead (scheduler, executor and timeline lock).
44 * Think of the job object as moving between these sub systems through-out
45 * its lifetime. Different part of the PP job struct is used by different
46 * subsystems. Accessor functions ensure that correct lock is taken.
47 * Do NOT access any data members directly from outside this module!
51 * These members are typically only set at creation,
52 * and only read later on.
53 * They do not require any lock protection.
55 _mali_uk_pp_start_job_s uargs; /**< Arguments from user space */
56 struct mali_session_data *session; /**< Session which submitted this job */
57 u32 pid; /**< Process ID of submitting process */
58 u32 tid; /**< Thread ID of submitting thread */
59 u32 id; /**< Identifier for this job in kernel space (sequential numbering) */
60 u32 cache_order; /**< Cache order used for L2 cache flushing (sequential numbering) */
61 struct mali_timeline_tracker tracker; /**< Timeline tracker for this job */
62 _mali_osk_notification_t *finished_notification; /**< Notification sent back to userspace on job complete */
63 u32 perf_counter_per_sub_job_count; /**< Number of values in the two arrays which is != MALI_HW_CORE_NO_COUNTER */
64 u32 perf_counter_per_sub_job_src0[_MALI_PP_MAX_SUB_JOBS]; /**< Per sub job counters src0 */
65 u32 perf_counter_per_sub_job_src1[_MALI_PP_MAX_SUB_JOBS]; /**< Per sub job counters src1 */
66 u32 sub_jobs_num; /**< Number of subjobs; set to 1 for Mali-450 if DLBU is used, otherwise equals number of PP cores */
68 pp_job_status swap_status; /**< Used to track each PP job swap status, if fail, we need to drop them in scheduler part */
69 mali_bool user_notification; /**< When we deferred delete PP job, we need to judge if we need to send job finish notification to user space */
70 u32 num_pp_cores_in_virtual; /**< How many PP cores we have when job finished */
73 * These members are used by both scheduler and executor.
74 * They are "protected" by atomic operations.
76 _mali_osk_atomic_t sub_jobs_completed; /**< Number of completed sub-jobs in this superjob */
77 _mali_osk_atomic_t sub_job_errors; /**< Bitfield with errors (errors for each single sub-job is or'ed together) */
80 * These members are used by scheduler, but only when no one else
81 * knows about this job object but the working function.
82 * No lock is thus needed for these.
84 u32 *memory_cookies; /**< Memory cookies attached to job */
87 * These members are used by the scheduler,
88 * protected by scheduler lock
90 _mali_osk_list_t list; /**< Used to link jobs together in the scheduler queue */
91 _mali_osk_list_t session_fb_lookup_list; /**< Used to link jobs together from the same frame builder in the session */
93 u32 sub_jobs_started; /**< Total number of sub-jobs started (always started in ascending order) */
96 * Set by executor/group on job completion, read by scheduler when
97 * returning job to user. Hold executor lock when setting,
98 * no lock needed when reading
100 u32 perf_counter_value0[_MALI_PP_MAX_SUB_JOBS]; /**< Value of performance counter 0 (to be returned to user space), one for each sub job */
101 u32 perf_counter_value1[_MALI_PP_MAX_SUB_JOBS]; /**< Value of performance counter 1 (to be returned to user space), one for each sub job */
103 #if defined(CONFIG_MALI_DMA_BUF_FENCE)
104 struct mali_dma_fence_context dma_fence_context; /**< The mali dma fence context to record dma fence waiters that this job wait for */
105 struct fence *rendered_dma_fence; /**< the new dma fence link to this job */
109 void mali_pp_job_initialize(void);
110 void mali_pp_job_terminate(void);
112 struct mali_pp_job *mali_pp_job_create(struct mali_session_data *session, _mali_uk_pp_start_job_s *uargs, u32 id);
113 void mali_pp_job_delete(struct mali_pp_job *job);
115 u32 mali_pp_job_get_perf_counter_src0(struct mali_pp_job *job, u32 sub_job);
116 u32 mali_pp_job_get_perf_counter_src1(struct mali_pp_job *job, u32 sub_job);
118 void mali_pp_job_set_pp_counter_global_src0(u32 counter);
119 void mali_pp_job_set_pp_counter_global_src1(u32 counter);
120 void mali_pp_job_set_pp_counter_sub_job_src0(u32 sub_job, u32 counter);
121 void mali_pp_job_set_pp_counter_sub_job_src1(u32 sub_job, u32 counter);
123 u32 mali_pp_job_get_pp_counter_global_src0(void);
124 u32 mali_pp_job_get_pp_counter_global_src1(void);
125 u32 mali_pp_job_get_pp_counter_sub_job_src0(u32 sub_job);
126 u32 mali_pp_job_get_pp_counter_sub_job_src1(u32 sub_job);
128 MALI_STATIC_INLINE u32 mali_pp_job_get_id(struct mali_pp_job *job)
130 MALI_DEBUG_ASSERT_POINTER(job);
131 return (NULL == job) ? 0 : job->id;
134 MALI_STATIC_INLINE void mali_pp_job_set_cache_order(struct mali_pp_job *job,
137 MALI_DEBUG_ASSERT_POINTER(job);
138 MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
139 job->cache_order = cache_order;
142 MALI_STATIC_INLINE u32 mali_pp_job_get_cache_order(struct mali_pp_job *job)
144 MALI_DEBUG_ASSERT_POINTER(job);
145 return (NULL == job) ? 0 : job->cache_order;
148 MALI_STATIC_INLINE u64 mali_pp_job_get_user_id(struct mali_pp_job *job)
150 MALI_DEBUG_ASSERT_POINTER(job);
151 return job->uargs.user_job_ptr;
154 MALI_STATIC_INLINE u32 mali_pp_job_get_frame_builder_id(struct mali_pp_job *job)
156 MALI_DEBUG_ASSERT_POINTER(job);
157 return job->uargs.frame_builder_id;
160 MALI_STATIC_INLINE u32 mali_pp_job_get_flush_id(struct mali_pp_job *job)
162 MALI_DEBUG_ASSERT_POINTER(job);
163 return job->uargs.flush_id;
166 MALI_STATIC_INLINE u32 mali_pp_job_get_pid(struct mali_pp_job *job)
168 MALI_DEBUG_ASSERT_POINTER(job);
172 MALI_STATIC_INLINE u32 mali_pp_job_get_tid(struct mali_pp_job *job)
174 MALI_DEBUG_ASSERT_POINTER(job);
178 MALI_STATIC_INLINE u32 *mali_pp_job_get_frame_registers(struct mali_pp_job *job)
180 MALI_DEBUG_ASSERT_POINTER(job);
181 return job->uargs.frame_registers;
184 MALI_STATIC_INLINE u32 *mali_pp_job_get_dlbu_registers(struct mali_pp_job *job)
186 MALI_DEBUG_ASSERT_POINTER(job);
187 return job->uargs.dlbu_registers;
190 MALI_STATIC_INLINE mali_bool mali_pp_job_is_virtual(struct mali_pp_job *job)
192 #if (defined(CONFIG_MALI450) || defined(CONFIG_MALI470))
193 MALI_DEBUG_ASSERT_POINTER(job);
194 return (0 == job->uargs.num_cores) ? MALI_TRUE : MALI_FALSE;
200 MALI_STATIC_INLINE u32 mali_pp_job_get_addr_frame(struct mali_pp_job *job, u32 sub_job)
202 MALI_DEBUG_ASSERT_POINTER(job);
204 if (mali_pp_job_is_virtual(job)) {
205 return MALI_DLBU_VIRT_ADDR;
206 } else if (0 == sub_job) {
207 return job->uargs.frame_registers[MALI200_REG_ADDR_FRAME / sizeof(u32)];
208 } else if (sub_job < _MALI_PP_MAX_SUB_JOBS) {
209 return job->uargs.frame_registers_addr_frame[sub_job - 1];
215 MALI_STATIC_INLINE u32 mali_pp_job_get_addr_stack(struct mali_pp_job *job, u32 sub_job)
217 MALI_DEBUG_ASSERT_POINTER(job);
220 return job->uargs.frame_registers[MALI200_REG_ADDR_STACK / sizeof(u32)];
221 } else if (sub_job < _MALI_PP_MAX_SUB_JOBS) {
222 return job->uargs.frame_registers_addr_stack[sub_job - 1];
228 void mali_pp_job_list_add(struct mali_pp_job *job, _mali_osk_list_t *list);
230 MALI_STATIC_INLINE void mali_pp_job_list_addtail(struct mali_pp_job *job,
231 _mali_osk_list_t *list)
233 _mali_osk_list_addtail(&job->list, list);
236 MALI_STATIC_INLINE void mali_pp_job_list_move(struct mali_pp_job *job,
237 _mali_osk_list_t *list)
239 MALI_DEBUG_ASSERT_POINTER(job);
240 MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
241 MALI_DEBUG_ASSERT(!_mali_osk_list_empty(&job->list));
242 _mali_osk_list_move(&job->list, list);
245 MALI_STATIC_INLINE void mali_pp_job_list_remove(struct mali_pp_job *job)
247 MALI_DEBUG_ASSERT_POINTER(job);
248 MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
249 _mali_osk_list_delinit(&job->list);
252 MALI_STATIC_INLINE u32 *mali_pp_job_get_wb0_registers(struct mali_pp_job *job)
254 MALI_DEBUG_ASSERT_POINTER(job);
255 return job->uargs.wb0_registers;
258 MALI_STATIC_INLINE u32 *mali_pp_job_get_wb1_registers(struct mali_pp_job *job)
260 MALI_DEBUG_ASSERT_POINTER(job);
261 return job->uargs.wb1_registers;
264 MALI_STATIC_INLINE u32 *mali_pp_job_get_wb2_registers(struct mali_pp_job *job)
266 MALI_DEBUG_ASSERT_POINTER(job);
267 return job->uargs.wb2_registers;
270 MALI_STATIC_INLINE u32 mali_pp_job_get_wb0_source_addr(struct mali_pp_job *job)
272 MALI_DEBUG_ASSERT_POINTER(job);
273 return job->uargs.wb0_registers[MALI200_REG_ADDR_WB_SOURCE_ADDR / sizeof(u32)];
276 MALI_STATIC_INLINE u32 mali_pp_job_get_wb1_source_addr(struct mali_pp_job *job)
278 MALI_DEBUG_ASSERT_POINTER(job);
279 return job->uargs.wb1_registers[MALI200_REG_ADDR_WB_SOURCE_ADDR / sizeof(u32)];
282 MALI_STATIC_INLINE u32 mali_pp_job_get_wb2_source_addr(struct mali_pp_job *job)
284 MALI_DEBUG_ASSERT_POINTER(job);
285 return job->uargs.wb2_registers[MALI200_REG_ADDR_WB_SOURCE_ADDR / sizeof(u32)];
288 MALI_STATIC_INLINE void mali_pp_job_disable_wb0(struct mali_pp_job *job)
290 MALI_DEBUG_ASSERT_POINTER(job);
291 job->uargs.wb0_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] = 0;
294 MALI_STATIC_INLINE void mali_pp_job_disable_wb1(struct mali_pp_job *job)
296 MALI_DEBUG_ASSERT_POINTER(job);
297 job->uargs.wb1_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] = 0;
300 MALI_STATIC_INLINE void mali_pp_job_disable_wb2(struct mali_pp_job *job)
302 MALI_DEBUG_ASSERT_POINTER(job);
303 job->uargs.wb2_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] = 0;
306 MALI_STATIC_INLINE mali_bool mali_pp_job_all_writeback_unit_disabled(struct mali_pp_job *job)
308 MALI_DEBUG_ASSERT_POINTER(job);
310 if (job->uargs.wb0_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] ||
311 job->uargs.wb1_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT] ||
312 job->uargs.wb2_registers[MALI200_REG_ADDR_WB_SOURCE_SELECT]
314 /* At least one output unit active */
318 /* All outputs are disabled - we can abort the job */
322 MALI_STATIC_INLINE void mali_pp_job_fb_lookup_add(struct mali_pp_job *job)
326 MALI_DEBUG_ASSERT_POINTER(job);
327 MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
329 fb_lookup_id = MALI_PP_JOB_FB_LOOKUP_LIST_MASK & job->uargs.frame_builder_id;
331 MALI_DEBUG_ASSERT(MALI_PP_JOB_FB_LOOKUP_LIST_SIZE > fb_lookup_id);
333 _mali_osk_list_addtail(&job->session_fb_lookup_list,
334 &job->session->pp_job_fb_lookup_list[fb_lookup_id]);
337 MALI_STATIC_INLINE void mali_pp_job_fb_lookup_remove(struct mali_pp_job *job)
339 MALI_DEBUG_ASSERT_POINTER(job);
340 MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
341 _mali_osk_list_delinit(&job->session_fb_lookup_list);
344 MALI_STATIC_INLINE struct mali_session_data *mali_pp_job_get_session(struct mali_pp_job *job)
346 MALI_DEBUG_ASSERT_POINTER(job);
350 MALI_STATIC_INLINE mali_bool mali_pp_job_has_started_sub_jobs(struct mali_pp_job *job)
352 MALI_DEBUG_ASSERT_POINTER(job);
353 MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
354 return (0 < job->sub_jobs_started) ? MALI_TRUE : MALI_FALSE;
357 MALI_STATIC_INLINE mali_bool mali_pp_job_has_unstarted_sub_jobs(struct mali_pp_job *job)
359 MALI_DEBUG_ASSERT_POINTER(job);
360 MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
361 return (job->sub_jobs_started < job->sub_jobs_num) ? MALI_TRUE : MALI_FALSE;
364 /* Function used when we are terminating a session with jobs. Return TRUE if it has a rendering job.
365 Makes sure that no new subjobs are started. */
366 MALI_STATIC_INLINE void mali_pp_job_mark_unstarted_failed(struct mali_pp_job *job)
371 MALI_DEBUG_ASSERT_POINTER(job);
372 MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
374 jobs_remaining = job->sub_jobs_num - job->sub_jobs_started;
375 job->sub_jobs_started += jobs_remaining;
377 /* Not the most optimal way, but this is only used in error cases */
378 for (i = 0; i < jobs_remaining; i++) {
379 _mali_osk_atomic_inc(&job->sub_jobs_completed);
380 _mali_osk_atomic_inc(&job->sub_job_errors);
384 MALI_STATIC_INLINE mali_bool mali_pp_job_is_complete(struct mali_pp_job *job)
386 MALI_DEBUG_ASSERT_POINTER(job);
387 return (job->sub_jobs_num ==
388 _mali_osk_atomic_read(&job->sub_jobs_completed)) ?
389 MALI_TRUE : MALI_FALSE;
392 MALI_STATIC_INLINE u32 mali_pp_job_get_first_unstarted_sub_job(struct mali_pp_job *job)
394 MALI_DEBUG_ASSERT_POINTER(job);
395 MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
396 return job->sub_jobs_started;
399 MALI_STATIC_INLINE u32 mali_pp_job_get_sub_job_count(struct mali_pp_job *job)
401 MALI_DEBUG_ASSERT_POINTER(job);
402 return job->sub_jobs_num;
405 MALI_STATIC_INLINE u32 mali_pp_job_unstarted_sub_job_count(struct mali_pp_job *job)
407 MALI_DEBUG_ASSERT_POINTER(job);
408 MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
409 MALI_DEBUG_ASSERT(job->sub_jobs_num >= job->sub_jobs_started);
410 return (job->sub_jobs_num - job->sub_jobs_started);
413 MALI_STATIC_INLINE u32 mali_pp_job_num_memory_cookies(struct mali_pp_job *job)
415 MALI_DEBUG_ASSERT_POINTER(job);
416 return job->uargs.num_memory_cookies;
419 MALI_STATIC_INLINE u32 mali_pp_job_get_memory_cookie(
420 struct mali_pp_job *job, u32 index)
422 MALI_DEBUG_ASSERT_POINTER(job);
423 MALI_DEBUG_ASSERT(index < job->uargs.num_memory_cookies);
424 MALI_DEBUG_ASSERT_POINTER(job->memory_cookies);
425 return job->memory_cookies[index];
428 MALI_STATIC_INLINE mali_bool mali_pp_job_needs_dma_buf_mapping(struct mali_pp_job *job)
430 MALI_DEBUG_ASSERT_POINTER(job);
432 if (0 < job->uargs.num_memory_cookies) {
439 MALI_STATIC_INLINE void mali_pp_job_mark_sub_job_started(struct mali_pp_job *job, u32 sub_job)
441 MALI_DEBUG_ASSERT_POINTER(job);
442 MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
444 /* Assert that we are marking the "first unstarted sub job" as started */
445 MALI_DEBUG_ASSERT(job->sub_jobs_started == sub_job);
447 job->sub_jobs_started++;
450 MALI_STATIC_INLINE void mali_pp_job_mark_sub_job_completed(struct mali_pp_job *job, mali_bool success)
452 MALI_DEBUG_ASSERT_POINTER(job);
454 _mali_osk_atomic_inc(&job->sub_jobs_completed);
455 if (MALI_FALSE == success) {
456 _mali_osk_atomic_inc(&job->sub_job_errors);
460 MALI_STATIC_INLINE mali_bool mali_pp_job_was_success(struct mali_pp_job *job)
462 MALI_DEBUG_ASSERT_POINTER(job);
463 if (0 == _mali_osk_atomic_read(&job->sub_job_errors)) {
469 MALI_STATIC_INLINE mali_bool mali_pp_job_use_no_notification(
470 struct mali_pp_job *job)
472 MALI_DEBUG_ASSERT_POINTER(job);
473 return (job->uargs.flags & _MALI_PP_JOB_FLAG_NO_NOTIFICATION) ?
474 MALI_TRUE : MALI_FALSE;
477 MALI_STATIC_INLINE mali_bool mali_pp_job_is_pilot_job(struct mali_pp_job *job)
480 * A pilot job is currently identified as jobs which
481 * require no callback notification.
483 return mali_pp_job_use_no_notification(job);
486 MALI_STATIC_INLINE _mali_osk_notification_t *
487 mali_pp_job_get_finished_notification(struct mali_pp_job *job)
489 _mali_osk_notification_t *notification;
491 MALI_DEBUG_ASSERT_POINTER(job);
492 MALI_DEBUG_ASSERT_POINTER(job->finished_notification);
494 notification = job->finished_notification;
495 job->finished_notification = NULL;
500 MALI_STATIC_INLINE mali_bool mali_pp_job_is_window_surface(
501 struct mali_pp_job *job)
503 MALI_DEBUG_ASSERT_POINTER(job);
504 return (job->uargs.flags & _MALI_PP_JOB_FLAG_IS_WINDOW_SURFACE)
505 ? MALI_TRUE : MALI_FALSE;
508 MALI_STATIC_INLINE mali_bool mali_pp_job_is_protected_job(struct mali_pp_job *job)
510 MALI_DEBUG_ASSERT_POINTER(job);
511 return (job->uargs.flags & _MALI_PP_JOB_FLAG_PROTECTED)
512 ? MALI_TRUE : MALI_FALSE;
515 MALI_STATIC_INLINE u32 mali_pp_job_get_perf_counter_flag(struct mali_pp_job *job)
517 MALI_DEBUG_ASSERT_POINTER(job);
518 return job->uargs.perf_counter_flag;
521 MALI_STATIC_INLINE u32 mali_pp_job_get_perf_counter_value0(struct mali_pp_job *job, u32 sub_job)
523 MALI_DEBUG_ASSERT_POINTER(job);
524 return job->perf_counter_value0[sub_job];
527 MALI_STATIC_INLINE u32 mali_pp_job_get_perf_counter_value1(struct mali_pp_job *job, u32 sub_job)
529 MALI_DEBUG_ASSERT_POINTER(job);
530 return job->perf_counter_value1[sub_job];
533 MALI_STATIC_INLINE void mali_pp_job_set_perf_counter_value0(struct mali_pp_job *job, u32 sub_job, u32 value)
535 MALI_DEBUG_ASSERT_POINTER(job);
536 MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
537 job->perf_counter_value0[sub_job] = value;
540 MALI_STATIC_INLINE void mali_pp_job_set_perf_counter_value1(struct mali_pp_job *job, u32 sub_job, u32 value)
542 MALI_DEBUG_ASSERT_POINTER(job);
543 MALI_DEBUG_ASSERT_EXECUTOR_LOCK_HELD();
544 job->perf_counter_value1[sub_job] = value;
547 MALI_STATIC_INLINE _mali_osk_errcode_t mali_pp_job_check(struct mali_pp_job *job)
549 MALI_DEBUG_ASSERT_POINTER(job);
550 if (mali_pp_job_is_virtual(job) && job->sub_jobs_num != 1) {
551 return _MALI_OSK_ERR_FAULT;
553 return _MALI_OSK_ERR_OK;
557 * Returns MALI_TRUE if this job has more than two sub jobs and all sub jobs are unstarted.
559 * @param job Job to check.
560 * @return MALI_TRUE if job has more than two sub jobs and all sub jobs are unstarted, MALI_FALSE if not.
562 MALI_STATIC_INLINE mali_bool mali_pp_job_is_large_and_unstarted(struct mali_pp_job *job)
564 MALI_DEBUG_ASSERT_POINTER(job);
565 MALI_DEBUG_ASSERT_SCHEDULER_LOCK_HELD();
566 MALI_DEBUG_ASSERT(!mali_pp_job_is_virtual(job));
568 return (0 == job->sub_jobs_started && 2 < job->sub_jobs_num);
572 * Get PP job's Timeline tracker.
575 * @return Pointer to Timeline tracker for the job.
577 MALI_STATIC_INLINE struct mali_timeline_tracker *mali_pp_job_get_tracker(struct mali_pp_job *job)
579 MALI_DEBUG_ASSERT_POINTER(job);
580 return &(job->tracker);
583 MALI_STATIC_INLINE u32 *mali_pp_job_get_timeline_point_ptr(
584 struct mali_pp_job *job)
586 MALI_DEBUG_ASSERT_POINTER(job);
587 return (u32 __user *)(uintptr_t)job->uargs.timeline_point_ptr;
591 #endif /* __MALI_PP_JOB_H__ */