3 * (C) COPYRIGHT ARM Limited. All rights reserved.
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
21 * @file mali_kbase_js.h
28 #include <malisw/mali_malisw.h>
30 #include "mali_kbase_js_defs.h"
31 #include "mali_kbase_js_policy.h"
32 #include "mali_kbase_defs.h"
33 #include "mali_kbase_debug.h"
35 #include "mali_kbase_js_ctx_attr.h"
38 * @addtogroup base_api
43 * @addtogroup base_kbase_api
48 * @addtogroup kbase_js Job Scheduler Internal APIs
51 * These APIs are Internal to KBase and are available for use by the
52 * @ref kbase_js_policy "Job Scheduler Policy APIs"
56 * @brief Initialize the Job Scheduler
58 * The kbasep_js_device_data sub-structure of \a kbdev must be zero
59 * initialized before passing to the kbasep_js_devdata_init() function. This is
60 * to give efficient error path code.
62 mali_error kbasep_js_devdata_init(kbase_device * const kbdev);
65 * @brief Halt the Job Scheduler.
67 * It is safe to call this on \a kbdev even if it the kbasep_js_device_data
68 * sub-structure was never initialized/failed initialization, to give efficient
71 * For this to work, the kbasep_js_device_data sub-structure of \a kbdev must
72 * be zero initialized before passing to the kbasep_js_devdata_init()
73 * function. This is to give efficient error path code.
75 * It is a Programming Error to call this whilst there are still kbase_context
76 * structures registered with this scheduler.
79 void kbasep_js_devdata_halt(kbase_device *kbdev);
82 * @brief Terminate the Job Scheduler
84 * It is safe to call this on \a kbdev even if it the kbasep_js_device_data
85 * sub-structure was never initialized/failed initialization, to give efficient
88 * For this to work, the kbasep_js_device_data sub-structure of \a kbdev must
89 * be zero initialized before passing to the kbasep_js_devdata_init()
90 * function. This is to give efficient error path code.
92 * It is a Programming Error to call this whilst there are still kbase_context
93 * structures registered with this scheduler.
95 void kbasep_js_devdata_term(kbase_device *kbdev);
98 * @brief Initialize the Scheduling Component of a kbase_context on the Job Scheduler.
100 * This effectively registers a kbase_context with a Job Scheduler.
102 * It does not register any jobs owned by the kbase_context with the scheduler.
103 * Those must be separately registered by kbasep_js_add_job().
105 * The kbase_context must be zero intitialized before passing to the
106 * kbase_js_init() function. This is to give efficient error path code.
108 mali_error kbasep_js_kctx_init(kbase_context * const kctx);
111 * @brief Terminate the Scheduling Component of a kbase_context on the Job Scheduler
113 * This effectively de-registers a kbase_context from its Job Scheduler
115 * It is safe to call this on a kbase_context that has never had or failed
116 * initialization of its jctx.sched_info member, to give efficient error-path
119 * For this to work, the kbase_context must be zero intitialized before passing
120 * to the kbase_js_init() function.
122 * It is a Programming Error to call this whilst there are still jobs
123 * registered with this context.
125 void kbasep_js_kctx_term(kbase_context *kctx);
128 * @brief Add a job chain to the Job Scheduler, and take necessary actions to
129 * schedule the context/run the job.
131 * This atomically does the following:
132 * - Update the numbers of jobs information
133 * - Add the job to the run pool if necessary (part of init_job)
135 * Once this is done, then an appropriate action is taken:
136 * - If the ctx is scheduled, it attempts to start the next job (which might be
138 * - Otherwise, and if this is the first job on the context, it enqueues it on
141 * The Policy's Queue can be updated by this in the following ways:
142 * - In the above case that this is the first job on the context
143 * - If the job is high priority and the context is not scheduled, then it
144 * could cause the Policy to schedule out a low-priority context, allowing
145 * this context to be scheduled in.
147 * If the context is already scheduled on the RunPool, then adding a job to it
148 * is guarenteed not to update the Policy Queue. And so, the caller is
149 * guarenteed to not need to try scheduling a context from the Run Pool - it
150 * can safely assert that the result is MALI_FALSE.
152 * It is a programming error to have more than U32_MAX jobs in flight at a time.
154 * The following locking conditions are made on the caller:
155 * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex.
156 * - it must \em not hold kbasep_js_device_data::runpool_irq::lock (as this will be
157 * obtained internally)
158 * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be
159 * obtained internally)
160 * - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used internally).
162 * @return MALI_TRUE indicates that the Policy Queue was updated, and so the
163 * caller will need to try scheduling a context onto the Run Pool.
164 * @return MALI_FALSE indicates that no updates were made to the Policy Queue,
165 * so no further action is required from the caller. This is \b always returned
166 * when the context is currently scheduled.
168 mali_bool kbasep_js_add_job(kbase_context *kctx, kbase_jd_atom *atom);
171 * @brief Remove a job chain from the Job Scheduler, except for its 'retained state'.
173 * Completely removing a job requires several calls:
174 * - kbasep_js_copy_atom_retained_state(), to capture the 'retained state' of
176 * - kbasep_js_remove_job(), to partially remove the atom from the Job Scheduler
177 * - kbasep_js_runpool_release_ctx_and_katom_retained_state(), to release the
178 * remaining state held as part of the job having been run.
180 * In the common case of atoms completing normally, this set of actions is more optimal for spinlock purposes than having kbasep_js_remove_job() handle all of the actions.
182 * In the case of cancelling atoms, it is easier to call kbasep_js_remove_cancelled_job(), which handles all the necessary actions.
184 * It is a programming error to call this when:
185 * - \a atom is not a job belonging to kctx.
186 * - \a atom has already been removed from the Job Scheduler.
187 * - \a atom is still in the runpool:
188 * - it has not been removed with kbasep_js_policy_dequeue_job()
189 * - or, it has not been removed with kbasep_js_policy_dequeue_job_irq()
191 * Do not use this for removing jobs being killed by kbase_jd_cancel() - use
192 * kbasep_js_remove_cancelled_job() instead.
194 * The following locking conditions are made on the caller:
195 * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex.
198 void kbasep_js_remove_job(kbase_device *kbdev, kbase_context *kctx, kbase_jd_atom *atom);
201 * @brief Completely remove a job chain from the Job Scheduler, in the case
202 * where the job chain was cancelled.
204 * This is a variant of kbasep_js_remove_job() that takes care of removing all
205 * of the retained state too. This is generally useful for cancelled atoms,
206 * which need not be handled in an optimal way.
208 * It is a programming error to call this when:
209 * - \a atom is not a job belonging to kctx.
210 * - \a atom has already been removed from the Job Scheduler.
211 * - \a atom is still in the runpool:
212 * - it is not being killed with kbasep_jd_cancel()
213 * - or, it has not been removed with kbasep_js_policy_dequeue_job()
214 * - or, it has not been removed with kbasep_js_policy_dequeue_job_irq()
216 * The following locking conditions are made on the caller:
217 * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex.
218 * - it must \em not hold the kbasep_js_device_data::runpool_irq::lock, (as this will be
219 * obtained internally)
220 * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this could be
221 * obtained internally)
223 void kbasep_js_remove_cancelled_job(kbase_device *kbdev, kbase_context *kctx, kbase_jd_atom *katom);
226 * @brief Refcount a context as being busy, preventing it from being scheduled
229 * @note This function can safely be called from IRQ context.
231 * The following locking conditions are made on the caller:
232 * - it must \em not hold the kbasep_js_device_data::runpool_irq::lock, because
233 * it will be used internally.
235 * @return value != MALI_FALSE if the retain succeeded, and the context will not be scheduled out.
236 * @return MALI_FALSE if the retain failed (because the context is being/has been scheduled out).
238 mali_bool kbasep_js_runpool_retain_ctx(kbase_device *kbdev, kbase_context *kctx);
241 * @brief Refcount a context as being busy, preventing it from being scheduled
244 * @note This function can safely be called from IRQ context.
246 * The following locks must be held by the caller:
247 * - kbasep_js_device_data::runpool_irq::lock
249 * @return value != MALI_FALSE if the retain succeeded, and the context will not be scheduled out.
250 * @return MALI_FALSE if the retain failed (because the context is being/has been scheduled out).
252 mali_bool kbasep_js_runpool_retain_ctx_nolock(kbase_device *kbdev, kbase_context *kctx);
255 * @brief Lookup a context in the Run Pool based upon its current address space
256 * and ensure that is stays scheduled in.
258 * The context is refcounted as being busy to prevent it from scheduling
259 * out. It must be released with kbasep_js_runpool_release_ctx() when it is no
260 * longer required to stay scheduled in.
262 * @note This function can safely be called from IRQ context.
264 * The following locking conditions are made on the caller:
265 * - it must \em not hold the kbasep_js_device_data::runpoool_irq::lock, because
266 * it will be used internally.
268 * @return a valid kbase_context on success, which has been refcounted as being busy.
269 * @return NULL on failure, indicating that no context was found in \a as_nr
271 kbase_context *kbasep_js_runpool_lookup_ctx(kbase_device *kbdev, int as_nr);
274 * @brief Handling the requeuing/killing of a context that was evicted from the
275 * policy queue or runpool.
277 * This should be used whenever handing off a context that has been evicted
278 * from the policy queue or the runpool:
279 * - If the context is not dying and has jobs, it gets re-added to the policy
281 * - Otherwise, it is not added
283 * In addition, if the context is dying the jobs are killed asynchronously.
285 * In all cases, the Power Manager active reference is released
286 * (kbase_pm_context_idle()) whenever the has_pm_ref parameter is true. \a
287 * has_pm_ref must be set to false whenever the context was not previously in
288 * the runpool and does not hold a Power Manager active refcount. Note that
289 * contexts in a rollback of kbasep_js_try_schedule_head_ctx() might have an
290 * active refcount even though they weren't in the runpool.
292 * The following locking conditions are made on the caller:
293 * - it must hold kbasep_js_kctx_info::ctx::jsctx_mutex.
294 * - it must \em not hold kbasep_jd_device_data::queue_mutex (as this will be
295 * obtained internally)
297 void kbasep_js_runpool_requeue_or_kill_ctx(kbase_device *kbdev, kbase_context *kctx, mali_bool has_pm_ref);
300 * @brief Release a refcount of a context being busy, allowing it to be
303 * When the refcount reaches zero and the context \em might be scheduled out
304 * (depending on whether the Scheudling Policy has deemed it so, or if it has run
307 * If the context does get scheduled out, then The following actions will be
308 * taken as part of deschduling a context:
309 * - For the context being descheduled:
310 * - If the context is in the processing of dying (all the jobs are being
311 * removed from it), then descheduling also kills off any jobs remaining in the
313 * - If the context is not dying, and any jobs remain after descheduling the
314 * context then it is re-enqueued to the Policy's Queue.
315 * - Otherwise, the context is still known to the scheduler, but remains absent
316 * from the Policy Queue until a job is next added to it.
317 * - In all descheduling cases, the Power Manager active reference (obtained
318 * during kbasep_js_try_schedule_head_ctx()) is released (kbase_pm_context_idle()).
320 * Whilst the context is being descheduled, this also handles actions that
321 * cause more atoms to be run:
322 * - Attempt submitting atoms when the Context Attributes on the Runpool have
323 * changed. This is because the context being scheduled out could mean that
324 * there are more opportunities to run atoms.
325 * - Attempt submitting to a slot that was previously blocked due to affinity
326 * restrictions. This is usually only necessary when releasing a context
327 * happens as part of completing a previous job, but is harmless nonetheless.
328 * - Attempt scheduling in a new context (if one is available), and if necessary,
329 * running a job from that new context.
331 * Unlike retaining a context in the runpool, this function \b cannot be called
334 * It is a programming error to call this on a \a kctx that is not currently
335 * scheduled, or that already has a zero refcount.
337 * The following locking conditions are made on the caller:
338 * - it must \em not hold the kbasep_js_device_data::runpool_irq::lock, because
339 * it will be used internally.
340 * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex.
341 * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be
342 * obtained internally)
343 * - it must \em not hold the kbase_device::as[n].transaction_mutex (as this will be obtained internally)
344 * - it must \em not hold kbasep_jd_device_data::queue_mutex (as this will be
345 * obtained internally)
348 void kbasep_js_runpool_release_ctx(kbase_device *kbdev, kbase_context *kctx);
351 * @brief Variant of kbasep_js_runpool_release_ctx() that handles additional
352 * actions from completing an atom.
354 * This is usually called as part of completing an atom and releasing the
355 * refcount on the context held by the atom.
357 * Therefore, the extra actions carried out are part of handling actions queued
358 * on a completed atom, namely:
359 * - Releasing the atom's context attributes
360 * - Retrying the submission on a particular slot, because we couldn't submit
361 * on that slot from an IRQ handler.
363 * The locking conditions of this function are the same as those for
364 * kbasep_js_runpool_release_ctx()
366 void kbasep_js_runpool_release_ctx_and_katom_retained_state(kbase_device *kbdev, kbase_context *kctx, kbasep_js_atom_retained_state *katom_retained_state);
369 * @brief Try to submit the next job on a \b particular slot whilst in IRQ
370 * context, and whilst the caller already holds the runpool IRQ spinlock.
372 * \a *submit_count will be checked against
373 * KBASE_JS_MAX_JOB_SUBMIT_PER_SLOT_PER_IRQ to see whether too many jobs have
374 * been submitted. This is to prevent the IRQ handler looping over lots of GPU
375 * NULL jobs, which may complete whilst the IRQ handler is still processing. \a
376 * submit_count itself should point to kbase_device::slot_submit_count_irq[ \a js ],
377 * which is initialized to zero on entry to the IRQ handler.
379 * The following locks must be held by the caller:
380 * - kbasep_js_device_data::runpool_irq::lock
382 * @return truthful (i.e. != MALI_FALSE) if too many jobs were submitted from
383 * IRQ. Therefore, this indicates that submission should be retried from a
384 * work-queue, by using
385 * kbasep_js_try_run_next_job_on_slot_nolock()/kbase_js_try_run_jobs_on_slot().
386 * @return MALI_FALSE if submission had no problems: the GPU is either already
387 * full of jobs in the HEAD and NEXT registers, or we were able to get enough
388 * jobs from the Run Pool to fill the GPU's HEAD and NEXT registers.
390 mali_bool kbasep_js_try_run_next_job_on_slot_irq_nolock(kbase_device *kbdev, int js, s8 *submit_count);
393 * @brief Try to submit the next job on a particular slot, outside of IRQ context
395 * This obtains the Job Slot lock for the duration of the call only.
397 * Unlike kbasep_js_try_run_next_job_on_slot_irq_nolock(), there is no limit on
398 * submission, because eventually IRQ_THROTTLE will kick in to prevent us
399 * getting stuck in a loop of submitting GPU NULL jobs. This is because the IRQ
400 * handler will be delayed, and so this function will eventually fill up the
401 * space in our software 'submitted' slot (kbase_jm_slot::submitted).
403 * In addition, there's no return value - we'll run the maintenence functions
404 * on the Policy's Run Pool, but if there's nothing there after that, then the
405 * Run Pool is truely empty, and so no more action need be taken.
407 * The following locking conditions are made on the caller:
408 * - it must hold kbasep_js_device_data::runpool_mutex
409 * - it must hold kbasep_js_device_data::runpool_irq::lock
411 * This must only be called whilst the GPU is powered - for example, when
412 * kbdev->jsdata.nr_user_contexts_running > 0.
414 * @note The caller \em might be holding one of the
415 * kbasep_js_kctx_info::ctx::jsctx_mutex locks.
418 void kbasep_js_try_run_next_job_on_slot_nolock(kbase_device *kbdev, int js);
421 * @brief Try to submit the next job for each slot in the system, outside of IRQ context
423 * This will internally call kbasep_js_try_run_next_job_on_slot_nolock(), so similar
424 * locking conditions on the caller are required.
426 * The following locking conditions are made on the caller:
427 * - it must hold kbasep_js_device_data::runpool_mutex
428 * - it must hold kbasep_js_device_data::runpool_irq::lock
430 * @note The caller \em might be holding one of the
431 * kbasep_js_kctx_info::ctx::jsctx_mutex locks.
434 void kbasep_js_try_run_next_job_nolock(kbase_device *kbdev);
437 * @brief Try to schedule the next context onto the Run Pool
439 * This checks whether there's space in the Run Pool to accommodate a new
440 * context. If so, it attempts to dequeue a context from the Policy Queue, and
441 * submit this to the Run Pool.
443 * If the scheduling succeeds, then it also makes a call to
444 * kbasep_js_try_run_next_job_nolock(), in case the new context has jobs
445 * matching the job slot requirements, but no other currently scheduled context
448 * Whilst attempting to obtain a context from the policy queue, or add a
449 * context to the runpool, this function takes a Power Manager active
450 * reference. If for any reason a context cannot be added to the runpool, any
451 * reference obtained is released once the context is safely back in the policy
452 * queue. If no context was available on the policy queue, any reference
453 * obtained is released too.
455 * Only if the context gets placed in the runpool does the Power Manager active
456 * reference stay held (and is effectively now owned by the
457 * context/runpool). It is only released once the context is removed
458 * completely, or added back to the policy queue
459 * (e.g. kbasep_js_runpool_release_ctx(),
460 * kbasep_js_runpool_requeue_or_kill_ctx(), etc)
462 * If any of these actions fail (Run Pool Full, Policy Queue empty, can't get
463 * PM active reference due to a suspend, etc) then any actions taken are rolled
464 * back and the function just returns normally.
466 * The following locking conditions are made on the caller:
467 * - it must \em not hold the kbasep_js_device_data::runpool_irq::lock, because
468 * it will be used internally.
469 * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be
470 * obtained internally)
471 * - it must \em not hold the kbase_device::as[n].transaction_mutex (as this will be obtained internally)
472 * - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used internally).
473 * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex, because it will
474 * be used internally.
477 void kbasep_js_try_schedule_head_ctx(kbase_device *kbdev);
480 * @brief Schedule in a privileged context
482 * This schedules a context in regardless of the context priority.
483 * If the runpool is full, a context will be forced out of the runpool and the function will wait
484 * for the new context to be scheduled in.
485 * The context will be kept scheduled in (and the corresponding address space reserved) until
486 * kbasep_js_release_privileged_ctx is called).
488 * The following locking conditions are made on the caller:
489 * - it must \em not hold the kbasep_js_device_data::runpool_irq::lock, because
490 * it will be used internally.
491 * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be
492 * obtained internally)
493 * - it must \em not hold the kbase_device::as[n].transaction_mutex (as this will be obtained internally)
494 * - it must \em not hold kbasep_jd_device_data::queue_mutex (again, it's used internally).
495 * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex, because it will
496 * be used internally.
499 void kbasep_js_schedule_privileged_ctx(kbase_device *kbdev, kbase_context *kctx);
502 * @brief Release a privileged context, allowing it to be scheduled out.
504 * See kbasep_js_runpool_release_ctx for potential side effects.
506 * The following locking conditions are made on the caller:
507 * - it must \em not hold the kbasep_js_device_data::runpool_irq::lock, because
508 * it will be used internally.
509 * - it must \em not hold kbasep_js_kctx_info::ctx::jsctx_mutex.
510 * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be
511 * obtained internally)
512 * - it must \em not hold the kbase_device::as[n].transaction_mutex (as this will be obtained internally)
515 void kbasep_js_release_privileged_ctx(kbase_device *kbdev, kbase_context *kctx);
518 * @brief Handle the Job Scheduler component for the IRQ of a job finishing
520 * This does the following:
521 * -# Releases resources held by the atom
522 * -# if \a end_timestamp != NULL, updates the runpool's notion of time spent by a running ctx
523 * -# determines whether a context should be marked for scheduling out
524 * -# examines done_code to determine whether to submit the next job on the slot
525 * (picking from all ctxs in the runpool)
527 * In addition, if submission didn't happen (the submit-from-IRQ function
528 * failed or done_code didn't specify to start new jobs), then this sets a
529 * message on katom that submission needs to be retried from the worker thread.
531 * Normally, the time calculated from end_timestamp is rounded up to the
532 * minimum time precision. Therefore, to ensure the job is recorded as not
533 * spending any time, then set end_timestamp to NULL. For example, this is necessary when
534 * evicting jobs from JSn_HEAD_NEXT (because they didn't actually run).
536 * NOTE: It's possible to move the steps (2) and (3) (inc calculating job's time
537 * used) into the worker (outside of IRQ context), but this may allow a context
538 * to use up to twice as much timeslice as is allowed by the policy. For
539 * policies that order by time spent, this is not a problem for overall
540 * 'fairness', but can still increase latency between contexts.
542 * The following locking conditions are made on the caller:
543 * - it must hold kbasep_js_device_data::runpoool_irq::lock
545 void kbasep_js_job_done_slot_irq(kbase_jd_atom *katom, int slot_nr,
546 ktime_t *end_timestamp,
547 kbasep_js_atom_done_code done_code);
550 * @brief Try to submit the next job on each slot
552 * The following locks may be used:
553 * - kbasep_js_device_data::runpool_mutex
554 * - kbasep_js_device_data::runpool_irq::lock
556 void kbase_js_try_run_jobs(kbase_device *kbdev);
559 * @brief Try to submit the next job on a specfic slot
561 * The following locking conditions are made on the caller:
563 * - it must \em not hold kbasep_js_device_data::runpool_mutex (as this will be
564 * obtained internally)
565 * - it must \em not hold kbasep_js_device_data::runpool_irq::lock (as this
566 * will be obtained internally)
569 void kbase_js_try_run_jobs_on_slot(kbase_device *kbdev, int js);
572 * @brief Handle releasing cores for power management and affinity management,
573 * ensuring that cores are powered down and affinity tracking is updated.
575 * This must only be called on an atom that is not currently running, and has
576 * not been re-queued onto the context (and so does not need locking)
578 * This function enters at the following @ref kbase_atom_coreref_state states:
579 * - NO_CORES_REQUESTED
580 * - WAITING_FOR_REQUESTED_CORES
584 * It transitions the above states back to NO_CORES_REQUESTED by the end of the
585 * function call (possibly via intermediate states).
587 * No locks need be held by the caller, since this takes the necessary Power
588 * Management locks itself. The runpool_irq.lock is not taken (the work that
589 * requires it is handled by kbase_js_affinity_submit_to_blocked_slots() ).
591 * @note The corresponding kbasep_js_job_check_ref_cores() is private to the
592 * Job Scheduler, and is called automatically when running the next job.
594 void kbasep_js_job_check_deref_cores(kbase_device *kbdev, struct kbase_jd_atom *katom);
597 * @brief Suspend the job scheduler during a Power Management Suspend event.
599 * Causes all contexts to be removed from the runpool, and prevents any
600 * contexts from (re)entering the runpool.
602 * This does not handle suspending the one privileged context: the caller must
603 * instead do this by by suspending the GPU HW Counter Instrumentation.
605 * This will eventually cause all Power Management active references held by
606 * contexts on the runpool to be released, without running any more atoms.
608 * The caller must then wait for all Power Mangement active refcount to become
609 * zero before completing the suspend.
611 * The emptying mechanism may take some time to complete, since it can wait for
612 * jobs to complete naturally instead of forcing them to end quickly. However,
613 * this is bounded by the Job Scheduling Policy's Job Timeouts. Hence, this
614 * function is guaranteed to complete in a finite time whenever the Job
615 * Scheduling Policy implements Job Timeouts (such as those done by CFS).
617 void kbasep_js_suspend(kbase_device *kbdev);
620 * @brief Resume the Job Scheduler after a Power Management Resume event.
622 * This restores the actions from kbasep_js_suspend():
623 * - Schedules contexts back into the runpool
624 * - Resumes running atoms on the GPU
626 void kbasep_js_resume(kbase_device *kbdev);
634 * @brief Check that a context is allowed to submit jobs on this policy
636 * The purpose of this abstraction is to hide the underlying data size, and wrap up
637 * the long repeated line of code.
639 * As with any mali_bool, never test the return value with MALI_TRUE.
641 * The caller must hold kbasep_js_device_data::runpool_irq::lock.
643 static INLINE mali_bool kbasep_js_is_submit_allowed(kbasep_js_device_data *js_devdata, kbase_context *kctx)
647 /* Ensure context really is scheduled in */
648 KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
649 KBASE_DEBUG_ASSERT(kctx->jctx.sched_info.ctx.is_scheduled != MALI_FALSE);
651 test_bit = (u16) (1u << kctx->as_nr);
653 return (mali_bool) (js_devdata->runpool_irq.submit_allowed & test_bit);
657 * @brief Allow a context to submit jobs on this policy
659 * The purpose of this abstraction is to hide the underlying data size, and wrap up
660 * the long repeated line of code.
662 * The caller must hold kbasep_js_device_data::runpool_irq::lock.
664 static INLINE void kbasep_js_set_submit_allowed(kbasep_js_device_data *js_devdata, kbase_context *kctx)
668 /* Ensure context really is scheduled in */
669 KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
670 KBASE_DEBUG_ASSERT(kctx->jctx.sched_info.ctx.is_scheduled != MALI_FALSE);
672 set_bit = (u16) (1u << kctx->as_nr);
674 KBASE_LOG(3, kctx->kbdev->dev, "JS: Setting Submit Allowed on %p (as=%d)", kctx, kctx->as_nr);
676 js_devdata->runpool_irq.submit_allowed |= set_bit;
680 * @brief Prevent a context from submitting more jobs on this policy
682 * The purpose of this abstraction is to hide the underlying data size, and wrap up
683 * the long repeated line of code.
685 * The caller must hold kbasep_js_device_data::runpool_irq::lock.
687 static INLINE void kbasep_js_clear_submit_allowed(kbasep_js_device_data *js_devdata, kbase_context *kctx)
692 /* Ensure context really is scheduled in */
693 KBASE_DEBUG_ASSERT(kctx->as_nr != KBASEP_AS_NR_INVALID);
694 KBASE_DEBUG_ASSERT(kctx->jctx.sched_info.ctx.is_scheduled != MALI_FALSE);
696 clear_bit = (u16) (1u << kctx->as_nr);
697 clear_mask = ~clear_bit;
699 KBASE_LOG(3, kctx->kbdev->dev, "JS: Clearing Submit Allowed on %p (as=%d)", kctx, kctx->as_nr);
701 js_devdata->runpool_irq.submit_allowed &= clear_mask;
705 * @brief Manage the 'retry_submit_on_slot' part of a kbase_jd_atom
707 static INLINE void kbasep_js_clear_job_retry_submit(kbase_jd_atom *atom)
709 atom->retry_submit_on_slot = KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID;
713 * Mark a slot as requiring resubmission by carrying that information on a
716 * @note This can ASSERT in debug builds if the submit slot has been set to
717 * something other than the current value for @a js. This is because you might
718 * be unintentionally stopping more jobs being submitted on the old submit
719 * slot, and that might cause a scheduling-hang.
721 * @note If you can guarantee that the atoms for the original slot will be
722 * submitted on some other slot, then call kbasep_js_clear_job_retry_submit()
723 * first to silence the ASSERT.
725 static INLINE void kbasep_js_set_job_retry_submit_slot(kbase_jd_atom *atom, int js)
727 KBASE_DEBUG_ASSERT(0 <= js && js <= BASE_JM_MAX_NR_SLOTS);
728 KBASE_DEBUG_ASSERT(atom->retry_submit_on_slot == KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID
729 || atom->retry_submit_on_slot == js);
731 atom->retry_submit_on_slot = js;
735 * Create an initial 'invalid' atom retained state, that requires no
736 * atom-related work to be done on releasing with
737 * kbasep_js_runpool_release_ctx_and_katom_retained_state()
739 static INLINE void kbasep_js_atom_retained_state_init_invalid(kbasep_js_atom_retained_state *retained_state)
741 retained_state->event_code = BASE_JD_EVENT_NOT_STARTED;
742 retained_state->core_req = KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID;
743 retained_state->retry_submit_on_slot = KBASEP_JS_RETRY_SUBMIT_SLOT_INVALID;
747 * Copy atom state that can be made available after jd_done_nolock() is called
750 static INLINE void kbasep_js_atom_retained_state_copy(kbasep_js_atom_retained_state *retained_state, const kbase_jd_atom *katom)
752 retained_state->event_code = katom->event_code;
753 retained_state->core_req = katom->core_req;
754 retained_state->retry_submit_on_slot = katom->retry_submit_on_slot;
758 * @brief Determine whether an atom has finished (given its retained state),
759 * and so should be given back to userspace/removed from the system.
761 * Reasons for an atom not finishing include:
762 * - Being soft-stopped (and so, the atom should be resubmitted sometime later)
764 * @param[in] katom_retained_state the retained state of the atom to check
765 * @return MALI_FALSE if the atom has not finished
766 * @return !=MALI_FALSE if the atom has finished
768 static INLINE mali_bool kbasep_js_has_atom_finished(const kbasep_js_atom_retained_state *katom_retained_state)
770 return (mali_bool) (katom_retained_state->event_code != BASE_JD_EVENT_STOPPED && katom_retained_state->event_code != BASE_JD_EVENT_REMOVED_FROM_NEXT);
774 * @brief Determine whether a kbasep_js_atom_retained_state is valid
776 * An invalid kbasep_js_atom_retained_state is allowed, and indicates that the
777 * code should just ignore it.
779 * @param[in] katom_retained_state the atom's retained state to check
780 * @return MALI_FALSE if the retained state is invalid, and can be ignored
781 * @return !=MALI_FALSE if the retained state is valid
783 static INLINE mali_bool kbasep_js_atom_retained_state_is_valid(const kbasep_js_atom_retained_state *katom_retained_state)
785 return (mali_bool) (katom_retained_state->core_req != KBASEP_JS_ATOM_RETAINED_STATE_CORE_REQ_INVALID);
788 static INLINE mali_bool kbasep_js_get_atom_retry_submit_slot(const kbasep_js_atom_retained_state *katom_retained_state, int *res)
790 int js = katom_retained_state->retry_submit_on_slot;
792 return (mali_bool) (js >= 0);
795 #if KBASE_DEBUG_DISABLE_ASSERTS == 0
797 * Debug Check the refcount of a context. Only use within ASSERTs
799 * Obtains kbasep_js_device_data::runpool_irq::lock
801 * @return negative value if the context is not scheduled in
802 * @return current refcount of the context if it is scheduled in. The refcount
803 * is not guarenteed to be kept constant.
805 static INLINE int kbasep_js_debug_check_ctx_refcount(kbase_device *kbdev, kbase_context *kctx)
808 kbasep_js_device_data *js_devdata;
812 KBASE_DEBUG_ASSERT(kbdev != NULL);
813 KBASE_DEBUG_ASSERT(kctx != NULL);
814 js_devdata = &kbdev->js_data;
816 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
818 if (as_nr != KBASEP_AS_NR_INVALID)
819 result = js_devdata->runpool_irq.per_as_data[as_nr].as_busy_refcount;
821 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
825 #endif /* KBASE_DEBUG_DISABLE_ASSERTS == 0 */
828 * @brief Variant of kbasep_js_runpool_lookup_ctx() that can be used when the
829 * context is guarenteed to be already previously retained.
831 * It is a programming error to supply the \a as_nr of a context that has not
832 * been previously retained/has a busy refcount of zero. The only exception is
833 * when there is no ctx in \a as_nr (NULL returned).
835 * The following locking conditions are made on the caller:
836 * - it must \em not hold the kbasep_js_device_data::runpoool_irq::lock, because
837 * it will be used internally.
839 * @return a valid kbase_context on success, with a refcount that is guarenteed
840 * to be non-zero and unmodified by this function.
841 * @return NULL on failure, indicating that no context was found in \a as_nr
843 static INLINE kbase_context *kbasep_js_runpool_lookup_ctx_noretain(kbase_device *kbdev, int as_nr)
846 kbasep_js_device_data *js_devdata;
847 kbase_context *found_kctx;
848 kbasep_js_per_as_data *js_per_as_data;
850 KBASE_DEBUG_ASSERT(kbdev != NULL);
851 KBASE_DEBUG_ASSERT(0 <= as_nr && as_nr < BASE_MAX_NR_AS);
852 js_devdata = &kbdev->js_data;
853 js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
855 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
857 found_kctx = js_per_as_data->kctx;
858 KBASE_DEBUG_ASSERT(found_kctx == NULL || js_per_as_data->as_busy_refcount > 0);
860 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
866 * This will provide a conversion from time (us) to ticks of the gpu clock
867 * based on the minimum available gpu frequency.
868 * This is usually good to compute best/worst case (where the use of current
869 * frequency is not valid due to DVFS).
870 * e.g.: when you need the number of cycles to guarantee you won't wait for
871 * longer than 'us' time (you might have a shorter wait).
873 static INLINE u32 kbasep_js_convert_us_to_gpu_ticks_min_freq(kbase_device *kbdev, u32 us)
875 u32 gpu_freq = kbdev->gpu_props.props.core_props.gpu_freq_khz_min;
876 KBASE_DEBUG_ASSERT(0 != gpu_freq);
877 return us * (gpu_freq / 1000);
881 * This will provide a conversion from time (us) to ticks of the gpu clock
882 * based on the maximum available gpu frequency.
883 * This is usually good to compute best/worst case (where the use of current
884 * frequency is not valid due to DVFS).
885 * e.g.: When you need the number of cycles to guarantee you'll wait at least
886 * 'us' amount of time (but you might wait longer).
888 static INLINE u32 kbasep_js_convert_us_to_gpu_ticks_max_freq(kbase_device *kbdev, u32 us)
890 u32 gpu_freq = kbdev->gpu_props.props.core_props.gpu_freq_khz_max;
891 KBASE_DEBUG_ASSERT(0 != gpu_freq);
892 return us * (u32) (gpu_freq / 1000);
896 * This will provide a conversion from ticks of the gpu clock to time (us)
897 * based on the minimum available gpu frequency.
898 * This is usually good to compute best/worst case (where the use of current
899 * frequency is not valid due to DVFS).
900 * e.g.: When you need to know the worst-case wait that 'ticks' cycles will
901 * take (you guarantee that you won't wait any longer than this, but it may
904 static INLINE u32 kbasep_js_convert_gpu_ticks_to_us_min_freq(kbase_device *kbdev, u32 ticks)
906 u32 gpu_freq = kbdev->gpu_props.props.core_props.gpu_freq_khz_min;
907 KBASE_DEBUG_ASSERT(0 != gpu_freq);
908 return ticks / gpu_freq * 1000;
912 * This will provide a conversion from ticks of the gpu clock to time (us)
913 * based on the maximum available gpu frequency.
914 * This is usually good to compute best/worst case (where the use of current
915 * frequency is not valid due to DVFS).
916 * e.g.: When you need to know the best-case wait for 'tick' cycles (you
917 * guarantee to be waiting for at least this long, but it may be longer).
919 static INLINE u32 kbasep_js_convert_gpu_ticks_to_us_max_freq(kbase_device *kbdev, u32 ticks)
921 u32 gpu_freq = kbdev->gpu_props.props.core_props.gpu_freq_khz_max;
922 KBASE_DEBUG_ASSERT(0 != gpu_freq);
923 return ticks / gpu_freq * 1000;
926 /** @} *//* end group kbase_js */
927 /** @} *//* end group base_kbase_api */
928 /** @} *//* end group base_api */
930 #endif /* _KBASE_JS_H_ */