get_compat_pointer(struct kbase_context *kctx, const union kbase_pointer *p)
{
#ifdef CONFIG_COMPAT
- if (kctx->is_compat)
+ if (kbase_ctx_flag(kctx, KCTX_COMPAT))
return compat_ptr(p->compat_value);
#endif
return p->value;
dep_count = atomic_read(&dep_atom->dma_fence.dep_count);
if (likely(dep_count == -1)) {
dep_satisfied = true;
- } else if (dep_count == 0) {
+ } else {
/*
- * All fences for this atom has signaled, but
- * the worker that will queue the atom has not
- * yet run.
+ * There are either still active callbacks, or
+ * all fences for this @dep_atom has signaled,
+ * but the worker that will queue the atom has
+ * not yet run.
*
- * Mark the atom as handled by setting
- * dep_count to -1 so that the worker doesn't
- * queue the atom again.
- */
- atomic_set(&dep_atom->dma_fence.dep_count, -1);
- /*
- * Remove the atom from the list of dma-fence
- * waiting atoms.
+ * Wait for the fences to signal and the fence
+ * worker to run and handle @dep_atom. If
+ * @dep_atom was completed due to error on
+ * @katom, then the fence worker will pick up
+ * the complete status and error code set on
+ * @dep_atom above.
*/
- kbase_dma_fence_waiters_remove(dep_atom);
- dep_satisfied = true;
- } else {
dep_satisfied = false;
}
#endif /* CONFIG_MALI_DMA_FENCE */
}
#endif
+/**
+ * is_dep_valid - Validate that a dependency is valid for early dependency
+ * submission
+ * @katom: Dependency atom to validate
+ *
+ * A dependency is valid if any of the following are true :
+ * - It does not exist (a non-existent dependency does not block submission)
+ * - It is in the job scheduler
+ * - It has completed, does not have a failure event code, and has not been
+ * marked to fail in the future
+ *
+ * Return: true if valid, false otherwise
+ */
+static bool is_dep_valid(struct kbase_jd_atom *katom)
+{
+ /* If there's no dependency then this is 'valid' from the perspective of
+ * early dependency submission */
+ if (!katom)
+ return true;
+
+ /* Dependency must have reached the job scheduler */
+ if (katom->status < KBASE_JD_ATOM_STATE_IN_JS)
+ return false;
+
+ /* If dependency has completed and has failed or will fail then it is
+ * not valid */
+ if (katom->status >= KBASE_JD_ATOM_STATE_HW_COMPLETED &&
+ (katom->event_code != BASE_JD_EVENT_DONE ||
+ katom->will_fail_event_code))
+ return false;
+
+ return true;
+}
+
static void jd_try_submitting_deps(struct list_head *out_list,
struct kbase_jd_atom *node)
{
if (IS_GPU_ATOM(dep_atom) && !dep_atom->in_jd_list) {
/*Check if atom deps look sane*/
- bool dep0_valid = !dep_atom->dep[0].atom ||
- (dep_atom->dep[0].atom->status
- >= KBASE_JD_ATOM_STATE_IN_JS);
- bool dep1_valid = !dep_atom->dep[1].atom ||
- (dep_atom->dep[1].atom->status
- >= KBASE_JD_ATOM_STATE_IN_JS);
-
- if (dep0_valid && dep1_valid) {
+ bool dep0_valid = is_dep_valid(
+ dep_atom->dep[0].atom);
+ bool dep1_valid = is_dep_valid(
+ dep_atom->dep[1].atom);
+ bool dep_satisfied = true;
+#ifdef CONFIG_MALI_DMA_FENCE
+ int dep_count;
+
+ dep_count = atomic_read(
+ &dep_atom->dma_fence.dep_count);
+ if (likely(dep_count == -1)) {
+ dep_satisfied = true;
+ } else {
+ /*
+ * There are either still active callbacks, or
+ * all fences for this @dep_atom has signaled,
+ * but the worker that will queue the atom has
+ * not yet run.
+ *
+ * Wait for the fences to signal and the fence
+ * worker to run and handle @dep_atom. If
+ * @dep_atom was completed due to error on
+ * @katom, then the fence worker will pick up
+ * the complete status and error code set on
+ * @dep_atom above.
+ */
+ dep_satisfied = false;
+ }
+#endif /* CONFIG_MALI_DMA_FENCE */
+#ifdef CONFIG_KDS
+ dep_satisfied = dep_satisfied &&
+ dep_atom->kds_dep_satisfied;
+#endif
+
+ if (dep0_valid && dep1_valid && dep_satisfied) {
dep_atom->in_jd_list = true;
list_add(&dep_atom->jd_item, out_list);
}
for (i = 0; i < 2; i++)
jd_resolve_dep(&runnable_jobs, katom, i,
- kctx->jctx.sched_info.ctx.is_dying);
+ kbase_ctx_flag(kctx, KCTX_DYING));
if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES)
kbase_jd_post_external_resources(katom);
KBASE_DEBUG_ASSERT(node->status != KBASE_JD_ATOM_STATE_UNUSED);
if (node->status != KBASE_JD_ATOM_STATE_COMPLETED &&
- !kctx->jctx.sched_info.ctx.is_dying) {
+ !kbase_ctx_flag(kctx, KCTX_DYING)) {
need_to_try_schedule_context |= jd_run_atom(node);
} else {
node->event_code = katom->event_code;
katom->x_pre_dep = NULL;
katom->x_post_dep = NULL;
katom->will_fail_event_code = BASE_JD_EVENT_NOT_STARTED;
- katom->exit_protected_state = KBASE_ATOM_EXIT_PROTECTED_CHECK;
+
+ /* Implicitly sets katom->protected_state.enter as well. */
+ katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK;
+
katom->age = kctx->age_count++;
INIT_LIST_HEAD(&katom->jd_item);
atomic_set(&katom->dma_fence.dep_count, -1);
#endif
+ kbase_tlstream_tl_attrib_atom_state(katom, TL_ATOM_STATE_IDLE);
+
/* Don't do anything if there is a mess up with dependencies.
This is done in a separate cycle to check both the dependencies at ones, otherwise
it will be extra complexity to deal with 1st dependency ( just added to the list )
katom->status = KBASE_JD_ATOM_STATE_QUEUED;
}
+ /* For invalid priority, be most lenient and choose the default */
+ sched_prio = kbasep_js_atom_prio_to_sched_prio(user_atom->prio);
+ if (sched_prio == KBASE_JS_ATOM_SCHED_PRIO_INVALID)
+ sched_prio = KBASE_JS_ATOM_SCHED_PRIO_DEFAULT;
+ katom->sched_priority = sched_prio;
+
/* Create a new atom recording all dependencies it was set up with. */
kbase_tlstream_tl_new_atom(
katom,
kbase_jd_atom_id(kctx, katom));
+ kbase_tlstream_tl_attrib_atom_priority(katom, katom->sched_priority);
kbase_tlstream_tl_ret_atom_ctx(katom, kctx);
for (i = 0; i < 2; i++)
if (BASE_JD_DEP_TYPE_INVALID != kbase_jd_katom_dep_type(
goto out;
}
- /* For invalid priority, be most lenient and choose the default */
- sched_prio = kbasep_js_atom_prio_to_sched_prio(user_atom->prio);
- if (sched_prio == KBASE_JS_ATOM_SCHED_PRIO_INVALID)
- sched_prio = KBASE_JS_ATOM_SCHED_PRIO_DEFAULT;
- katom->sched_priority = sched_prio;
-
if (katom->core_req & BASE_JD_REQ_EXTERNAL_RESOURCES) {
/* handle what we need to do to access the external resources */
if (kbase_jd_pre_external_resources(katom, user_atom) != 0) {
beenthere(kctx, "%s", "Enter");
- if ((kctx->jctx.sched_info.ctx.flags & KBASE_CTX_FLAG_SUBMIT_DISABLED) != 0) {
+ if (kbase_ctx_flag(kctx, KCTX_SUBMIT_DISABLED)) {
dev_err(kbdev->dev, "Attempt to submit to a context that has SUBMIT_DISABLED set on it");
return -EINVAL;
}
struct kbasep_js_device_data *js_devdata;
u64 cache_jc = katom->jc;
struct kbasep_js_atom_retained_state katom_retained_state;
- bool schedule = false;
bool context_idle;
base_jd_core_req core_req = katom->core_req;
u64 affinity = katom->affinity;
* Begin transaction on JD context and JS context
*/
mutex_lock(&jctx->lock);
+ kbase_tlstream_tl_attrib_atom_state(katom, TL_ATOM_STATE_DONE);
mutex_lock(&js_devdata->queue_mutex);
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
* because it only happens in response to an IRQ from a job that was
* running.
*/
- KBASE_DEBUG_ASSERT(js_kctx_info->ctx.is_scheduled);
+ KBASE_DEBUG_ASSERT(kbase_ctx_flag(kctx, KCTX_SCHEDULED));
if (katom->event_code == BASE_JD_EVENT_STOPPED) {
/* Atom has been promoted to stopped */
mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
mutex_unlock(&js_devdata->queue_mutex);
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
katom->status = KBASE_JD_ATOM_STATE_IN_JS;
kbase_js_unpull(kctx, katom);
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
mutex_unlock(&jctx->lock);
return;
/* Retain state before the katom disappears */
kbasep_js_atom_retained_state_copy(&katom_retained_state, katom);
- if (!kbasep_js_has_atom_finished(&katom_retained_state)) {
- mutex_lock(&js_devdata->runpool_mutex);
- kbasep_js_clear_job_retry_submit(katom);
- /* An atom that has been hard-stopped might have previously
- * been soft-stopped and has just finished before the hard-stop
- * occurred. For this reason, clear the hard-stopped flag */
- katom->atom_flags &= ~(KBASE_KATOM_FLAG_BEEN_HARD_STOPPED);
- mutex_unlock(&js_devdata->runpool_mutex);
- }
-
- if (kbasep_js_has_atom_finished(&katom_retained_state))
- schedule = true;
-
context_idle = kbase_js_complete_atom_wq(kctx, katom);
KBASE_DEBUG_ASSERT(kbasep_js_has_atom_finished(&katom_retained_state));
mutex_unlock(&js_devdata->queue_mutex);
katom->atom_flags &= ~KBASE_KATOM_FLAG_HOLDING_CTX_REF;
/* jd_done_nolock() requires the jsctx_mutex lock to be dropped */
- schedule |= jd_done_nolock(katom, &kctx->completed_jobs);
+ jd_done_nolock(katom, &kctx->completed_jobs);
/* katom may have been freed now, do not use! */
if (context_idle) {
unsigned long flags;
+ context_idle = false;
mutex_lock(&js_devdata->queue_mutex);
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
/* If kbase_sched() has scheduled this context back in then
- * ctx_active will have been set after we marked it as inactive,
- * and another pm reference will have been taken, so drop our
- * reference. But do not call kbase_jm_idle_ctx(), as the
- * context is active and fast-starting is allowed.
+ * KCTX_ACTIVE will have been set after we marked it as
+ * inactive, and another pm reference will have been taken, so
+ * drop our reference. But do not call kbase_jm_idle_ctx(), as
+ * the context is active and fast-starting is allowed.
*
* If an atom has been fast-started then kctx->atoms_pulled will
- * be non-zero but ctx_active will still be false (as the
+ * be non-zero but KCTX_ACTIVE will still be false (as the
* previous pm reference has been inherited). Do NOT drop our
* reference, as it has been re-used, and leave the context as
* active.
*
- * If no new atoms have been started then ctx_active will still
+ * If no new atoms have been started then KCTX_ACTIVE will still
* be false and atoms_pulled will be zero, so drop the reference
* and call kbase_jm_idle_ctx().
*
* As the checks are done under both the queue_mutex and
- * runpool_irq.lock is should be impossible for this to race
+ * hwaccess_lock is should be impossible for this to race
* with the scheduler code.
*/
- if (kctx->ctx_active || !atomic_read(&kctx->atoms_pulled)) {
+ if (kbase_ctx_flag(kctx, KCTX_ACTIVE) ||
+ !atomic_read(&kctx->atoms_pulled)) {
/* Calling kbase_jm_idle_ctx() here will ensure that
* atoms are not fast-started when we drop the
- * runpool_irq.lock. This is not performed if ctx_active
- * is set as in that case another pm reference has been
- * taken and a fast-start would be valid.
+ * hwaccess_lock. This is not performed if
+ * KCTX_ACTIVE is set as in that case another pm
+ * reference has been taken and a fast-start would be
+ * valid.
*/
- if (!kctx->ctx_active)
+ if (!kbase_ctx_flag(kctx, KCTX_ACTIVE))
kbase_jm_idle_ctx(kbdev, kctx);
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock,
- flags);
-
- kbase_pm_context_idle(kbdev);
+ context_idle = true;
} else {
- kctx->ctx_active = true;
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock,
- flags);
+ kbase_ctx_flag_set(kctx, KCTX_ACTIVE);
}
+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
mutex_unlock(&js_devdata->queue_mutex);
}
kbasep_js_runpool_release_ctx_and_katom_retained_state(kbdev, kctx, &katom_retained_state);
- if (schedule)
- kbase_js_sched_all(kbdev);
+ kbase_js_sched_all(kbdev);
if (!atomic_dec_return(&kctx->work_count)) {
/* If worker now idle then post all events that jd_done_nolock()
kbase_backend_complete_wq_post_sched(kbdev, core_req, affinity,
coreref_state);
+ if (context_idle)
+ kbase_pm_context_idle(kbdev);
+
KBASE_TRACE_ADD(kbdev, JD_DONE_WORKER_END, kctx, NULL, cache_jc, 0);
}
* any), nor must we try to schedule out the context (it's already
* scheduled out).
*/
- KBASE_DEBUG_ASSERT(!js_kctx_info->ctx.is_scheduled);
+ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED));
/* Scheduler: Remove the job from the system */
mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
*
* Context:
* This can be called safely from atomic context.
- * The caller must hold kbasep_js_device_data.runpool_irq.lock
+ * The caller must hold kbdev->hwaccess_lock
*/
void kbase_jd_done(struct kbase_jd_atom *katom, int slot_nr,
ktime_t *end_timestamp, kbasep_js_atom_done_code done_code)
KBASE_TRACE_ADD(kbdev, JD_CANCEL, kctx, katom, katom->jc, 0);
/* This should only be done from a context that is not scheduled */
- KBASE_DEBUG_ASSERT(!js_kctx_info->ctx.is_scheduled);
+ KBASE_DEBUG_ASSERT(!kbase_ctx_flag(kctx, KCTX_SCHEDULED));
WARN_ON(work_pending(&katom->work));
projects / firefly-linux-kernel-4.4.55.git / blobdiff