/*
*
- * (C) COPYRIGHT ARM Limited. All rights reserved.
+ * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved.
*
* This program is free software and is provided to you under the terms of the
* GNU General Public License version 2 as published by the Free Software
*/
#include <mali_kbase.h>
-#include <mali_kbase_jm.h>
#include <mali_kbase_js.h>
#include <mali_kbase_js_policy_cfs.h>
+#include <linux/version.h>
#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
#include <linux/sched/rt.h>
#endif
*/
#define CINSTR_DUMPING_ENABLED (2 == MALI_INSTRUMENTATION_LEVEL)
-/** Fixed point constants used for runtime weight calculations */
+/* Fixed point constants used for runtime weight calculations */
#define WEIGHT_FIXEDPOINT_SHIFT 10
#define WEIGHT_TABLE_SIZE 40
#define WEIGHT_0_NICE (WEIGHT_TABLE_SIZE/2)
#define WEIGHT_0_VAL (1 << WEIGHT_FIXEDPOINT_SHIFT)
-#define LOOKUP_VARIANT_MASK ((1u<<KBASEP_JS_MAX_NR_CORE_REQ_VARIANTS) - 1u)
-
#define PROCESS_PRIORITY_MIN (-20)
#define PROCESS_PRIORITY_MAX (19)
-/** Core requirements that all the variants support */
-#define JS_CORE_REQ_ALL_OTHERS \
- (BASE_JD_REQ_CF | BASE_JD_REQ_V | BASE_JD_REQ_PERMON | BASE_JD_REQ_EXTERNAL_RESOURCES | BASEP_JD_REQ_EVENT_NEVER)
-
-/** Context requirements the all the variants support */
-
-/* In HW issue 8987 workaround, restrict Compute-only contexts and Compute jobs onto job slot[2],
- * which will ensure their affinity does not intersect GLES jobs */
-#define JS_CTX_REQ_ALL_OTHERS_8987 \
- (KBASE_CTX_FLAG_PRIVILEGED)
-#define JS_CORE_REQ_COMPUTE_SLOT_8987 \
- (BASE_JD_REQ_CS)
-#define JS_CORE_REQ_ONLY_COMPUTE_SLOT_8987 \
- (BASE_JD_REQ_ONLY_COMPUTE)
-
-/* Otherwise, compute-only contexts/compute jobs can use any job slot */
-#define JS_CTX_REQ_ALL_OTHERS \
- (KBASE_CTX_FLAG_PRIVILEGED | KBASE_CTX_FLAG_HINT_ONLY_COMPUTE)
-#define JS_CORE_REQ_COMPUTE_SLOT \
- (BASE_JD_REQ_CS | BASE_JD_REQ_ONLY_COMPUTE)
-
-/* core_req variants are ordered by least restrictive first, so that our
- * algorithm in cached_variant_idx_init picks the least restrictive variant for
- * each job . Note that coherent_group requirement is added to all CS variants as the
- * selection of job-slot does not depend on the coherency requirement. */
-static const struct kbasep_atom_req core_req_variants[] = {
- {
- /* 0: Fragment variant */
- (JS_CORE_REQ_ALL_OTHERS | BASE_JD_REQ_FS | BASE_JD_REQ_FS_AFBC |
- BASE_JD_REQ_COHERENT_GROUP),
- (JS_CTX_REQ_ALL_OTHERS),
- 0},
- {
- /* 1: Compute variant, can use all coregroups */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_COMPUTE_SLOT),
- (JS_CTX_REQ_ALL_OTHERS),
- 0},
- {
- /* 2: Compute variant, uses only coherent coregroups */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_COMPUTE_SLOT | BASE_JD_REQ_COHERENT_GROUP),
- (JS_CTX_REQ_ALL_OTHERS),
- 0},
- {
- /* 3: Compute variant, might only use coherent coregroup, and must use tiling */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_COMPUTE_SLOT | BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_T),
- (JS_CTX_REQ_ALL_OTHERS),
- 0},
-
- {
- /* 4: Unused */
- 0,
- 0,
- 0},
-
- {
- /* 5: Compute variant for specific-coherent-group targetting CoreGroup 0 */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_COMPUTE_SLOT | BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP),
- (JS_CTX_REQ_ALL_OTHERS),
- 0 /* device_nr */
- },
- {
- /* 6: Compute variant for specific-coherent-group targetting CoreGroup 1 */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_COMPUTE_SLOT | BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP),
- (JS_CTX_REQ_ALL_OTHERS),
- 1 /* device_nr */
- },
-
- /* Unused core_req variants, to bring the total up to a power of 2 */
- {
- /* 7 */
- 0,
- 0,
- 0},
-};
-
-static const struct kbasep_atom_req core_req_variants_8987[] = {
- {
- /* 0: Fragment variant */
- (JS_CORE_REQ_ALL_OTHERS | BASE_JD_REQ_FS | BASE_JD_REQ_COHERENT_GROUP),
- (JS_CTX_REQ_ALL_OTHERS_8987),
- 0},
- {
- /* 1: Compute variant, can use all coregroups */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_COMPUTE_SLOT_8987),
- (JS_CTX_REQ_ALL_OTHERS_8987),
- 0},
- {
- /* 2: Compute variant, uses only coherent coregroups */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_COMPUTE_SLOT_8987 | BASE_JD_REQ_COHERENT_GROUP),
- (JS_CTX_REQ_ALL_OTHERS_8987),
- 0},
- {
- /* 3: Compute variant, might only use coherent coregroup, and must use tiling */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_COMPUTE_SLOT_8987 | BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_T),
- (JS_CTX_REQ_ALL_OTHERS_8987),
- 0},
-
- {
- /* 4: Variant guarenteed to support Compute contexts/atoms
- *
- * In the case of a context that's specified as 'Only Compute', it'll
- * not allow Tiler or Fragment atoms, and so those get rejected */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_ONLY_COMPUTE_SLOT_8987 | BASE_JD_REQ_COHERENT_GROUP),
- (JS_CTX_REQ_ALL_OTHERS_8987 | KBASE_CTX_FLAG_HINT_ONLY_COMPUTE),
- 0},
-
- {
- /* 5: Compute variant for specific-coherent-group targetting CoreGroup 0
- * Specifically, this only allows 'Only Compute' contexts/atoms */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_ONLY_COMPUTE_SLOT_8987 | BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP),
- (JS_CTX_REQ_ALL_OTHERS_8987 | KBASE_CTX_FLAG_HINT_ONLY_COMPUTE),
- 0 /* device_nr */
- },
- {
- /* 6: Compute variant for specific-coherent-group targetting CoreGroup 1
- * Specifically, this only allows 'Only Compute' contexts/atoms */
- (JS_CORE_REQ_ALL_OTHERS | JS_CORE_REQ_ONLY_COMPUTE_SLOT_8987 | BASE_JD_REQ_COHERENT_GROUP | BASE_JD_REQ_SPECIFIC_COHERENT_GROUP),
- (JS_CTX_REQ_ALL_OTHERS_8987 | KBASE_CTX_FLAG_HINT_ONLY_COMPUTE),
- 1 /* device_nr */
- },
- /* Unused core_req variants, to bring the total up to a power of 2 */
- {
- /* 7 */
- 0,
- 0,
- 0},
-};
-
-#define CORE_REQ_VARIANT_FRAGMENT 0
-#define CORE_REQ_VARIANT_COMPUTE_ALL_CORES 1
-#define CORE_REQ_VARIANT_COMPUTE_ONLY_COHERENT_GROUP 2
-#define CORE_REQ_VARIANT_COMPUTE_OR_TILING 3
-#define CORE_REQ_VARIANT_COMPUTE_SPECIFIC_COHERENT_0 5
-#define CORE_REQ_VARIANT_COMPUTE_SPECIFIC_COHERENT_1 6
-
-#define CORE_REQ_VARIANT_ONLY_COMPUTE_8987 4
-#define CORE_REQ_VARIANT_ONLY_COMPUTE_8987_SPECIFIC_COHERENT_0 5
-#define CORE_REQ_VARIANT_ONLY_COMPUTE_8987_SPECIFIC_COHERENT_1 6
-
-#define NUM_CORE_REQ_VARIANTS NELEMS(core_req_variants)
-#define NUM_CORE_REQ_VARIANTS_8987 NELEMS(core_req_variants_8987)
-
-/** Mappings between job slot and variant lists for Soft-Stoppable State */
-static const u32 variants_supported_ss_state[] = {
- /* js[0] uses Fragment only */
- (1u << CORE_REQ_VARIANT_FRAGMENT),
-
- /* js[1] uses: Compute-all-cores, Compute-only-coherent, Compute-or-Tiling,
- * compute-specific-coregroup-0 */
- (1u << CORE_REQ_VARIANT_COMPUTE_ALL_CORES)
- | (1u << CORE_REQ_VARIANT_COMPUTE_ONLY_COHERENT_GROUP)
- | (1u << CORE_REQ_VARIANT_COMPUTE_OR_TILING)
- | (1u << CORE_REQ_VARIANT_COMPUTE_SPECIFIC_COHERENT_0),
-
- /* js[2] uses: Compute-only-coherent, compute-specific-coregroup-1 */
- (1u << CORE_REQ_VARIANT_COMPUTE_ONLY_COHERENT_GROUP)
- | (1u << CORE_REQ_VARIANT_COMPUTE_SPECIFIC_COHERENT_1)
-};
-
-/** Mappings between job slot and variant lists for Soft-Stoppable State, when
- * we have atoms that can use all the cores (KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES)
- * and there's more than one coregroup */
-static const u32 variants_supported_ss_allcore_state[] = {
- /* js[0] uses Fragment only */
- (1u << CORE_REQ_VARIANT_FRAGMENT),
-
- /* js[1] uses: Compute-all-cores, Compute-only-coherent, Compute-or-Tiling,
- * compute-specific-coregroup-0, compute-specific-coregroup-1 */
- (1u << CORE_REQ_VARIANT_COMPUTE_ALL_CORES)
- | (1u << CORE_REQ_VARIANT_COMPUTE_ONLY_COHERENT_GROUP)
- | (1u << CORE_REQ_VARIANT_COMPUTE_OR_TILING)
- | (1u << CORE_REQ_VARIANT_COMPUTE_SPECIFIC_COHERENT_0)
- | (1u << CORE_REQ_VARIANT_COMPUTE_SPECIFIC_COHERENT_1),
-
- /* js[2] not used */
- 0
-};
-
-/** Mappings between job slot and variant lists for Soft-Stoppable State for
- * BASE_HW_ISSUE_8987
- *
- * @note There is no 'allcores' variant of this, because this HW issue forces all
- * atoms with BASE_JD_CORE_REQ_SPECIFIC_COHERENT_GROUP to use slot 2 anyway -
- * hence regardless of whether a specific coregroup is targetted, those atoms
- * still make progress. */
-static const u32 variants_supported_ss_state_8987[] = {
- /* js[0] uses Fragment only */
- (1u << CORE_REQ_VARIANT_FRAGMENT),
-
- /* js[1] uses: Compute-all-cores, Compute-only-coherent, Compute-or-Tiling */
- (1u << CORE_REQ_VARIANT_COMPUTE_ALL_CORES)
- | (1u << CORE_REQ_VARIANT_COMPUTE_ONLY_COHERENT_GROUP)
- | (1u << CORE_REQ_VARIANT_COMPUTE_OR_TILING),
-
- /* js[2] uses: All Only-compute atoms (including those targetting a
- * specific coregroup), and nothing else. This is because their affinity
- * must not intersect with non-only-compute atoms.
- *
- * As a side effect, this causes the 'device_nr' for atoms targetting a
- * specific coregroup to be ignored */
- (1u << CORE_REQ_VARIANT_ONLY_COMPUTE_8987)
- | (1u << CORE_REQ_VARIANT_ONLY_COMPUTE_8987_SPECIFIC_COHERENT_0)
- | (1u << CORE_REQ_VARIANT_ONLY_COMPUTE_8987_SPECIFIC_COHERENT_1)
-};
-
/* Defines for easy asserts 'is scheduled'/'is queued'/'is neither queued norscheduled' */
#define KBASEP_JS_CHECKFLAG_QUEUED (1u << 0) /**< Check the queued state */
#define KBASEP_JS_CHECKFLAG_SCHEDULED (1u << 1) /**< Check the scheduled state */
* Private Functions
*/
-/* Table autogenerated using util built from: midgard/scripts/gen_cfs_weight_of_prio.c */
+/* Table autogenerated using util built from: base/tools/gen_cfs_weight_of_prio/ */
/* weight = 1.25 */
static const int weight_of_priority[] = {
/* 16 */ 36388, 45485, 56856, 71070
};
-/**
- * @note There is nothing to stop the priority of the ctx containing \a
+/*
+ * Note: There is nothing to stop the priority of the ctx containing
* ctx_info changing during or immediately after this function is called
* (because its jsctx_mutex cannot be held during IRQ). Therefore, this
* function should only be seen as a heuristic guide as to the priority weight
* of the context.
*/
-STATIC u64 priority_weight(struct kbasep_js_policy_cfs_ctx *ctx_info, u64 time_us)
+static u64 priority_weight(struct kbasep_js_policy_cfs_ctx *ctx_info, u64 time_us)
{
u64 time_delta_us;
int priority;
- priority = ctx_info->process_priority + ctx_info->bag_priority;
+ priority = ctx_info->process_priority;
/* Adjust runtime_us using priority weight if required */
if (priority != 0 && time_us != 0) {
}
#if KBASE_TRACE_ENABLE
-STATIC int kbasep_js_policy_trace_get_refcnt_nolock(struct kbase_device *kbdev, struct kbase_context *kctx)
+static int kbasep_js_policy_trace_get_refcnt_nolock(struct kbase_device *kbdev, struct kbase_context *kctx)
{
struct kbasep_js_device_data *js_devdata;
int as_nr;
as_nr = kctx->as_nr;
if (as_nr != KBASEP_AS_NR_INVALID) {
struct kbasep_js_per_as_data *js_per_as_data;
+
js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
refcnt = js_per_as_data->as_busy_refcount;
return refcnt;
}
-STATIC INLINE int kbasep_js_policy_trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx)
+static inline int kbasep_js_policy_trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx)
{
unsigned long flags;
struct kbasep_js_device_data *js_devdata;
js_devdata = &kbdev->js_data;
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
+ spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
refcnt = kbasep_js_policy_trace_get_refcnt_nolock(kbdev, kctx);
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
+ spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
return refcnt;
}
#else /* KBASE_TRACE_ENABLE */
-STATIC int kbasep_js_policy_trace_get_refcnt_nolock(struct kbase_device *kbdev, struct kbase_context *kctx)
-{
- CSTD_UNUSED(kbdev);
- CSTD_UNUSED(kctx);
- return 0;
-}
-
-STATIC INLINE int kbasep_js_policy_trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx)
+static inline int kbasep_js_policy_trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx)
{
CSTD_UNUSED(kbdev);
CSTD_UNUSED(kctx);
}
#endif /* KBASE_TRACE_ENABLE */
-#ifdef CONFIG_MALI_DEBUG
-STATIC void kbasep_js_debug_check(struct kbasep_js_policy_cfs *policy_info, struct kbase_context *kctx, kbasep_js_check check_flag)
-{
- /* This function uses the ternary operator and non-explicit comparisons,
- * because it makes for much shorter, easier to read code */
-
- if (check_flag & KBASEP_JS_CHECKFLAG_QUEUED) {
- mali_bool is_queued;
- mali_bool expect_queued;
-
- is_queued = (kbasep_list_member_of(
- &policy_info->ctx_queue_head,
- &kctx->jctx.sched_info.runpool.policy_ctx.cfs.list)) ?
- MALI_TRUE : MALI_FALSE;
-
- if (!is_queued)
- is_queued = (kbasep_list_member_of(&policy_info->ctx_rt_queue_head, &kctx->jctx.sched_info.runpool.policy_ctx.cfs.list)) ? MALI_TRUE : MALI_FALSE;
-
- expect_queued = (check_flag & KBASEP_JS_CHECKFLAG_IS_QUEUED) ? MALI_TRUE : MALI_FALSE;
-
- KBASE_DEBUG_ASSERT_MSG(expect_queued == is_queued, "Expected context %p to be %s but it was %s\n", kctx, (expect_queued) ? "queued" : "not queued", (is_queued) ? "queued" : "not queued");
-
- }
-
- if (check_flag & KBASEP_JS_CHECKFLAG_SCHEDULED) {
- mali_bool is_scheduled;
- mali_bool expect_scheduled;
-
- is_scheduled = (kbasep_list_member_of(
- &policy_info->scheduled_ctxs_head,
- &kctx->jctx.sched_info.runpool.policy_ctx.cfs.list)) ?
- MALI_TRUE : MALI_FALSE;
-
- expect_scheduled = (check_flag & KBASEP_JS_CHECKFLAG_IS_SCHEDULED) ? MALI_TRUE : MALI_FALSE;
- KBASE_DEBUG_ASSERT_MSG(expect_scheduled == is_scheduled, "Expected context %p to be %s but it was %s\n", kctx, (expect_scheduled) ? "scheduled" : "not scheduled", (is_scheduled) ? "scheduled" : "not scheduled");
- }
-
-}
-#else /* CONFIG_MALI_DEBUG */
-STATIC void kbasep_js_debug_check(struct kbasep_js_policy_cfs *policy_info, struct kbase_context *kctx, kbasep_js_check check_flag)
-{
- CSTD_UNUSED(policy_info);
- CSTD_UNUSED(kctx);
- CSTD_UNUSED(check_flag);
-}
-#endif /* CONFIG_MALI_DEBUG */
-
-STATIC INLINE void set_slot_to_variant_lookup(u32 *bit_array, u32 slot_idx, u32 variants_supported)
-{
- u32 overall_bit_idx = slot_idx * KBASEP_JS_MAX_NR_CORE_REQ_VARIANTS;
- u32 word_idx = overall_bit_idx / 32;
- u32 bit_idx = overall_bit_idx % 32;
-
- KBASE_DEBUG_ASSERT(slot_idx < BASE_JM_MAX_NR_SLOTS);
- KBASE_DEBUG_ASSERT((variants_supported & ~LOOKUP_VARIANT_MASK) == 0);
-
- bit_array[word_idx] |= variants_supported << bit_idx;
-}
-
-STATIC INLINE u32 get_slot_to_variant_lookup(u32 *bit_array, u32 slot_idx)
-{
- u32 overall_bit_idx = slot_idx * KBASEP_JS_MAX_NR_CORE_REQ_VARIANTS;
- u32 word_idx = overall_bit_idx / 32;
- u32 bit_idx = overall_bit_idx % 32;
-
- u32 res;
-
- KBASE_DEBUG_ASSERT(slot_idx < BASE_JM_MAX_NR_SLOTS);
-
- res = bit_array[word_idx] >> bit_idx;
- res &= LOOKUP_VARIANT_MASK;
-
- return res;
-}
-
-/* Check the core_req_variants: make sure that every job slot is satisifed by
- * one of the variants. This checks that cached_variant_idx_init will produce a
- * valid result for jobs that make maximum use of the job slots.
- *
- * @note The checks are limited to the job slots - this does not check that
- * every context requirement is covered (because some are intentionally not
- * supported, such as KBASE_CTX_FLAG_SUBMIT_DISABLED) */
-#ifdef CONFIG_MALI_DEBUG
-STATIC void debug_check_core_req_variants(struct kbase_device *kbdev, struct kbasep_js_policy_cfs *policy_info)
-{
- struct kbasep_js_device_data *js_devdata;
- u32 i;
- int j;
-
- js_devdata = &kbdev->js_data;
-
- for (j = 0; j < kbdev->gpu_props.num_job_slots; ++j) {
- base_jd_core_req job_core_req;
- mali_bool found = MALI_FALSE;
-
- job_core_req = js_devdata->js_reqs[j];
- for (i = 0; i < policy_info->num_core_req_variants; ++i) {
- base_jd_core_req var_core_req;
-
- var_core_req = policy_info->core_req_variants[i].core_req;
-
- if ((var_core_req & job_core_req) == job_core_req) {
- found = MALI_TRUE;
- break;
- }
- }
-
- /* Early-out on any failure */
- KBASE_DEBUG_ASSERT_MSG(found != MALI_FALSE, "Job slot %d features 0x%x not matched by core_req_variants. " "Rework core_req_variants and vairants_supported_<...>_state[] to match\n", j, job_core_req);
- }
-}
-#endif
-
-STATIC void build_core_req_variants(struct kbase_device *kbdev, struct kbasep_js_policy_cfs *policy_info)
-{
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(policy_info != NULL);
- CSTD_UNUSED(kbdev);
-
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987)) {
- KBASE_DEBUG_ASSERT(NUM_CORE_REQ_VARIANTS_8987 <= KBASEP_JS_MAX_NR_CORE_REQ_VARIANTS);
-
- /* Assume a static set of variants */
- memcpy(policy_info->core_req_variants, core_req_variants_8987, sizeof(core_req_variants_8987));
-
- policy_info->num_core_req_variants = NUM_CORE_REQ_VARIANTS_8987;
- } else {
- KBASE_DEBUG_ASSERT(NUM_CORE_REQ_VARIANTS <= KBASEP_JS_MAX_NR_CORE_REQ_VARIANTS);
-
- /* Assume a static set of variants */
- memcpy(policy_info->core_req_variants, core_req_variants, sizeof(core_req_variants));
-
- policy_info->num_core_req_variants = NUM_CORE_REQ_VARIANTS;
- }
-
- KBASE_DEBUG_CODE(debug_check_core_req_variants(kbdev, policy_info));
-}
-
-STATIC void build_slot_lookups(struct kbase_device *kbdev, struct kbasep_js_policy_cfs *policy_info)
-{
- u8 i;
- const u32 *variants_supported_ss_for_this_hw = variants_supported_ss_state;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(policy_info != NULL);
-
- KBASE_DEBUG_ASSERT(kbdev->gpu_props.num_job_slots <= NELEMS(variants_supported_ss_state));
- KBASE_DEBUG_ASSERT(kbdev->gpu_props.num_job_slots <= NELEMS(variants_supported_ss_allcore_state));
- KBASE_DEBUG_ASSERT(kbdev->gpu_props.num_job_slots <= NELEMS(variants_supported_ss_state_8987));
-
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
- variants_supported_ss_for_this_hw = variants_supported_ss_state_8987;
-
- /* Given the static set of variants, provide a static set of lookups */
- for (i = 0; i < kbdev->gpu_props.num_job_slots; ++i) {
- set_slot_to_variant_lookup(policy_info->slot_to_variant_lookup_ss_state, i, variants_supported_ss_for_this_hw[i]);
-
- set_slot_to_variant_lookup(policy_info->slot_to_variant_lookup_ss_allcore_state, i, variants_supported_ss_allcore_state[i]);
- }
-
-}
-
-STATIC mali_error cached_variant_idx_init(const struct kbasep_js_policy_cfs *policy_info, const struct kbase_context *kctx, struct kbase_jd_atom *atom)
-{
- struct kbasep_js_policy_cfs_job *job_info;
- u32 i;
- base_jd_core_req job_core_req;
- u32 job_device_nr;
- kbase_context_flags ctx_flags;
- const struct kbasep_js_kctx_info *js_kctx_info;
- const struct kbase_device *kbdev;
-
- KBASE_DEBUG_ASSERT(policy_info != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
- KBASE_DEBUG_ASSERT(atom != NULL);
-
- kbdev = container_of(policy_info, const struct kbase_device, js_data.policy.cfs);
- job_info = &atom->sched_info.cfs;
- job_core_req = atom->core_req;
- job_device_nr = atom->device_nr;
- js_kctx_info = &kctx->jctx.sched_info;
- ctx_flags = js_kctx_info->ctx.flags;
-
- /* Initial check for atoms targetting a specific coregroup */
- if ((job_core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) != MALI_FALSE && job_device_nr >= kbdev->gpu_props.num_core_groups) {
- /* device_nr exceeds the number of coregroups - not allowed by
- * @ref struct base_jd_atom_v2 API contract */
- return MALI_ERROR_FUNCTION_FAILED;
- }
-
- /* Pick a core_req variant that matches us. Since they're ordered by least
- * restrictive first, it picks the least restrictive variant */
- for (i = 0; i < policy_info->num_core_req_variants; ++i) {
- base_jd_core_req var_core_req;
- kbase_context_flags var_ctx_req;
- u32 var_device_nr;
- var_core_req = policy_info->core_req_variants[i].core_req;
- var_ctx_req = policy_info->core_req_variants[i].ctx_req;
- var_device_nr = policy_info->core_req_variants[i].device_nr;
-
- if ((var_core_req & job_core_req) == job_core_req && (var_ctx_req & ctx_flags) == ctx_flags && ((var_core_req & BASE_JD_REQ_SPECIFIC_COHERENT_GROUP) == MALI_FALSE || var_device_nr == job_device_nr)) {
- job_info->cached_variant_idx = i;
- return MALI_ERROR_NONE;
- }
- }
-
- /* Could not find a matching requirement, this should only be caused by an
- * attempt to attack the driver. */
- return MALI_ERROR_FUNCTION_FAILED;
-}
-
-STATIC mali_bool dequeue_job(struct kbase_device *kbdev,
- struct kbase_context *kctx,
- u32 variants_supported,
- struct kbase_jd_atom ** const katom_ptr,
- int job_slot_idx)
-{
- struct kbasep_js_device_data *js_devdata;
- struct kbasep_js_policy_cfs *policy_info;
- struct kbasep_js_policy_cfs_ctx *ctx_info;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(katom_ptr != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- js_devdata = &kbdev->js_data;
- policy_info = &js_devdata->policy.cfs;
- ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;
-
- /* Only submit jobs from contexts that are allowed */
- if (kbasep_js_is_submit_allowed(js_devdata, kctx) != MALI_FALSE) {
- /* Check each variant in turn */
- while (variants_supported != 0) {
- long variant_idx;
- struct list_head *job_list;
-
- variant_idx = ffs(variants_supported) - 1;
- job_list = &ctx_info->job_list_head[variant_idx];
-
- if (!list_empty(job_list)) {
- /* Found a context with a matching job */
- {
- struct kbase_jd_atom *front_atom =
- list_entry(job_list->next, struct kbase_jd_atom, sched_info.cfs.list);
-
- KBASE_TRACE_ADD_SLOT(kbdev, JS_POLICY_DEQUEUE_JOB, front_atom->kctx, front_atom, front_atom->jc, job_slot_idx);
- }
- *katom_ptr = list_entry(job_list->next, struct kbase_jd_atom, sched_info.cfs.list);
- list_del(job_list->next);
-
- (*katom_ptr)->sched_info.cfs.ticks = 0;
-
- /* Put this context at the back of the Run Pool */
- list_del(&kctx->jctx.sched_info.runpool.policy_ctx.cfs.list);
- list_add_tail(&kctx->jctx.sched_info.runpool.policy_ctx.cfs.list, &policy_info->scheduled_ctxs_head);
-
- return MALI_TRUE;
- }
-
- variants_supported &= ~(1u << variant_idx);
- }
- /* All variants checked by here */
- }
-
- /* The context does not have a matching job */
-
- return MALI_FALSE;
-}
-
-/**
- * Hold the runpool_irq spinlock for this
- */
-STATIC INLINE mali_bool timer_callback_should_run(struct kbase_device *kbdev)
-{
- struct kbasep_js_device_data *js_devdata;
- s8 nr_running_ctxs;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- js_devdata = &kbdev->js_data;
-
- /* nr_user_contexts_running is updated with the runpool_mutex. However, the
- * locking in the caller gives us a barrier that ensures nr_user_contexts is
- * up-to-date for reading */
- nr_running_ctxs = js_devdata->nr_user_contexts_running;
-
-#ifdef CONFIG_MALI_DEBUG
- if (js_devdata->softstop_always && nr_running_ctxs > 0) {
- /* Debug support for allowing soft-stop on a single context */
- return MALI_TRUE;
- }
-#endif /* CONFIG_MALI_DEBUG */
-
- if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9435)) {
- /* Timeouts would have to be 4x longer (due to micro-architectural design)
- * to support OpenCL conformance tests, so only run the timer when there's:
- * - 2 or more CL contexts
- * - 1 or more GLES contexts
- *
- * NOTE: We will treat a context that has both Compute and Non-Compute jobs
- * will be treated as an OpenCL context (hence, we don't check
- * KBASEP_JS_CTX_ATTR_NON_COMPUTE).
- */
- {
- s8 nr_compute_ctxs = kbasep_js_ctx_attr_count_on_runpool(kbdev, KBASEP_JS_CTX_ATTR_COMPUTE);
- s8 nr_noncompute_ctxs = nr_running_ctxs - nr_compute_ctxs;
-
- return (mali_bool) (nr_compute_ctxs >= 2 || nr_noncompute_ctxs > 0);
- }
- } else {
- /* Run the timer callback whenever you have at least 1 context */
- return (mali_bool) (nr_running_ctxs > 0);
- }
-}
-
-static enum hrtimer_restart timer_callback(struct hrtimer *timer)
-{
- unsigned long flags;
- struct kbase_device *kbdev;
- struct kbasep_js_device_data *js_devdata;
- struct kbasep_js_policy_cfs *policy_info;
- int s;
- mali_bool reset_needed = MALI_FALSE;
-
- KBASE_DEBUG_ASSERT(timer != NULL);
-
- policy_info = container_of(timer, struct kbasep_js_policy_cfs, scheduling_timer);
- kbdev = container_of(policy_info, struct kbase_device, js_data.policy.cfs);
- js_devdata = &kbdev->js_data;
-
- /* Loop through the slots */
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
- for (s = 0; s < kbdev->gpu_props.num_job_slots; s++) {
- struct kbase_jm_slot *slot = &kbdev->jm_slots[s];
- struct kbase_jd_atom *atom = NULL;
-
- if (kbasep_jm_nr_jobs_submitted(slot) > 0) {
- atom = kbasep_jm_peek_idx_submit_slot(slot, 0);
- KBASE_DEBUG_ASSERT(atom != NULL);
-
- if (kbasep_jm_is_dummy_workaround_job(kbdev, atom) != MALI_FALSE) {
- /* Prevent further use of the atom - never cause a soft-stop, hard-stop, or a GPU reset due to it. */
- atom = NULL;
- }
- }
-
- if (atom != NULL) {
- /* The current version of the model doesn't support Soft-Stop */
- if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) {
- u32 ticks = atom->sched_info.cfs.ticks++;
-
-#if !CINSTR_DUMPING_ENABLED
- u32 soft_stop_ticks, hard_stop_ticks, gpu_reset_ticks;
- if (atom->core_req & BASE_JD_REQ_ONLY_COMPUTE) {
- soft_stop_ticks = js_devdata->soft_stop_ticks_cl;
- hard_stop_ticks = js_devdata->hard_stop_ticks_cl;
- gpu_reset_ticks = js_devdata->gpu_reset_ticks_cl;
- } else {
- soft_stop_ticks = js_devdata->soft_stop_ticks;
- hard_stop_ticks = js_devdata->hard_stop_ticks_ss;
- gpu_reset_ticks = js_devdata->gpu_reset_ticks_ss;
- }
-
- /* Job is Soft-Stoppable */
- if (ticks == soft_stop_ticks) {
- int disjoint_threshold =
- KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD;
- u32 softstop_flags = 0u;
- /* Job has been scheduled for at least js_devdata->soft_stop_ticks ticks.
- * Soft stop the slot so we can run other jobs.
- */
- dev_dbg(kbdev->dev, "Soft-stop");
-
-#if !KBASE_DISABLE_SCHEDULING_SOFT_STOPS
- /* nr_user_contexts_running is updated with the runpool_mutex,
- * but we can't take that here.
- *
- * However, if it's about to be increased then the new context
- * can't run any jobs until they take the runpool_irq lock, so
- * it's OK to observe the older value.
- *
- * Similarly, if it's about to be decreased, the last job from
- * another context has already finished, so it's not too bad
- * that we observe the older value and register a disjoint
- * event when we try soft-stopping */
- if (js_devdata->nr_user_contexts_running >= disjoint_threshold)
- softstop_flags |= JS_COMMAND_SW_CAUSES_DISJOINT;
- kbase_job_slot_softstop_swflags(kbdev,
- s, atom, softstop_flags);
-#endif
- } else if (ticks == hard_stop_ticks) {
- /* Job has been scheduled for at least js_devdata->hard_stop_ticks_ss ticks.
- * It should have been soft-stopped by now. Hard stop the slot.
- */
-#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS
- dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", (unsigned long)ticks, (unsigned long)(js_devdata->scheduling_tick_ns / 1000000u));
- kbase_job_slot_hardstop(atom->kctx, s, atom);
-#endif
- } else if (ticks == gpu_reset_ticks) {
- /* Job has been scheduled for at least js_devdata->gpu_reset_ticks_ss ticks.
- * It should have left the GPU by now. Signal that the GPU needs to be reset.
- */
- reset_needed = MALI_TRUE;
- }
-#else /* !CINSTR_DUMPING_ENABLED */
- /* NOTE: During CINSTR_DUMPING_ENABLED, we use the alternate timeouts, which
- * makes the hard-stop and GPU reset timeout much longer. We also ensure that
- * we don't soft-stop at all. */
- if (ticks == js_devdata->soft_stop_ticks) {
- /* Job has been scheduled for at least js_devdata->soft_stop_ticks.
- * We do not soft-stop during CINSTR_DUMPING_ENABLED, however.
- */
- dev_dbg(kbdev->dev, "Soft-stop");
- } else if (ticks == js_devdata->hard_stop_ticks_nss) {
- /* Job has been scheduled for at least js_devdata->hard_stop_ticks_nss ticks.
- * Hard stop the slot.
- */
-#if !KBASE_DISABLE_SCHEDULING_HARD_STOPS
- dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)", (unsigned long)ticks, (unsigned long)(js_devdata->scheduling_tick_ns / 1000000u));
- kbase_job_slot_hardstop(atom->kctx, s, atom);
-#endif
- } else if (ticks == js_devdata->gpu_reset_ticks_nss) {
- /* Job has been scheduled for at least js_devdata->gpu_reset_ticks_nss ticks.
- * It should have left the GPU by now. Signal that the GPU needs to be reset.
- */
- reset_needed = MALI_TRUE;
- }
-#endif /* !CINSTR_DUMPING_ENABLED */
- }
- }
- }
-#if KBASE_GPU_RESET_EN
- if (reset_needed) {
- dev_err(kbdev->dev, "JS: Job has been on the GPU for too long (KBASE_CONFIG_ATTR_JS_RESET_TICKS_SS/NSS timeout hit). Issueing GPU soft-reset to resolve.");
-
- if (kbase_prepare_to_reset_gpu_locked(kbdev))
- kbase_reset_gpu_locked(kbdev);
- }
-#endif /* KBASE_GPU_RESET_EN */
- /* the timer is re-issued if there is contexts in the run-pool */
-
- if (timer_callback_should_run(kbdev) != MALI_FALSE) {
- hrtimer_start(&policy_info->scheduling_timer, HR_TIMER_DELAY_NSEC(js_devdata->scheduling_tick_ns), HRTIMER_MODE_REL);
- } else {
- KBASE_TRACE_ADD(kbdev, JS_POLICY_TIMER_END, NULL, NULL, 0u, 0u);
- /* timer_running state is updated by kbasep_js_policy_runpool_timers_sync() */
- }
-
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
-
- return HRTIMER_NORESTART;
-}
/*
* Non-private functions
*/
-mali_error kbasep_js_policy_init(struct kbase_device *kbdev)
+int kbasep_js_policy_init(struct kbase_device *kbdev)
{
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_policy_cfs *policy_info;
js_devdata = &kbdev->js_data;
policy_info = &js_devdata->policy.cfs;
- INIT_LIST_HEAD(&policy_info->ctx_queue_head);
- INIT_LIST_HEAD(&policy_info->scheduled_ctxs_head);
- INIT_LIST_HEAD(&policy_info->ctx_rt_queue_head);
-
atomic64_set(&policy_info->least_runtime_us, KBASEP_JS_RUNTIME_EMPTY);
atomic64_set(&policy_info->rt_least_runtime_us, KBASEP_JS_RUNTIME_EMPTY);
- hrtimer_init(&policy_info->scheduling_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
- policy_info->scheduling_timer.function = timer_callback;
-
- policy_info->timer_running = MALI_FALSE;
policy_info->head_runtime_us = 0;
- /* Build up the core_req variants */
- build_core_req_variants(kbdev, policy_info);
- /* Build the slot to variant lookups */
- build_slot_lookups(kbdev, policy_info);
-
- return MALI_ERROR_NONE;
+ return 0;
}
void kbasep_js_policy_term(union kbasep_js_policy *js_policy)
{
- struct kbasep_js_policy_cfs *policy_info;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- policy_info = &js_policy->cfs;
-
- /* ASSERT that there are no contexts queued */
- KBASE_DEBUG_ASSERT(list_empty(&policy_info->ctx_queue_head));
- KBASE_DEBUG_ASSERT(KBASEP_JS_RUNTIME_EMPTY == atomic64_read(&policy_info->least_runtime_us));
-
- /* ASSERT that there are no contexts scheduled */
- KBASE_DEBUG_ASSERT(list_empty(&policy_info->scheduled_ctxs_head));
-
- /* ASSERT that there are no contexts queued */
- KBASE_DEBUG_ASSERT(list_empty(&policy_info->ctx_rt_queue_head));
- KBASE_DEBUG_ASSERT(KBASEP_JS_RUNTIME_EMPTY == atomic64_read(&policy_info->rt_least_runtime_us));
-
- hrtimer_cancel(&policy_info->scheduling_timer);
+ CSTD_UNUSED(js_policy);
}
-mali_error kbasep_js_policy_init_ctx(struct kbase_device *kbdev, struct kbase_context *kctx)
+int kbasep_js_policy_init_ctx(struct kbase_device *kbdev, struct kbase_context *kctx)
{
struct kbasep_js_device_data *js_devdata;
struct kbasep_js_policy_cfs_ctx *ctx_info;
struct kbasep_js_policy_cfs *policy_info;
- u32 i;
int policy;
KBASE_DEBUG_ASSERT(kbdev != NULL);
KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_INIT_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx));
- for (i = 0; i < policy_info->num_core_req_variants; ++i)
- INIT_LIST_HEAD(&ctx_info->job_list_head[i]);
-
policy = current->policy;
if (policy == SCHED_FIFO || policy == SCHED_RR) {
- ctx_info->process_rt_policy = MALI_TRUE;
+ ctx_info->process_rt_policy = true;
ctx_info->process_priority = (((MAX_RT_PRIO - 1) - current->rt_priority) / 5) - 20;
} else {
- ctx_info->process_rt_policy = MALI_FALSE;
+ ctx_info->process_rt_policy = false;
ctx_info->process_priority = (current->static_prio - MAX_RT_PRIO) - 20;
}
- ctx_info->bag_total_priority = 0;
- ctx_info->bag_total_nr_atoms = 0;
-
/* Initial runtime (relative to least-run context runtime)
*
* This uses the Policy Queue's most up-to-date head_runtime_us by using the
* head_runtime_us occur strictly after this context is initialized */
mutex_lock(&js_devdata->queue_mutex);
- /* No need to hold the the runpool_irq.lock here, because we're initializing
+ /* No need to hold the the hwaccess_lock here, because we're initializing
* the value, and the context is definitely not being updated in the
* runpool at this point. The queue_mutex ensures the memory barrier. */
ctx_info->runtime_us = policy_info->head_runtime_us + priority_weight(ctx_info, (u64) js_devdata->cfs_ctx_runtime_init_slices * (u64) (js_devdata->ctx_timeslice_ns / 1000u));
mutex_unlock(&js_devdata->queue_mutex);
- return MALI_ERROR_NONE;
+ return 0;
}
void kbasep_js_policy_term_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx)
{
- struct kbasep_js_policy_cfs_ctx *ctx_info;
- struct kbasep_js_policy_cfs *policy_info;
- u32 i;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- policy_info = &js_policy->cfs;
- ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;
-
- {
- struct kbase_device *kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_TERM_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx));
- }
-
- /* ASSERT that no jobs are present */
- for (i = 0; i < policy_info->num_core_req_variants; ++i)
- KBASE_DEBUG_ASSERT(list_empty(&ctx_info->job_list_head[i]));
-
- /* No work to do */
-}
-
-/*
- * Context Management
- */
-
-void kbasep_js_policy_enqueue_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx)
-{
- struct kbasep_js_policy_cfs *policy_info;
- struct kbasep_js_policy_cfs_ctx *ctx_info;
- struct kbase_context *head_ctx;
- struct kbase_context *list_kctx = NULL;
- struct kbasep_js_device_data *js_devdata;
- struct list_head *queue_head;
- struct list_head *pos;
- struct kbase_device *kbdev;
- atomic64_t *least_runtime_us;
- u64 head_runtime;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- policy_info = &js_policy->cfs;
- ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;
- kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
- js_devdata = &kbdev->js_data;
-
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_ENQUEUE_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx));
-
- /* ASSERT about scheduled-ness/queued-ness */
- kbasep_js_debug_check(policy_info, kctx, KBASEP_JS_CHECK_NOTQUEUED);
-
- /* Clamp the runtime to prevent DoS attacks through "stored-up" runtime */
- if (policy_info->head_runtime_us > ctx_info->runtime_us + (u64) js_devdata->cfs_ctx_runtime_min_slices * (u64) (js_devdata->ctx_timeslice_ns / 1000u)) {
- /* No need to hold the the runpool_irq.lock here, because we're essentially
- * initializing the value, and the context is definitely not being updated in the
- * runpool at this point. The queue_mutex held by the caller ensures the memory
- * barrier. */
- ctx_info->runtime_us = policy_info->head_runtime_us - (u64) js_devdata->cfs_ctx_runtime_min_slices * (u64) (js_devdata->ctx_timeslice_ns / 1000u);
- }
-
- /* Find the position where the context should be enqueued */
- if (ctx_info->process_rt_policy) {
- queue_head = &policy_info->ctx_rt_queue_head;
- least_runtime_us = &policy_info->rt_least_runtime_us;
- } else {
- queue_head = &policy_info->ctx_queue_head;
- least_runtime_us = &policy_info->least_runtime_us;
- }
-
- if (list_empty(queue_head)) {
- list_add_tail(&kctx->jctx.sched_info.runpool.policy_ctx.cfs.list, queue_head);
- } else {
- list_for_each(pos, queue_head) {
- struct kbasep_js_policy_cfs_ctx *list_ctx_info;
-
- list_kctx = list_entry(pos, struct kbase_context, jctx.sched_info.runpool.policy_ctx.cfs.list);
- list_ctx_info = &list_kctx->jctx.sched_info.runpool.policy_ctx.cfs;
-
- if ((kctx->jctx.sched_info.ctx.flags & KBASE_CTX_FLAG_PRIVILEGED) != 0)
- break;
-
- if ((list_ctx_info->runtime_us > ctx_info->runtime_us) && ((list_kctx->jctx.sched_info.ctx.flags & KBASE_CTX_FLAG_PRIVILEGED) == 0))
- break;
- }
-
- /* Add the context to the queue */
- list_add_tail(&kctx->jctx.sched_info.runpool.policy_ctx.cfs.list, &list_kctx->jctx.sched_info.runpool.policy_ctx.cfs.list);
- }
-
- /* Ensure least_runtime_us is up to date*/
- head_ctx = list_entry(queue_head->next, struct kbase_context, jctx.sched_info.runpool.policy_ctx.cfs.list);
- head_runtime = head_ctx->jctx.sched_info.runpool.policy_ctx.cfs.runtime_us;
- atomic64_set(least_runtime_us, head_runtime);
-}
-
-mali_bool kbasep_js_policy_dequeue_head_ctx(union kbasep_js_policy *js_policy, struct kbase_context ** const kctx_ptr)
-{
- struct kbasep_js_policy_cfs *policy_info;
- struct kbase_context *head_ctx;
- struct list_head *queue_head;
- atomic64_t *least_runtime_us;
- struct kbase_device *kbdev;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(kctx_ptr != NULL);
-
- policy_info = &js_policy->cfs;
- kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
-
- /* attempt to dequeue from the 'realttime' queue first */
- if (list_empty(&policy_info->ctx_rt_queue_head)) {
- if (list_empty(&policy_info->ctx_queue_head)) {
- /* Nothing to dequeue */
- return MALI_FALSE;
- } else {
- queue_head = &policy_info->ctx_queue_head;
- least_runtime_us = &policy_info->least_runtime_us;
- }
- } else {
- queue_head = &policy_info->ctx_rt_queue_head;
- least_runtime_us = &policy_info->rt_least_runtime_us;
- }
-
- /* Contexts are dequeued from the front of the queue */
- *kctx_ptr = list_entry(queue_head->next, struct kbase_context, jctx.sched_info.runpool.policy_ctx.cfs.list);
- /* If dequeuing will empty the list, then set least_runtime_us prior to deletion */
- if (queue_head->next->next == queue_head)
- atomic64_set(least_runtime_us, KBASEP_JS_RUNTIME_EMPTY);
- list_del(queue_head->next);
-
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_DEQUEUE_HEAD_CTX, *kctx_ptr, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, *kctx_ptr));
-
- /* Update the head runtime */
- if (!list_empty(queue_head)) {
- u64 head_runtime;
-
- head_ctx = list_entry(queue_head->next, struct kbase_context, jctx.sched_info.runpool.policy_ctx.cfs.list);
-
- /* No need to hold the the runpool_irq.lock here for reading - the
- * context is definitely not being updated in the runpool at this
- * point. The queue_mutex held by the caller ensures the memory barrier. */
- head_runtime = head_ctx->jctx.sched_info.runpool.policy_ctx.cfs.runtime_us;
-
- if (head_runtime > policy_info->head_runtime_us)
- policy_info->head_runtime_us = head_runtime;
-
- atomic64_set(least_runtime_us, head_runtime);
- }
-
- return MALI_TRUE;
-}
-
-mali_bool kbasep_js_policy_try_evict_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx)
-{
- struct kbasep_js_policy_cfs_ctx *ctx_info;
- struct kbasep_js_policy_cfs *policy_info;
- mali_bool is_present;
- struct list_head *queue_head;
- atomic64_t *least_runtime_us;
- struct list_head *qhead;
- struct kbase_device *kbdev;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- policy_info = &js_policy->cfs;
- ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;
- kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
-
- if (ctx_info->process_rt_policy) {
- queue_head = &policy_info->ctx_rt_queue_head;
- least_runtime_us = &policy_info->rt_least_runtime_us;
- } else {
- queue_head = &policy_info->ctx_queue_head;
- least_runtime_us = &policy_info->least_runtime_us;
- }
-
- qhead = queue_head;
-
- is_present = kbasep_list_member_of(qhead, &kctx->jctx.sched_info.runpool.policy_ctx.cfs.list);
-
- KBASE_TRACE_ADD_REFCOUNT_INFO(kbdev, JS_POLICY_TRY_EVICT_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx), is_present);
-
- if (is_present != MALI_FALSE) {
- struct kbase_context *head_ctx;
-
- qhead = queue_head;
-
- /* If dequeuing will empty the list, then set least_runtime_us prior to deletion */
- if (queue_head->next->next == queue_head)
- atomic64_set(least_runtime_us, KBASEP_JS_RUNTIME_EMPTY);
-
- /* Remove the context */
- list_del(&kctx->jctx.sched_info.runpool.policy_ctx.cfs.list);
-
- qhead = queue_head;
- /* Update the head runtime */
- if (!list_empty(qhead)) {
- u64 head_runtime;
-
- head_ctx = list_entry(qhead->next, struct kbase_context, jctx.sched_info.runpool.policy_ctx.cfs.list);
-
- /* No need to hold the the runpool_irq.lock here for reading - the
- * context is definitely not being updated in the runpool at this
- * point. The queue_mutex held by the caller ensures the memory barrier. */
- head_runtime = head_ctx->jctx.sched_info.runpool.policy_ctx.cfs.runtime_us;
-
- if (head_runtime > policy_info->head_runtime_us)
- policy_info->head_runtime_us = head_runtime;
-
- atomic64_set(least_runtime_us, head_runtime);
- }
- }
-
- return is_present;
-}
-
-void kbasep_js_policy_foreach_ctx_job(union kbasep_js_policy *js_policy, struct kbase_context *kctx,
- kbasep_js_policy_ctx_job_cb callback, mali_bool detach_jobs)
-{
- struct kbasep_js_policy_cfs *policy_info;
- struct kbasep_js_policy_cfs_ctx *ctx_info;
struct kbase_device *kbdev;
- u32 i;
KBASE_DEBUG_ASSERT(js_policy != NULL);
KBASE_DEBUG_ASSERT(kctx != NULL);
kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
- policy_info = &js_policy->cfs;
- ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;
-
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_FOREACH_CTX_JOBS, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx));
-
- /* Invoke callback on jobs on each variant in turn */
- for (i = 0; i < policy_info->num_core_req_variants; ++i) {
- struct list_head *job_list;
- struct kbase_jd_atom *atom;
- struct kbase_jd_atom *tmp_iter;
- job_list = &ctx_info->job_list_head[i];
- /* Invoke callback on all kbase_jd_atoms in this list, optionally
- * removing them from the list */
- list_for_each_entry_safe(atom, tmp_iter, job_list, sched_info.cfs.list) {
- if (detach_jobs)
- list_del(&atom->sched_info.cfs.list);
- callback(kbdev, atom);
- }
- }
-
-}
-
-void kbasep_js_policy_runpool_add_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx)
-{
- struct kbasep_js_policy_cfs *policy_info;
- struct kbasep_js_device_data *js_devdata;
- struct kbase_device *kbdev;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- policy_info = &js_policy->cfs;
- js_devdata = container_of(js_policy, struct kbasep_js_device_data, policy);
-
- kbdev = kctx->kbdev;
-
- {
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_RUNPOOL_ADD_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt_nolock(kbdev, kctx));
- }
-
- /* ASSERT about scheduled-ness/queued-ness */
- kbasep_js_debug_check(policy_info, kctx, KBASEP_JS_CHECK_NOTSCHEDULED);
-
- /* All enqueued contexts go to the back of the runpool */
- list_add_tail(&kctx->jctx.sched_info.runpool.policy_ctx.cfs.list, &policy_info->scheduled_ctxs_head);
-
- if (timer_callback_should_run(kbdev) != MALI_FALSE && policy_info->timer_running == MALI_FALSE) {
- hrtimer_start(&policy_info->scheduling_timer, HR_TIMER_DELAY_NSEC(js_devdata->scheduling_tick_ns), HRTIMER_MODE_REL);
-
- KBASE_TRACE_ADD(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u, 0u);
- policy_info->timer_running = MALI_TRUE;
- }
-}
-
-void kbasep_js_policy_runpool_remove_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx)
-{
- struct kbasep_js_policy_cfs *policy_info;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- policy_info = &js_policy->cfs;
-
- {
- struct kbase_device *kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
-
- KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_RUNPOOL_REMOVE_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt_nolock(kbdev, kctx));
- }
-
- /* ASSERT about scheduled-ness/queued-ness */
- kbasep_js_debug_check(policy_info, kctx, KBASEP_JS_CHECK_SCHEDULED);
+ KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_TERM_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx));
- /* No searching or significant list maintenance required to remove this context */
- list_del(&kctx->jctx.sched_info.runpool.policy_ctx.cfs.list);
-}
-
-mali_bool kbasep_js_policy_should_remove_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx)
-{
- struct kbasep_js_policy_cfs_ctx *ctx_info;
- struct kbasep_js_policy_cfs *policy_info;
- struct kbasep_js_device_data *js_devdata;
- u64 least_runtime_us;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- policy_info = &js_policy->cfs;
- ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;
- js_devdata = container_of(js_policy, struct kbasep_js_device_data, policy);
-
- if (ctx_info->process_rt_policy)
- least_runtime_us = atomic64_read(&policy_info->rt_least_runtime_us);
- else
- least_runtime_us = atomic64_read(&policy_info->least_runtime_us);
-
- if (KBASEP_JS_RUNTIME_EMPTY == least_runtime_us) {
- /* Queue is empty */
- return MALI_FALSE;
- }
-
- if ((least_runtime_us + priority_weight(ctx_info, (u64) (js_devdata->ctx_timeslice_ns / 1000u)))
- < ctx_info->runtime_us) {
- /* The context is scheduled out if it's not the least-run context anymore.
- * The "real" head runtime is used instead of the cached runtime so the current
- * context is not scheduled out when there is less contexts than address spaces.
- */
- return MALI_TRUE;
- }
-
- return MALI_FALSE;
-}
-
-void kbasep_js_policy_runpool_timers_sync(union kbasep_js_policy *js_policy)
-{
- struct kbasep_js_policy_cfs *policy_info;
- struct kbase_device *kbdev;
- struct kbasep_js_device_data *js_devdata;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
-
- policy_info = &js_policy->cfs;
- kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
- js_devdata = &kbdev->js_data;
-
- if (!timer_callback_should_run(kbdev)) {
- unsigned long flags;
-
- /* If the timer is running now, synchronize with it by
- * locking/unlocking its spinlock, to ensure it's not using an old value
- * from timer_callback_should_run() */
- spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
- spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
-
- /* From now on, return value of timer_callback_should_run() will also
- * cause the timer to not requeue itself. Its return value cannot
- * change, because it depends on variables updated with the
- * runpool_mutex held, which the caller of this must also hold */
- hrtimer_cancel(&policy_info->scheduling_timer);
-
- policy_info->timer_running = MALI_FALSE;
- }
+ /* No work to do */
}
/*
* Job Chain Management
*/
-mali_error kbasep_js_policy_init_job(const union kbasep_js_policy *js_policy, const struct kbase_context *kctx, struct kbase_jd_atom *katom)
-{
- const struct kbasep_js_policy_cfs *policy_info;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(katom != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- policy_info = &js_policy->cfs;
-
- /* Determine the job's index into the job list head, will return error if the
- * atom is malformed and so is reported. */
- return cached_variant_idx_init(policy_info, kctx, katom);
-}
-
-void kbasep_js_policy_register_job(union kbasep_js_policy *js_policy, struct kbase_context *kctx, struct kbase_jd_atom *katom)
-{
- struct kbasep_js_policy_cfs_ctx *ctx_info;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(katom != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;
-
- /* Adjust context priority to include the new job */
- ctx_info->bag_total_nr_atoms++;
- ctx_info->bag_total_priority += katom->nice_prio;
-
- /* Get average priority and convert to NICE range -20..19 */
- if (ctx_info->bag_total_nr_atoms)
- ctx_info->bag_priority = (ctx_info->bag_total_priority / ctx_info->bag_total_nr_atoms) - 20;
-}
-
-void kbasep_js_policy_deregister_job(union kbasep_js_policy *js_policy, struct kbase_context *kctx, struct kbase_jd_atom *katom)
-{
- struct kbasep_js_policy_cfs_ctx *ctx_info;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- CSTD_UNUSED(js_policy);
- KBASE_DEBUG_ASSERT(katom != NULL);
- KBASE_DEBUG_ASSERT(kctx != NULL);
-
- ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;
-
- /* Adjust context priority to no longer include removed job */
- KBASE_DEBUG_ASSERT(ctx_info->bag_total_nr_atoms > 0);
- ctx_info->bag_total_nr_atoms--;
- ctx_info->bag_total_priority -= katom->nice_prio;
- KBASE_DEBUG_ASSERT(ctx_info->bag_total_priority >= 0);
-
- /* Get average priority and convert to NICE range -20..19 */
- if (ctx_info->bag_total_nr_atoms)
- ctx_info->bag_priority = (ctx_info->bag_total_priority / ctx_info->bag_total_nr_atoms) - 20;
-}
-KBASE_EXPORT_TEST_API(kbasep_js_policy_deregister_job)
-
-mali_bool kbasep_js_policy_dequeue_job(struct kbase_device *kbdev,
- int job_slot_idx,
- struct kbase_jd_atom ** const katom_ptr)
-{
- struct kbasep_js_device_data *js_devdata;
- struct kbasep_js_policy_cfs *policy_info;
- struct kbase_context *kctx;
- u32 variants_supported;
- struct list_head *pos;
-
- KBASE_DEBUG_ASSERT(kbdev != NULL);
- KBASE_DEBUG_ASSERT(katom_ptr != NULL);
- KBASE_DEBUG_ASSERT(job_slot_idx < BASE_JM_MAX_NR_SLOTS);
-
- js_devdata = &kbdev->js_data;
- policy_info = &js_devdata->policy.cfs;
-
- /* Get the variants for this slot */
- if (kbdev->gpu_props.num_core_groups > 1 && kbasep_js_ctx_attr_is_attr_on_runpool(kbdev, KBASEP_JS_CTX_ATTR_COMPUTE_ALL_CORES) != MALI_FALSE) {
- /* SS-allcore state, and there's more than one coregroup */
- variants_supported = get_slot_to_variant_lookup(policy_info->slot_to_variant_lookup_ss_allcore_state, job_slot_idx);
- } else {
- /* SS-state */
- variants_supported = get_slot_to_variant_lookup(policy_info->slot_to_variant_lookup_ss_state, job_slot_idx);
- }
-
- /* First pass through the runpool we consider the realtime priority jobs */
- list_for_each(pos, &policy_info->scheduled_ctxs_head) {
- kctx = list_entry(pos, struct kbase_context, jctx.sched_info.runpool.policy_ctx.cfs.list);
- if (kctx->jctx.sched_info.runpool.policy_ctx.cfs.process_rt_policy) {
- if (dequeue_job(kbdev, kctx, variants_supported, katom_ptr, job_slot_idx)) {
- /* Realtime policy job matched */
- return MALI_TRUE;
- }
- }
- }
-
- /* Second pass through the runpool we consider the non-realtime priority jobs */
- list_for_each(pos, &policy_info->scheduled_ctxs_head) {
- kctx = list_entry(pos, struct kbase_context, jctx.sched_info.runpool.policy_ctx.cfs.list);
- if (kctx->jctx.sched_info.runpool.policy_ctx.cfs.process_rt_policy == MALI_FALSE) {
- if (dequeue_job(kbdev, kctx, variants_supported, katom_ptr, job_slot_idx)) {
- /* Non-realtime policy job matched */
- return MALI_TRUE;
- }
- }
- }
-
- /* By this point, no contexts had a matching job */
- return MALI_FALSE;
-}
-
-void kbasep_js_policy_enqueue_job(union kbasep_js_policy *js_policy, struct kbase_jd_atom *katom)
-{
- struct kbasep_js_policy_cfs_job *job_info;
- struct kbasep_js_policy_cfs_ctx *ctx_info;
- struct kbase_context *parent_ctx;
-
- KBASE_DEBUG_ASSERT(js_policy != NULL);
- KBASE_DEBUG_ASSERT(katom != NULL);
- parent_ctx = katom->kctx;
- KBASE_DEBUG_ASSERT(parent_ctx != NULL);
-
- job_info = &katom->sched_info.cfs;
- ctx_info = &parent_ctx->jctx.sched_info.runpool.policy_ctx.cfs;
-
- {
- struct kbase_device *kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
- KBASE_TRACE_ADD(kbdev, JS_POLICY_ENQUEUE_JOB, katom->kctx, katom, katom->jc, 0);
- }
- list_add_tail(&katom->sched_info.cfs.list, &ctx_info->job_list_head[job_info->cached_variant_idx]);
-}
-
void kbasep_js_policy_log_job_result(union kbasep_js_policy *js_policy, struct kbase_jd_atom *katom, u64 time_spent_us)
{
struct kbasep_js_policy_cfs_ctx *ctx_info;
struct kbase_context *parent_ctx;
+
KBASE_DEBUG_ASSERT(js_policy != NULL);
KBASE_DEBUG_ASSERT(katom != NULL);
CSTD_UNUSED(js_policy);
katom->time_spent_us += time_spent_us;
}
-mali_bool kbasep_js_policy_ctx_has_priority(union kbasep_js_policy *js_policy, struct kbase_context *current_ctx, struct kbase_context *new_ctx)
+bool kbasep_js_policy_ctx_has_priority(union kbasep_js_policy *js_policy, struct kbase_context *current_ctx, struct kbase_context *new_ctx)
{
struct kbasep_js_policy_cfs_ctx *current_ctx_info;
struct kbasep_js_policy_cfs_ctx *new_ctx_info;
current_ctx_info = ¤t_ctx->jctx.sched_info.runpool.policy_ctx.cfs;
new_ctx_info = &new_ctx->jctx.sched_info.runpool.policy_ctx.cfs;
- if ((current_ctx_info->process_rt_policy == MALI_FALSE) && (new_ctx_info->process_rt_policy == MALI_TRUE))
- return MALI_TRUE;
+ if (!current_ctx_info->process_rt_policy && new_ctx_info->process_rt_policy)
+ return true;
- if ((current_ctx_info->process_rt_policy == new_ctx_info->process_rt_policy) && (current_ctx_info->bag_priority > new_ctx_info->bag_priority))
- return MALI_TRUE;
+ if (current_ctx_info->process_rt_policy ==
+ new_ctx_info->process_rt_policy)
+ return true;
- return MALI_FALSE;
+ return false;
}
projects / firefly-linux-kernel-4.4.55.git / blobdiff