3 * (C) COPYRIGHT 2011-2016 ARM Limited. All rights reserved.
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
21 * Job Scheduler: Completely Fair Policy Implementation
24 #include <mali_kbase.h>
25 #include <mali_kbase_js.h>
26 #include <mali_kbase_js_policy_cfs.h>
27 #include <linux/version.h>
28 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
29 #include <linux/sched/rt.h>
33 * Define for when dumping is enabled.
34 * This should not be based on the instrumentation level as whether dumping is enabled for a particular level is down to the integrator.
35 * However this is being used for now as otherwise the cinstr headers would be needed.
37 #define CINSTR_DUMPING_ENABLED (2 == MALI_INSTRUMENTATION_LEVEL)
39 /* Fixed point constants used for runtime weight calculations */
40 #define WEIGHT_FIXEDPOINT_SHIFT 10
41 #define WEIGHT_TABLE_SIZE 40
42 #define WEIGHT_0_NICE (WEIGHT_TABLE_SIZE/2)
43 #define WEIGHT_0_VAL (1 << WEIGHT_FIXEDPOINT_SHIFT)
45 #define PROCESS_PRIORITY_MIN (-20)
46 #define PROCESS_PRIORITY_MAX (19)
48 /* Defines for easy asserts 'is scheduled'/'is queued'/'is neither queued norscheduled' */
49 #define KBASEP_JS_CHECKFLAG_QUEUED (1u << 0) /**< Check the queued state */
50 #define KBASEP_JS_CHECKFLAG_SCHEDULED (1u << 1) /**< Check the scheduled state */
51 #define KBASEP_JS_CHECKFLAG_IS_QUEUED (1u << 2) /**< Expect queued state to be set */
52 #define KBASEP_JS_CHECKFLAG_IS_SCHEDULED (1u << 3) /**< Expect scheduled state to be set */
55 KBASEP_JS_CHECK_NOTQUEUED = KBASEP_JS_CHECKFLAG_QUEUED,
56 KBASEP_JS_CHECK_NOTSCHEDULED = KBASEP_JS_CHECKFLAG_SCHEDULED,
57 KBASEP_JS_CHECK_QUEUED = KBASEP_JS_CHECKFLAG_QUEUED | KBASEP_JS_CHECKFLAG_IS_QUEUED,
58 KBASEP_JS_CHECK_SCHEDULED = KBASEP_JS_CHECKFLAG_SCHEDULED | KBASEP_JS_CHECKFLAG_IS_SCHEDULED
61 typedef u32 kbasep_js_check;
67 /* Table autogenerated using util built from: base/tools/gen_cfs_weight_of_prio/ */
70 static const int weight_of_priority[] = {
71 /* -20 */ 11, 14, 18, 23,
72 /* -16 */ 29, 36, 45, 56,
73 /* -12 */ 70, 88, 110, 137,
74 /* -8 */ 171, 214, 268, 335,
75 /* -4 */ 419, 524, 655, 819,
76 /* 0 */ 1024, 1280, 1600, 2000,
77 /* 4 */ 2500, 3125, 3906, 4883,
78 /* 8 */ 6104, 7630, 9538, 11923,
79 /* 12 */ 14904, 18630, 23288, 29110,
80 /* 16 */ 36388, 45485, 56856, 71070
84 * Note: There is nothing to stop the priority of the ctx containing
85 * ctx_info changing during or immediately after this function is called
86 * (because its jsctx_mutex cannot be held during IRQ). Therefore, this
87 * function should only be seen as a heuristic guide as to the priority weight
90 static u64 priority_weight(struct kbasep_js_policy_cfs_ctx *ctx_info, u64 time_us)
95 priority = ctx_info->process_priority;
97 /* Adjust runtime_us using priority weight if required */
98 if (priority != 0 && time_us != 0) {
101 /* Clamp values to min..max weights */
102 if (priority > PROCESS_PRIORITY_MAX)
103 clamped_priority = PROCESS_PRIORITY_MAX;
104 else if (priority < PROCESS_PRIORITY_MIN)
105 clamped_priority = PROCESS_PRIORITY_MIN;
107 clamped_priority = priority;
109 /* Fixed point multiplication */
110 time_delta_us = (time_us * weight_of_priority[WEIGHT_0_NICE + clamped_priority]);
111 /* Remove fraction */
112 time_delta_us = time_delta_us >> WEIGHT_FIXEDPOINT_SHIFT;
113 /* Make sure the time always increases */
114 if (0 == time_delta_us)
117 time_delta_us = time_us;
120 return time_delta_us;
123 #if KBASE_TRACE_ENABLE
124 static int kbasep_js_policy_trace_get_refcnt_nolock(struct kbase_device *kbdev, struct kbase_context *kctx)
126 struct kbasep_js_device_data *js_devdata;
130 js_devdata = &kbdev->js_data;
133 if (as_nr != KBASEP_AS_NR_INVALID) {
134 struct kbasep_js_per_as_data *js_per_as_data;
136 js_per_as_data = &js_devdata->runpool_irq.per_as_data[as_nr];
138 refcnt = js_per_as_data->as_busy_refcount;
144 static inline int kbasep_js_policy_trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx)
147 struct kbasep_js_device_data *js_devdata;
150 js_devdata = &kbdev->js_data;
152 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
153 refcnt = kbasep_js_policy_trace_get_refcnt_nolock(kbdev, kctx);
154 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
158 #else /* KBASE_TRACE_ENABLE */
159 static inline int kbasep_js_policy_trace_get_refcnt(struct kbase_device *kbdev, struct kbase_context *kctx)
165 #endif /* KBASE_TRACE_ENABLE */
169 * Non-private functions
172 int kbasep_js_policy_init(struct kbase_device *kbdev)
174 struct kbasep_js_device_data *js_devdata;
175 struct kbasep_js_policy_cfs *policy_info;
177 KBASE_DEBUG_ASSERT(kbdev != NULL);
178 js_devdata = &kbdev->js_data;
179 policy_info = &js_devdata->policy.cfs;
181 atomic64_set(&policy_info->least_runtime_us, KBASEP_JS_RUNTIME_EMPTY);
182 atomic64_set(&policy_info->rt_least_runtime_us, KBASEP_JS_RUNTIME_EMPTY);
184 policy_info->head_runtime_us = 0;
189 void kbasep_js_policy_term(union kbasep_js_policy *js_policy)
191 CSTD_UNUSED(js_policy);
194 int kbasep_js_policy_init_ctx(struct kbase_device *kbdev, struct kbase_context *kctx)
196 struct kbasep_js_device_data *js_devdata;
197 struct kbasep_js_policy_cfs_ctx *ctx_info;
198 struct kbasep_js_policy_cfs *policy_info;
201 KBASE_DEBUG_ASSERT(kbdev != NULL);
202 KBASE_DEBUG_ASSERT(kctx != NULL);
204 js_devdata = &kbdev->js_data;
205 policy_info = &kbdev->js_data.policy.cfs;
206 ctx_info = &kctx->jctx.sched_info.runpool.policy_ctx.cfs;
208 KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_INIT_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx));
210 policy = current->policy;
211 if (policy == SCHED_FIFO || policy == SCHED_RR) {
212 ctx_info->process_rt_policy = true;
213 ctx_info->process_priority = (((MAX_RT_PRIO - 1) - current->rt_priority) / 5) - 20;
215 ctx_info->process_rt_policy = false;
216 ctx_info->process_priority = (current->static_prio - MAX_RT_PRIO) - 20;
219 /* Initial runtime (relative to least-run context runtime)
221 * This uses the Policy Queue's most up-to-date head_runtime_us by using the
222 * queue mutex to issue memory barriers - also ensure future updates to
223 * head_runtime_us occur strictly after this context is initialized */
224 mutex_lock(&js_devdata->queue_mutex);
226 /* No need to hold the the runpool_irq.lock here, because we're initializing
227 * the value, and the context is definitely not being updated in the
228 * runpool at this point. The queue_mutex ensures the memory barrier. */
229 ctx_info->runtime_us = policy_info->head_runtime_us + priority_weight(ctx_info, (u64) js_devdata->cfs_ctx_runtime_init_slices * (u64) (js_devdata->ctx_timeslice_ns / 1000u));
231 mutex_unlock(&js_devdata->queue_mutex);
236 void kbasep_js_policy_term_ctx(union kbasep_js_policy *js_policy, struct kbase_context *kctx)
238 struct kbase_device *kbdev;
240 KBASE_DEBUG_ASSERT(js_policy != NULL);
241 KBASE_DEBUG_ASSERT(kctx != NULL);
243 kbdev = container_of(js_policy, struct kbase_device, js_data.policy);
244 KBASE_TRACE_ADD_REFCOUNT(kbdev, JS_POLICY_TERM_CTX, kctx, NULL, 0u, kbasep_js_policy_trace_get_refcnt(kbdev, kctx));
250 * Job Chain Management
253 void kbasep_js_policy_log_job_result(union kbasep_js_policy *js_policy, struct kbase_jd_atom *katom, u64 time_spent_us)
255 struct kbasep_js_policy_cfs_ctx *ctx_info;
256 struct kbase_context *parent_ctx;
258 KBASE_DEBUG_ASSERT(js_policy != NULL);
259 KBASE_DEBUG_ASSERT(katom != NULL);
260 CSTD_UNUSED(js_policy);
262 parent_ctx = katom->kctx;
263 KBASE_DEBUG_ASSERT(parent_ctx != NULL);
265 ctx_info = &parent_ctx->jctx.sched_info.runpool.policy_ctx.cfs;
267 ctx_info->runtime_us += priority_weight(ctx_info, time_spent_us);
269 katom->time_spent_us += time_spent_us;
272 bool kbasep_js_policy_ctx_has_priority(union kbasep_js_policy *js_policy, struct kbase_context *current_ctx, struct kbase_context *new_ctx)
274 struct kbasep_js_policy_cfs_ctx *current_ctx_info;
275 struct kbasep_js_policy_cfs_ctx *new_ctx_info;
277 KBASE_DEBUG_ASSERT(current_ctx != NULL);
278 KBASE_DEBUG_ASSERT(new_ctx != NULL);
279 CSTD_UNUSED(js_policy);
281 current_ctx_info = ¤t_ctx->jctx.sched_info.runpool.policy_ctx.cfs;
282 new_ctx_info = &new_ctx->jctx.sched_info.runpool.policy_ctx.cfs;
284 if (!current_ctx_info->process_rt_policy && new_ctx_info->process_rt_policy)
287 if (current_ctx_info->process_rt_policy ==
288 new_ctx_info->process_rt_policy)