3 * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
20 * Register-based HW access backend specific job scheduler APIs
23 #include <mali_kbase.h>
24 #include <mali_kbase_hwaccess_jm.h>
25 #include <backend/gpu/mali_kbase_jm_internal.h>
26 #include <backend/gpu/mali_kbase_js_internal.h>
29 * Define for when dumping is enabled.
30 * This should not be based on the instrumentation level as whether dumping is
31 * enabled for a particular level is down to the integrator. However this is
32 * being used for now as otherwise the cinstr headers would be needed.
34 #define CINSTR_DUMPING_ENABLED (2 == MALI_INSTRUMENTATION_LEVEL)
37 * Hold the runpool_mutex for this
39 static inline bool timer_callback_should_run(struct kbase_device *kbdev)
41 struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
44 lockdep_assert_held(&kbdev->js_data.runpool_mutex);
46 /* Timer must stop if we are suspending */
47 if (backend->suspend_timer)
50 /* nr_contexts_pullable is updated with the runpool_mutex. However, the
51 * locking in the caller gives us a barrier that ensures
52 * nr_contexts_pullable is up-to-date for reading */
53 nr_running_ctxs = atomic_read(&kbdev->js_data.nr_contexts_runnable);
55 #ifdef CONFIG_MALI_DEBUG
56 if (kbdev->js_data.softstop_always) {
57 /* Debug support for allowing soft-stop on a single context */
60 #endif /* CONFIG_MALI_DEBUG */
62 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9435)) {
63 /* Timeouts would have to be 4x longer (due to micro-
64 * architectural design) to support OpenCL conformance tests, so
65 * only run the timer when there's:
66 * - 2 or more CL contexts
67 * - 1 or more GLES contexts
69 * NOTE: We will treat a context that has both Compute and Non-
70 * Compute jobs will be treated as an OpenCL context (hence, we
71 * don't check KBASEP_JS_CTX_ATTR_NON_COMPUTE).
75 kbasep_js_ctx_attr_count_on_runpool(kbdev,
76 KBASEP_JS_CTX_ATTR_COMPUTE);
77 s8 nr_noncompute_ctxs = nr_running_ctxs -
80 return (bool) (nr_compute_ctxs >= 2 ||
81 nr_noncompute_ctxs > 0);
84 /* Run the timer callback whenever you have at least 1 context
86 return (bool) (nr_running_ctxs > 0);
90 static enum hrtimer_restart timer_callback(struct hrtimer *timer)
93 struct kbase_device *kbdev;
94 struct kbasep_js_device_data *js_devdata;
95 struct kbase_backend_data *backend;
97 bool reset_needed = false;
99 KBASE_DEBUG_ASSERT(timer != NULL);
101 backend = container_of(timer, struct kbase_backend_data,
103 kbdev = container_of(backend, struct kbase_device, hwaccess.backend);
104 js_devdata = &kbdev->js_data;
106 /* Loop through the slots */
107 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
108 for (s = 0; s < kbdev->gpu_props.num_job_slots; s++) {
109 struct kbase_jd_atom *atom = NULL;
111 if (kbase_backend_nr_atoms_on_slot(kbdev, s) > 0) {
112 atom = kbase_gpu_inspect(kbdev, s, 0);
113 KBASE_DEBUG_ASSERT(atom != NULL);
117 /* The current version of the model doesn't support
119 if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) {
120 u32 ticks = atom->sched_info.cfs.ticks++;
122 #if !CINSTR_DUMPING_ENABLED
123 u32 soft_stop_ticks, hard_stop_ticks,
125 if (atom->core_req & BASE_JD_REQ_ONLY_COMPUTE) {
127 js_devdata->soft_stop_ticks_cl;
129 js_devdata->hard_stop_ticks_cl;
131 js_devdata->gpu_reset_ticks_cl;
134 js_devdata->soft_stop_ticks;
136 js_devdata->hard_stop_ticks_ss;
138 js_devdata->gpu_reset_ticks_ss;
141 /* If timeouts have been changed then ensure
142 * that atom tick count is not greater than the
143 * new soft_stop timeout. This ensures that
144 * atoms do not miss any of the timeouts due to
145 * races between this worker and the thread
146 * changing the timeouts. */
147 if (backend->timeouts_updated &&
148 ticks > soft_stop_ticks)
149 ticks = atom->sched_info.cfs.ticks =
152 /* Job is Soft-Stoppable */
153 if (ticks == soft_stop_ticks) {
154 int disjoint_threshold =
155 KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD;
156 u32 softstop_flags = 0u;
157 /* Job has been scheduled for at least
158 * js_devdata->soft_stop_ticks ticks.
159 * Soft stop the slot so we can run
162 dev_dbg(kbdev->dev, "Soft-stop");
163 #if !KBASE_DISABLE_SCHEDULING_SOFT_STOPS
164 /* nr_user_contexts_running is updated
165 * with the runpool_mutex, but we can't
168 * However, if it's about to be
169 * increased then the new context can't
170 * run any jobs until they take the
171 * runpool_irq lock, so it's OK to
172 * observe the older value.
174 * Similarly, if it's about to be
175 * decreased, the last job from another
176 * context has already finished, so it's
177 * not too bad that we observe the older
178 * value and register a disjoint event
179 * when we try soft-stopping */
180 if (js_devdata->nr_user_contexts_running
181 >= disjoint_threshold)
183 JS_COMMAND_SW_CAUSES_DISJOINT;
185 kbase_job_slot_softstop_swflags(kbdev,
186 s, atom, softstop_flags);
188 } else if (ticks == hard_stop_ticks) {
189 /* Job has been scheduled for at least
190 * js_devdata->hard_stop_ticks_ss ticks.
191 * It should have been soft-stopped by
192 * now. Hard stop the slot.
194 #if !KBASE_DISABLE_SCHEDULING_HARD_STOPS
196 js_devdata->scheduling_period_ns
198 dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)",
199 (unsigned long)ticks,
201 kbase_job_slot_hardstop(atom->kctx, s,
204 } else if (ticks == gpu_reset_ticks) {
205 /* Job has been scheduled for at least
206 * js_devdata->gpu_reset_ticks_ss ticks.
207 * It should have left the GPU by now.
208 * Signal that the GPU needs to be
213 #else /* !CINSTR_DUMPING_ENABLED */
214 /* NOTE: During CINSTR_DUMPING_ENABLED, we use
215 * the alternate timeouts, which makes the hard-
216 * stop and GPU reset timeout much longer. We
217 * also ensure that we don't soft-stop at all.
219 if (ticks == js_devdata->soft_stop_ticks) {
220 /* Job has been scheduled for at least
221 * js_devdata->soft_stop_ticks. We do
222 * not soft-stop during
223 * CINSTR_DUMPING_ENABLED, however.
225 dev_dbg(kbdev->dev, "Soft-stop");
227 js_devdata->hard_stop_ticks_dumping) {
228 /* Job has been scheduled for at least
229 * js_devdata->hard_stop_ticks_dumping
230 * ticks. Hard stop the slot.
232 #if !KBASE_DISABLE_SCHEDULING_HARD_STOPS
234 js_devdata->scheduling_period_ns
236 dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)",
237 (unsigned long)ticks,
239 kbase_job_slot_hardstop(atom->kctx, s,
243 js_devdata->gpu_reset_ticks_dumping) {
244 /* Job has been scheduled for at least
245 * js_devdata->gpu_reset_ticks_dumping
246 * ticks. It should have left the GPU by
247 * now. Signal that the GPU needs to be
252 #endif /* !CINSTR_DUMPING_ENABLED */
256 #if KBASE_GPU_RESET_EN
258 dev_err(kbdev->dev, "JS: Job has been on the GPU for too long (JS_RESET_TICKS_SS/DUMPING timeout hit). Issueing GPU soft-reset to resolve.");
260 if (kbase_prepare_to_reset_gpu_locked(kbdev))
261 kbase_reset_gpu_locked(kbdev);
263 #endif /* KBASE_GPU_RESET_EN */
264 /* the timer is re-issued if there is contexts in the run-pool */
266 if (backend->timer_running)
267 hrtimer_start(&backend->scheduling_timer,
268 HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns),
271 backend->timeouts_updated = false;
273 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
275 return HRTIMER_NORESTART;
278 void kbase_backend_ctx_count_changed(struct kbase_device *kbdev)
280 struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
281 struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
284 lockdep_assert_held(&js_devdata->runpool_mutex);
286 if (!timer_callback_should_run(kbdev)) {
287 /* Take spinlock to force synchronisation with timer */
288 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
289 backend->timer_running = false;
290 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
291 /* From now on, return value of timer_callback_should_run() will
292 * also cause the timer to not requeue itself. Its return value
293 * cannot change, because it depends on variables updated with
294 * the runpool_mutex held, which the caller of this must also
296 hrtimer_cancel(&backend->scheduling_timer);
299 if (timer_callback_should_run(kbdev) && !backend->timer_running) {
300 /* Take spinlock to force synchronisation with timer */
301 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
302 backend->timer_running = true;
303 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
304 hrtimer_start(&backend->scheduling_timer,
305 HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns),
308 KBASE_TRACE_ADD(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u,
313 int kbase_backend_timer_init(struct kbase_device *kbdev)
315 struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
317 hrtimer_init(&backend->scheduling_timer, CLOCK_MONOTONIC,
319 backend->scheduling_timer.function = timer_callback;
321 backend->timer_running = false;
326 void kbase_backend_timer_term(struct kbase_device *kbdev)
328 struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
330 hrtimer_cancel(&backend->scheduling_timer);
333 void kbase_backend_timer_suspend(struct kbase_device *kbdev)
335 struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
337 backend->suspend_timer = true;
339 kbase_backend_ctx_count_changed(kbdev);
342 void kbase_backend_timer_resume(struct kbase_device *kbdev)
344 struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
346 backend->suspend_timer = false;
348 kbase_backend_ctx_count_changed(kbdev);
351 void kbase_backend_timeouts_changed(struct kbase_device *kbdev)
353 struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
355 backend->timeouts_updated = true;