3 * (C) COPYRIGHT ARM Limited. All rights reserved.
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
21 * @file mali_kbase_jm.c
22 * Base kernel job manager APIs
25 #include <mali_kbase.h>
26 #include <mali_midg_regmap.h>
27 #include <mali_kbase_gator.h>
28 #include <mali_kbase_js_affinity.h>
29 #include <mali_kbase_hw.h>
31 #include "mali_kbase_jm.h"
33 #define beenthere(kctx, f, a...) dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a)
35 #ifdef CONFIG_MALI_DEBUG_SHADER_SPLIT_FS
36 u64 mali_js0_affinity_mask = 0xFFFFFFFFFFFFFFFFULL;
37 u64 mali_js1_affinity_mask = 0xFFFFFFFFFFFFFFFFULL;
38 u64 mali_js2_affinity_mask = 0xFFFFFFFFFFFFFFFFULL;
42 static void kbasep_try_reset_gpu_early(kbase_device *kbdev);
44 #ifdef CONFIG_GPU_TRACEPOINTS
45 static char *kbasep_make_job_slot_string(int js, char *js_string)
47 sprintf(js_string, "job_slot_%i", js);
52 static void kbase_job_hw_submit(kbase_device *kbdev, kbase_jd_atom *katom, int js)
56 u64 jc_head = katom->jc;
58 KBASE_DEBUG_ASSERT(kbdev);
59 KBASE_DEBUG_ASSERT(katom);
63 /* Command register must be available */
64 KBASE_DEBUG_ASSERT(kbasep_jm_is_js_free(kbdev, js, kctx));
65 /* Affinity is not violating */
66 kbase_js_debug_log_current_affinities(kbdev);
67 KBASE_DEBUG_ASSERT(!kbase_js_affinity_would_violate(kbdev, js, katom->affinity));
69 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JSn_HEAD_NEXT_LO), jc_head & 0xFFFFFFFF, kctx);
70 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JSn_HEAD_NEXT_HI), jc_head >> 32, kctx);
72 #ifdef CONFIG_MALI_DEBUG_SHADER_SPLIT_FS
79 mask = mali_js0_affinity_mask;
83 mask = mali_js1_affinity_mask;
87 mask = mali_js2_affinity_mask;
90 value = katom->affinity & (mask & 0xFFFFFFFF);
92 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JSn_AFFINITY_NEXT_LO), value, kctx);
94 value = (katom->affinity >> 32) & ((mask>>32) & 0xFFFFFFFF);
95 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JSn_AFFINITY_NEXT_HI), value, kctx);
98 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JSn_AFFINITY_NEXT_LO), katom->affinity & 0xFFFFFFFF, kctx);
99 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JSn_AFFINITY_NEXT_HI), katom->affinity >> 32, kctx);
102 /* start MMU, medium priority, cache clean/flush on end, clean/flush on start */
103 cfg = kctx->as_nr | JSn_CONFIG_END_FLUSH_CLEAN_INVALIDATE | JSn_CONFIG_START_MMU | JSn_CONFIG_START_FLUSH_CLEAN_INVALIDATE | JSn_CONFIG_THREAD_PRI(8);
105 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
106 if (!kbdev->jm_slots[js].job_chain_flag) {
107 cfg |= JSn_CONFIG_JOB_CHAIN_FLAG;
108 katom->atom_flags |= KBASE_KATOM_FLAGS_JOBCHAIN;
109 kbdev->jm_slots[js].job_chain_flag = MALI_TRUE;
111 katom->atom_flags &= ~KBASE_KATOM_FLAGS_JOBCHAIN;
112 kbdev->jm_slots[js].job_chain_flag = MALI_FALSE;
116 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JSn_CONFIG_NEXT), cfg, kctx);
118 /* Write an approximate start timestamp.
119 * It's approximate because there might be a job in the HEAD register. In
120 * such cases, we'll try to make a better approximation in the IRQ handler
121 * (up to the KBASE_JS_IRQ_THROTTLE_TIME_US). */
122 katom->start_timestamp = ktime_get();
125 dev_dbg(kbdev->dev, "JS: Submitting atom %p from ctx %p to js[%d] with head=0x%llx, affinity=0x%llx", katom, kctx, js, jc_head, katom->affinity);
127 KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js, (u32) katom->affinity);
129 #ifdef CONFIG_MALI_GATOR_SUPPORT
130 kbase_trace_mali_job_slots_event(GATOR_MAKE_EVENT(GATOR_JOB_SLOT_START, js), kctx, kbase_jd_atom_id(kctx, katom));
131 #endif /* CONFIG_MALI_GATOR_SUPPORT */
132 #ifdef CONFIG_GPU_TRACEPOINTS
133 if (kbasep_jm_nr_jobs_submitted(&kbdev->jm_slots[js]) == 1)
135 /* If this is the only job on the slot, trace it as starting */
137 trace_gpu_sched_switch(kbasep_make_job_slot_string(js, js_string), ktime_to_ns(katom->start_timestamp), (u32)katom->kctx, 0, katom->work_id);
138 kbdev->jm_slots[js].last_context = katom->kctx;
141 kbase_timeline_job_slot_submit(kbdev, kctx, katom, js);
143 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JSn_COMMAND_NEXT), JSn_COMMAND_START, katom->kctx);
146 void kbase_job_submit_nolock(kbase_device *kbdev, kbase_jd_atom *katom, int js)
148 kbase_jm_slot *jm_slots;
150 KBASE_DEBUG_ASSERT(kbdev);
152 jm_slots = kbdev->jm_slots;
156 * - one job already done (pending interrupt),
158 * - one ready to be run.
159 * Hence a maximum of 3 inflight jobs. We have a 4 job
160 * queue, which I hope will be enough...
162 kbasep_jm_enqueue_submit_slot(&jm_slots[js], katom);
163 kbase_job_hw_submit(kbdev, katom, js);
166 void kbase_job_done_slot(kbase_device *kbdev, int s, u32 completion_code, u64 job_tail, ktime_t *end_timestamp)
169 kbase_jd_atom *katom;
173 KBASE_DEBUG_ASSERT(kbdev);
175 if (completion_code != BASE_JD_EVENT_DONE && completion_code != BASE_JD_EVENT_STOPPED)
176 dev_err(kbdev->dev, "t6xx: GPU fault 0x%02lx from job slot %d\n", (unsigned long)completion_code, s);
178 /* IMPORTANT: this function must only contain work necessary to complete a
179 * job from a Real IRQ (and not 'fake' completion, e.g. from
180 * Soft-stop). For general work that must happen no matter how the job was
181 * removed from the hardware, place it in kbase_jd_done() */
183 slot = &kbdev->jm_slots[s];
184 katom = kbasep_jm_dequeue_submit_slot(slot);
186 /* If the katom completed is because it's a dummy job for HW workarounds, then take no further action */
187 if (kbasep_jm_is_dummy_workaround_job(kbdev, katom)) {
188 KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, NULL, NULL, 0, s, completion_code);
195 KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, jc_head, s, completion_code);
197 if (completion_code != BASE_JD_EVENT_DONE && completion_code != BASE_JD_EVENT_STOPPED) {
199 #if KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR != 0
200 KBASE_TRACE_DUMP(kbdev);
204 mali_bool was_updated = (job_tail != jc_head);
205 /* Some of the job has been executed, so we update the job chain address to where we should resume from */
206 katom->jc = job_tail;
208 KBASE_TRACE_ADD_SLOT(kbdev, JM_UPDATE_HEAD, kctx, katom, job_tail, s);
211 /* Only update the event code for jobs that weren't cancelled */
212 if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED)
213 katom->event_code = (base_jd_event_code) completion_code;
215 kbase_device_trace_register_access(kctx, REG_WRITE, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 1 << s);
217 /* Complete the job, and start new ones
219 * Also defer remaining work onto the workqueue:
220 * - Re-queue Soft-stopped jobs
221 * - For any other jobs, queue the job back into the dependency system
222 * - Schedule out the parent context if necessary, and schedule a new one in.
224 #ifdef CONFIG_GPU_TRACEPOINTS
225 if (kbasep_jm_nr_jobs_submitted(slot) != 0) {
226 kbase_jd_atom *katom;
228 katom = kbasep_jm_peek_idx_submit_slot(slot, 0); /* The atom in the HEAD */
229 trace_gpu_sched_switch(kbasep_make_job_slot_string(s, js_string), ktime_to_ns(*end_timestamp), (u32)katom->kctx, 0, katom->work_id);
230 slot->last_context = katom->kctx;
233 trace_gpu_sched_switch(kbasep_make_job_slot_string(s, js_string), ktime_to_ns(ktime_get()), 0, 0, 0);
234 slot->last_context = 0;
237 kbase_jd_done(katom, s, end_timestamp, KBASE_JS_ATOM_DONE_START_NEW_ATOMS);
241 * Update the start_timestamp of the job currently in the HEAD, based on the
242 * fact that we got an IRQ for the previous set of completed jobs.
244 * The estimate also takes into account the KBASE_JS_IRQ_THROTTLE_TIME_US and
245 * the time the job was submitted, to work out the best estimate (which might
246 * still result in an over-estimate to the calculated time spent)
248 STATIC void kbasep_job_slot_update_head_start_timestamp(kbase_device *kbdev, kbase_jm_slot *slot, ktime_t end_timestamp)
250 KBASE_DEBUG_ASSERT(slot);
252 if (kbasep_jm_nr_jobs_submitted(slot) > 0) {
253 kbase_jd_atom *katom;
254 ktime_t new_timestamp;
255 ktime_t timestamp_diff;
256 katom = kbasep_jm_peek_idx_submit_slot(slot, 0); /* The atom in the HEAD */
258 KBASE_DEBUG_ASSERT(katom != NULL);
260 if (kbasep_jm_is_dummy_workaround_job(kbdev, katom) != MALI_FALSE) {
261 /* Don't access the members of HW workaround 'dummy' jobs */
265 /* Account for any IRQ Throttle time - makes an overestimate of the time spent by the job */
266 new_timestamp = ktime_sub_ns(end_timestamp, KBASE_JS_IRQ_THROTTLE_TIME_US * 1000);
267 timestamp_diff = ktime_sub(new_timestamp, katom->start_timestamp);
268 if (ktime_to_ns(timestamp_diff) >= 0) {
269 /* Only update the timestamp if it's a better estimate than what's currently stored.
270 * This is because our estimate that accounts for the throttle time may be too much
271 * of an overestimate */
272 katom->start_timestamp = new_timestamp;
277 void kbase_job_done(kbase_device *kbdev, u32 done)
282 ktime_t end_timestamp = ktime_get();
283 kbasep_js_device_data *js_devdata;
285 KBASE_DEBUG_ASSERT(kbdev);
286 js_devdata = &kbdev->js_data;
288 KBASE_TRACE_ADD(kbdev, JM_IRQ, NULL, NULL, 0, done);
290 memset(&kbdev->slot_submit_count_irq[0], 0, sizeof(kbdev->slot_submit_count_irq));
292 /* write irq throttle register, this will prevent irqs from occurring until
293 * the given number of gpu clock cycles have passed */
295 int irq_throttle_cycles = atomic_read(&kbdev->irq_throttle_cycles);
296 kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_THROTTLE), irq_throttle_cycles, NULL);
299 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
303 u32 failed = done >> 16;
305 /* treat failed slots as finished slots */
306 u32 finished = (done & 0xFFFF) | failed;
308 /* Note: This is inherently unfair, as we always check
309 * for lower numbered interrupts before the higher
311 i = ffs(finished) - 1;
312 KBASE_DEBUG_ASSERT(i >= 0);
314 slot = &kbdev->jm_slots[i];
319 u32 completion_code = BASE_JD_EVENT_DONE; /* assume OK */
322 if (failed & (1u << i)) {
323 /* read out the job slot status code if the job slot reported failure */
324 completion_code = kbase_reg_read(kbdev, JOB_SLOT_REG(i, JSn_STATUS), NULL);
326 switch (completion_code) {
327 case BASE_JD_EVENT_STOPPED:
328 #ifdef CONFIG_MALI_GATOR_SUPPORT
329 kbase_trace_mali_job_slots_event(GATOR_MAKE_EVENT(GATOR_JOB_SLOT_SOFT_STOPPED, i), NULL, 0);
330 #endif /* CONFIG_MALI_GATOR_SUPPORT */
331 /* Soft-stopped job - read the value of JS<n>_TAIL so that the job chain can be resumed */
332 job_tail = (u64) kbase_reg_read(kbdev, JOB_SLOT_REG(i, JSn_TAIL_LO), NULL) | ((u64) kbase_reg_read(kbdev, JOB_SLOT_REG(i, JSn_TAIL_HI), NULL) << 32);
334 case BASE_JD_EVENT_NOT_STARTED:
335 /* PRLAM-10673 can cause a TERMINATED job to come back as NOT_STARTED, but the error interrupt helps us detect it */
336 completion_code = BASE_JD_EVENT_TERMINATED;
339 dev_warn(kbdev->dev, "error detected from slot %d, job status 0x%08x (%s)", i, completion_code, kbase_exception_name(completion_code));
343 kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), done & ((1 << i) | (1 << (i + 16))), NULL);
344 active = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_JS_STATE), NULL);
346 if (((active >> i) & 1) == 0 && (((done >> (i + 16)) & 1) == 0)) {
347 /* There is a potential race we must work around:
349 * 1. A job slot has a job in both current and next registers
350 * 2. The job in current completes successfully, the IRQ handler reads RAWSTAT
351 * and calls this function with the relevant bit set in "done"
352 * 3. The job in the next registers becomes the current job on the GPU
353 * 4. Sometime before the JOB_IRQ_CLEAR line above the job on the GPU _fails_
354 * 5. The IRQ_CLEAR clears the done bit but not the failed bit. This atomically sets
355 * JOB_IRQ_JS_STATE. However since both jobs have now completed the relevant bits
356 * for the slot are set to 0.
358 * If we now did nothing then we'd incorrectly assume that _both_ jobs had completed
359 * successfully (since we haven't yet observed the fail bit being set in RAWSTAT).
361 * So at this point if there are no active jobs left we check to see if RAWSTAT has a failure
362 * bit set for the job slot. If it does we know that there has been a new failure that we
363 * didn't previously know about, so we make sure that we record this in active (but we wait
364 * for the next loop to deal with it).
366 * If we were handling a job failure (i.e. done has the relevant high bit set) then we know that
367 * the value read back from JOB_IRQ_JS_STATE is the correct number of remaining jobs because
368 * the failed job will have prevented any futher jobs from starting execution.
370 u32 rawstat = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL);
372 if ((rawstat >> (i + 16)) & 1) {
373 /* There is a failed job that we've missed - add it back to active */
378 dev_dbg(kbdev->dev, "Job ended with status 0x%08X\n", completion_code);
380 nr_done = kbasep_jm_nr_jobs_submitted(slot);
381 nr_done -= (active >> i) & 1;
382 nr_done -= (active >> (i + 16)) & 1;
385 dev_warn(kbdev->dev, "Spurious interrupt on slot %d", i);
393 kbase_job_done_slot(kbdev, i, completion_code, job_tail, &end_timestamp);
395 /* More than one job has completed. Since this is not the last job being reported this time it
396 * must have passed. This is because the hardware will not allow further jobs in a job slot to
397 * complete until the faile job is cleared from the IRQ status.
399 kbase_job_done_slot(kbdev, i, BASE_JD_EVENT_DONE, 0, &end_timestamp);
405 done = kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL);
407 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10883)) {
408 /* Workaround for missing interrupt caused by PRLAM-10883 */
409 if (((active >> i) & 1) && (0 == kbase_reg_read(kbdev, JOB_SLOT_REG(i, JSn_STATUS), NULL))) {
410 /* Force job slot to be processed again */
416 finished = (done & 0xFFFF) | failed;
417 } while (finished & (1 << i));
419 kbasep_job_slot_update_head_start_timestamp(kbdev, slot, end_timestamp);
421 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
423 if (atomic_read(&kbdev->reset_gpu) == KBASE_RESET_GPU_COMMITTED) {
424 /* If we're trying to reset the GPU then we might be able to do it early
425 * (without waiting for a timeout) because some jobs have completed
427 kbasep_try_reset_gpu_early(kbdev);
430 KBASE_TRACE_ADD(kbdev, JM_IRQ_END, NULL, NULL, 0, count);
432 KBASE_EXPORT_TEST_API(kbase_job_done)
434 static mali_bool kbasep_soft_stop_allowed(kbase_device *kbdev, u16 core_reqs)
436 mali_bool soft_stops_allowed = MALI_TRUE;
438 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) {
439 if ((core_reqs & BASE_JD_REQ_T) != 0)
440 soft_stops_allowed = MALI_FALSE;
442 return soft_stops_allowed;
445 static mali_bool kbasep_hard_stop_allowed(kbase_device *kbdev, u16 core_reqs)
447 mali_bool hard_stops_allowed = MALI_TRUE;
449 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8394)) {
450 if ((core_reqs & BASE_JD_REQ_T) != 0)
451 hard_stops_allowed = MALI_FALSE;
453 return hard_stops_allowed;
456 static void kbasep_job_slot_soft_or_hard_stop_do_action(kbase_device *kbdev, int js, u32 action, u16 core_reqs, kbase_jd_atom * target_katom )
458 kbase_context *kctx = target_katom->kctx;
459 #if KBASE_TRACE_ENABLE
460 u32 status_reg_before;
461 u64 job_in_head_before;
462 u32 status_reg_after;
464 /* Check the head pointer */
465 job_in_head_before = ((u64) kbase_reg_read(kbdev, JOB_SLOT_REG(js, JSn_HEAD_LO), NULL))
466 | (((u64) kbase_reg_read(kbdev, JOB_SLOT_REG(js, JSn_HEAD_HI), NULL)) << 32);
467 status_reg_before = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JSn_STATUS), NULL);
470 if (action == JSn_COMMAND_SOFT_STOP) {
471 mali_bool soft_stop_allowed = kbasep_soft_stop_allowed(kbdev, core_reqs);
472 if (!soft_stop_allowed) {
473 #ifdef CONFIG_MALI_DEBUG
474 dev_dbg(kbdev->dev, "Attempt made to soft-stop a job that cannot be soft-stopped. core_reqs = 0x%X", (unsigned int)core_reqs);
475 #endif /* CONFIG_MALI_DEBUG */
479 /* We are about to issue a soft stop, so mark the atom as having been soft stopped */
480 target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED;
483 if (action == JSn_COMMAND_HARD_STOP) {
484 mali_bool hard_stop_allowed = kbasep_hard_stop_allowed(kbdev, core_reqs);
485 if (!hard_stop_allowed) {
486 /* Jobs can be hard-stopped for the following reasons:
487 * * CFS decides the job has been running too long (and soft-stop has not occurred).
488 * In this case the GPU will be reset by CFS if the job remains on the GPU.
490 * * The context is destroyed, kbase_jd_zap_context will attempt to hard-stop the job. However
491 * it also has a watchdog which will cause the GPU to be reset if the job remains on the GPU.
493 * * An (unhandled) MMU fault occurred. As long as BASE_HW_ISSUE_8245 is defined then
494 * the GPU will be reset.
496 * All three cases result in the GPU being reset if the hard-stop fails,
497 * so it is safe to just return and ignore the hard-stop request.
499 dev_warn(kbdev->dev, "Attempt made to hard-stop a job that cannot be hard-stopped. core_reqs = 0x%X", (unsigned int)core_reqs);
504 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316) && action == JSn_COMMAND_SOFT_STOP) {
507 slot = &kbdev->jm_slots[js];
509 for (i = 0; i < kbasep_jm_nr_jobs_submitted(slot); i++) {
510 kbase_jd_atom *katom;
512 katom = kbasep_jm_peek_idx_submit_slot(slot, i);
514 KBASE_DEBUG_ASSERT(katom);
516 if (kbasep_jm_is_dummy_workaround_job(kbdev, katom) != MALI_FALSE) {
517 /* Don't access the members of HW workaround 'dummy' jobs
519 * This assumes that such jobs can't cause HW_ISSUE_8316, and could only be blocked
520 * by other jobs causing HW_ISSUE_8316 (which will get poked/or eventually get killed) */
524 /* For HW_ISSUE_8316, only 'bad' jobs attacking the system can
525 * cause this issue: normally, all memory should be allocated in
526 * multiples of 4 pages, and growable memory should be changed size
527 * in multiples of 4 pages.
529 * Whilst such 'bad' jobs can be cleared by a GPU reset, the
530 * locking up of a uTLB entry caused by the bad job could also
531 * stall other ASs, meaning that other ASs' jobs don't complete in
532 * the 'grace' period before the reset. We don't want to lose other
533 * ASs' jobs when they would normally complete fine, so we must
534 * 'poke' the MMU regularly to help other ASs complete */
535 kbase_as_poking_timer_retain_atom(kbdev, katom->kctx, katom);
539 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
540 if (action == JSn_COMMAND_SOFT_STOP)
541 action = (target_katom->atom_flags & KBASE_KATOM_FLAGS_JOBCHAIN) ?
542 JSn_COMMAND_SOFT_STOP_1:
543 JSn_COMMAND_SOFT_STOP_0;
545 action = (target_katom->atom_flags & KBASE_KATOM_FLAGS_JOBCHAIN) ?
546 JSn_COMMAND_HARD_STOP_1:
547 JSn_COMMAND_HARD_STOP_0;
550 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JSn_COMMAND), action, kctx);
552 #if KBASE_TRACE_ENABLE
553 status_reg_after = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JSn_STATUS), NULL);
554 if (status_reg_after == BASE_JD_EVENT_ACTIVE) {
557 kbase_context *head_kctx;
559 slot = &kbdev->jm_slots[js];
560 head = kbasep_jm_peek_idx_submit_slot(slot, slot->submitted_nr - 1);
561 head_kctx = head->kctx;
563 /* We don't need to check kbasep_jm_is_dummy_workaround_job( head ) here:
564 * - Members are not indirected through
565 * - The members will all be zero anyway
567 if (status_reg_before == BASE_JD_EVENT_ACTIVE)
568 KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, head_kctx, head, job_in_head_before, js);
570 KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, 0, js);
573 case JSn_COMMAND_SOFT_STOP:
574 KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, head_kctx, head, head->jc, js);
576 case JSn_COMMAND_SOFT_STOP_0:
577 KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx, head, head->jc, js);
579 case JSn_COMMAND_SOFT_STOP_1:
580 KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx, head, head->jc, js);
582 case JSn_COMMAND_HARD_STOP:
583 KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, head_kctx, head, head->jc, js);
585 case JSn_COMMAND_HARD_STOP_0:
586 KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, head_kctx, head, head->jc, js);
588 case JSn_COMMAND_HARD_STOP_1:
589 KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, head_kctx, head, head->jc, js);
596 if (status_reg_before == BASE_JD_EVENT_ACTIVE)
597 KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, job_in_head_before, js);
599 KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL, 0, js);
602 case JSn_COMMAND_SOFT_STOP:
603 KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0, js);
605 case JSn_COMMAND_SOFT_STOP_0:
606 KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL, 0, js);
608 case JSn_COMMAND_SOFT_STOP_1:
609 KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL, 0, js);
611 case JSn_COMMAND_HARD_STOP:
612 KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0, js);
614 case JSn_COMMAND_HARD_STOP_0:
615 KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL, 0, js);
617 case JSn_COMMAND_HARD_STOP_1:
618 KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL, 0, js);
628 /* Helper macros used by kbasep_job_slot_soft_or_hard_stop */
629 #define JM_SLOT_MAX_JOB_SUBMIT_REGS 2
630 #define JM_JOB_IS_CURRENT_JOB_INDEX(n) (1 == n) /* Index of the last job to process */
631 #define JM_JOB_IS_NEXT_JOB_INDEX(n) (2 == n) /* Index of the prior to last job to process */
633 /** Soft or hard-stop a slot
635 * This function safely ensures that the correct job is either hard or soft-stopped.
636 * It deals with evicting jobs from the next registers where appropriate.
638 * This does not attempt to stop or evict jobs that are 'dummy' jobs for HW workarounds.
640 * @param kbdev The kbase device
641 * @param kctx The context to soft/hard-stop job(s) from (or NULL is all jobs should be targeted)
642 * @param js The slot that the job(s) are on
643 * @param target_katom The atom that should be targeted (or NULL if all jobs from the context should be targeted)
644 * @param action The action to perform, either JSn_COMMAND_HARD_STOP or JSn_COMMAND_SOFT_STOP
646 static void kbasep_job_slot_soft_or_hard_stop(kbase_device *kbdev, kbase_context *kctx, int js, kbase_jd_atom *target_katom, u32 action)
648 kbase_jd_atom *katom;
653 kbasep_js_device_data *js_devdata;
654 mali_bool can_safely_stop = kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION);
656 KBASE_DEBUG_ASSERT(action == JSn_COMMAND_HARD_STOP || action == JSn_COMMAND_SOFT_STOP);
657 KBASE_DEBUG_ASSERT(kbdev);
658 js_devdata = &kbdev->js_data;
660 slot = &kbdev->jm_slots[js];
661 KBASE_DEBUG_ASSERT(slot);
662 lockdep_assert_held(&js_devdata->runpool_irq.lock);
664 jobs_submitted = kbasep_jm_nr_jobs_submitted(slot);
666 KBASE_TIMELINE_TRY_SOFT_STOP(kctx, js, 1);
667 KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_SLOT_SOFT_OR_HARD_STOP, kctx, NULL, 0u, js, jobs_submitted);
669 if (jobs_submitted > JM_SLOT_MAX_JOB_SUBMIT_REGS)
670 i = jobs_submitted - JM_SLOT_MAX_JOB_SUBMIT_REGS;
674 /* Loop through all jobs that have been submitted to the slot and haven't completed */
675 for (; i < jobs_submitted; i++) {
676 katom = kbasep_jm_peek_idx_submit_slot(slot, i);
678 if (kctx && katom->kctx != kctx)
681 if (target_katom && katom != target_katom)
684 if (kbasep_jm_is_dummy_workaround_job(kbdev, katom))
687 core_reqs = katom->core_req;
689 if (JM_JOB_IS_CURRENT_JOB_INDEX(jobs_submitted - i)) {
690 /* The last job in the slot, check if there is a job in the next register */
691 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JSn_COMMAND_NEXT), NULL) == 0)
692 kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, action, core_reqs, katom);
694 /* The job is in the next registers */
695 beenthere(kctx, "clearing job from next registers on slot %d", js);
696 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JSn_COMMAND_NEXT), JSn_COMMAND_NOP, NULL);
697 /* Check to see if we did remove a job from the next registers */
698 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JSn_HEAD_NEXT_LO), NULL) != 0 || kbase_reg_read(kbdev, JOB_SLOT_REG(js, JSn_HEAD_NEXT_HI), NULL) != 0) {
699 /* The job was successfully cleared from the next registers, requeue it */
700 kbase_jd_atom *dequeued_katom = kbasep_jm_dequeue_tail_submit_slot(slot);
701 KBASE_DEBUG_ASSERT(dequeued_katom == katom);
704 /* Set the next registers to NULL */
705 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JSn_HEAD_NEXT_LO), 0, NULL);
706 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JSn_HEAD_NEXT_HI), 0, NULL);
708 /* As the job is removed from the next registers we undo the associated
709 * update to the job_chain_flag for the job slot. */
711 slot->job_chain_flag = !slot->job_chain_flag;
713 KBASE_TRACE_ADD_SLOT(kbdev, JM_SLOT_EVICT, dequeued_katom->kctx, dequeued_katom, dequeued_katom->jc, js);
715 /* Complete the job, indicate it took no time, but don't submit any more at this point */
716 kbase_jd_done(dequeued_katom, js, NULL, KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT);
718 /* The job transitioned into the current registers before we managed to evict it,
719 * in this case we fall back to soft/hard-stopping the job */
720 beenthere(kctx, "missed job in next register, soft/hard-stopping slot %d", js);
721 kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, action, core_reqs, katom);
724 } else if (JM_JOB_IS_NEXT_JOB_INDEX(jobs_submitted - i)) {
725 /* There's a job after this one, check to see if that job is in the next registers.
726 * If so, we need to pay attention to not accidently stop that one when issueing
727 * the command to stop the one pointed to by the head registers (as the one in the head
728 * may finish in the mean time and the one in the next moves to the head). Either the hardware
729 * has support for this using job chain disambiguation or we need to evict the job
730 * from the next registers first to ensure we can safely stop the one pointed to by
731 * the head registers. */
732 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JSn_COMMAND_NEXT), NULL) != 0) {
733 kbase_jd_atom *check_next_atom;
734 /* It is - we should remove that job and soft/hard-stop the slot */
736 /* Only proceed when the next job isn't a HW workaround 'dummy' job
738 * This can't be an ASSERT due to MMU fault code:
739 * - This first hard-stops the job that caused the fault
740 * - Under HW Issue 8245, it will then reset the GPU
741 * - This causes a Soft-stop to occur on all slots
742 * - By the time of the soft-stop, we may (depending on timing) still have:
743 * - The original job in HEAD, if it's not finished the hard-stop
744 * - The dummy workaround job in NEXT
746 * Other cases could be coded in future that cause back-to-back Soft/Hard
747 * stops with dummy workaround jobs in place, e.g. MMU handler code and Job
748 * Scheduler watchdog timer running in parallel.
750 * Note, the index i+1 is valid to peek from: i == jobs_submitted-2, therefore
751 * i+1 == jobs_submitted-1 */
752 check_next_atom = kbasep_jm_peek_idx_submit_slot(slot, i + 1);
753 if (kbasep_jm_is_dummy_workaround_job(kbdev, check_next_atom) != MALI_FALSE)
756 if (!can_safely_stop) {
757 beenthere(kctx, "clearing job from next registers on slot %d", js);
758 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JSn_COMMAND_NEXT), JSn_COMMAND_NOP, NULL);
760 /* Check to see if we did remove a job from the next registers */
761 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JSn_HEAD_NEXT_LO), NULL) != 0 || kbase_reg_read(kbdev, JOB_SLOT_REG(js, JSn_HEAD_NEXT_HI), NULL) != 0) {
762 /* We did remove a job from the next registers, requeue it */
763 kbase_jd_atom *dequeued_katom = kbasep_jm_dequeue_tail_submit_slot(slot);
764 KBASE_DEBUG_ASSERT(dequeued_katom != NULL);
767 /* Set the next registers to NULL */
768 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JSn_HEAD_NEXT_LO), 0, NULL);
769 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JSn_HEAD_NEXT_HI), 0, NULL);
771 KBASE_TRACE_ADD_SLOT(kbdev, JM_SLOT_EVICT, dequeued_katom->kctx, dequeued_katom, dequeued_katom->jc, js);
773 /* Complete the job, indicate it took no time, but don't submit any more at this point */
774 kbase_jd_done(dequeued_katom, js, NULL, KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT);
776 /* We missed the job, that means the job we're interested in left the hardware before
777 * we managed to do anything, so we can proceed to the next job */
782 /* Next is now free, so we can soft/hard-stop the slot */
783 beenthere(kctx, "soft/hard-stopped slot %d (there was a job in next which was successfully cleared)\n", js);
784 kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, action, core_reqs, katom);
786 /* If there was no job in the next registers, then the job we were
787 * interested in has finished, so we need not take any action
792 KBASE_TIMELINE_TRY_SOFT_STOP(kctx, js, 0);
795 void kbase_job_kill_jobs_from_context(kbase_context *kctx)
799 kbasep_js_device_data *js_devdata;
802 KBASE_DEBUG_ASSERT(kctx != NULL);
804 KBASE_DEBUG_ASSERT(kbdev != NULL);
805 js_devdata = &kbdev->js_data;
807 /* Cancel any remaining running jobs for this kctx */
808 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
810 /* Invalidate all jobs in context, to prevent re-submitting */
811 for (i = 0; i < BASE_JD_ATOM_COUNT; i++)
812 kctx->jctx.atoms[i].event_code = BASE_JD_EVENT_JOB_CANCELLED;
814 for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
815 kbase_job_slot_hardstop(kctx, i, NULL);
817 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
820 void kbase_job_zap_context(kbase_context *kctx)
823 kbasep_js_device_data *js_devdata;
824 kbasep_js_kctx_info *js_kctx_info;
826 mali_bool evict_success;
828 KBASE_DEBUG_ASSERT(kctx != NULL);
830 KBASE_DEBUG_ASSERT(kbdev != NULL);
831 js_devdata = &kbdev->js_data;
832 js_kctx_info = &kctx->jctx.sched_info;
835 * Critical assumption: No more submission is possible outside of the
836 * workqueue. This is because the OS *must* prevent U/K calls (IOCTLs)
837 * whilst the kbase_context is terminating.
840 /* First, atomically do the following:
841 * - mark the context as dying
842 * - try to evict it from the policy queue */
844 mutex_lock(&js_kctx_info->ctx.jsctx_mutex);
845 js_kctx_info->ctx.is_dying = MALI_TRUE;
847 dev_dbg(kbdev->dev, "Zap: Try Evict Ctx %p", kctx);
848 mutex_lock(&js_devdata->queue_mutex);
849 evict_success = kbasep_js_policy_try_evict_ctx(&js_devdata->policy, kctx);
850 mutex_unlock(&js_devdata->queue_mutex);
853 * At this point we know:
854 * - If eviction succeeded, it was in the policy queue, but now no longer is
855 * - We must cancel the jobs here. No Power Manager active reference to
857 * - This happens asynchronously - kbase_jd_zap_context() will wait for
858 * those jobs to be killed.
859 * - If eviction failed, then it wasn't in the policy queue. It is one of
861 * - a. it didn't have any jobs, and so is not in the Policy Queue or the
862 * Run Pool (not scheduled)
863 * - Hence, no more work required to cancel jobs. No Power Manager active
864 * reference to release.
865 * - b. it was in the middle of a scheduling transaction (and thus must
866 * have at least 1 job). This can happen from a syscall or a kernel thread.
867 * We still hold the jsctx_mutex, and so the thread must be waiting inside
868 * kbasep_js_try_schedule_head_ctx(), before checking whether the runpool
869 * is full. That thread will continue after we drop the mutex, and will
870 * notice the context is dying. It will rollback the transaction, killing
871 * all jobs at the same time. kbase_jd_zap_context() will wait for those
873 * - Hence, no more work required to cancel jobs, or to release the Power
874 * Manager active reference.
875 * - c. it is scheduled, and may or may not be running jobs
876 * - We must cause it to leave the runpool by stopping it from submitting
877 * any more jobs. When it finally does leave,
878 * kbasep_js_runpool_requeue_or_kill_ctx() will kill all remaining jobs
879 * (because it is dying), release the Power Manager active reference, and
880 * will not requeue the context in the policy queue. kbase_jd_zap_context()
881 * will wait for those jobs to be killed.
882 * - Hence, work required just to make it leave the runpool. Cancelling
883 * jobs and releasing the Power manager active reference will be handled
884 * when it leaves the runpool.
887 if (evict_success != MALI_FALSE || js_kctx_info->ctx.is_scheduled == MALI_FALSE) {
888 /* The following events require us to kill off remaining jobs and
889 * update PM book-keeping:
890 * - we evicted it correctly (it must have jobs to be in the Policy Queue)
892 * These events need no action, but take this path anyway:
893 * - Case a: it didn't have any jobs, and was never in the Queue
894 * - Case b: scheduling transaction will be partially rolled-back (this
895 * already cancels the jobs)
898 KBASE_TRACE_ADD(kbdev, JM_ZAP_NON_SCHEDULED, kctx, NULL, 0u, js_kctx_info->ctx.is_scheduled);
900 dev_dbg(kbdev->dev, "Zap: Ctx %p evict_success=%d, scheduled=%d", kctx, evict_success, js_kctx_info->ctx.is_scheduled);
902 if (evict_success != MALI_FALSE) {
903 /* Only cancel jobs when we evicted from the policy queue. No Power
904 * Manager active reference was held.
906 * Having is_dying set ensures that this kills, and doesn't requeue */
907 kbasep_js_runpool_requeue_or_kill_ctx(kbdev, kctx, MALI_FALSE);
909 mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
912 mali_bool was_retained;
913 /* Case c: didn't evict, but it is scheduled - it's in the Run Pool */
914 KBASE_TRACE_ADD(kbdev, JM_ZAP_SCHEDULED, kctx, NULL, 0u, js_kctx_info->ctx.is_scheduled);
915 dev_dbg(kbdev->dev, "Zap: Ctx %p is in RunPool", kctx);
917 /* Disable the ctx from submitting any more jobs */
918 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
919 kbasep_js_clear_submit_allowed(js_devdata, kctx);
921 /* Retain and (later) release the context whilst it is is now disallowed from submitting
922 * jobs - ensures that someone somewhere will be removing the context later on */
923 was_retained = kbasep_js_runpool_retain_ctx_nolock(kbdev, kctx);
925 /* Since it's scheduled and we have the jsctx_mutex, it must be retained successfully */
926 KBASE_DEBUG_ASSERT(was_retained != MALI_FALSE);
928 dev_dbg(kbdev->dev, "Zap: Ctx %p Kill Any Running jobs", kctx);
929 /* Cancel any remaining running jobs for this kctx - if any. Submit is disallowed
930 * which takes effect immediately, so no more new jobs will appear after we do this. */
931 for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
932 kbase_job_slot_hardstop(kctx, i, NULL);
934 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
935 mutex_unlock(&js_kctx_info->ctx.jsctx_mutex);
937 dev_dbg(kbdev->dev, "Zap: Ctx %p Release (may or may not schedule out immediately)", kctx);
938 kbasep_js_runpool_release_ctx(kbdev, kctx);
940 KBASE_TRACE_ADD(kbdev, JM_ZAP_DONE, kctx, NULL, 0u, 0u);
942 /* After this, you must wait on both the kbase_jd_context::zero_jobs_wait
943 * and the kbasep_js_kctx_info::ctx::is_scheduled_waitq - to wait for the
944 * jobs to be destroyed, and the context to be de-scheduled (if it was on
947 * kbase_jd_zap_context() will do this. */
949 KBASE_EXPORT_TEST_API(kbase_job_zap_context)
951 mali_error kbase_job_slot_init(kbase_device *kbdev)
954 KBASE_DEBUG_ASSERT(kbdev);
956 for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
957 kbasep_jm_init_submit_slot(&kbdev->jm_slots[i]);
959 return MALI_ERROR_NONE;
961 KBASE_EXPORT_TEST_API(kbase_job_slot_init)
963 void kbase_job_slot_halt(kbase_device *kbdev)
968 void kbase_job_slot_term(kbase_device *kbdev)
970 KBASE_DEBUG_ASSERT(kbdev);
972 KBASE_EXPORT_TEST_API(kbase_job_slot_term)
975 * Soft-stop the specified job slot
977 * The job slot lock must be held when calling this function.
978 * The job slot must not already be in the process of being soft-stopped.
980 * Where possible any job in the next register is evicted before the soft-stop.
982 * @param kbdev The kbase device
983 * @param js The job slot to soft-stop
984 * @param target_katom The job that should be soft-stopped (or NULL for any job)
986 void kbase_job_slot_softstop(kbase_device *kbdev, int js, kbase_jd_atom *target_katom)
988 kbasep_job_slot_soft_or_hard_stop(kbdev, NULL, js, target_katom, JSn_COMMAND_SOFT_STOP);
992 * Hard-stop the specified job slot
994 * The job slot lock must be held when calling this function.
996 * @param kctx The kbase context that contains the job(s) that should
998 * @param js The job slot to hard-stop
999 * @param target_katom The job that should be hard-stopped (or NULL for all
1000 * jobs from the context)
1002 void kbase_job_slot_hardstop(kbase_context *kctx, int js,
1003 kbase_jd_atom *target_katom)
1005 kbase_device *kbdev = kctx->kbdev;
1007 kbasep_job_slot_soft_or_hard_stop(kbdev, kctx, js, target_katom,
1008 JSn_COMMAND_HARD_STOP);
1009 if (kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_8401) ||
1010 kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_9510) ||
1011 (kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_T76X_3542) &&
1012 (target_katom == NULL || target_katom->core_req & BASE_JD_REQ_FS_AFBC))) {
1013 /* MIDBASE-2916 if a fragment job with AFBC encoding is
1014 * hardstopped, ensure to do a soft reset also in order to
1015 * clear the GPU status.
1016 * Workaround for HW issue 8401 has an issue,so after
1017 * hard-stopping just reset the GPU. This will ensure that the
1018 * jobs leave the GPU.*/
1019 if (kbase_prepare_to_reset_gpu_locked(kbdev)) {
1020 dev_err(kbdev->dev, "Issueing GPU\
1021 soft-reset after hard stopping due to hardware issue");
1022 kbase_reset_gpu_locked(kbdev);
1028 void kbase_debug_dump_registers(kbase_device *kbdev)
1031 dev_err(kbdev->dev, "Register state:");
1032 dev_err(kbdev->dev, " GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x",
1033 kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL),
1034 kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS), NULL));
1035 dev_err(kbdev->dev, " JOB_IRQ_RAWSTAT=0x%08x JOB_IRQ_JS_STATE=0x%08x JOB_IRQ_THROTTLE=0x%08x",
1036 kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL),
1037 kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_JS_STATE), NULL),
1038 kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_THROTTLE), NULL));
1039 for (i = 0; i < 3; i++) {
1040 dev_err(kbdev->dev, " JS%d_STATUS=0x%08x JS%d_HEAD_LO=0x%08x",
1041 i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JSn_STATUS),
1043 i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JSn_HEAD_LO),
1046 dev_err(kbdev->dev, " MMU_IRQ_RAWSTAT=0x%08x GPU_FAULTSTATUS=0x%08x",
1047 kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_RAWSTAT), NULL),
1048 kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS), NULL));
1049 dev_err(kbdev->dev, " GPU_IRQ_MASK=0x%08x JOB_IRQ_MASK=0x%08x MMU_IRQ_MASK=0x%08x",
1050 kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL),
1051 kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), NULL),
1052 kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL));
1053 dev_err(kbdev->dev, " PWR_OVERRIDE0=0x%08x PWR_OVERRIDE1=0x%08x",
1054 kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE0), NULL),
1055 kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE1), NULL));
1056 dev_err(kbdev->dev, " SHADER_CONFIG=0x%08x L2_MMU_CONFIG=0x%08x",
1057 kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), NULL),
1058 kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), NULL));
1061 void kbasep_reset_timeout_worker(struct work_struct *data)
1063 unsigned long flags;
1064 kbase_device *kbdev;
1066 ktime_t end_timestamp = ktime_get();
1067 kbasep_js_device_data *js_devdata;
1068 kbase_uk_hwcnt_setup hwcnt_setup = { {0} };
1069 kbase_instr_state bckp_state;
1071 KBASE_DEBUG_ASSERT(data);
1073 kbdev = container_of(data, kbase_device, reset_work);
1075 KBASE_DEBUG_ASSERT(kbdev);
1076 js_devdata = &kbdev->js_data;
1078 KBASE_TRACE_ADD(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0);
1080 /* Make sure the timer has completed - this cannot be done from interrupt context,
1081 * so this cannot be done within kbasep_try_reset_gpu_early. */
1082 hrtimer_cancel(&kbdev->reset_timer);
1084 if (kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
1085 /* This would re-activate the GPU. Since it's already idle, there's no
1086 * need to reset it */
1087 atomic_set(&kbdev->reset_gpu, KBASE_RESET_GPU_NOT_PENDING);
1088 wake_up(&kbdev->reset_wait);
1092 mutex_lock(&kbdev->pm.lock);
1093 /* We hold the pm lock, so there ought to be a current policy */
1094 KBASE_DEBUG_ASSERT(kbdev->pm.pm_current_policy);
1096 /* All slot have been soft-stopped and we've waited SOFT_STOP_RESET_TIMEOUT for the slots to clear, at this point
1097 * we assume that anything that is still left on the GPU is stuck there and we'll kill it when we reset the GPU */
1099 dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)", RESET_TIMEOUT);
1101 spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
1103 if (kbdev->hwcnt.state == KBASE_INSTR_STATE_RESETTING) { /*the same interrupt handler preempted itself */
1104 /* GPU is being reset */
1105 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
1106 wait_event(kbdev->hwcnt.wait, kbdev->hwcnt.triggered != 0);
1107 spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
1109 /* Save the HW counters setup */
1110 if (kbdev->hwcnt.kctx != NULL) {
1111 kbase_context *kctx = kbdev->hwcnt.kctx;
1112 hwcnt_setup.dump_buffer = kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), kctx) & 0xffffffff;
1113 hwcnt_setup.dump_buffer |= (mali_addr64) kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), kctx) << 32;
1114 hwcnt_setup.jm_bm = kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN), kctx);
1115 hwcnt_setup.shader_bm = kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN), kctx);
1116 hwcnt_setup.tiler_bm = kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), kctx);
1117 hwcnt_setup.l3_cache_bm = kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_L3_CACHE_EN), kctx);
1118 hwcnt_setup.mmu_l2_bm = kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN), kctx);
1121 /* Output the state of some interesting registers to help in the
1122 * debugging of GPU resets */
1123 kbase_debug_dump_registers(kbdev);
1125 bckp_state = kbdev->hwcnt.state;
1126 kbdev->hwcnt.state = KBASE_INSTR_STATE_RESETTING;
1127 kbdev->hwcnt.triggered = 0;
1128 /* Disable IRQ to avoid IRQ handlers to kick in after releaseing the spinlock;
1129 * this also clears any outstanding interrupts */
1130 kbase_pm_disable_interrupts(kbdev);
1131 /* Ensure that any IRQ handlers have finished */
1132 kbase_synchronize_irqs(kbdev);
1133 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
1136 kbase_pm_init_hw(kbdev, MALI_TRUE);
1137 /* IRQs were re-enabled by kbase_pm_init_hw, and GPU is still powered */
1139 spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
1140 /* Restore the HW counters setup */
1141 if (kbdev->hwcnt.kctx != NULL) {
1142 kbase_context *kctx = kbdev->hwcnt.kctx;
1143 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_OFF, kctx);
1144 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), hwcnt_setup.dump_buffer & 0xFFFFFFFF, kctx);
1145 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), hwcnt_setup.dump_buffer >> 32, kctx);
1146 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN), hwcnt_setup.jm_bm, kctx);
1147 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN), hwcnt_setup.shader_bm, kctx);
1148 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_L3_CACHE_EN), hwcnt_setup.l3_cache_bm, kctx);
1149 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN), hwcnt_setup.mmu_l2_bm, kctx);
1151 /* Due to PRLAM-8186 we need to disable the Tiler before we enable the HW counter dump. */
1152 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
1153 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), 0, kctx);
1155 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), hwcnt_setup.tiler_bm, kctx);
1157 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG), (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) | PRFCNT_CONFIG_MODE_MANUAL, kctx);
1159 /* If HW has PRLAM-8186 we can now re-enable the tiler HW counters dump */
1160 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
1161 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), hwcnt_setup.tiler_bm, kctx);
1163 kbdev->hwcnt.state = bckp_state;
1164 switch(kbdev->hwcnt.state) {
1165 /* Cases for waking kbasep_cache_clean_worker worker */
1166 case KBASE_INSTR_STATE_CLEANED:
1167 /* Cache-clean IRQ occurred, but we reset:
1168 * Wakeup incase the waiter saw RESETTING */
1169 case KBASE_INSTR_STATE_REQUEST_CLEAN:
1170 /* After a clean was requested, but before the regs were written:
1171 * Wakeup incase the waiter saw RESETTING */
1172 wake_up(&kbdev->hwcnt.cache_clean_wait);
1174 case KBASE_INSTR_STATE_CLEANING:
1176 * 1) We've not got the Cache-clean IRQ yet: it was lost, or:
1177 * 2) We got it whilst resetting: it was voluntarily lost
1179 * So, move to the next state and wakeup: */
1180 kbdev->hwcnt.state = KBASE_INSTR_STATE_CLEANED;
1181 wake_up(&kbdev->hwcnt.cache_clean_wait);
1184 /* Cases for waking anyone else */
1185 case KBASE_INSTR_STATE_DUMPING:
1186 /* If dumping, abort the dump, because we may've lost the IRQ */
1187 kbdev->hwcnt.state = KBASE_INSTR_STATE_IDLE;
1188 kbdev->hwcnt.triggered = 1;
1189 wake_up(&kbdev->hwcnt.wait);
1191 case KBASE_INSTR_STATE_DISABLED:
1192 case KBASE_INSTR_STATE_IDLE:
1193 case KBASE_INSTR_STATE_FAULT:
1194 /* Every other reason: wakeup in that state */
1195 kbdev->hwcnt.triggered = 1;
1196 wake_up(&kbdev->hwcnt.wait);
1199 /* Unhandled cases */
1200 case KBASE_INSTR_STATE_RESETTING:
1205 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
1207 /* Complete any jobs that were still on the GPU */
1208 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1209 for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) {
1211 kbase_jm_slot *slot = &kbdev->jm_slots[i];
1213 nr_done = kbasep_jm_nr_jobs_submitted(slot);
1215 dev_err(kbdev->dev, "Job stuck in slot %d on the GPU was cancelled", i);
1216 kbase_job_done_slot(kbdev, i, BASE_JD_EVENT_JOB_CANCELLED, 0, &end_timestamp);
1220 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1222 mutex_lock(&js_devdata->runpool_mutex);
1224 /* Reprogram the GPU's MMU */
1225 for (i = 0; i < BASE_MAX_NR_AS; i++) {
1226 if (js_devdata->runpool_irq.per_as_data[i].kctx) {
1227 kbase_as *as = &kbdev->as[i];
1228 mutex_lock(&as->transaction_mutex);
1229 kbase_mmu_update(js_devdata->runpool_irq.per_as_data[i].kctx);
1230 mutex_unlock(&as->transaction_mutex);
1234 atomic_set(&kbdev->reset_gpu, KBASE_RESET_GPU_NOT_PENDING);
1235 wake_up(&kbdev->reset_wait);
1236 dev_err(kbdev->dev, "Reset complete");
1238 /* Find out what cores are required now */
1239 kbase_pm_update_cores_state(kbdev);
1241 /* Synchronously request and wait for those cores, because if
1242 * instrumentation is enabled it would need them immediately. */
1243 kbase_pm_check_transitions_sync(kbdev);
1245 /* Try submitting some jobs to restart processing */
1246 if (js_devdata->nr_user_contexts_running > 0) {
1247 KBASE_TRACE_ADD(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u, 0);
1249 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1250 kbasep_js_try_run_next_job_nolock(kbdev);
1251 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1253 mutex_unlock(&js_devdata->runpool_mutex);
1254 mutex_unlock(&kbdev->pm.lock);
1256 kbase_pm_context_idle(kbdev);
1257 KBASE_TRACE_ADD(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0);
1260 enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer)
1262 kbase_device *kbdev = container_of(timer, kbase_device, reset_timer);
1264 KBASE_DEBUG_ASSERT(kbdev);
1266 /* Reset still pending? */
1267 if (atomic_cmpxchg(&kbdev->reset_gpu, KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) == KBASE_RESET_GPU_COMMITTED)
1268 queue_work(kbdev->reset_workq, &kbdev->reset_work);
1270 return HRTIMER_NORESTART;
1274 * If all jobs are evicted from the GPU then we can reset the GPU
1275 * immediately instead of waiting for the timeout to elapse
1278 static void kbasep_try_reset_gpu_early_locked(kbase_device *kbdev)
1281 int pending_jobs = 0;
1283 KBASE_DEBUG_ASSERT(kbdev);
1285 /* Count the number of jobs */
1286 for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) {
1287 kbase_jm_slot *slot = &kbdev->jm_slots[i];
1288 pending_jobs += kbasep_jm_nr_jobs_submitted(slot);
1291 if (pending_jobs > 0) {
1292 /* There are still jobs on the GPU - wait */
1296 /* Check that the reset has been committed to (i.e. kbase_reset_gpu has been called), and that no other
1297 * thread beat this thread to starting the reset */
1298 if (atomic_cmpxchg(&kbdev->reset_gpu, KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) != KBASE_RESET_GPU_COMMITTED) {
1299 /* Reset has already occurred */
1302 queue_work(kbdev->reset_workq, &kbdev->reset_work);
1305 static void kbasep_try_reset_gpu_early(kbase_device *kbdev)
1307 unsigned long flags;
1308 kbasep_js_device_data *js_devdata;
1310 js_devdata = &kbdev->js_data;
1311 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1312 kbasep_try_reset_gpu_early_locked(kbdev);
1313 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1317 * Prepare for resetting the GPU.
1318 * This function just soft-stops all the slots to ensure that as many jobs as possible are saved.
1320 * The function returns a boolean which should be interpreted as follows:
1321 * - MALI_TRUE - Prepared for reset, kbase_reset_gpu should be called.
1322 * - MALI_FALSE - Another thread is performing a reset, kbase_reset_gpu should not be called.
1324 * @return See description
1326 mali_bool kbase_prepare_to_reset_gpu_locked(kbase_device *kbdev)
1330 KBASE_DEBUG_ASSERT(kbdev);
1332 if (atomic_cmpxchg(&kbdev->reset_gpu, KBASE_RESET_GPU_NOT_PENDING, KBASE_RESET_GPU_PREPARED) != KBASE_RESET_GPU_NOT_PENDING) {
1333 /* Some other thread is already resetting the GPU */
1337 for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
1338 kbase_job_slot_softstop(kbdev, i, NULL);
1343 mali_bool kbase_prepare_to_reset_gpu(kbase_device *kbdev)
1345 unsigned long flags;
1347 kbasep_js_device_data *js_devdata;
1349 js_devdata = &kbdev->js_data;
1350 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1351 ret = kbase_prepare_to_reset_gpu_locked(kbdev);
1352 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1356 KBASE_EXPORT_TEST_API(kbase_prepare_to_reset_gpu)
1359 * This function should be called after kbase_prepare_to_reset_gpu iff it returns MALI_TRUE.
1360 * It should never be called without a corresponding call to kbase_prepare_to_reset_gpu.
1362 * After this function is called (or not called if kbase_prepare_to_reset_gpu returned MALI_FALSE),
1363 * the caller should wait for kbdev->reset_waitq to be signalled to know when the reset has completed.
1365 void kbase_reset_gpu(kbase_device *kbdev)
1369 KBASE_DEBUG_ASSERT(kbdev);
1371 /* Note this is an assert/atomic_set because it is a software issue for a race to be occuring here */
1372 KBASE_DEBUG_ASSERT(atomic_read(&kbdev->reset_gpu) == KBASE_RESET_GPU_PREPARED);
1373 atomic_set(&kbdev->reset_gpu, KBASE_RESET_GPU_COMMITTED);
1375 timeout_ms = kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS);
1376 dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", timeout_ms);
1377 hrtimer_start(&kbdev->reset_timer, HR_TIMER_DELAY_MSEC(timeout_ms), HRTIMER_MODE_REL);
1379 /* Try resetting early */
1380 kbasep_try_reset_gpu_early(kbdev);
1382 KBASE_EXPORT_TEST_API(kbase_reset_gpu)
1384 void kbase_reset_gpu_locked(kbase_device *kbdev)
1388 KBASE_DEBUG_ASSERT(kbdev);
1390 /* Note this is an assert/atomic_set because it is a software issue for a race to be occuring here */
1391 KBASE_DEBUG_ASSERT(atomic_read(&kbdev->reset_gpu) == KBASE_RESET_GPU_PREPARED);
1392 atomic_set(&kbdev->reset_gpu, KBASE_RESET_GPU_COMMITTED);
1394 timeout_ms = kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_JS_RESET_TIMEOUT_MS);
1395 dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n", timeout_ms);
1396 hrtimer_start(&kbdev->reset_timer, HR_TIMER_DELAY_MSEC(timeout_ms), HRTIMER_MODE_REL);
1398 /* Try resetting early */
1399 kbasep_try_reset_gpu_early_locked(kbdev);