3 * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
20 * Register-based HW access backend specific APIs
23 #include <mali_kbase.h>
24 #include <mali_kbase_hwaccess_jm.h>
25 #include <mali_kbase_jm.h>
26 #include <mali_kbase_js.h>
27 #include <mali_kbase_tlstream.h>
28 #include <mali_kbase_10969_workaround.h>
29 #include <backend/gpu/mali_kbase_device_internal.h>
30 #include <backend/gpu/mali_kbase_jm_internal.h>
31 #include <backend/gpu/mali_kbase_js_affinity.h>
32 #include <backend/gpu/mali_kbase_pm_internal.h>
34 /* Return whether the specified ringbuffer is empty. HW access lock must be
36 #define SLOT_RB_EMPTY(rb) (rb->write_idx == rb->read_idx)
37 /* Return number of atoms currently in the specified ringbuffer. HW access lock
39 #define SLOT_RB_ENTRIES(rb) (int)(s8)(rb->write_idx - rb->read_idx)
41 static void kbase_gpu_release_atom(struct kbase_device *kbdev,
42 struct kbase_jd_atom *katom,
43 ktime_t *end_timestamp);
46 * kbase_gpu_enqueue_atom - Enqueue an atom in the HW access ringbuffer
47 * @kbdev: Device pointer
48 * @katom: Atom to enqueue
50 * Context: Caller must hold the HW access lock
52 static void kbase_gpu_enqueue_atom(struct kbase_device *kbdev,
53 struct kbase_jd_atom *katom)
55 struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[katom->slot_nr];
57 WARN_ON(SLOT_RB_ENTRIES(rb) >= SLOT_RB_SIZE);
59 lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
61 rb->entries[rb->write_idx & SLOT_RB_MASK].katom = katom;
64 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED;
68 * kbase_gpu_dequeue_atom - Remove an atom from the HW access ringbuffer, once
69 * it has been completed
70 * @kbdev: Device pointer
71 * @js: Job slot to remove atom from
72 * @end_timestamp: Pointer to timestamp of atom completion. May be NULL, in
73 * which case current time will be used.
75 * Context: Caller must hold the HW access lock
77 * Return: Atom removed from ringbuffer
79 static struct kbase_jd_atom *kbase_gpu_dequeue_atom(struct kbase_device *kbdev,
81 ktime_t *end_timestamp)
83 struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
84 struct kbase_jd_atom *katom;
86 if (SLOT_RB_EMPTY(rb)) {
87 WARN(1, "GPU ringbuffer unexpectedly empty\n");
91 lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
93 katom = rb->entries[rb->read_idx & SLOT_RB_MASK].katom;
95 kbase_gpu_release_atom(kbdev, katom, end_timestamp);
99 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB;
101 kbase_js_debug_log_current_affinities(kbdev);
106 struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js,
109 struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
111 lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
113 if ((SLOT_RB_ENTRIES(rb) - 1) < idx)
114 return NULL; /* idx out of range */
116 return rb->entries[(rb->read_idx + idx) & SLOT_RB_MASK].katom;
119 struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev,
122 return kbase_gpu_inspect(kbdev, js, 0);
125 struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev,
128 struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
130 if (SLOT_RB_EMPTY(rb))
133 return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom;
137 * kbase_gpu_atoms_submitted - Inspect whether a slot has any atoms currently
139 * @kbdev: Device pointer
140 * @js: Job slot to inspect
142 * Return: true if there are atoms on the GPU for slot js,
145 static bool kbase_gpu_atoms_submitted(struct kbase_device *kbdev, int js)
149 lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
151 for (i = 0; i < SLOT_RB_SIZE; i++) {
152 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
156 if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED ||
157 katom->gpu_rb_state == KBASE_ATOM_GPU_RB_READY)
165 * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms
166 * currently on the GPU
167 * @kbdev: Device pointer
169 * Return: true if there are any atoms on the GPU, false otherwise
171 static bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev)
176 lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
178 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
179 for (i = 0; i < SLOT_RB_SIZE; i++) {
180 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
182 if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED)
189 int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js)
194 lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
196 for (i = 0; i < SLOT_RB_SIZE; i++) {
197 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
199 if (katom && (katom->gpu_rb_state ==
200 KBASE_ATOM_GPU_RB_SUBMITTED))
207 int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js)
212 lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
214 for (i = 0; i < SLOT_RB_SIZE; i++) {
215 if (kbase_gpu_inspect(kbdev, js, i))
222 static int kbase_gpu_nr_atoms_on_slot_min(struct kbase_device *kbdev, int js,
223 enum kbase_atom_gpu_rb_state min_rb_state)
228 lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
230 for (i = 0; i < SLOT_RB_SIZE; i++) {
231 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
233 if (katom && (katom->gpu_rb_state >= min_rb_state))
240 int kbase_backend_slot_free(struct kbase_device *kbdev, int js)
242 if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) !=
243 KBASE_RESET_GPU_NOT_PENDING) {
244 /* The GPU is being reset - so prevent submission */
248 return SLOT_RB_SIZE - kbase_backend_nr_atoms_on_slot(kbdev, js);
252 static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev,
253 struct kbase_jd_atom *katom);
255 static bool kbasep_js_job_check_ref_cores(struct kbase_device *kbdev,
257 struct kbase_jd_atom *katom)
259 /* The most recently checked affinity. Having this at this scope allows
260 * us to guarantee that we've checked the affinity in this function
263 u64 recently_chosen_affinity = 0;
264 bool chosen_affinity = false;
270 /* NOTE: The following uses a number of FALLTHROUGHs to optimize
271 * the calls to this function. Ending of the function is
272 * indicated by BREAK OUT */
273 switch (katom->coreref_state) {
274 /* State when job is first attempted to be run */
275 case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
276 KBASE_DEBUG_ASSERT(katom->affinity == 0);
278 /* Compute affinity */
279 if (false == kbase_js_choose_affinity(
280 &recently_chosen_affinity, kbdev, katom,
282 /* No cores are currently available */
283 /* *** BREAK OUT: No state transition *** */
287 chosen_affinity = true;
289 /* Request the cores */
290 kbase_pm_request_cores(kbdev,
291 katom->core_req & BASE_JD_REQ_T,
292 recently_chosen_affinity);
294 katom->affinity = recently_chosen_affinity;
296 /* Proceed to next state */
297 katom->coreref_state =
298 KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
300 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
302 case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
304 enum kbase_pm_cores_ready cores_ready;
306 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
307 (katom->core_req & BASE_JD_REQ_T));
309 cores_ready = kbase_pm_register_inuse_cores(
311 katom->core_req & BASE_JD_REQ_T,
313 if (cores_ready == KBASE_NEW_AFFINITY) {
314 /* Affinity no longer valid - return to
316 kbasep_js_job_check_deref_cores(kbdev,
318 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
319 JS_CORE_REF_REGISTER_INUSE_FAILED,
322 (u32) katom->affinity);
323 /* *** BREAK OUT: Return to previous
324 * state, retry *** */
328 if (cores_ready == KBASE_CORES_NOT_READY) {
329 /* Stay in this state and return, to
330 * retry at this state later */
331 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
332 JS_CORE_REF_REGISTER_INUSE_FAILED,
335 (u32) katom->affinity);
336 /* *** BREAK OUT: No state transition
340 /* Proceed to next state */
341 katom->coreref_state =
342 KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
345 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
347 case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
348 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
349 (katom->core_req & BASE_JD_REQ_T));
351 /* Optimize out choosing the affinity twice in the same
353 if (chosen_affinity == false) {
354 /* See if the affinity changed since a previous
356 if (false == kbase_js_choose_affinity(
357 &recently_chosen_affinity,
359 /* No cores are currently available */
360 kbasep_js_job_check_deref_cores(kbdev,
362 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
363 JS_CORE_REF_REQUEST_ON_RECHECK_FAILED,
366 (u32) recently_chosen_affinity);
367 /* *** BREAK OUT: Transition to lower
371 chosen_affinity = true;
374 /* Now see if this requires a different set of cores */
375 if (recently_chosen_affinity != katom->affinity) {
376 enum kbase_pm_cores_ready cores_ready;
378 kbase_pm_request_cores(kbdev,
379 katom->core_req & BASE_JD_REQ_T,
380 recently_chosen_affinity);
382 /* Register new cores whilst we still hold the
383 * old ones, to minimize power transitions */
385 kbase_pm_register_inuse_cores(kbdev,
386 katom->core_req & BASE_JD_REQ_T,
387 recently_chosen_affinity);
388 kbasep_js_job_check_deref_cores(kbdev, katom);
390 /* Fixup the state that was reduced by
392 katom->coreref_state =
393 KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
394 katom->affinity = recently_chosen_affinity;
395 if (cores_ready == KBASE_NEW_AFFINITY) {
396 /* Affinity no longer valid - return to
398 katom->coreref_state =
399 KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
401 kbasep_js_job_check_deref_cores(kbdev,
404 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
405 JS_CORE_REF_REGISTER_INUSE_FAILED,
408 (u32) katom->affinity);
409 /* *** BREAK OUT: Return to previous
410 * state, retry *** */
414 /* Now might be waiting for powerup again, with
416 if (cores_ready == KBASE_CORES_NOT_READY) {
417 /* Return to previous state */
418 katom->coreref_state =
419 KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
420 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
421 JS_CORE_REF_REGISTER_ON_RECHECK_FAILED,
424 (u32) katom->affinity);
425 /* *** BREAK OUT: Transition to lower
430 /* Proceed to next state */
431 katom->coreref_state =
432 KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS;
434 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
435 case KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS:
436 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
437 (katom->core_req & BASE_JD_REQ_T));
438 KBASE_DEBUG_ASSERT(katom->affinity ==
439 recently_chosen_affinity);
441 /* Note: this is where the caller must've taken the
442 * runpool_irq.lock */
444 /* Check for affinity violations - if there are any,
445 * then we just ask the caller to requeue and try again
447 if (kbase_js_affinity_would_violate(kbdev, js,
448 katom->affinity) != false) {
449 /* Return to previous state */
450 katom->coreref_state =
451 KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
452 /* *** BREAK OUT: Transition to lower state ***
454 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
455 JS_CORE_REF_AFFINITY_WOULD_VIOLATE,
456 katom->kctx, katom, katom->jc, js,
457 (u32) katom->affinity);
461 /* No affinity violations would result, so the cores are
463 katom->coreref_state = KBASE_ATOM_COREREF_STATE_READY;
464 /* *** BREAK OUT: Cores Ready *** */
468 KBASE_DEBUG_ASSERT_MSG(false,
469 "Unhandled kbase_atom_coreref_state %d",
470 katom->coreref_state);
473 } while (retry != false);
475 return (katom->coreref_state == KBASE_ATOM_COREREF_STATE_READY);
478 static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev,
479 struct kbase_jd_atom *katom)
481 KBASE_DEBUG_ASSERT(kbdev != NULL);
482 KBASE_DEBUG_ASSERT(katom != NULL);
484 switch (katom->coreref_state) {
485 case KBASE_ATOM_COREREF_STATE_READY:
486 /* State where atom was submitted to the HW - just proceed to
488 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
489 (katom->core_req & BASE_JD_REQ_T));
491 /* *** FALLTHROUGH *** */
493 case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
494 /* State where cores were registered */
495 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
496 (katom->core_req & BASE_JD_REQ_T));
497 kbase_pm_release_cores(kbdev, katom->core_req & BASE_JD_REQ_T,
502 case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
503 /* State where cores were requested, but not registered */
504 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
505 (katom->core_req & BASE_JD_REQ_T));
506 kbase_pm_unrequest_cores(kbdev, katom->core_req & BASE_JD_REQ_T,
510 case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
511 /* Initial state - nothing required */
512 KBASE_DEBUG_ASSERT(katom->affinity == 0);
516 KBASE_DEBUG_ASSERT_MSG(false,
517 "Unhandled coreref_state: %d",
518 katom->coreref_state);
523 katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
526 static void kbasep_js_job_check_deref_cores_nokatom(struct kbase_device *kbdev,
527 base_jd_core_req core_req, u64 affinity,
528 enum kbase_atom_coreref_state coreref_state)
530 KBASE_DEBUG_ASSERT(kbdev != NULL);
532 switch (coreref_state) {
533 case KBASE_ATOM_COREREF_STATE_READY:
534 /* State where atom was submitted to the HW - just proceed to
536 KBASE_DEBUG_ASSERT(affinity != 0 ||
537 (core_req & BASE_JD_REQ_T));
539 /* *** FALLTHROUGH *** */
541 case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
542 /* State where cores were registered */
543 KBASE_DEBUG_ASSERT(affinity != 0 ||
544 (core_req & BASE_JD_REQ_T));
545 kbase_pm_release_cores(kbdev, core_req & BASE_JD_REQ_T,
550 case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
551 /* State where cores were requested, but not registered */
552 KBASE_DEBUG_ASSERT(affinity != 0 ||
553 (core_req & BASE_JD_REQ_T));
554 kbase_pm_unrequest_cores(kbdev, core_req & BASE_JD_REQ_T,
558 case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
559 /* Initial state - nothing required */
560 KBASE_DEBUG_ASSERT(affinity == 0);
564 KBASE_DEBUG_ASSERT_MSG(false,
565 "Unhandled coreref_state: %d",
571 static void kbase_gpu_release_atom(struct kbase_device *kbdev,
572 struct kbase_jd_atom *katom,
573 ktime_t *end_timestamp)
575 switch (katom->gpu_rb_state) {
576 case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB:
577 /* Should be impossible */
578 WARN(1, "Attempting to release atom not in ringbuffer\n");
581 case KBASE_ATOM_GPU_RB_SUBMITTED:
582 /* Inform power management at start/finish of atom so it can
583 * update its GPU utilisation metrics. Mark atom as not
584 * submitted beforehand. */
585 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY;
586 kbase_pm_metrics_update(kbdev, end_timestamp);
588 if (katom->core_req & BASE_JD_REQ_PERMON)
589 kbase_pm_release_gpu_cycle_counter(kbdev);
590 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
592 case KBASE_ATOM_GPU_RB_READY:
593 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
595 case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_ENTRY:
596 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
598 case KBASE_ATOM_GPU_RB_WAITING_AFFINITY:
599 kbase_js_affinity_release_slot_cores(kbdev, katom->slot_nr,
601 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
603 case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE:
606 case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_EXIT:
607 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
609 case KBASE_ATOM_GPU_RB_WAITING_BLOCKED:
610 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
612 case KBASE_ATOM_GPU_RB_RETURN_TO_JS:
616 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED;
619 static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev,
620 struct kbase_jd_atom *katom)
622 kbase_gpu_release_atom(kbdev, katom, NULL);
623 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS;
626 static inline bool kbase_gpu_rmu_workaround(struct kbase_device *kbdev, int js)
628 struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
631 if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
633 slot_busy[0] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 0,
634 KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
635 slot_busy[1] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 1,
636 KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
637 slot_busy[2] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 2,
638 KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
640 if ((js == 2 && !(slot_busy[0] || slot_busy[1])) ||
641 (js != 2 && !slot_busy[2]))
644 /* Don't submit slot 2 atom while GPU has jobs on slots 0/1 */
645 if (js == 2 && (kbase_gpu_atoms_submitted(kbdev, 0) ||
646 kbase_gpu_atoms_submitted(kbdev, 1) ||
647 backend->rmu_workaround_flag))
650 /* Don't submit slot 0/1 atom while GPU has jobs on slot 2 */
651 if (js != 2 && (kbase_gpu_atoms_submitted(kbdev, 2) ||
652 !backend->rmu_workaround_flag))
655 backend->rmu_workaround_flag = !backend->rmu_workaround_flag;
660 static inline bool kbase_gpu_in_protected_mode(struct kbase_device *kbdev)
662 return kbdev->protected_mode;
665 static int kbase_gpu_protected_mode_enter(struct kbase_device *kbdev)
669 lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
671 WARN_ONCE(!kbdev->protected_ops,
672 "Cannot enter protected mode: protected callbacks not specified.\n");
674 if (kbdev->protected_ops) {
675 /* Switch GPU to protected mode */
676 err = kbdev->protected_ops->protected_mode_enter(kbdev);
679 dev_warn(kbdev->dev, "Failed to enable protected mode: %d\n",
682 kbdev->protected_mode = true;
688 static int kbase_gpu_protected_mode_reset(struct kbase_device *kbdev)
690 lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
692 WARN_ONCE(!kbdev->protected_ops,
693 "Cannot exit protected mode: protected callbacks not specified.\n");
695 if (!kbdev->protected_ops)
698 kbdev->protected_mode_transition = true;
699 kbase_reset_gpu_silent(kbdev);
704 static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev,
705 struct kbase_jd_atom **katom, int idx, int js)
709 switch (katom[idx]->exit_protected_state) {
710 case KBASE_ATOM_EXIT_PROTECTED_CHECK:
712 * If the atom ahead of this one hasn't got to being
713 * submitted yet then bail.
716 (katom[0]->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED &&
717 katom[0]->gpu_rb_state != KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB))
720 /* If we're not exiting protected mode then we're done here. */
721 if (!(kbase_gpu_in_protected_mode(kbdev) &&
722 !kbase_jd_katom_is_protected(katom[idx])))
726 * If there is a transition in progress, or work still
727 * on the GPU try again later.
729 if (kbdev->protected_mode_transition ||
730 kbase_gpu_atoms_submitted_any(kbdev))
734 * Exiting protected mode requires a reset, but first the L2
735 * needs to be powered down to ensure it's not active when the
738 katom[idx]->exit_protected_state =
739 KBASE_ATOM_EXIT_PROTECTED_IDLE_L2;
741 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
743 case KBASE_ATOM_EXIT_PROTECTED_IDLE_L2:
744 if (kbase_pm_get_active_cores(kbdev, KBASE_PM_CORE_L2) ||
745 kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) {
747 * The L2 is still powered, wait for all the users to
748 * finish with it before doing the actual reset.
752 katom[idx]->exit_protected_state =
753 KBASE_ATOM_EXIT_PROTECTED_RESET;
755 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
757 case KBASE_ATOM_EXIT_PROTECTED_RESET:
758 /* Issue the reset to the GPU */
759 err = kbase_gpu_protected_mode_reset(kbdev);
761 /* Failed to exit protected mode, fail atom */
762 katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID;
763 kbase_gpu_mark_atom_for_return(kbdev, katom[idx]);
764 /* Only return if head atom or previous atom
765 * already removed - as atoms must be returned
767 if (idx == 0 || katom[0]->gpu_rb_state ==
768 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
769 kbase_gpu_dequeue_atom(kbdev, js, NULL);
770 kbase_jm_return_atom_to_js(kbdev, katom[idx]);
773 kbase_vinstr_resume(kbdev->vinstr_ctx);
778 katom[idx]->exit_protected_state =
779 KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT;
781 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
783 case KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT:
784 if (kbase_reset_gpu_active(kbdev))
787 /* protected mode sanity checks */
788 KBASE_DEBUG_ASSERT_MSG(
789 kbase_jd_katom_is_protected(katom[idx]) == kbase_gpu_in_protected_mode(kbdev),
790 "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)",
791 kbase_jd_katom_is_protected(katom[idx]), kbase_gpu_in_protected_mode(kbdev));
792 KBASE_DEBUG_ASSERT_MSG(
793 (kbase_jd_katom_is_protected(katom[idx]) && js == 0) ||
794 !kbase_jd_katom_is_protected(katom[idx]),
795 "Protected atom on JS%d not supported", js);
801 void kbase_gpu_slot_update(struct kbase_device *kbdev)
805 lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
807 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
808 struct kbase_jd_atom *katom[2];
811 katom[0] = kbase_gpu_inspect(kbdev, js, 0);
812 katom[1] = kbase_gpu_inspect(kbdev, js, 1);
813 WARN_ON(katom[1] && !katom[0]);
815 for (idx = 0; idx < SLOT_RB_SIZE; idx++) {
822 switch (katom[idx]->gpu_rb_state) {
823 case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB:
824 /* Should be impossible */
825 WARN(1, "Attempting to update atom not in ringbuffer\n");
828 case KBASE_ATOM_GPU_RB_WAITING_BLOCKED:
829 if (katom[idx]->atom_flags &
830 KBASE_KATOM_FLAG_X_DEP_BLOCKED)
833 katom[idx]->gpu_rb_state =
834 KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_EXIT;
836 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
838 case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_EXIT:
840 * Exiting protected mode must be done before
841 * the references on the cores are taken as
842 * a power down the L2 is required which
843 * can't happen after the references for this
846 ret = kbase_jm_exit_protected_mode(kbdev,
851 katom[idx]->gpu_rb_state =
852 KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE;
854 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
856 case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE:
857 if (katom[idx]->will_fail_event_code) {
858 kbase_gpu_mark_atom_for_return(kbdev,
860 /* Set EVENT_DONE so this atom will be
861 completed, not unpulled. */
862 katom[idx]->event_code =
864 /* Only return if head atom or previous
865 * atom already removed - as atoms must
866 * be returned in order. */
867 if (idx == 0 || katom[0]->gpu_rb_state ==
868 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
869 kbase_gpu_dequeue_atom(kbdev, js, NULL);
870 kbase_jm_return_atom_to_js(kbdev, katom[idx]);
877 kbasep_js_job_check_ref_cores(kbdev, js,
880 if (katom[idx]->event_code ==
881 BASE_JD_EVENT_PM_EVENT) {
882 katom[idx]->gpu_rb_state =
883 KBASE_ATOM_GPU_RB_RETURN_TO_JS;
890 kbase_js_affinity_retain_slot_cores(kbdev, js,
891 katom[idx]->affinity);
892 katom[idx]->gpu_rb_state =
893 KBASE_ATOM_GPU_RB_WAITING_AFFINITY;
895 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
897 case KBASE_ATOM_GPU_RB_WAITING_AFFINITY:
898 if (!kbase_gpu_rmu_workaround(kbdev, js))
901 katom[idx]->gpu_rb_state =
902 KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_ENTRY;
904 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
906 case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_ENTRY:
908 /* Only submit if head atom or previous atom
909 * already submitted */
911 (katom[0]->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED &&
912 katom[0]->gpu_rb_state != KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB))
916 * If the GPU is transitioning protected mode
917 * then bail now and we'll be called when the
918 * new state has settled.
920 if (kbdev->protected_mode_transition)
923 if (!kbase_gpu_in_protected_mode(kbdev) && kbase_jd_katom_is_protected(katom[idx])) {
926 /* Not in correct mode, take action */
927 if (kbase_gpu_atoms_submitted_any(kbdev)) {
929 * We are not in the correct
930 * GPU mode for this job, and
931 * we can't switch now because
932 * there are jobs already
937 if (kbase_vinstr_try_suspend(kbdev->vinstr_ctx) < 0) {
939 * We can't switch now because
940 * the vinstr core state switch
945 /* Once reaching this point GPU must be
946 * switched to protected mode or vinstr
949 /* No jobs running, so we can switch GPU mode right now */
950 err = kbase_gpu_protected_mode_enter(kbdev);
953 * Failed to switch into protected mode, resume
954 * vinstr core and fail atom.
956 kbase_vinstr_resume(kbdev->vinstr_ctx);
957 katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID;
958 kbase_gpu_mark_atom_for_return(kbdev, katom[idx]);
959 /* Only return if head atom or previous atom
960 * already removed - as atoms must be returned
962 if (idx == 0 || katom[0]->gpu_rb_state ==
963 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
964 kbase_gpu_dequeue_atom(kbdev, js, NULL);
965 kbase_jm_return_atom_to_js(kbdev, katom[idx]);
971 /* Protected mode sanity checks */
972 KBASE_DEBUG_ASSERT_MSG(
973 kbase_jd_katom_is_protected(katom[idx]) == kbase_gpu_in_protected_mode(kbdev),
974 "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)",
975 kbase_jd_katom_is_protected(katom[idx]), kbase_gpu_in_protected_mode(kbdev));
976 katom[idx]->gpu_rb_state =
977 KBASE_ATOM_GPU_RB_READY;
979 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
981 case KBASE_ATOM_GPU_RB_READY:
983 /* Only submit if head atom or previous atom
984 * already submitted */
986 (katom[0]->gpu_rb_state !=
987 KBASE_ATOM_GPU_RB_SUBMITTED &&
988 katom[0]->gpu_rb_state !=
989 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB))
992 /* Check if this job needs the cycle counter
993 * enabled before submission */
994 if (katom[idx]->core_req & BASE_JD_REQ_PERMON)
995 kbase_pm_request_gpu_cycle_counter_l2_is_on(
998 kbase_job_hw_submit(kbdev, katom[idx], js);
999 katom[idx]->gpu_rb_state =
1000 KBASE_ATOM_GPU_RB_SUBMITTED;
1002 /* Inform power management at start/finish of
1003 * atom so it can update its GPU utilisation
1005 kbase_pm_metrics_update(kbdev,
1006 &katom[idx]->start_timestamp);
1008 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
1010 case KBASE_ATOM_GPU_RB_SUBMITTED:
1011 /* Atom submitted to HW, nothing else to do */
1014 case KBASE_ATOM_GPU_RB_RETURN_TO_JS:
1015 /* Only return if head atom or previous atom
1016 * already removed - as atoms must be returned
1018 if (idx == 0 || katom[0]->gpu_rb_state ==
1019 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
1020 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1021 kbase_jm_return_atom_to_js(kbdev,
1029 /* Warn if PRLAM-8987 affinity restrictions are violated */
1030 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
1031 WARN_ON((kbase_gpu_atoms_submitted(kbdev, 0) ||
1032 kbase_gpu_atoms_submitted(kbdev, 1)) &&
1033 kbase_gpu_atoms_submitted(kbdev, 2));
1037 void kbase_backend_run_atom(struct kbase_device *kbdev,
1038 struct kbase_jd_atom *katom)
1040 lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1042 kbase_gpu_enqueue_atom(kbdev, katom);
1043 kbase_gpu_slot_update(kbdev);
1046 bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js)
1048 struct kbase_jd_atom *katom;
1049 struct kbase_jd_atom *next_katom;
1051 lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1053 katom = kbase_gpu_inspect(kbdev, js, 0);
1054 next_katom = kbase_gpu_inspect(kbdev, js, 1);
1056 if (next_katom && katom->kctx == next_katom->kctx &&
1057 next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED &&
1058 (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL)
1060 kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL)
1062 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT),
1063 JS_COMMAND_NOP, NULL);
1064 next_katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY;
1071 void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
1072 u32 completion_code,
1074 ktime_t *end_timestamp)
1076 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0);
1077 struct kbase_context *kctx = katom->kctx;
1079 lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1081 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6787) &&
1082 completion_code != BASE_JD_EVENT_DONE &&
1083 !(completion_code & BASE_JD_SW_EVENT)) {
1084 katom->need_cache_flush_cores_retained = katom->affinity;
1085 kbase_pm_request_cores(kbdev, false, katom->affinity);
1086 } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10676)) {
1087 if (kbdev->gpu_props.num_core_groups > 1 &&
1089 kbdev->gpu_props.props.coherency_info.group[0].core_mask
1092 kbdev->gpu_props.props.coherency_info.group[1].core_mask
1094 dev_info(kbdev->dev, "JD: Flushing cache due to PRLAM-10676\n");
1095 katom->need_cache_flush_cores_retained =
1097 kbase_pm_request_cores(kbdev, false,
1102 katom = kbase_gpu_dequeue_atom(kbdev, js, end_timestamp);
1103 kbase_timeline_job_slot_done(kbdev, katom->kctx, katom, js, 0);
1104 kbase_tlstream_tl_nret_atom_lpu(
1106 &kbdev->gpu_props.props.raw_props.js_features[
1108 kbase_tlstream_tl_nret_atom_as(katom, &kbdev->as[kctx->as_nr]);
1109 kbase_tlstream_tl_nret_ctx_lpu(
1111 &kbdev->gpu_props.props.raw_props.js_features[
1114 if (completion_code == BASE_JD_EVENT_STOPPED) {
1115 struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js,
1119 * Dequeue next atom from ringbuffers on same slot if required.
1120 * This atom will already have been removed from the NEXT
1121 * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that
1122 * the atoms on this slot are returned in the correct order.
1124 if (next_katom && katom->kctx == next_katom->kctx) {
1125 kbase_gpu_dequeue_atom(kbdev, js, end_timestamp);
1126 kbase_jm_return_atom_to_js(kbdev, next_katom);
1128 } else if (completion_code != BASE_JD_EVENT_DONE) {
1129 struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1132 #if KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR != 0
1133 KBASE_TRACE_DUMP(kbdev);
1135 kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
1138 * Remove all atoms on the same context from ringbuffers. This
1139 * will not remove atoms that are already on the GPU, as these
1140 * are guaranteed not to have fail dependencies on the failed
1143 for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) {
1144 struct kbase_jd_atom *katom_idx0 =
1145 kbase_gpu_inspect(kbdev, i, 0);
1146 struct kbase_jd_atom *katom_idx1 =
1147 kbase_gpu_inspect(kbdev, i, 1);
1149 if (katom_idx0 && katom_idx0->kctx == katom->kctx &&
1150 katom_idx0->gpu_rb_state !=
1151 KBASE_ATOM_GPU_RB_SUBMITTED) {
1152 /* Dequeue katom_idx0 from ringbuffer */
1153 kbase_gpu_dequeue_atom(kbdev, i, end_timestamp);
1156 katom_idx1->kctx == katom->kctx &&
1157 katom_idx0->gpu_rb_state !=
1158 KBASE_ATOM_GPU_RB_SUBMITTED) {
1159 /* Dequeue katom_idx1 from ringbuffer */
1160 kbase_gpu_dequeue_atom(kbdev, i,
1163 katom_idx1->event_code =
1164 BASE_JD_EVENT_STOPPED;
1165 kbase_jm_return_atom_to_js(kbdev,
1168 katom_idx0->event_code = BASE_JD_EVENT_STOPPED;
1169 kbase_jm_return_atom_to_js(kbdev, katom_idx0);
1171 } else if (katom_idx1 &&
1172 katom_idx1->kctx == katom->kctx &&
1173 katom_idx1->gpu_rb_state !=
1174 KBASE_ATOM_GPU_RB_SUBMITTED) {
1175 /* Can not dequeue this atom yet - will be
1176 * dequeued when atom at idx0 completes */
1177 katom_idx1->event_code = BASE_JD_EVENT_STOPPED;
1178 kbase_gpu_mark_atom_for_return(kbdev,
1184 KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc,
1185 js, completion_code);
1187 if (job_tail != 0 && job_tail != katom->jc) {
1188 bool was_updated = (job_tail != katom->jc);
1190 /* Some of the job has been executed, so we update the job chain
1191 * address to where we should resume from */
1192 katom->jc = job_tail;
1194 KBASE_TRACE_ADD_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx,
1195 katom, job_tail, js);
1198 /* Only update the event code for jobs that weren't cancelled */
1199 if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED)
1200 katom->event_code = (base_jd_event_code)completion_code;
1202 kbase_device_trace_register_access(kctx, REG_WRITE,
1203 JOB_CONTROL_REG(JOB_IRQ_CLEAR),
1206 /* Complete the job, and start new ones
1208 * Also defer remaining work onto the workqueue:
1209 * - Re-queue Soft-stopped jobs
1210 * - For any other jobs, queue the job back into the dependency system
1211 * - Schedule out the parent context if necessary, and schedule a new
1214 #ifdef CONFIG_GPU_TRACEPOINTS
1216 /* The atom in the HEAD */
1217 struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js,
1220 if (next_katom && next_katom->gpu_rb_state ==
1221 KBASE_ATOM_GPU_RB_SUBMITTED) {
1224 trace_gpu_sched_switch(kbasep_make_job_slot_string(js,
1226 ktime_to_ns(*end_timestamp),
1227 (u32)next_katom->kctx->id, 0,
1228 next_katom->work_id);
1229 kbdev->hwaccess.backend.slot_rb[js].last_context =
1234 trace_gpu_sched_switch(kbasep_make_job_slot_string(js,
1236 ktime_to_ns(ktime_get()), 0, 0,
1238 kbdev->hwaccess.backend.slot_rb[js].last_context = 0;
1243 if (completion_code == BASE_JD_EVENT_STOPPED)
1244 kbase_jm_return_atom_to_js(kbdev, katom);
1246 kbase_jm_complete(kbdev, katom, end_timestamp);
1248 /* Job completion may have unblocked other atoms. Try to update all job
1250 kbase_gpu_slot_update(kbdev);
1253 void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp)
1257 lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1259 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
1262 for (idx = 0; idx < 2; idx++) {
1263 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev,
1265 bool keep_in_jm_rb = false;
1270 if (katom->gpu_rb_state < KBASE_ATOM_GPU_RB_SUBMITTED)
1271 keep_in_jm_rb = true;
1273 kbase_gpu_release_atom(kbdev, katom, NULL);
1276 * If the atom wasn't on HW when the reset was issued
1277 * then leave it in the RB and next time we're kicked
1278 * it will be processed again from the starting state.
1280 if (keep_in_jm_rb) {
1281 katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
1282 katom->exit_protected_state = KBASE_ATOM_EXIT_PROTECTED_CHECK;
1287 * The atom was on the HW when the reset was issued
1288 * all we can do is fail the atom.
1290 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1291 katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
1292 kbase_jm_complete(kbdev, katom, end_timestamp);
1297 static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev,
1299 struct kbase_jd_atom *katom,
1302 struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1303 u32 hw_action = action & JS_COMMAND_MASK;
1305 kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom);
1306 kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action,
1307 katom->core_req, katom);
1308 kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
1311 static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev,
1312 struct kbase_jd_atom *katom,
1316 struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1318 katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT;
1319 kbase_gpu_mark_atom_for_return(kbdev, katom);
1320 kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
1323 kbase_job_check_enter_disjoint(kbdev, action, katom->core_req,
1327 static int should_stop_x_dep_slot(struct kbase_jd_atom *katom)
1329 if (katom->x_post_dep) {
1330 struct kbase_jd_atom *dep_atom = katom->x_post_dep;
1332 if (dep_atom->gpu_rb_state !=
1333 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB &&
1334 dep_atom->gpu_rb_state !=
1335 KBASE_ATOM_GPU_RB_RETURN_TO_JS)
1336 return dep_atom->slot_nr;
1341 static void kbase_job_evicted(struct kbase_jd_atom *katom)
1343 kbase_timeline_job_slot_done(katom->kctx->kbdev, katom->kctx, katom,
1344 katom->slot_nr, KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT);
1347 bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev,
1348 struct kbase_context *kctx,
1350 struct kbase_jd_atom *katom,
1353 struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1355 struct kbase_jd_atom *katom_idx0;
1356 struct kbase_jd_atom *katom_idx1;
1358 bool katom_idx0_valid, katom_idx1_valid;
1362 int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1;
1364 lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1366 katom_idx0 = kbase_gpu_inspect(kbdev, js, 0);
1367 katom_idx1 = kbase_gpu_inspect(kbdev, js, 1);
1370 katom_idx0_valid = (katom_idx0 == katom);
1371 /* If idx0 is to be removed and idx1 is on the same context,
1372 * then idx1 must also be removed otherwise the atoms might be
1373 * returned out of order */
1375 katom_idx1_valid = (katom_idx1 == katom) ||
1376 (katom_idx0_valid &&
1377 (katom_idx0->kctx ==
1380 katom_idx1_valid = false;
1382 katom_idx0_valid = (katom_idx0 &&
1383 (!kctx || katom_idx0->kctx == kctx));
1384 katom_idx1_valid = (katom_idx1 &&
1385 (!kctx || katom_idx1->kctx == kctx));
1388 if (katom_idx0_valid)
1389 stop_x_dep_idx0 = should_stop_x_dep_slot(katom_idx0);
1390 if (katom_idx1_valid)
1391 stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1);
1393 if (katom_idx0_valid) {
1394 if (katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) {
1395 /* Simple case - just dequeue and return */
1396 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1397 if (katom_idx1_valid) {
1398 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1399 katom_idx1->event_code =
1400 BASE_JD_EVENT_REMOVED_FROM_NEXT;
1401 kbase_jm_return_atom_to_js(kbdev, katom_idx1);
1402 kbasep_js_clear_submit_allowed(js_devdata,
1406 katom_idx0->event_code =
1407 BASE_JD_EVENT_REMOVED_FROM_NEXT;
1408 kbase_jm_return_atom_to_js(kbdev, katom_idx0);
1409 kbasep_js_clear_submit_allowed(js_devdata,
1412 /* katom_idx0 is on GPU */
1413 if (katom_idx1 && katom_idx1->gpu_rb_state ==
1414 KBASE_ATOM_GPU_RB_SUBMITTED) {
1415 /* katom_idx0 and katom_idx1 are on GPU */
1417 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1418 JS_COMMAND_NEXT), NULL) == 0) {
1419 /* idx0 has already completed - stop
1421 if (katom_idx1_valid) {
1422 kbase_gpu_stop_atom(kbdev, js,
1428 /* idx1 is in NEXT registers - attempt
1430 kbase_reg_write(kbdev,
1433 JS_COMMAND_NOP, NULL);
1435 if (kbase_reg_read(kbdev,
1437 JS_HEAD_NEXT_LO), NULL)
1439 kbase_reg_read(kbdev,
1441 JS_HEAD_NEXT_HI), NULL)
1443 /* idx1 removed successfully,
1444 * will be handled in IRQ */
1445 kbase_job_evicted(katom_idx1);
1446 kbase_gpu_remove_atom(kbdev,
1450 should_stop_x_dep_slot(katom_idx1);
1452 /* stop idx0 if still on GPU */
1453 kbase_gpu_stop_atom(kbdev, js,
1457 } else if (katom_idx1_valid) {
1458 /* idx0 has already completed,
1459 * stop idx1 if needed */
1460 kbase_gpu_stop_atom(kbdev, js,
1466 } else if (katom_idx1_valid) {
1467 /* idx1 not on GPU but must be dequeued*/
1469 /* idx1 will be handled in IRQ */
1470 kbase_gpu_remove_atom(kbdev, katom_idx1, action,
1473 /* This will be repeated for anything removed
1474 * from the next registers, since their normal
1475 * flow was also interrupted, and this function
1476 * might not enter disjoint state e.g. if we
1477 * don't actually do a hard stop on the head
1479 kbase_gpu_stop_atom(kbdev, js, katom_idx0,
1483 /* no atom in idx1 */
1484 /* just stop idx0 */
1485 kbase_gpu_stop_atom(kbdev, js, katom_idx0,
1490 } else if (katom_idx1_valid) {
1491 if (katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) {
1492 /* Mark for return */
1493 /* idx1 will be returned once idx0 completes */
1494 kbase_gpu_remove_atom(kbdev, katom_idx1, action,
1497 /* idx1 is on GPU */
1498 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1499 JS_COMMAND_NEXT), NULL) == 0) {
1500 /* idx0 has already completed - stop idx1 */
1501 kbase_gpu_stop_atom(kbdev, js, katom_idx1,
1505 /* idx1 is in NEXT registers - attempt to
1507 kbase_reg_write(kbdev, JOB_SLOT_REG(js,
1509 JS_COMMAND_NOP, NULL);
1511 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1512 JS_HEAD_NEXT_LO), NULL) != 0 ||
1513 kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1514 JS_HEAD_NEXT_HI), NULL) != 0) {
1515 /* idx1 removed successfully, will be
1516 * handled in IRQ once idx0 completes */
1517 kbase_job_evicted(katom_idx1);
1518 kbase_gpu_remove_atom(kbdev, katom_idx1,
1522 /* idx0 has already completed - stop
1524 kbase_gpu_stop_atom(kbdev, js,
1534 if (stop_x_dep_idx0 != -1)
1535 kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx0,
1538 if (stop_x_dep_idx1 != -1)
1539 kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx1,
1545 void kbase_gpu_cacheclean(struct kbase_device *kbdev,
1546 struct kbase_jd_atom *katom)
1548 /* Limit the number of loops to avoid a hang if the interrupt is missed
1550 u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS;
1552 mutex_lock(&kbdev->cacheclean_lock);
1554 /* use GPU_COMMAND completion solution */
1555 /* clean & invalidate the caches */
1556 KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0);
1557 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1558 GPU_COMMAND_CLEAN_INV_CACHES, NULL);
1560 /* wait for cache flush to complete before continuing */
1561 while (--max_loops &&
1562 (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) &
1563 CLEAN_CACHES_COMPLETED) == 0)
1566 /* clear the CLEAN_CACHES_COMPLETED irq */
1567 KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u,
1568 CLEAN_CACHES_COMPLETED);
1569 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR),
1570 CLEAN_CACHES_COMPLETED, NULL);
1571 KBASE_DEBUG_ASSERT_MSG(kbdev->hwcnt.backend.state !=
1572 KBASE_INSTR_STATE_CLEANING,
1573 "Instrumentation code was cleaning caches, but Job Management code cleared their IRQ - Instrumentation code will now hang.");
1575 mutex_unlock(&kbdev->cacheclean_lock);
1577 kbase_pm_unrequest_cores(kbdev, false,
1578 katom->need_cache_flush_cores_retained);
1581 void kbase_backend_complete_wq(struct kbase_device *kbdev,
1582 struct kbase_jd_atom *katom)
1585 * If cache flush required due to HW workaround then perform the flush
1588 if (katom->need_cache_flush_cores_retained) {
1589 kbase_gpu_cacheclean(kbdev, katom);
1590 katom->need_cache_flush_cores_retained = 0;
1593 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10969) &&
1594 (katom->core_req & BASE_JD_REQ_FS) &&
1595 katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT &&
1596 (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED) &&
1597 !(katom->atom_flags & KBASE_KATOM_FLAGS_RERUN)) {
1598 dev_dbg(kbdev->dev, "Soft-stopped fragment shader job got a TILE_RANGE_FAULT. Possible HW issue, trying SW workaround\n");
1599 if (kbasep_10969_workaround_clamp_coordinates(katom)) {
1600 /* The job had a TILE_RANGE_FAULT after was soft-stopped
1601 * Due to an HW issue we try to execute the job again.
1604 "Clamping has been executed, try to rerun the job\n"
1606 katom->event_code = BASE_JD_EVENT_STOPPED;
1607 katom->atom_flags |= KBASE_KATOM_FLAGS_RERUN;
1611 /* Clear the coreref_state now - while check_deref_cores() may not have
1612 * been called yet, the caller will have taken a copy of this field. If
1613 * this is not done, then if the atom is re-scheduled (following a soft
1614 * stop) then the core reference would not be retaken. */
1615 katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
1616 katom->affinity = 0;
1619 void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev,
1620 base_jd_core_req core_req, u64 affinity,
1621 enum kbase_atom_coreref_state coreref_state)
1623 kbasep_js_job_check_deref_cores_nokatom(kbdev, core_req, affinity,
1626 if (!kbdev->pm.active_count) {
1627 mutex_lock(&kbdev->js_data.runpool_mutex);
1628 mutex_lock(&kbdev->pm.lock);
1629 kbase_pm_update_active(kbdev);
1630 mutex_unlock(&kbdev->pm.lock);
1631 mutex_unlock(&kbdev->js_data.runpool_mutex);
1635 void kbase_gpu_dump_slots(struct kbase_device *kbdev)
1637 struct kbasep_js_device_data *js_devdata;
1638 unsigned long flags;
1641 js_devdata = &kbdev->js_data;
1643 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1645 dev_info(kbdev->dev, "kbase_gpu_dump_slots:\n");
1647 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
1650 for (idx = 0; idx < SLOT_RB_SIZE; idx++) {
1651 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev,
1656 dev_info(kbdev->dev,
1657 " js%d idx%d : katom=%p gpu_rb_state=%d\n",
1658 js, idx, katom, katom->gpu_rb_state);
1660 dev_info(kbdev->dev, " js%d idx%d : empty\n",
1665 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);