3 * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
20 * Register-based HW access backend specific APIs
23 #include <mali_kbase.h>
24 #include <mali_kbase_hwaccess_jm.h>
25 #include <mali_kbase_jm.h>
26 #include <mali_kbase_js.h>
27 #include <mali_kbase_tlstream.h>
28 #include <mali_kbase_10969_workaround.h>
29 #include <backend/gpu/mali_kbase_cache_policy_backend.h>
30 #include <backend/gpu/mali_kbase_device_internal.h>
31 #include <backend/gpu/mali_kbase_jm_internal.h>
32 #include <backend/gpu/mali_kbase_js_affinity.h>
33 #include <backend/gpu/mali_kbase_pm_internal.h>
35 /* Return whether the specified ringbuffer is empty. HW access lock must be
37 #define SLOT_RB_EMPTY(rb) (rb->write_idx == rb->read_idx)
38 /* Return number of atoms currently in the specified ringbuffer. HW access lock
40 #define SLOT_RB_ENTRIES(rb) (int)(s8)(rb->write_idx - rb->read_idx)
42 static void kbase_gpu_release_atom(struct kbase_device *kbdev,
43 struct kbase_jd_atom *katom,
44 ktime_t *end_timestamp);
47 * kbase_gpu_enqueue_atom - Enqueue an atom in the HW access ringbuffer
48 * @kbdev: Device pointer
49 * @katom: Atom to enqueue
51 * Context: Caller must hold the HW access lock
53 static void kbase_gpu_enqueue_atom(struct kbase_device *kbdev,
54 struct kbase_jd_atom *katom)
56 struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[katom->slot_nr];
58 WARN_ON(SLOT_RB_ENTRIES(rb) >= SLOT_RB_SIZE);
60 lockdep_assert_held(&kbdev->hwaccess_lock);
62 rb->entries[rb->write_idx & SLOT_RB_MASK].katom = katom;
65 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED;
69 * kbase_gpu_dequeue_atom - Remove an atom from the HW access ringbuffer, once
70 * it has been completed
71 * @kbdev: Device pointer
72 * @js: Job slot to remove atom from
73 * @end_timestamp: Pointer to timestamp of atom completion. May be NULL, in
74 * which case current time will be used.
76 * Context: Caller must hold the HW access lock
78 * Return: Atom removed from ringbuffer
80 static struct kbase_jd_atom *kbase_gpu_dequeue_atom(struct kbase_device *kbdev,
82 ktime_t *end_timestamp)
84 struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
85 struct kbase_jd_atom *katom;
87 if (SLOT_RB_EMPTY(rb)) {
88 WARN(1, "GPU ringbuffer unexpectedly empty\n");
92 lockdep_assert_held(&kbdev->hwaccess_lock);
94 katom = rb->entries[rb->read_idx & SLOT_RB_MASK].katom;
96 kbase_gpu_release_atom(kbdev, katom, end_timestamp);
100 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB;
102 kbase_js_debug_log_current_affinities(kbdev);
107 struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js,
110 struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
112 lockdep_assert_held(&kbdev->hwaccess_lock);
114 if ((SLOT_RB_ENTRIES(rb) - 1) < idx)
115 return NULL; /* idx out of range */
117 return rb->entries[(rb->read_idx + idx) & SLOT_RB_MASK].katom;
120 struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev,
123 return kbase_gpu_inspect(kbdev, js, 0);
126 struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev,
129 struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
131 if (SLOT_RB_EMPTY(rb))
134 return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom;
138 * kbase_gpu_atoms_submitted - Inspect whether a slot has any atoms currently
140 * @kbdev: Device pointer
141 * @js: Job slot to inspect
143 * Return: true if there are atoms on the GPU for slot js,
146 static bool kbase_gpu_atoms_submitted(struct kbase_device *kbdev, int js)
150 lockdep_assert_held(&kbdev->hwaccess_lock);
152 for (i = 0; i < SLOT_RB_SIZE; i++) {
153 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
157 if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED ||
158 katom->gpu_rb_state == KBASE_ATOM_GPU_RB_READY)
166 * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms
167 * currently on the GPU
168 * @kbdev: Device pointer
170 * Return: true if there are any atoms on the GPU, false otherwise
172 static bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev)
177 lockdep_assert_held(&kbdev->hwaccess_lock);
179 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
180 for (i = 0; i < SLOT_RB_SIZE; i++) {
181 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
183 if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED)
190 int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js)
195 lockdep_assert_held(&kbdev->hwaccess_lock);
197 for (i = 0; i < SLOT_RB_SIZE; i++) {
198 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
200 if (katom && (katom->gpu_rb_state ==
201 KBASE_ATOM_GPU_RB_SUBMITTED))
208 int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js)
213 lockdep_assert_held(&kbdev->hwaccess_lock);
215 for (i = 0; i < SLOT_RB_SIZE; i++) {
216 if (kbase_gpu_inspect(kbdev, js, i))
223 static int kbase_gpu_nr_atoms_on_slot_min(struct kbase_device *kbdev, int js,
224 enum kbase_atom_gpu_rb_state min_rb_state)
229 lockdep_assert_held(&kbdev->hwaccess_lock);
231 for (i = 0; i < SLOT_RB_SIZE; i++) {
232 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
234 if (katom && (katom->gpu_rb_state >= min_rb_state))
242 * check_secure_atom - Check if the given atom is in the given secure state and
243 * has a ringbuffer state of at least
244 * KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION
245 * @katom: Atom pointer
246 * @secure: Desired secure state
248 * Return: true if atom is in the given state, false otherwise
250 static bool check_secure_atom(struct kbase_jd_atom *katom, bool secure)
252 if (katom->gpu_rb_state >=
253 KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION &&
254 ((kbase_jd_katom_is_protected(katom) && secure) ||
255 (!kbase_jd_katom_is_protected(katom) && !secure)))
262 * kbase_gpu_check_secure_atoms - Check if there are any atoms in the given
263 * secure state in the ringbuffers of at least
265 * KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE
266 * @kbdev: Device pointer
267 * @secure: Desired secure state
269 * Return: true if any atoms are in the given state, false otherwise
271 static bool kbase_gpu_check_secure_atoms(struct kbase_device *kbdev,
276 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
277 for (i = 0; i < SLOT_RB_SIZE; i++) {
278 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev,
282 if (check_secure_atom(katom, secure))
291 int kbase_backend_slot_free(struct kbase_device *kbdev, int js)
293 if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) !=
294 KBASE_RESET_GPU_NOT_PENDING) {
295 /* The GPU is being reset - so prevent submission */
299 return SLOT_RB_SIZE - kbase_backend_nr_atoms_on_slot(kbdev, js);
303 static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev,
304 struct kbase_jd_atom *katom);
306 static bool kbasep_js_job_check_ref_cores(struct kbase_device *kbdev,
308 struct kbase_jd_atom *katom)
310 /* The most recently checked affinity. Having this at this scope allows
311 * us to guarantee that we've checked the affinity in this function
314 u64 recently_chosen_affinity = 0;
315 bool chosen_affinity = false;
321 /* NOTE: The following uses a number of FALLTHROUGHs to optimize
322 * the calls to this function. Ending of the function is
323 * indicated by BREAK OUT */
324 switch (katom->coreref_state) {
325 /* State when job is first attempted to be run */
326 case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
327 KBASE_DEBUG_ASSERT(katom->affinity == 0);
329 /* Compute affinity */
330 if (false == kbase_js_choose_affinity(
331 &recently_chosen_affinity, kbdev, katom,
333 /* No cores are currently available */
334 /* *** BREAK OUT: No state transition *** */
338 chosen_affinity = true;
340 /* Request the cores */
341 kbase_pm_request_cores(kbdev,
342 katom->core_req & BASE_JD_REQ_T,
343 recently_chosen_affinity);
345 katom->affinity = recently_chosen_affinity;
347 /* Proceed to next state */
348 katom->coreref_state =
349 KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
351 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
353 case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
355 enum kbase_pm_cores_ready cores_ready;
357 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
358 (katom->core_req & BASE_JD_REQ_T));
360 cores_ready = kbase_pm_register_inuse_cores(
362 katom->core_req & BASE_JD_REQ_T,
364 if (cores_ready == KBASE_NEW_AFFINITY) {
365 /* Affinity no longer valid - return to
367 kbasep_js_job_check_deref_cores(kbdev,
369 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
370 JS_CORE_REF_REGISTER_INUSE_FAILED,
373 (u32) katom->affinity);
374 /* *** BREAK OUT: Return to previous
375 * state, retry *** */
379 if (cores_ready == KBASE_CORES_NOT_READY) {
380 /* Stay in this state and return, to
381 * retry at this state later */
382 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
383 JS_CORE_REF_REGISTER_INUSE_FAILED,
386 (u32) katom->affinity);
387 /* *** BREAK OUT: No state transition
391 /* Proceed to next state */
392 katom->coreref_state =
393 KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
396 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
398 case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
399 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
400 (katom->core_req & BASE_JD_REQ_T));
402 /* Optimize out choosing the affinity twice in the same
404 if (chosen_affinity == false) {
405 /* See if the affinity changed since a previous
407 if (false == kbase_js_choose_affinity(
408 &recently_chosen_affinity,
410 /* No cores are currently available */
411 kbasep_js_job_check_deref_cores(kbdev,
413 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
414 JS_CORE_REF_REQUEST_ON_RECHECK_FAILED,
417 (u32) recently_chosen_affinity);
418 /* *** BREAK OUT: Transition to lower
422 chosen_affinity = true;
425 /* Now see if this requires a different set of cores */
426 if (recently_chosen_affinity != katom->affinity) {
427 enum kbase_pm_cores_ready cores_ready;
429 kbase_pm_request_cores(kbdev,
430 katom->core_req & BASE_JD_REQ_T,
431 recently_chosen_affinity);
433 /* Register new cores whilst we still hold the
434 * old ones, to minimize power transitions */
436 kbase_pm_register_inuse_cores(kbdev,
437 katom->core_req & BASE_JD_REQ_T,
438 recently_chosen_affinity);
439 kbasep_js_job_check_deref_cores(kbdev, katom);
441 /* Fixup the state that was reduced by
443 katom->coreref_state =
444 KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
445 katom->affinity = recently_chosen_affinity;
446 if (cores_ready == KBASE_NEW_AFFINITY) {
447 /* Affinity no longer valid - return to
449 katom->coreref_state =
450 KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
452 kbasep_js_job_check_deref_cores(kbdev,
455 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
456 JS_CORE_REF_REGISTER_INUSE_FAILED,
459 (u32) katom->affinity);
460 /* *** BREAK OUT: Return to previous
461 * state, retry *** */
465 /* Now might be waiting for powerup again, with
467 if (cores_ready == KBASE_CORES_NOT_READY) {
468 /* Return to previous state */
469 katom->coreref_state =
470 KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
471 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
472 JS_CORE_REF_REGISTER_ON_RECHECK_FAILED,
475 (u32) katom->affinity);
476 /* *** BREAK OUT: Transition to lower
481 /* Proceed to next state */
482 katom->coreref_state =
483 KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS;
485 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
486 case KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS:
487 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
488 (katom->core_req & BASE_JD_REQ_T));
489 KBASE_DEBUG_ASSERT(katom->affinity ==
490 recently_chosen_affinity);
492 /* Note: this is where the caller must've taken the
495 /* Check for affinity violations - if there are any,
496 * then we just ask the caller to requeue and try again
498 if (kbase_js_affinity_would_violate(kbdev, js,
499 katom->affinity) != false) {
500 /* Return to previous state */
501 katom->coreref_state =
502 KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
503 /* *** BREAK OUT: Transition to lower state ***
505 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
506 JS_CORE_REF_AFFINITY_WOULD_VIOLATE,
507 katom->kctx, katom, katom->jc, js,
508 (u32) katom->affinity);
512 /* No affinity violations would result, so the cores are
514 katom->coreref_state = KBASE_ATOM_COREREF_STATE_READY;
515 /* *** BREAK OUT: Cores Ready *** */
519 KBASE_DEBUG_ASSERT_MSG(false,
520 "Unhandled kbase_atom_coreref_state %d",
521 katom->coreref_state);
524 } while (retry != false);
526 return (katom->coreref_state == KBASE_ATOM_COREREF_STATE_READY);
529 static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev,
530 struct kbase_jd_atom *katom)
532 KBASE_DEBUG_ASSERT(kbdev != NULL);
533 KBASE_DEBUG_ASSERT(katom != NULL);
535 switch (katom->coreref_state) {
536 case KBASE_ATOM_COREREF_STATE_READY:
537 /* State where atom was submitted to the HW - just proceed to
539 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
540 (katom->core_req & BASE_JD_REQ_T));
542 /* *** FALLTHROUGH *** */
544 case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
545 /* State where cores were registered */
546 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
547 (katom->core_req & BASE_JD_REQ_T));
548 kbase_pm_release_cores(kbdev, katom->core_req & BASE_JD_REQ_T,
553 case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
554 /* State where cores were requested, but not registered */
555 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
556 (katom->core_req & BASE_JD_REQ_T));
557 kbase_pm_unrequest_cores(kbdev, katom->core_req & BASE_JD_REQ_T,
561 case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
562 /* Initial state - nothing required */
563 KBASE_DEBUG_ASSERT(katom->affinity == 0);
567 KBASE_DEBUG_ASSERT_MSG(false,
568 "Unhandled coreref_state: %d",
569 katom->coreref_state);
574 katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
577 static void kbasep_js_job_check_deref_cores_nokatom(struct kbase_device *kbdev,
578 base_jd_core_req core_req, u64 affinity,
579 enum kbase_atom_coreref_state coreref_state)
581 KBASE_DEBUG_ASSERT(kbdev != NULL);
583 switch (coreref_state) {
584 case KBASE_ATOM_COREREF_STATE_READY:
585 /* State where atom was submitted to the HW - just proceed to
587 KBASE_DEBUG_ASSERT(affinity != 0 ||
588 (core_req & BASE_JD_REQ_T));
590 /* *** FALLTHROUGH *** */
592 case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
593 /* State where cores were registered */
594 KBASE_DEBUG_ASSERT(affinity != 0 ||
595 (core_req & BASE_JD_REQ_T));
596 kbase_pm_release_cores(kbdev, core_req & BASE_JD_REQ_T,
601 case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
602 /* State where cores were requested, but not registered */
603 KBASE_DEBUG_ASSERT(affinity != 0 ||
604 (core_req & BASE_JD_REQ_T));
605 kbase_pm_unrequest_cores(kbdev, core_req & BASE_JD_REQ_T,
609 case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
610 /* Initial state - nothing required */
611 KBASE_DEBUG_ASSERT(affinity == 0);
615 KBASE_DEBUG_ASSERT_MSG(false,
616 "Unhandled coreref_state: %d",
622 static void kbase_gpu_release_atom(struct kbase_device *kbdev,
623 struct kbase_jd_atom *katom,
624 ktime_t *end_timestamp)
626 switch (katom->gpu_rb_state) {
627 case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB:
628 /* Should be impossible */
629 WARN(1, "Attempting to release atom not in ringbuffer\n");
632 case KBASE_ATOM_GPU_RB_SUBMITTED:
633 /* Inform power management at start/finish of atom so it can
634 * update its GPU utilisation metrics. Mark atom as not
635 * submitted beforehand. */
636 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY;
637 kbase_pm_metrics_update(kbdev, end_timestamp);
639 if (katom->core_req & BASE_JD_REQ_PERMON)
640 kbase_pm_release_gpu_cycle_counter_nolock(kbdev);
641 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
643 case KBASE_ATOM_GPU_RB_READY:
644 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
646 case KBASE_ATOM_GPU_RB_WAITING_AFFINITY:
647 kbase_js_affinity_release_slot_cores(kbdev, katom->slot_nr,
649 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
651 case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE:
654 case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION:
655 if (katom->protected_state.enter !=
656 KBASE_ATOM_ENTER_PROTECTED_CHECK ||
657 katom->protected_state.exit !=
658 KBASE_ATOM_EXIT_PROTECTED_CHECK)
659 kbdev->protected_mode_transition = false;
661 if (kbase_jd_katom_is_protected(katom) &&
662 (katom->protected_state.enter ==
663 KBASE_ATOM_ENTER_PROTECTED_IDLE_L2))
664 kbase_vinstr_resume(kbdev->vinstr_ctx);
666 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
668 case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV:
669 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
671 case KBASE_ATOM_GPU_RB_WAITING_BLOCKED:
672 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
674 case KBASE_ATOM_GPU_RB_RETURN_TO_JS:
678 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED;
681 static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev,
682 struct kbase_jd_atom *katom)
684 kbase_gpu_release_atom(kbdev, katom, NULL);
685 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS;
688 static inline bool kbase_gpu_rmu_workaround(struct kbase_device *kbdev, int js)
690 struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
693 if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
695 slot_busy[0] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 0,
696 KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
697 slot_busy[1] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 1,
698 KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
699 slot_busy[2] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 2,
700 KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
702 if ((js == 2 && !(slot_busy[0] || slot_busy[1])) ||
703 (js != 2 && !slot_busy[2]))
706 /* Don't submit slot 2 atom while GPU has jobs on slots 0/1 */
707 if (js == 2 && (kbase_gpu_atoms_submitted(kbdev, 0) ||
708 kbase_gpu_atoms_submitted(kbdev, 1) ||
709 backend->rmu_workaround_flag))
712 /* Don't submit slot 0/1 atom while GPU has jobs on slot 2 */
713 if (js != 2 && (kbase_gpu_atoms_submitted(kbdev, 2) ||
714 !backend->rmu_workaround_flag))
717 backend->rmu_workaround_flag = !backend->rmu_workaround_flag;
722 static inline bool kbase_gpu_in_protected_mode(struct kbase_device *kbdev)
724 return kbdev->protected_mode;
727 static int kbase_gpu_protected_mode_enter(struct kbase_device *kbdev)
731 lockdep_assert_held(&kbdev->hwaccess_lock);
733 WARN_ONCE(!kbdev->protected_ops,
734 "Cannot enter protected mode: protected callbacks not specified.\n");
737 * When entering into protected mode, we must ensure that the
738 * GPU is not operating in coherent mode as well. This is to
739 * ensure that no protected memory can be leaked.
741 if (kbdev->system_coherency == COHERENCY_ACE)
742 kbase_cache_set_coherency_mode(kbdev, COHERENCY_ACE_LITE);
744 if (kbdev->protected_ops) {
745 /* Switch GPU to protected mode */
746 err = kbdev->protected_ops->protected_mode_enter(kbdev);
749 dev_warn(kbdev->dev, "Failed to enable protected mode: %d\n",
752 kbdev->protected_mode = true;
758 static int kbase_gpu_protected_mode_reset(struct kbase_device *kbdev)
760 lockdep_assert_held(&kbdev->hwaccess_lock);
762 WARN_ONCE(!kbdev->protected_ops,
763 "Cannot exit protected mode: protected callbacks not specified.\n");
765 if (!kbdev->protected_ops)
768 kbase_reset_gpu_silent(kbdev);
773 static int kbase_jm_enter_protected_mode(struct kbase_device *kbdev,
774 struct kbase_jd_atom **katom, int idx, int js)
778 switch (katom[idx]->protected_state.enter) {
779 case KBASE_ATOM_ENTER_PROTECTED_CHECK:
780 /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV
781 * should ensure that we are not already transitiong, and that
782 * there are no atoms currently on the GPU. */
783 WARN_ON(kbdev->protected_mode_transition);
784 WARN_ON(kbase_gpu_atoms_submitted_any(kbdev));
786 kbdev->protected_mode_transition = true;
787 katom[idx]->protected_state.enter =
788 KBASE_ATOM_ENTER_PROTECTED_VINSTR;
790 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
792 case KBASE_ATOM_ENTER_PROTECTED_VINSTR:
793 if (kbase_vinstr_try_suspend(kbdev->vinstr_ctx) < 0) {
795 * We can't switch now because
796 * the vinstr core state switch
802 /* Once reaching this point GPU must be
803 * switched to protected mode or vinstr
807 * Not in correct mode, begin protected mode switch.
808 * Entering protected mode requires us to power down the L2,
809 * and drop out of fully coherent mode.
811 katom[idx]->protected_state.enter =
812 KBASE_ATOM_ENTER_PROTECTED_IDLE_L2;
814 kbase_pm_update_cores_state_nolock(kbdev);
816 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
818 case KBASE_ATOM_ENTER_PROTECTED_IDLE_L2:
819 /* Avoid unnecessary waiting on non-ACE platforms. */
820 if (kbdev->current_gpu_coherency_mode == COHERENCY_ACE) {
821 if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) ||
822 kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) {
824 * The L2 is still powered, wait for all the users to
825 * finish with it before doing the actual reset.
831 katom[idx]->protected_state.enter =
832 KBASE_ATOM_ENTER_PROTECTED_FINISHED;
834 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
836 case KBASE_ATOM_ENTER_PROTECTED_FINISHED:
838 /* No jobs running, so we can switch GPU mode right now. */
839 err = kbase_gpu_protected_mode_enter(kbdev);
842 * Regardless of result, we are no longer transitioning
845 kbdev->protected_mode_transition = false;
849 * Failed to switch into protected mode, resume
850 * vinstr core and fail atom.
852 kbase_vinstr_resume(kbdev->vinstr_ctx);
853 katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID;
854 kbase_gpu_mark_atom_for_return(kbdev, katom[idx]);
855 /* Only return if head atom or previous atom
856 * already removed - as atoms must be returned
858 if (idx == 0 || katom[0]->gpu_rb_state ==
859 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
860 kbase_gpu_dequeue_atom(kbdev, js, NULL);
861 kbase_jm_return_atom_to_js(kbdev, katom[idx]);
866 /* Protected mode sanity checks. */
867 KBASE_DEBUG_ASSERT_MSG(
868 kbase_jd_katom_is_protected(katom[idx]) ==
869 kbase_gpu_in_protected_mode(kbdev),
870 "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)",
871 kbase_jd_katom_is_protected(katom[idx]),
872 kbase_gpu_in_protected_mode(kbdev));
873 katom[idx]->gpu_rb_state =
874 KBASE_ATOM_GPU_RB_READY;
880 static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev,
881 struct kbase_jd_atom **katom, int idx, int js)
886 switch (katom[idx]->protected_state.exit) {
887 case KBASE_ATOM_EXIT_PROTECTED_CHECK:
888 /* The checks in KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV
889 * should ensure that we are not already transitiong, and that
890 * there are no atoms currently on the GPU. */
891 WARN_ON(kbdev->protected_mode_transition);
892 WARN_ON(kbase_gpu_atoms_submitted_any(kbdev));
895 * Exiting protected mode requires a reset, but first the L2
896 * needs to be powered down to ensure it's not active when the
899 katom[idx]->protected_state.exit =
900 KBASE_ATOM_EXIT_PROTECTED_IDLE_L2;
902 kbdev->protected_mode_transition = true;
903 kbase_pm_update_cores_state_nolock(kbdev);
905 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
906 case KBASE_ATOM_EXIT_PROTECTED_IDLE_L2:
907 if (kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2) ||
908 kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) {
910 * The L2 is still powered, wait for all the users to
911 * finish with it before doing the actual reset.
915 katom[idx]->protected_state.exit =
916 KBASE_ATOM_EXIT_PROTECTED_RESET;
918 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
920 case KBASE_ATOM_EXIT_PROTECTED_RESET:
921 /* Issue the reset to the GPU */
922 err = kbase_gpu_protected_mode_reset(kbdev);
925 kbdev->protected_mode_transition = false;
927 /* Failed to exit protected mode, fail atom */
928 katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID;
929 kbase_gpu_mark_atom_for_return(kbdev, katom[idx]);
930 /* Only return if head atom or previous atom
931 * already removed - as atoms must be returned
933 if (idx == 0 || katom[0]->gpu_rb_state ==
934 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
935 kbase_gpu_dequeue_atom(kbdev, js, NULL);
936 kbase_jm_return_atom_to_js(kbdev, katom[idx]);
939 kbase_vinstr_resume(kbdev->vinstr_ctx);
944 katom[idx]->protected_state.exit =
945 KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT;
947 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
949 case KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT:
950 if (kbase_reset_gpu_active(kbdev))
953 kbdev->protected_mode_transition = false;
954 kbdev->protected_mode = false;
956 /* protected mode sanity checks */
957 KBASE_DEBUG_ASSERT_MSG(
958 kbase_jd_katom_is_protected(katom[idx]) == kbase_gpu_in_protected_mode(kbdev),
959 "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)",
960 kbase_jd_katom_is_protected(katom[idx]), kbase_gpu_in_protected_mode(kbdev));
961 KBASE_DEBUG_ASSERT_MSG(
962 (kbase_jd_katom_is_protected(katom[idx]) && js == 0) ||
963 !kbase_jd_katom_is_protected(katom[idx]),
964 "Protected atom on JS%d not supported", js);
970 void kbase_backend_slot_update(struct kbase_device *kbdev)
974 lockdep_assert_held(&kbdev->hwaccess_lock);
976 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
977 struct kbase_jd_atom *katom[2];
980 katom[0] = kbase_gpu_inspect(kbdev, js, 0);
981 katom[1] = kbase_gpu_inspect(kbdev, js, 1);
982 WARN_ON(katom[1] && !katom[0]);
984 for (idx = 0; idx < SLOT_RB_SIZE; idx++) {
991 switch (katom[idx]->gpu_rb_state) {
992 case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB:
993 /* Should be impossible */
994 WARN(1, "Attempting to update atom not in ringbuffer\n");
997 case KBASE_ATOM_GPU_RB_WAITING_BLOCKED:
998 if (katom[idx]->atom_flags &
999 KBASE_KATOM_FLAG_X_DEP_BLOCKED)
1002 katom[idx]->gpu_rb_state =
1003 KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV;
1005 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
1007 case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_PREV:
1008 if (kbase_gpu_check_secure_atoms(kbdev,
1009 !kbase_jd_katom_is_protected(
1013 if (kbdev->protected_mode_transition)
1016 katom[idx]->gpu_rb_state =
1017 KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION;
1019 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
1021 case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_TRANSITION:
1024 * Exiting protected mode must be done before
1025 * the references on the cores are taken as
1026 * a power down the L2 is required which
1027 * can't happen after the references for this
1031 if (!kbase_gpu_in_protected_mode(kbdev) &&
1032 kbase_jd_katom_is_protected(katom[idx])) {
1033 /* Atom needs to transition into protected mode. */
1034 ret = kbase_jm_enter_protected_mode(kbdev,
1038 } else if (kbase_gpu_in_protected_mode(kbdev) &&
1039 !kbase_jd_katom_is_protected(katom[idx])) {
1040 /* Atom needs to transition out of protected mode. */
1041 ret = kbase_jm_exit_protected_mode(kbdev,
1046 katom[idx]->protected_state.exit =
1047 KBASE_ATOM_EXIT_PROTECTED_CHECK;
1049 /* Atom needs no protected mode transition. */
1051 katom[idx]->gpu_rb_state =
1052 KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE;
1054 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
1056 case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE:
1057 if (katom[idx]->will_fail_event_code) {
1058 kbase_gpu_mark_atom_for_return(kbdev,
1060 /* Set EVENT_DONE so this atom will be
1061 completed, not unpulled. */
1062 katom[idx]->event_code =
1064 /* Only return if head atom or previous
1065 * atom already removed - as atoms must
1066 * be returned in order. */
1067 if (idx == 0 || katom[0]->gpu_rb_state ==
1068 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
1069 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1070 kbase_jm_return_atom_to_js(kbdev, katom[idx]);
1076 kbasep_js_job_check_ref_cores(kbdev, js,
1079 if (katom[idx]->event_code ==
1080 BASE_JD_EVENT_PM_EVENT) {
1081 katom[idx]->gpu_rb_state =
1082 KBASE_ATOM_GPU_RB_RETURN_TO_JS;
1089 kbase_js_affinity_retain_slot_cores(kbdev, js,
1090 katom[idx]->affinity);
1091 katom[idx]->gpu_rb_state =
1092 KBASE_ATOM_GPU_RB_WAITING_AFFINITY;
1094 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
1096 case KBASE_ATOM_GPU_RB_WAITING_AFFINITY:
1097 if (!kbase_gpu_rmu_workaround(kbdev, js))
1100 katom[idx]->gpu_rb_state =
1101 KBASE_ATOM_GPU_RB_READY;
1103 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
1105 case KBASE_ATOM_GPU_RB_READY:
1107 /* Only submit if head atom or previous atom
1108 * already submitted */
1110 (katom[0]->gpu_rb_state !=
1111 KBASE_ATOM_GPU_RB_SUBMITTED &&
1112 katom[0]->gpu_rb_state !=
1113 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB))
1116 /* Check if this job needs the cycle counter
1117 * enabled before submission */
1118 if (katom[idx]->core_req & BASE_JD_REQ_PERMON)
1119 kbase_pm_request_gpu_cycle_counter_l2_is_on(
1122 kbase_job_hw_submit(kbdev, katom[idx], js);
1123 katom[idx]->gpu_rb_state =
1124 KBASE_ATOM_GPU_RB_SUBMITTED;
1126 /* Inform power management at start/finish of
1127 * atom so it can update its GPU utilisation
1129 kbase_pm_metrics_update(kbdev,
1130 &katom[idx]->start_timestamp);
1132 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
1134 case KBASE_ATOM_GPU_RB_SUBMITTED:
1135 /* Atom submitted to HW, nothing else to do */
1138 case KBASE_ATOM_GPU_RB_RETURN_TO_JS:
1139 /* Only return if head atom or previous atom
1140 * already removed - as atoms must be returned
1142 if (idx == 0 || katom[0]->gpu_rb_state ==
1143 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
1144 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1145 kbase_jm_return_atom_to_js(kbdev,
1153 /* Warn if PRLAM-8987 affinity restrictions are violated */
1154 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
1155 WARN_ON((kbase_gpu_atoms_submitted(kbdev, 0) ||
1156 kbase_gpu_atoms_submitted(kbdev, 1)) &&
1157 kbase_gpu_atoms_submitted(kbdev, 2));
1161 void kbase_backend_run_atom(struct kbase_device *kbdev,
1162 struct kbase_jd_atom *katom)
1164 lockdep_assert_held(&kbdev->hwaccess_lock);
1165 kbase_gpu_enqueue_atom(kbdev, katom);
1166 kbase_backend_slot_update(kbdev);
1169 bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js)
1171 struct kbase_jd_atom *katom;
1172 struct kbase_jd_atom *next_katom;
1174 lockdep_assert_held(&kbdev->hwaccess_lock);
1176 katom = kbase_gpu_inspect(kbdev, js, 0);
1177 next_katom = kbase_gpu_inspect(kbdev, js, 1);
1179 if (next_katom && katom->kctx == next_katom->kctx &&
1180 next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED &&
1181 (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL)
1183 kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL)
1185 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT),
1186 JS_COMMAND_NOP, NULL);
1187 next_katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY;
1194 void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
1195 u32 completion_code,
1197 ktime_t *end_timestamp)
1199 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0);
1200 struct kbase_context *kctx = katom->kctx;
1202 lockdep_assert_held(&kbdev->hwaccess_lock);
1204 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6787) &&
1205 completion_code != BASE_JD_EVENT_DONE &&
1206 !(completion_code & BASE_JD_SW_EVENT)) {
1207 katom->need_cache_flush_cores_retained = katom->affinity;
1208 kbase_pm_request_cores(kbdev, false, katom->affinity);
1209 } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10676)) {
1210 if (kbdev->gpu_props.num_core_groups > 1 &&
1212 kbdev->gpu_props.props.coherency_info.group[0].core_mask
1215 kbdev->gpu_props.props.coherency_info.group[1].core_mask
1217 dev_info(kbdev->dev, "JD: Flushing cache due to PRLAM-10676\n");
1218 katom->need_cache_flush_cores_retained =
1220 kbase_pm_request_cores(kbdev, false,
1225 katom = kbase_gpu_dequeue_atom(kbdev, js, end_timestamp);
1226 kbase_timeline_job_slot_done(kbdev, katom->kctx, katom, js, 0);
1227 kbase_tlstream_tl_nret_atom_lpu(
1229 &kbdev->gpu_props.props.raw_props.js_features[
1231 kbase_tlstream_tl_nret_atom_as(katom, &kbdev->as[kctx->as_nr]);
1232 kbase_tlstream_tl_nret_ctx_lpu(
1234 &kbdev->gpu_props.props.raw_props.js_features[
1237 if (completion_code == BASE_JD_EVENT_STOPPED) {
1238 struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js,
1242 * Dequeue next atom from ringbuffers on same slot if required.
1243 * This atom will already have been removed from the NEXT
1244 * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that
1245 * the atoms on this slot are returned in the correct order.
1247 if (next_katom && katom->kctx == next_katom->kctx) {
1248 kbase_gpu_dequeue_atom(kbdev, js, end_timestamp);
1249 kbase_jm_return_atom_to_js(kbdev, next_katom);
1251 } else if (completion_code != BASE_JD_EVENT_DONE) {
1252 struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1255 #if KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR != 0
1256 KBASE_TRACE_DUMP(kbdev);
1258 kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
1261 * Remove all atoms on the same context from ringbuffers. This
1262 * will not remove atoms that are already on the GPU, as these
1263 * are guaranteed not to have fail dependencies on the failed
1266 for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) {
1267 struct kbase_jd_atom *katom_idx0 =
1268 kbase_gpu_inspect(kbdev, i, 0);
1269 struct kbase_jd_atom *katom_idx1 =
1270 kbase_gpu_inspect(kbdev, i, 1);
1272 if (katom_idx0 && katom_idx0->kctx == katom->kctx &&
1273 katom_idx0->gpu_rb_state !=
1274 KBASE_ATOM_GPU_RB_SUBMITTED) {
1275 /* Dequeue katom_idx0 from ringbuffer */
1276 kbase_gpu_dequeue_atom(kbdev, i, end_timestamp);
1279 katom_idx1->kctx == katom->kctx &&
1280 katom_idx0->gpu_rb_state !=
1281 KBASE_ATOM_GPU_RB_SUBMITTED) {
1282 /* Dequeue katom_idx1 from ringbuffer */
1283 kbase_gpu_dequeue_atom(kbdev, i,
1286 katom_idx1->event_code =
1287 BASE_JD_EVENT_STOPPED;
1288 kbase_jm_return_atom_to_js(kbdev,
1291 katom_idx0->event_code = BASE_JD_EVENT_STOPPED;
1292 kbase_jm_return_atom_to_js(kbdev, katom_idx0);
1294 } else if (katom_idx1 &&
1295 katom_idx1->kctx == katom->kctx &&
1296 katom_idx1->gpu_rb_state !=
1297 KBASE_ATOM_GPU_RB_SUBMITTED) {
1298 /* Can not dequeue this atom yet - will be
1299 * dequeued when atom at idx0 completes */
1300 katom_idx1->event_code = BASE_JD_EVENT_STOPPED;
1301 kbase_gpu_mark_atom_for_return(kbdev,
1307 KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc,
1308 js, completion_code);
1310 if (job_tail != 0 && job_tail != katom->jc) {
1311 bool was_updated = (job_tail != katom->jc);
1313 /* Some of the job has been executed, so we update the job chain
1314 * address to where we should resume from */
1315 katom->jc = job_tail;
1317 KBASE_TRACE_ADD_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx,
1318 katom, job_tail, js);
1321 /* Only update the event code for jobs that weren't cancelled */
1322 if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED)
1323 katom->event_code = (base_jd_event_code)completion_code;
1325 kbase_device_trace_register_access(kctx, REG_WRITE,
1326 JOB_CONTROL_REG(JOB_IRQ_CLEAR),
1329 /* Complete the job, and start new ones
1331 * Also defer remaining work onto the workqueue:
1332 * - Re-queue Soft-stopped jobs
1333 * - For any other jobs, queue the job back into the dependency system
1334 * - Schedule out the parent context if necessary, and schedule a new
1337 #ifdef CONFIG_GPU_TRACEPOINTS
1339 /* The atom in the HEAD */
1340 struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js,
1343 if (next_katom && next_katom->gpu_rb_state ==
1344 KBASE_ATOM_GPU_RB_SUBMITTED) {
1347 trace_gpu_sched_switch(kbasep_make_job_slot_string(js,
1349 ktime_to_ns(*end_timestamp),
1350 (u32)next_katom->kctx->id, 0,
1351 next_katom->work_id);
1352 kbdev->hwaccess.backend.slot_rb[js].last_context =
1357 trace_gpu_sched_switch(kbasep_make_job_slot_string(js,
1359 ktime_to_ns(ktime_get()), 0, 0,
1361 kbdev->hwaccess.backend.slot_rb[js].last_context = 0;
1366 if (completion_code == BASE_JD_EVENT_STOPPED)
1367 katom = kbase_jm_return_atom_to_js(kbdev, katom);
1369 katom = kbase_jm_complete(kbdev, katom, end_timestamp);
1372 /* Cross-slot dependency has now become runnable. Try to submit
1375 /* Check if there are lower priority jobs to soft stop */
1376 kbase_job_slot_ctx_priority_check_locked(kctx, katom);
1378 kbase_jm_try_kick(kbdev, 1 << katom->slot_nr);
1381 /* Job completion may have unblocked other atoms. Try to update all job
1383 kbase_backend_slot_update(kbdev);
1386 void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp)
1390 lockdep_assert_held(&kbdev->hwaccess_lock);
1392 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
1396 for (idx = 0; idx < SLOT_RB_SIZE; idx++) {
1397 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev,
1399 bool keep_in_jm_rb = false;
1404 if (katom->gpu_rb_state < KBASE_ATOM_GPU_RB_SUBMITTED)
1405 keep_in_jm_rb = true;
1407 kbase_gpu_release_atom(kbdev, katom, NULL);
1410 * If the atom wasn't on HW when the reset was issued
1411 * then leave it in the RB and next time we're kicked
1412 * it will be processed again from the starting state.
1414 if (keep_in_jm_rb) {
1415 katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
1416 katom->affinity = 0;
1417 katom->protected_state.exit = KBASE_ATOM_EXIT_PROTECTED_CHECK;
1418 /* As the atom was not removed, increment the
1419 * index so that we read the correct atom in the
1420 * next iteration. */
1426 * The atom was on the HW when the reset was issued
1427 * all we can do is fail the atom.
1429 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1430 katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
1431 kbase_jm_complete(kbdev, katom, end_timestamp);
1435 kbdev->protected_mode_transition = false;
1436 kbdev->protected_mode = false;
1439 static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev,
1441 struct kbase_jd_atom *katom,
1444 struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1445 u32 hw_action = action & JS_COMMAND_MASK;
1447 kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom);
1448 kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action,
1449 katom->core_req, katom);
1450 kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
1453 static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev,
1454 struct kbase_jd_atom *katom,
1458 struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1460 katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT;
1461 kbase_gpu_mark_atom_for_return(kbdev, katom);
1462 kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
1465 kbase_job_check_enter_disjoint(kbdev, action, katom->core_req,
1469 static int should_stop_x_dep_slot(struct kbase_jd_atom *katom)
1471 if (katom->x_post_dep) {
1472 struct kbase_jd_atom *dep_atom = katom->x_post_dep;
1474 if (dep_atom->gpu_rb_state !=
1475 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB &&
1476 dep_atom->gpu_rb_state !=
1477 KBASE_ATOM_GPU_RB_RETURN_TO_JS)
1478 return dep_atom->slot_nr;
1483 static void kbase_job_evicted(struct kbase_jd_atom *katom)
1485 kbase_timeline_job_slot_done(katom->kctx->kbdev, katom->kctx, katom,
1486 katom->slot_nr, KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT);
1489 bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev,
1490 struct kbase_context *kctx,
1492 struct kbase_jd_atom *katom,
1495 struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1497 struct kbase_jd_atom *katom_idx0;
1498 struct kbase_jd_atom *katom_idx1;
1500 bool katom_idx0_valid, katom_idx1_valid;
1504 int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1;
1506 lockdep_assert_held(&kbdev->hwaccess_lock);
1508 katom_idx0 = kbase_gpu_inspect(kbdev, js, 0);
1509 katom_idx1 = kbase_gpu_inspect(kbdev, js, 1);
1512 katom_idx0_valid = (katom_idx0 == katom);
1513 /* If idx0 is to be removed and idx1 is on the same context,
1514 * then idx1 must also be removed otherwise the atoms might be
1515 * returned out of order */
1517 katom_idx1_valid = (katom_idx1 == katom) ||
1518 (katom_idx0_valid &&
1519 (katom_idx0->kctx ==
1522 katom_idx1_valid = false;
1524 katom_idx0_valid = (katom_idx0 &&
1525 (!kctx || katom_idx0->kctx == kctx));
1526 katom_idx1_valid = (katom_idx1 &&
1527 (!kctx || katom_idx1->kctx == kctx));
1530 if (katom_idx0_valid)
1531 stop_x_dep_idx0 = should_stop_x_dep_slot(katom_idx0);
1532 if (katom_idx1_valid)
1533 stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1);
1535 if (katom_idx0_valid) {
1536 if (katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) {
1537 /* Simple case - just dequeue and return */
1538 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1539 if (katom_idx1_valid) {
1540 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1541 katom_idx1->event_code =
1542 BASE_JD_EVENT_REMOVED_FROM_NEXT;
1543 kbase_jm_return_atom_to_js(kbdev, katom_idx1);
1544 kbasep_js_clear_submit_allowed(js_devdata,
1548 katom_idx0->event_code =
1549 BASE_JD_EVENT_REMOVED_FROM_NEXT;
1550 kbase_jm_return_atom_to_js(kbdev, katom_idx0);
1551 kbasep_js_clear_submit_allowed(js_devdata,
1554 /* katom_idx0 is on GPU */
1555 if (katom_idx1 && katom_idx1->gpu_rb_state ==
1556 KBASE_ATOM_GPU_RB_SUBMITTED) {
1557 /* katom_idx0 and katom_idx1 are on GPU */
1559 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1560 JS_COMMAND_NEXT), NULL) == 0) {
1561 /* idx0 has already completed - stop
1563 if (katom_idx1_valid) {
1564 kbase_gpu_stop_atom(kbdev, js,
1570 /* idx1 is in NEXT registers - attempt
1572 kbase_reg_write(kbdev,
1575 JS_COMMAND_NOP, NULL);
1577 if (kbase_reg_read(kbdev,
1579 JS_HEAD_NEXT_LO), NULL)
1581 kbase_reg_read(kbdev,
1583 JS_HEAD_NEXT_HI), NULL)
1585 /* idx1 removed successfully,
1586 * will be handled in IRQ */
1587 kbase_job_evicted(katom_idx1);
1588 kbase_gpu_remove_atom(kbdev,
1592 should_stop_x_dep_slot(katom_idx1);
1594 /* stop idx0 if still on GPU */
1595 kbase_gpu_stop_atom(kbdev, js,
1599 } else if (katom_idx1_valid) {
1600 /* idx0 has already completed,
1601 * stop idx1 if needed */
1602 kbase_gpu_stop_atom(kbdev, js,
1608 } else if (katom_idx1_valid) {
1609 /* idx1 not on GPU but must be dequeued*/
1611 /* idx1 will be handled in IRQ */
1612 kbase_gpu_remove_atom(kbdev, katom_idx1, action,
1615 /* This will be repeated for anything removed
1616 * from the next registers, since their normal
1617 * flow was also interrupted, and this function
1618 * might not enter disjoint state e.g. if we
1619 * don't actually do a hard stop on the head
1621 kbase_gpu_stop_atom(kbdev, js, katom_idx0,
1625 /* no atom in idx1 */
1626 /* just stop idx0 */
1627 kbase_gpu_stop_atom(kbdev, js, katom_idx0,
1632 } else if (katom_idx1_valid) {
1633 if (katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) {
1634 /* Mark for return */
1635 /* idx1 will be returned once idx0 completes */
1636 kbase_gpu_remove_atom(kbdev, katom_idx1, action,
1639 /* idx1 is on GPU */
1640 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1641 JS_COMMAND_NEXT), NULL) == 0) {
1642 /* idx0 has already completed - stop idx1 */
1643 kbase_gpu_stop_atom(kbdev, js, katom_idx1,
1647 /* idx1 is in NEXT registers - attempt to
1649 kbase_reg_write(kbdev, JOB_SLOT_REG(js,
1651 JS_COMMAND_NOP, NULL);
1653 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1654 JS_HEAD_NEXT_LO), NULL) != 0 ||
1655 kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1656 JS_HEAD_NEXT_HI), NULL) != 0) {
1657 /* idx1 removed successfully, will be
1658 * handled in IRQ once idx0 completes */
1659 kbase_job_evicted(katom_idx1);
1660 kbase_gpu_remove_atom(kbdev, katom_idx1,
1664 /* idx0 has already completed - stop
1666 kbase_gpu_stop_atom(kbdev, js,
1676 if (stop_x_dep_idx0 != -1)
1677 kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx0,
1680 if (stop_x_dep_idx1 != -1)
1681 kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx1,
1687 void kbase_gpu_cacheclean(struct kbase_device *kbdev,
1688 struct kbase_jd_atom *katom)
1690 /* Limit the number of loops to avoid a hang if the interrupt is missed
1692 u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS;
1693 unsigned long flags;
1695 mutex_lock(&kbdev->cacheclean_lock);
1697 /* use GPU_COMMAND completion solution */
1698 /* clean & invalidate the caches */
1699 KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0);
1700 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1701 GPU_COMMAND_CLEAN_INV_CACHES, NULL);
1703 /* wait for cache flush to complete before continuing */
1704 while (--max_loops &&
1705 (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) &
1706 CLEAN_CACHES_COMPLETED) == 0)
1709 /* clear the CLEAN_CACHES_COMPLETED irq */
1710 KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u,
1711 CLEAN_CACHES_COMPLETED);
1712 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR),
1713 CLEAN_CACHES_COMPLETED, NULL);
1714 KBASE_DEBUG_ASSERT_MSG(kbdev->hwcnt.backend.state !=
1715 KBASE_INSTR_STATE_CLEANING,
1716 "Instrumentation code was cleaning caches, but Job Management code cleared their IRQ - Instrumentation code will now hang.");
1718 mutex_unlock(&kbdev->cacheclean_lock);
1720 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1721 kbase_pm_unrequest_cores(kbdev, false,
1722 katom->need_cache_flush_cores_retained);
1723 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1726 void kbase_backend_complete_wq(struct kbase_device *kbdev,
1727 struct kbase_jd_atom *katom)
1730 * If cache flush required due to HW workaround then perform the flush
1733 if (katom->need_cache_flush_cores_retained) {
1734 kbase_gpu_cacheclean(kbdev, katom);
1735 katom->need_cache_flush_cores_retained = 0;
1738 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10969) &&
1739 (katom->core_req & BASE_JD_REQ_FS) &&
1740 katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT &&
1741 (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED) &&
1742 !(katom->atom_flags & KBASE_KATOM_FLAGS_RERUN)) {
1743 dev_dbg(kbdev->dev, "Soft-stopped fragment shader job got a TILE_RANGE_FAULT. Possible HW issue, trying SW workaround\n");
1744 if (kbasep_10969_workaround_clamp_coordinates(katom)) {
1745 /* The job had a TILE_RANGE_FAULT after was soft-stopped
1746 * Due to an HW issue we try to execute the job again.
1749 "Clamping has been executed, try to rerun the job\n"
1751 katom->event_code = BASE_JD_EVENT_STOPPED;
1752 katom->atom_flags |= KBASE_KATOM_FLAGS_RERUN;
1756 /* Clear the coreref_state now - while check_deref_cores() may not have
1757 * been called yet, the caller will have taken a copy of this field. If
1758 * this is not done, then if the atom is re-scheduled (following a soft
1759 * stop) then the core reference would not be retaken. */
1760 katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
1761 katom->affinity = 0;
1764 void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev,
1765 base_jd_core_req core_req, u64 affinity,
1766 enum kbase_atom_coreref_state coreref_state)
1768 unsigned long flags;
1770 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1771 kbasep_js_job_check_deref_cores_nokatom(kbdev, core_req, affinity,
1773 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
1775 if (!kbdev->pm.active_count) {
1776 mutex_lock(&kbdev->js_data.runpool_mutex);
1777 mutex_lock(&kbdev->pm.lock);
1778 kbase_pm_update_active(kbdev);
1779 mutex_unlock(&kbdev->pm.lock);
1780 mutex_unlock(&kbdev->js_data.runpool_mutex);
1784 void kbase_gpu_dump_slots(struct kbase_device *kbdev)
1786 struct kbasep_js_device_data *js_devdata;
1787 unsigned long flags;
1790 js_devdata = &kbdev->js_data;
1792 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
1794 dev_info(kbdev->dev, "kbase_gpu_dump_slots:\n");
1796 for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
1799 for (idx = 0; idx < SLOT_RB_SIZE; idx++) {
1800 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev,
1805 dev_info(kbdev->dev,
1806 " js%d idx%d : katom=%p gpu_rb_state=%d\n",
1807 js, idx, katom, katom->gpu_rb_state);
1809 dev_info(kbdev->dev, " js%d idx%d : empty\n",
1814 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);