drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2014-2015 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18
  19 /*
  20  * Register-based HW access backend specific APIs
  21  */
  22
  23 #include <mali_kbase.h>
  24 #include <mali_kbase_hwaccess_jm.h>
  25 #include <mali_kbase_jm.h>
  26 #include <mali_kbase_js.h>
  27 #include <mali_kbase_10969_workaround.h>
  28 #include <backend/gpu/mali_kbase_device_internal.h>
  29 #include <backend/gpu/mali_kbase_jm_internal.h>
  30 #include <backend/gpu/mali_kbase_js_affinity.h>
  31 #include <backend/gpu/mali_kbase_js_internal.h>
  32 #include <backend/gpu/mali_kbase_pm_internal.h>
  33
  34 /* Return whether the specified ringbuffer is empty. HW access lock must be
  35  * held */
  36 #define SLOT_RB_EMPTY(rb)   (rb->write_idx == rb->read_idx)
  37 /* Return number of atoms currently in the specified ringbuffer. HW access lock
  38  * must be held */
  39 #define SLOT_RB_ENTRIES(rb) (int)(s8)(rb->write_idx - rb->read_idx)
  40
  41 static void kbase_gpu_release_atom(struct kbase_device *kbdev,
  42                                         struct kbase_jd_atom *katom);
  43
  44 /**
  45  * kbase_gpu_enqueue_atom - Enqueue an atom in the HW access ringbuffer
  46  * @kbdev: Device pointer
  47  * @katom: Atom to enqueue
  48  *
  49  * Context: Caller must hold the HW access lock
  50  */
  51 static void kbase_gpu_enqueue_atom(struct kbase_device *kbdev,
  52                                         struct kbase_jd_atom *katom)
  53 {
  54         struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[katom->slot_nr];
  55
  56         WARN_ON(SLOT_RB_ENTRIES(rb) >= SLOT_RB_SIZE);
  57
  58         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
  59
  60         rb->entries[rb->write_idx & SLOT_RB_MASK].katom = katom;
  61         rb->write_idx++;
  62
  63         katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED;
  64 }
  65
  66 /**
  67  * kbase_gpu_dequeue_atom - Remove an atom from the HW access ringbuffer, once
  68  * it has been completed
  69  * @kbdev: Device pointer
  70  * @js:    Job slot to remove atom from
  71  *
  72  * Context: Caller must hold the HW access lock
  73  *
  74  * Return: Atom removed from ringbuffer
  75  */
  76 static struct kbase_jd_atom *kbase_gpu_dequeue_atom(struct kbase_device *kbdev,
  77                                                         int js)
  78 {
  79         struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
  80         struct kbase_jd_atom *katom;
  81
  82         if (SLOT_RB_EMPTY(rb)) {
  83                 WARN(1, "GPU ringbuffer unexpectedly empty\n");
  84                 return NULL;
  85         }
  86
  87         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
  88
  89         katom = rb->entries[rb->read_idx & SLOT_RB_MASK].katom;
  90
  91         kbase_gpu_release_atom(kbdev, katom);
  92
  93         rb->read_idx++;
  94
  95         katom->gpu_rb_state = KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB;
  96
  97         kbase_js_debug_log_current_affinities(kbdev);
  98
  99         return katom;
 100 }
 101
 102 struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js,
 103                                         int idx)
 104 {
 105         struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
 106
 107         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 108
 109         if ((SLOT_RB_ENTRIES(rb) - 1) < idx)
 110                 return NULL; /* idx out of range */
 111
 112         return rb->entries[(rb->read_idx + idx) & SLOT_RB_MASK].katom;
 113 }
 114
 115 struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev,
 116                                         int js)
 117 {
 118         return kbase_gpu_inspect(kbdev, js, 0);
 119 }
 120
 121 struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev,
 122                                         int js)
 123 {
 124         struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
 125
 126         if (SLOT_RB_EMPTY(rb))
 127                 return NULL;
 128
 129         return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom;
 130 }
 131
 132 /**
 133  * kbase_gpu_atoms_submitted - Inspect whether a slot has any atoms currently
 134  * on the GPU
 135  * @kbdev:  Device pointer
 136  * @js:     Job slot to inspect
 137  *
 138  * Return: true if there are atoms on the GPU for slot js,
 139  *         false otherwise
 140  */
 141 static bool kbase_gpu_atoms_submitted(struct kbase_device *kbdev, int js)
 142 {
 143         int i;
 144
 145         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 146
 147         for (i = 0; i < SLOT_RB_SIZE; i++) {
 148                 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
 149
 150                 if (!katom)
 151                         return false;
 152                 if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED ||
 153                                 katom->gpu_rb_state == KBASE_ATOM_GPU_RB_READY)
 154                         return true;
 155         }
 156
 157         return false;
 158 }
 159
 160 /**
 161  * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms
 162  * currently on the GPU
 163  * @kbdev:  Device pointer
 164  *
 165  * Return: true if there are any atoms on the GPU, false otherwise
 166  */
 167 static bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev)
 168 {
 169         int js;
 170         int i;
 171
 172         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 173
 174         for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
 175                 for (i = 0; i < SLOT_RB_SIZE; i++) {
 176                         struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
 177
 178                         if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED)
 179                                 return true;
 180                 }
 181         }
 182         return false;
 183 }
 184
 185 int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js)
 186 {
 187         int nr = 0;
 188         int i;
 189
 190         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 191
 192         for (i = 0; i < SLOT_RB_SIZE; i++) {
 193                 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
 194
 195                 if (katom && (katom->gpu_rb_state ==
 196                                                 KBASE_ATOM_GPU_RB_SUBMITTED))
 197                         nr++;
 198         }
 199
 200         return nr;
 201 }
 202
 203 int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js)
 204 {
 205         int nr = 0;
 206         int i;
 207
 208         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 209
 210         for (i = 0; i < SLOT_RB_SIZE; i++) {
 211                 if (kbase_gpu_inspect(kbdev, js, i))
 212                         nr++;
 213         }
 214
 215         return nr;
 216 }
 217
 218 static int kbase_gpu_nr_atoms_on_slot_min(struct kbase_device *kbdev, int js,
 219                                 enum kbase_atom_gpu_rb_state min_rb_state)
 220 {
 221         int nr = 0;
 222         int i;
 223
 224         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 225
 226         for (i = 0; i < SLOT_RB_SIZE; i++) {
 227                 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
 228
 229                 if (katom && (katom->gpu_rb_state >= min_rb_state))
 230                         nr++;
 231         }
 232
 233         return nr;
 234 }
 235
 236 int kbase_backend_slot_free(struct kbase_device *kbdev, int js)
 237 {
 238         if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) !=
 239                                                 KBASE_RESET_GPU_NOT_PENDING) {
 240                 /* The GPU is being reset - so prevent submission */
 241                 return 0;
 242         }
 243
 244         return SLOT_RB_SIZE - kbase_backend_nr_atoms_on_slot(kbdev, js);
 245 }
 246
 247
 248 static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev,
 249                                                 struct kbase_jd_atom *katom);
 250
 251 static bool kbasep_js_job_check_ref_cores(struct kbase_device *kbdev,
 252                                                 int js,
 253                                                 struct kbase_jd_atom *katom)
 254 {
 255         /* The most recently checked affinity. Having this at this scope allows
 256          * us to guarantee that we've checked the affinity in this function
 257          * call.
 258          */
 259         u64 recently_chosen_affinity = 0;
 260         bool chosen_affinity = false;
 261         bool retry;
 262
 263         do {
 264                 retry = false;
 265
 266                 /* NOTE: The following uses a number of FALLTHROUGHs to optimize
 267                  * the calls to this function. Ending of the function is
 268                  * indicated by BREAK OUT */
 269                 switch (katom->coreref_state) {
 270                         /* State when job is first attempted to be run */
 271                 case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
 272                         KBASE_DEBUG_ASSERT(katom->affinity == 0);
 273
 274                         /* Compute affinity */
 275                         if (false == kbase_js_choose_affinity(
 276                                         &recently_chosen_affinity, kbdev, katom,
 277                                                                         js)) {
 278                                 /* No cores are currently available */
 279                                 /* *** BREAK OUT: No state transition *** */
 280                                 break;
 281                         }
 282
 283                         chosen_affinity = true;
 284
 285                         /* Request the cores */
 286                         kbase_pm_request_cores(kbdev,
 287                                         katom->core_req & BASE_JD_REQ_T,
 288                                                 recently_chosen_affinity);
 289
 290                         katom->affinity = recently_chosen_affinity;
 291
 292                         /* Proceed to next state */
 293                         katom->coreref_state =
 294                         KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
 295
 296                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 297
 298                 case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
 299                         {
 300                                 enum kbase_pm_cores_ready cores_ready;
 301
 302                                 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
 303                                         (katom->core_req & BASE_JD_REQ_T));
 304
 305                                 cores_ready = kbase_pm_register_inuse_cores(
 306                                                 kbdev,
 307                                                 katom->core_req & BASE_JD_REQ_T,
 308                                                 katom->affinity);
 309                                 if (cores_ready == KBASE_NEW_AFFINITY) {
 310                                         /* Affinity no longer valid - return to
 311                                          * previous state */
 312                                         kbasep_js_job_check_deref_cores(kbdev,
 313                                                                         katom);
 314                                         KBASE_TRACE_ADD_SLOT_INFO(kbdev,
 315                                         JS_CORE_REF_REGISTER_INUSE_FAILED,
 316                                                         katom->kctx, katom,
 317                                                         katom->jc, js,
 318                                                         (u32) katom->affinity);
 319                                         /* *** BREAK OUT: Return to previous
 320                                          * state, retry *** */
 321                                         retry = true;
 322                                         break;
 323                                 }
 324                                 if (cores_ready == KBASE_CORES_NOT_READY) {
 325                                         /* Stay in this state and return, to
 326                                          * retry at this state later */
 327                                         KBASE_TRACE_ADD_SLOT_INFO(kbdev,
 328                                         JS_CORE_REF_REGISTER_INUSE_FAILED,
 329                                                         katom->kctx, katom,
 330                                                         katom->jc, js,
 331                                                         (u32) katom->affinity);
 332                                         /* *** BREAK OUT: No state transition
 333                                          * *** */
 334                                         break;
 335                                 }
 336                                 /* Proceed to next state */
 337                                 katom->coreref_state =
 338                                 KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
 339                         }
 340
 341                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 342
 343                 case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
 344                         KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
 345                                         (katom->core_req & BASE_JD_REQ_T));
 346
 347                         /* Optimize out choosing the affinity twice in the same
 348                          * function call */
 349                         if (chosen_affinity == false) {
 350                                 /* See if the affinity changed since a previous
 351                                  * call. */
 352                                 if (false == kbase_js_choose_affinity(
 353                                                 &recently_chosen_affinity,
 354                                                         kbdev, katom, js)) {
 355                                         /* No cores are currently available */
 356                                         kbasep_js_job_check_deref_cores(kbdev,
 357                                                                         katom);
 358                                         KBASE_TRACE_ADD_SLOT_INFO(kbdev,
 359                                         JS_CORE_REF_REQUEST_ON_RECHECK_FAILED,
 360                                                 katom->kctx, katom,
 361                                                 katom->jc, js,
 362                                                 (u32) recently_chosen_affinity);
 363                                         /* *** BREAK OUT: Transition to lower
 364                                          * state *** */
 365                                         break;
 366                                 }
 367                                 chosen_affinity = true;
 368                         }
 369
 370                         /* Now see if this requires a different set of cores */
 371                         if (recently_chosen_affinity != katom->affinity) {
 372                                 enum kbase_pm_cores_ready cores_ready;
 373
 374                                 kbase_pm_request_cores(kbdev,
 375                                                 katom->core_req & BASE_JD_REQ_T,
 376                                                 recently_chosen_affinity);
 377
 378                                 /* Register new cores whilst we still hold the
 379                                  * old ones, to minimize power transitions */
 380                                 cores_ready =
 381                                         kbase_pm_register_inuse_cores(kbdev,
 382                                                 katom->core_req & BASE_JD_REQ_T,
 383                                                 recently_chosen_affinity);
 384                                 kbasep_js_job_check_deref_cores(kbdev, katom);
 385
 386                                 /* Fixup the state that was reduced by
 387                                  * deref_cores: */
 388                                 katom->coreref_state =
 389                                 KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
 390                                 katom->affinity = recently_chosen_affinity;
 391                                 if (cores_ready == KBASE_NEW_AFFINITY) {
 392                                         /* Affinity no longer valid - return to
 393                                          * previous state */
 394                                         katom->coreref_state =
 395                                         KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
 396
 397                                         kbasep_js_job_check_deref_cores(kbdev,
 398                                                                         katom);
 399
 400                                         KBASE_TRACE_ADD_SLOT_INFO(kbdev,
 401                                         JS_CORE_REF_REGISTER_INUSE_FAILED,
 402                                                         katom->kctx, katom,
 403                                                         katom->jc, js,
 404                                                         (u32) katom->affinity);
 405                                         /* *** BREAK OUT: Return to previous
 406                                          * state, retry *** */
 407                                         retry = true;
 408                                         break;
 409                                 }
 410                                 /* Now might be waiting for powerup again, with
 411                                  * a new affinity */
 412                                 if (cores_ready == KBASE_CORES_NOT_READY) {
 413                                         /* Return to previous state */
 414                                         katom->coreref_state =
 415                                         KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
 416                                         KBASE_TRACE_ADD_SLOT_INFO(kbdev,
 417                                         JS_CORE_REF_REGISTER_ON_RECHECK_FAILED,
 418                                                         katom->kctx, katom,
 419                                                         katom->jc, js,
 420                                                         (u32) katom->affinity);
 421                                         /* *** BREAK OUT: Transition to lower
 422                                          * state *** */
 423                                         break;
 424                                 }
 425                         }
 426                         /* Proceed to next state */
 427                         katom->coreref_state =
 428                         KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS;
 429
 430                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 431                 case KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS:
 432                         KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
 433                                         (katom->core_req & BASE_JD_REQ_T));
 434                         KBASE_DEBUG_ASSERT(katom->affinity ==
 435                                                 recently_chosen_affinity);
 436
 437                         /* Note: this is where the caller must've taken the
 438                          * runpool_irq.lock */
 439
 440                         /* Check for affinity violations - if there are any,
 441                          * then we just ask the caller to requeue and try again
 442                          * later */
 443                         if (kbase_js_affinity_would_violate(kbdev, js,
 444                                         katom->affinity) != false) {
 445                                 /* Return to previous state */
 446                                 katom->coreref_state =
 447                                 KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
 448                                 /* *** BREAK OUT: Transition to lower state ***
 449                                  */
 450                                 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
 451                                         JS_CORE_REF_AFFINITY_WOULD_VIOLATE,
 452                                         katom->kctx, katom, katom->jc, js,
 453                                         (u32) katom->affinity);
 454                                 break;
 455                         }
 456
 457                         /* No affinity violations would result, so the cores are
 458                          * ready */
 459                         katom->coreref_state = KBASE_ATOM_COREREF_STATE_READY;
 460                         /* *** BREAK OUT: Cores Ready *** */
 461                         break;
 462
 463                 default:
 464                         KBASE_DEBUG_ASSERT_MSG(false,
 465                                         "Unhandled kbase_atom_coreref_state %d",
 466                                                         katom->coreref_state);
 467                         break;
 468                 }
 469         } while (retry != false);
 470
 471         return (katom->coreref_state == KBASE_ATOM_COREREF_STATE_READY);
 472 }
 473
 474 static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev,
 475                                                 struct kbase_jd_atom *katom)
 476 {
 477         KBASE_DEBUG_ASSERT(kbdev != NULL);
 478         KBASE_DEBUG_ASSERT(katom != NULL);
 479
 480         switch (katom->coreref_state) {
 481         case KBASE_ATOM_COREREF_STATE_READY:
 482                 /* State where atom was submitted to the HW - just proceed to
 483                  * power-down */
 484                 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
 485                                         (katom->core_req & BASE_JD_REQ_T));
 486
 487                 /* *** FALLTHROUGH *** */
 488
 489         case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
 490                 /* State where cores were registered */
 491                 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
 492                                         (katom->core_req & BASE_JD_REQ_T));
 493                 kbase_pm_release_cores(kbdev, katom->core_req & BASE_JD_REQ_T,
 494                                                         katom->affinity);
 495
 496                 break;
 497
 498         case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
 499                 /* State where cores were requested, but not registered */
 500                 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
 501                                         (katom->core_req & BASE_JD_REQ_T));
 502                 kbase_pm_unrequest_cores(kbdev, katom->core_req & BASE_JD_REQ_T,
 503                                                         katom->affinity);
 504                 break;
 505
 506         case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
 507                 /* Initial state - nothing required */
 508                 KBASE_DEBUG_ASSERT(katom->affinity == 0);
 509                 break;
 510
 511         default:
 512                 KBASE_DEBUG_ASSERT_MSG(false,
 513                                                 "Unhandled coreref_state: %d",
 514                                                         katom->coreref_state);
 515                 break;
 516         }
 517
 518         katom->affinity = 0;
 519         katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
 520 }
 521
 522
 523 static void kbase_gpu_release_atom(struct kbase_device *kbdev,
 524                                         struct kbase_jd_atom *katom)
 525 {
 526         switch (katom->gpu_rb_state) {
 527         case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB:
 528                 /* Should be impossible */
 529                 WARN(1, "Attempting to release atom not in ringbuffer\n");
 530                 break;
 531
 532         case KBASE_ATOM_GPU_RB_SUBMITTED:
 533                 /* Inform power management at start/finish of atom
 534                  * so it can update its GPU utilisation metrics. */
 535                 kbase_pm_metrics_release_atom(kbdev, katom);
 536
 537                 if (katom->core_req & BASE_JD_REQ_PERMON)
 538                         kbase_pm_release_gpu_cycle_counter(kbdev);
 539                 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
 540
 541         case KBASE_ATOM_GPU_RB_READY:
 542                 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
 543
 544         case KBASE_ATOM_GPU_RB_WAITING_SECURE_MODE:
 545                 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
 546
 547         case KBASE_ATOM_GPU_RB_WAITING_AFFINITY:
 548                 kbase_js_affinity_release_slot_cores(kbdev, katom->slot_nr,
 549                                                         katom->affinity);
 550                 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
 551
 552         case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE:
 553                 kbasep_js_job_check_deref_cores(kbdev, katom);
 554                 break;
 555
 556         case KBASE_ATOM_GPU_RB_WAITING_BLOCKED:
 557                 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
 558
 559         case KBASE_ATOM_GPU_RB_RETURN_TO_JS:
 560                 break;
 561         }
 562
 563         katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED;
 564 }
 565
 566 static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev,
 567                                                 struct kbase_jd_atom *katom)
 568 {
 569         kbase_gpu_release_atom(kbdev, katom);
 570         katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS;
 571 }
 572
 573 static inline bool kbase_gpu_rmu_workaround(struct kbase_device *kbdev, int js)
 574 {
 575         struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
 576         bool slot_busy[3];
 577
 578         if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
 579                 return true;
 580         slot_busy[0] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 0,
 581                                         KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
 582         slot_busy[1] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 1,
 583                                         KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
 584         slot_busy[2] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 2,
 585                                         KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
 586
 587         if ((js == 2 && !(slot_busy[0] || slot_busy[1])) ||
 588                 (js != 2 && !slot_busy[2]))
 589                 return true;
 590
 591         /* Don't submit slot 2 atom while GPU has jobs on slots 0/1 */
 592         if (js == 2 && (kbase_gpu_atoms_submitted(kbdev, 0) ||
 593                         kbase_gpu_atoms_submitted(kbdev, 1) ||
 594                         backend->rmu_workaround_flag))
 595                 return false;
 596
 597         /* Don't submit slot 0/1 atom while GPU has jobs on slot 2 */
 598         if (js != 2 && (kbase_gpu_atoms_submitted(kbdev, 2) ||
 599                         !backend->rmu_workaround_flag))
 600                 return false;
 601
 602         backend->rmu_workaround_flag = !backend->rmu_workaround_flag;
 603
 604         return true;
 605 }
 606
 607 static bool kbase_gpu_in_secure_mode(struct kbase_device *kbdev)
 608 {
 609         return kbdev->js_data.runpool_irq.secure_mode;
 610 }
 611
 612 static int kbase_gpu_secure_mode_enable(struct kbase_device *kbdev)
 613 {
 614         int err = -EINVAL;
 615
 616         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 617
 618         WARN_ONCE(!kbdev->secure_ops,
 619                         "Cannot enable secure mode: secure callbacks not specified.\n");
 620
 621         if (kbdev->secure_ops) {
 622                 /* Switch GPU to secure mode */
 623                 err = kbdev->secure_ops->secure_mode_enable(kbdev);
 624
 625                 if (err)
 626                         dev_warn(kbdev->dev, "Failed to enable secure mode: %d\n", err);
 627                 else
 628                         kbdev->js_data.runpool_irq.secure_mode = true;
 629         }
 630
 631         return err;
 632 }
 633
 634 static int kbase_gpu_secure_mode_disable(struct kbase_device *kbdev)
 635 {
 636         int err = -EINVAL;
 637
 638         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 639
 640         WARN_ONCE(!kbdev->secure_ops,
 641                         "Cannot disable secure mode: secure callbacks not specified.\n");
 642
 643         if (kbdev->secure_ops) {
 644                 /* Switch GPU to non-secure mode */
 645                 err = kbdev->secure_ops->secure_mode_disable(kbdev);
 646
 647                 if (err)
 648                         dev_warn(kbdev->dev, "Failed to disable secure mode: %d\n", err);
 649                 else
 650                         kbdev->js_data.runpool_irq.secure_mode = false;
 651         }
 652
 653         return err;
 654 }
 655
 656 void kbase_gpu_slot_update(struct kbase_device *kbdev)
 657 {
 658         int js;
 659
 660         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 661
 662         for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
 663                 struct kbase_jd_atom *katom[2];
 664                 int idx;
 665
 666                 katom[0] = kbase_gpu_inspect(kbdev, js, 0);
 667                 katom[1] = kbase_gpu_inspect(kbdev, js, 1);
 668                 WARN_ON(katom[1] && !katom[0]);
 669
 670                 for (idx = 0; idx < SLOT_RB_SIZE; idx++) {
 671                         bool cores_ready;
 672
 673                         if (!katom[idx])
 674                                 continue;
 675
 676                         switch (katom[idx]->gpu_rb_state) {
 677                         case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB:
 678                                 /* Should be impossible */
 679                                 WARN(1, "Attempting to update atom not in ringbuffer\n");
 680                                 break;
 681
 682                         case KBASE_ATOM_GPU_RB_WAITING_BLOCKED:
 683                                 if (katom[idx]->atom_flags &
 684                                                 KBASE_KATOM_FLAG_X_DEP_BLOCKED)
 685                                         break;
 686
 687                                 katom[idx]->gpu_rb_state =
 688                                 KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE;
 689
 690                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 691                         case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE:
 692                                 cores_ready =
 693                                         kbasep_js_job_check_ref_cores(kbdev, js,
 694                                                                 katom[idx]);
 695
 696                                 if (katom[idx]->event_code ==
 697                                                 BASE_JD_EVENT_PM_EVENT) {
 698                                         katom[idx]->gpu_rb_state =
 699                                                 KBASE_ATOM_GPU_RB_RETURN_TO_JS;
 700                                         break;
 701                                 }
 702
 703                                 if (!cores_ready)
 704                                         break;
 705
 706                                 kbase_js_affinity_retain_slot_cores(kbdev, js,
 707                                                         katom[idx]->affinity);
 708                                 katom[idx]->gpu_rb_state =
 709                                         KBASE_ATOM_GPU_RB_WAITING_AFFINITY;
 710
 711                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 712
 713                         case KBASE_ATOM_GPU_RB_WAITING_AFFINITY:
 714                                 if (!kbase_gpu_rmu_workaround(kbdev, js))
 715                                         break;
 716
 717                                 katom[idx]->gpu_rb_state =
 718                                         KBASE_ATOM_GPU_RB_WAITING_SECURE_MODE;
 719
 720                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 721
 722                         case KBASE_ATOM_GPU_RB_WAITING_SECURE_MODE:
 723                                 if (kbase_gpu_in_secure_mode(kbdev) != kbase_jd_katom_is_secure(katom[idx])) {
 724                                         int err = 0;
 725
 726                                         /* Not in correct mode, take action */
 727                                         if (kbase_gpu_atoms_submitted_any(kbdev)) {
 728                                                 /*
 729                                                  * We are not in the correct
 730                                                  * GPU mode for this job, and
 731                                                  * we can't switch now because
 732                                                  * there are jobs already
 733                                                  * running.
 734                                                  */
 735                                                 break;
 736                                         }
 737
 738                                         /* No jobs running, so we can switch GPU mode right now */
 739                                         if (kbase_jd_katom_is_secure(katom[idx])) {
 740                                                 err = kbase_gpu_secure_mode_enable(kbdev);
 741                                         } else {
 742                                                 err = kbase_gpu_secure_mode_disable(kbdev);
 743                                         }
 744
 745                                         if (err) {
 746                                                 /* Failed to switch secure mode, fail atom */
 747                                                 katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID;
 748                                                 kbase_gpu_mark_atom_for_return(kbdev, katom[idx]);
 749                                                 break;
 750                                         }
 751                                 }
 752
 753                                 /* Secure mode sanity checks */
 754                                 KBASE_DEBUG_ASSERT_MSG(
 755                                         kbase_jd_katom_is_secure(katom[idx]) == kbase_gpu_in_secure_mode(kbdev),
 756                                         "Secure mode of atom (%d) doesn't match secure mode of GPU (%d)",
 757                                         kbase_jd_katom_is_secure(katom[idx]), kbase_gpu_in_secure_mode(kbdev));
 758                                 KBASE_DEBUG_ASSERT_MSG(
 759                                         (kbase_jd_katom_is_secure(katom[idx]) && js == 0) ||
 760                                         !kbase_jd_katom_is_secure(katom[idx]),
 761                                         "Secure atom on JS%d not supported", js);
 762
 763                                 katom[idx]->gpu_rb_state =
 764                                         KBASE_ATOM_GPU_RB_READY;
 765
 766                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 767
 768                         case KBASE_ATOM_GPU_RB_READY:
 769                                 /* Only submit if head atom or previous atom
 770                                  * already submitted */
 771                                 if (idx == 1 &&
 772                                         (katom[0]->gpu_rb_state !=
 773                                                 KBASE_ATOM_GPU_RB_SUBMITTED &&
 774                                         katom[0]->gpu_rb_state !=
 775                                         KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB))
 776                                         break;
 777
 778                                 /* Check if this job needs the cycle counter
 779                                  * enabled before submission */
 780                                 if (katom[idx]->core_req & BASE_JD_REQ_PERMON)
 781                                         kbase_pm_request_gpu_cycle_counter_l2_is_on(
 782                                                                         kbdev);
 783
 784                                 /* Inform power management at start/finish of
 785                                  * atom so it can update its GPU utilisation
 786                                  * metrics. */
 787                                 kbase_pm_metrics_run_atom(kbdev, katom[idx]);
 788
 789                                 kbase_job_hw_submit(kbdev, katom[idx], js);
 790                                 katom[idx]->gpu_rb_state =
 791                                                 KBASE_ATOM_GPU_RB_SUBMITTED;
 792
 793                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 794
 795                         case KBASE_ATOM_GPU_RB_SUBMITTED:
 796                                 /* Atom submitted to HW, nothing else to do */
 797                                 break;
 798
 799                         case KBASE_ATOM_GPU_RB_RETURN_TO_JS:
 800                                 /* Only return if head atom or previous atom
 801                                  * already removed - as atoms must be returned
 802                                  * in order */
 803                                 if (idx == 0 || katom[0]->gpu_rb_state ==
 804                                         KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
 805                                         kbase_gpu_dequeue_atom(kbdev, js);
 806                                         kbase_jm_return_atom_to_js(kbdev,
 807                                                                 katom[idx]);
 808                                 }
 809                                 break;
 810                         }
 811                 }
 812         }
 813
 814         /* Warn if PRLAM-8987 affinity restrictions are violated */
 815         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
 816                 WARN_ON((kbase_gpu_atoms_submitted(kbdev, 0) ||
 817                         kbase_gpu_atoms_submitted(kbdev, 1)) &&
 818                         kbase_gpu_atoms_submitted(kbdev, 2));
 819 }
 820
 821
 822 void kbase_backend_run_atom(struct kbase_device *kbdev,
 823                                 struct kbase_jd_atom *katom)
 824 {
 825         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 826
 827         kbase_gpu_enqueue_atom(kbdev, katom);
 828         kbase_gpu_slot_update(kbdev);
 829 }
 830
 831 bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js)
 832 {
 833         struct kbase_jd_atom *katom;
 834         struct kbase_jd_atom *next_katom;
 835
 836         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 837
 838         katom = kbase_gpu_inspect(kbdev, js, 0);
 839         next_katom = kbase_gpu_inspect(kbdev, js, 1);
 840
 841         if (next_katom && katom->kctx == next_katom->kctx &&
 842                 next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED &&
 843                 (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL)
 844                                                                         != 0 ||
 845                 kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL)
 846                                                                         != 0)) {
 847                 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT),
 848                                 JS_COMMAND_NOP, NULL);
 849                 next_katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY;
 850                 return true;
 851         }
 852
 853         return false;
 854 }
 855
 856 void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
 857                                 u32 completion_code,
 858                                 u64 job_tail,
 859                                 ktime_t *end_timestamp)
 860 {
 861         struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0);
 862         struct kbase_context *kctx = katom->kctx;
 863
 864         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 865
 866         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6787) &&
 867                         completion_code != BASE_JD_EVENT_DONE &&
 868                         !(completion_code & BASE_JD_SW_EVENT)) {
 869                 katom->need_cache_flush_cores_retained = katom->affinity;
 870                 kbase_pm_request_cores(kbdev, false, katom->affinity);
 871         } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10676)) {
 872                 if (kbdev->gpu_props.num_core_groups > 1 &&
 873                         !(katom->affinity &
 874                         kbdev->gpu_props.props.coherency_info.group[0].core_mask
 875                                                                         ) &&
 876                         (katom->affinity &
 877                         kbdev->gpu_props.props.coherency_info.group[1].core_mask
 878                                                                         )) {
 879                         dev_info(kbdev->dev, "JD: Flushing cache due to PRLAM-10676\n");
 880                         katom->need_cache_flush_cores_retained =
 881                                                                 katom->affinity;
 882                         kbase_pm_request_cores(kbdev, false,
 883                                                         katom->affinity);
 884                 }
 885         }
 886
 887         katom = kbase_gpu_dequeue_atom(kbdev, js);
 888
 889         kbase_timeline_job_slot_done(kbdev, katom->kctx, katom, js, 0);
 890
 891         if (completion_code == BASE_JD_EVENT_STOPPED) {
 892                 struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js,
 893                                                                         0);
 894
 895                 /*
 896                  * Dequeue next atom from ringbuffers on same slot if required.
 897                  * This atom will already have been removed from the NEXT
 898                  * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that
 899                  * the atoms on this slot are returned in the correct order.
 900                  */
 901                 if (next_katom && katom->kctx == next_katom->kctx) {
 902                         kbase_gpu_dequeue_atom(kbdev, js);
 903                         kbase_jm_return_atom_to_js(kbdev, next_katom);
 904                 }
 905         } else if (completion_code != BASE_JD_EVENT_DONE) {
 906                 struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
 907                 int i;
 908
 909 #if KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR != 0
 910                 KBASE_TRACE_DUMP(kbdev);
 911 #endif
 912                 kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
 913
 914                 /*
 915                  * Remove all atoms on the same context from ringbuffers. This
 916                  * will not remove atoms that are already on the GPU, as these
 917                  * are guaranteed not to have fail dependencies on the failed
 918                  * atom.
 919                  */
 920                 for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) {
 921                         struct kbase_jd_atom *katom_idx0 =
 922                                                 kbase_gpu_inspect(kbdev, i, 0);
 923                         struct kbase_jd_atom *katom_idx1 =
 924                                                 kbase_gpu_inspect(kbdev, i, 1);
 925
 926                         if (katom_idx0 && katom_idx0->kctx == katom->kctx &&
 927                                 katom_idx0->gpu_rb_state !=
 928                                 KBASE_ATOM_GPU_RB_SUBMITTED) {
 929                                 /* Dequeue katom_idx0 from ringbuffer */
 930                                 kbase_gpu_dequeue_atom(kbdev, i);
 931
 932                                 if (katom_idx1 &&
 933                                         katom_idx1->kctx == katom->kctx &&
 934                                         katom_idx0->gpu_rb_state !=
 935                                                 KBASE_ATOM_GPU_RB_SUBMITTED) {
 936                                         /* Dequeue katom_idx1 from ringbuffer */
 937                                         kbase_gpu_dequeue_atom(kbdev, i);
 938
 939                                         katom_idx1->event_code =
 940                                                         BASE_JD_EVENT_STOPPED;
 941                                         kbase_jm_return_atom_to_js(kbdev,
 942                                                                 katom_idx1);
 943                                 }
 944                                 katom_idx0->event_code = BASE_JD_EVENT_STOPPED;
 945                                 kbase_jm_return_atom_to_js(kbdev, katom_idx0);
 946
 947                         } else if (katom_idx1 &&
 948                                         katom_idx1->kctx == katom->kctx &&
 949                                         katom_idx1->gpu_rb_state !=
 950                                                 KBASE_ATOM_GPU_RB_SUBMITTED) {
 951                                 /* Can not dequeue this atom yet - will be
 952                                  * dequeued when atom at idx0 completes */
 953                                 katom_idx1->event_code = BASE_JD_EVENT_STOPPED;
 954                                 kbase_gpu_mark_atom_for_return(kbdev,
 955                                                                 katom_idx1);
 956                         }
 957                 }
 958         }
 959
 960         KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc,
 961                                         js, completion_code);
 962
 963         if (job_tail != 0 && job_tail != katom->jc) {
 964                 bool was_updated = (job_tail != katom->jc);
 965
 966                 /* Some of the job has been executed, so we update the job chain
 967                  * address to where we should resume from */
 968                 katom->jc = job_tail;
 969                 if (was_updated)
 970                         KBASE_TRACE_ADD_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx,
 971                                                 katom, job_tail, js);
 972         }
 973
 974         /* Only update the event code for jobs that weren't cancelled */
 975         if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED)
 976                 katom->event_code = (base_jd_event_code)completion_code;
 977
 978         kbase_device_trace_register_access(kctx, REG_WRITE,
 979                                                 JOB_CONTROL_REG(JOB_IRQ_CLEAR),
 980                                                 1 << js);
 981
 982         /* Complete the job, and start new ones
 983          *
 984          * Also defer remaining work onto the workqueue:
 985          * - Re-queue Soft-stopped jobs
 986          * - For any other jobs, queue the job back into the dependency system
 987          * - Schedule out the parent context if necessary, and schedule a new
 988          *   one in.
 989          */
 990 #ifdef CONFIG_GPU_TRACEPOINTS
 991         {
 992                 /* The atom in the HEAD */
 993                 struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js,
 994                                                                         0);
 995
 996                 if (next_katom && next_katom->gpu_rb_state ==
 997                                                 KBASE_ATOM_GPU_RB_SUBMITTED) {
 998                         char js_string[16];
 999
1000                         trace_gpu_sched_switch(kbasep_make_job_slot_string(js,
1001                                                                 js_string),
1002                                                 ktime_to_ns(*end_timestamp),
1003                                                 (u32)next_katom->kctx, 0,
1004                                                 next_katom->work_id);
1005                         kbdev->hwaccess.backend.slot_rb[js].last_context =
1006                                                         next_katom->kctx;
1007                 } else {
1008                         char js_string[16];
1009
1010                         trace_gpu_sched_switch(kbasep_make_job_slot_string(js,
1011                                                                 js_string),
1012                                                 ktime_to_ns(ktime_get()), 0, 0,
1013                                                 0);
1014                         kbdev->hwaccess.backend.slot_rb[js].last_context = 0;
1015                 }
1016         }
1017 #endif
1018
1019         if (completion_code == BASE_JD_EVENT_STOPPED)
1020                 kbase_jm_return_atom_to_js(kbdev, katom);
1021         else
1022                 kbase_jm_complete(kbdev, katom, end_timestamp);
1023
1024         /* Job completion may have unblocked other atoms. Try to update all job
1025          * slots */
1026         kbase_gpu_slot_update(kbdev);
1027 }
1028
1029 void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp)
1030 {
1031         int js;
1032
1033         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1034
1035         for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
1036                 int idx;
1037
1038                 for (idx = 0; idx < 2; idx++) {
1039                         struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev,
1040                                                                         js, 0);
1041
1042                         if (katom) {
1043                                 enum kbase_atom_gpu_rb_state gpu_rb_state =
1044                                                         katom->gpu_rb_state;
1045
1046                                 kbase_gpu_release_atom(kbdev, katom);
1047                                 kbase_gpu_dequeue_atom(kbdev, js);
1048
1049                                 if (gpu_rb_state ==
1050                                                 KBASE_ATOM_GPU_RB_SUBMITTED) {
1051                                         katom->event_code =
1052                                                 BASE_JD_EVENT_JOB_CANCELLED;
1053                                         kbase_jm_complete(kbdev, katom,
1054                                                                 end_timestamp);
1055                                 } else {
1056                                         katom->event_code =
1057                                                         BASE_JD_EVENT_STOPPED;
1058                                         kbase_jm_return_atom_to_js(kbdev,
1059                                                         katom);
1060                                 }
1061                         }
1062                 }
1063         }
1064 }
1065
1066 static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev,
1067                                         int js,
1068                                         struct kbase_jd_atom *katom,
1069                                         u32 action)
1070 {
1071         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1072         u32 hw_action = action & JS_COMMAND_MASK;
1073
1074         kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom);
1075         kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action,
1076                                                         katom->core_req, katom);
1077         kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
1078 }
1079
1080 static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev,
1081                                                 struct kbase_jd_atom *katom,
1082                                                 u32 action,
1083                                                 bool disjoint)
1084 {
1085         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1086
1087         katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT;
1088         kbase_gpu_mark_atom_for_return(kbdev, katom);
1089         kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
1090
1091         if (disjoint)
1092                 kbase_job_check_enter_disjoint(kbdev, action, katom->core_req,
1093                                                                         katom);
1094 }
1095
1096 static int should_stop_x_dep_slot(struct kbase_jd_atom *katom)
1097 {
1098         if (katom->x_post_dep) {
1099                 struct kbase_jd_atom *dep_atom = katom->x_post_dep;
1100
1101                 if (dep_atom->gpu_rb_state !=
1102                                         KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB &&
1103                         dep_atom->gpu_rb_state !=
1104                                         KBASE_ATOM_GPU_RB_RETURN_TO_JS)
1105                         return dep_atom->slot_nr;
1106         }
1107         return -1;
1108 }
1109
1110 bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev,
1111                                         struct kbase_context *kctx,
1112                                         int js,
1113                                         struct kbase_jd_atom *katom,
1114                                         u32 action)
1115 {
1116         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1117
1118         struct kbase_jd_atom *katom_idx0;
1119         struct kbase_jd_atom *katom_idx1;
1120
1121         bool katom_idx0_valid, katom_idx1_valid;
1122
1123         bool ret = false;
1124
1125         int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1;
1126
1127         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1128
1129         katom_idx0 = kbase_gpu_inspect(kbdev, js, 0);
1130         katom_idx1 = kbase_gpu_inspect(kbdev, js, 1);
1131
1132         if (katom) {
1133                 katom_idx0_valid = (katom_idx0 == katom);
1134                 /* If idx0 is to be removed and idx1 is on the same context,
1135                  * then idx1 must also be removed otherwise the atoms might be
1136                  * returned out of order */
1137                 if (katom_idx1)
1138                         katom_idx1_valid = (katom_idx1 == katom) ||
1139                                                 (katom_idx0_valid &&
1140                                                         (katom_idx0->kctx ==
1141                                                         katom_idx1->kctx));
1142                 else
1143                         katom_idx1_valid = false;
1144         } else {
1145                 katom_idx0_valid = (katom_idx0 &&
1146                                         (!kctx || katom_idx0->kctx == kctx));
1147                 katom_idx1_valid = (katom_idx1 &&
1148                                         (!kctx || katom_idx1->kctx == kctx));
1149         }
1150
1151         if (katom_idx0_valid)
1152                 stop_x_dep_idx0 = should_stop_x_dep_slot(katom_idx0);
1153         if (katom_idx1_valid)
1154                 stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1);
1155
1156         if (katom_idx0_valid) {
1157                 if (katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) {
1158                         /* Simple case - just dequeue and return */
1159                         kbase_gpu_dequeue_atom(kbdev, js);
1160                         if (katom_idx1_valid) {
1161                                 kbase_gpu_dequeue_atom(kbdev, js);
1162                                 katom_idx1->event_code =
1163                                                 BASE_JD_EVENT_REMOVED_FROM_NEXT;
1164                                 kbase_jm_return_atom_to_js(kbdev, katom_idx1);
1165                                 kbasep_js_clear_submit_allowed(js_devdata,
1166                                                         katom_idx1->kctx);
1167                         }
1168
1169                         katom_idx0->event_code =
1170                                                 BASE_JD_EVENT_REMOVED_FROM_NEXT;
1171                         kbase_jm_return_atom_to_js(kbdev, katom_idx0);
1172                         kbasep_js_clear_submit_allowed(js_devdata,
1173                                                         katom_idx0->kctx);
1174                 } else {
1175                         /* katom_idx0 is on GPU */
1176                         if (katom_idx1 && katom_idx1->gpu_rb_state ==
1177                                                 KBASE_ATOM_GPU_RB_SUBMITTED) {
1178                                 /* katom_idx0 and katom_idx1 are on GPU */
1179
1180                                 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1181                                                 JS_COMMAND_NEXT), NULL) == 0) {
1182                                         /* idx0 has already completed - stop
1183                                          * idx1 if needed*/
1184                                         if (katom_idx1_valid) {
1185                                                 kbase_gpu_stop_atom(kbdev, js,
1186                                                                 katom_idx1,
1187                                                                 action);
1188                                                 ret = true;
1189                                         }
1190                                 } else {
1191                                         /* idx1 is in NEXT registers - attempt
1192                                          * to remove */
1193                                         kbase_reg_write(kbdev,
1194                                                         JOB_SLOT_REG(js,
1195                                                         JS_COMMAND_NEXT),
1196                                                         JS_COMMAND_NOP, NULL);
1197
1198                                         if (kbase_reg_read(kbdev,
1199                                                         JOB_SLOT_REG(js,
1200                                                         JS_HEAD_NEXT_LO), NULL)
1201                                                                         != 0 ||
1202                                                 kbase_reg_read(kbdev,
1203                                                         JOB_SLOT_REG(js,
1204                                                         JS_HEAD_NEXT_HI), NULL)
1205                                                                         != 0) {
1206                                                 /* idx1 removed successfully,
1207                                                  * will be handled in IRQ */
1208                                                 kbase_gpu_remove_atom(kbdev,
1209                                                                 katom_idx1,
1210                                                                 action, true);
1211                                                 stop_x_dep_idx1 =
1212                                         should_stop_x_dep_slot(katom_idx1);
1213
1214                                                 /* stop idx0 if still on GPU */
1215                                                 kbase_gpu_stop_atom(kbdev, js,
1216                                                                 katom_idx0,
1217                                                                 action);
1218                                                 ret = true;
1219                                         } else if (katom_idx1_valid) {
1220                                                 /* idx0 has already completed,
1221                                                  * stop idx1 if needed */
1222                                                 kbase_gpu_stop_atom(kbdev, js,
1223                                                                 katom_idx1,
1224                                                                 action);
1225                                                 ret = true;
1226                                         }
1227                                 }
1228                         } else if (katom_idx1_valid) {
1229                                 /* idx1 not on GPU but must be dequeued*/
1230
1231                                 /* idx1 will be handled in IRQ */
1232                                 kbase_gpu_remove_atom(kbdev, katom_idx1, action,
1233                                                                         false);
1234                                 /* stop idx0 */
1235                                 /* This will be repeated for anything removed
1236                                  * from the next registers, since their normal
1237                                  * flow was also interrupted, and this function
1238                                  * might not enter disjoint state e.g. if we
1239                                  * don't actually do a hard stop on the head
1240                                  * atom */
1241                                 kbase_gpu_stop_atom(kbdev, js, katom_idx0,
1242                                                                         action);
1243                                 ret = true;
1244                         } else {
1245                                 /* no atom in idx1 */
1246                                 /* just stop idx0 */
1247                                 kbase_gpu_stop_atom(kbdev, js, katom_idx0,
1248                                                                         action);
1249                                 ret = true;
1250                         }
1251                 }
1252         } else if (katom_idx1_valid) {
1253                 if (katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) {
1254                         /* Mark for return */
1255                         /* idx1 will be returned once idx0 completes */
1256                         kbase_gpu_remove_atom(kbdev, katom_idx1, action,
1257                                                                         false);
1258                 } else {
1259                         /* idx1 is on GPU */
1260                         if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1261                                                 JS_COMMAND_NEXT), NULL) == 0) {
1262                                 /* idx0 has already completed - stop idx1 */
1263                                 kbase_gpu_stop_atom(kbdev, js, katom_idx1,
1264                                                                         action);
1265                                 ret = true;
1266                         } else {
1267                                 /* idx1 is in NEXT registers - attempt to
1268                                  * remove */
1269                                 kbase_reg_write(kbdev, JOB_SLOT_REG(js,
1270                                                         JS_COMMAND_NEXT),
1271                                                         JS_COMMAND_NOP, NULL);
1272
1273                                 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1274                                                 JS_HEAD_NEXT_LO), NULL) != 0 ||
1275                                     kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1276                                                 JS_HEAD_NEXT_HI), NULL) != 0) {
1277                                         /* idx1 removed successfully, will be
1278                                          * handled in IRQ once idx0 completes */
1279                                         kbase_gpu_remove_atom(kbdev, katom_idx1,
1280                                                                         action,
1281                                                                         false);
1282                                 } else {
1283                                         /* idx0 has already completed - stop
1284                                          * idx1 */
1285                                         kbase_gpu_stop_atom(kbdev, js,
1286                                                                 katom_idx1,
1287                                                                 action);
1288                                         ret = true;
1289                                 }
1290                         }
1291                 }
1292         }
1293
1294
1295         if (stop_x_dep_idx0 != -1)
1296                 kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx0,
1297                                                                 NULL, action);
1298
1299         if (stop_x_dep_idx1 != -1)
1300                 kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx1,
1301                                                                 NULL, action);
1302
1303         return ret;
1304 }
1305
1306 static void kbasep_gpu_cacheclean(struct kbase_device *kbdev,
1307                                         struct kbase_jd_atom *katom)
1308 {
1309         /* Limit the number of loops to avoid a hang if the interrupt is missed
1310          */
1311         u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS;
1312
1313         mutex_lock(&kbdev->cacheclean_lock);
1314
1315         /* use GPU_COMMAND completion solution */
1316         /* clean & invalidate the caches */
1317         KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0);
1318         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1319                                         GPU_COMMAND_CLEAN_INV_CACHES, NULL);
1320
1321         /* wait for cache flush to complete before continuing */
1322         while (--max_loops &&
1323                 (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) &
1324                                                 CLEAN_CACHES_COMPLETED) == 0)
1325                 ;
1326
1327         /* clear the CLEAN_CACHES_COMPLETED irq */
1328         KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u,
1329                                                         CLEAN_CACHES_COMPLETED);
1330         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR),
1331                                                 CLEAN_CACHES_COMPLETED, NULL);
1332         KBASE_DEBUG_ASSERT_MSG(kbdev->hwcnt.backend.state !=
1333                                                 KBASE_INSTR_STATE_CLEANING,
1334             "Instrumentation code was cleaning caches, but Job Management code cleared their IRQ - Instrumentation code will now hang.");
1335
1336         mutex_unlock(&kbdev->cacheclean_lock);
1337
1338         kbase_pm_unrequest_cores(kbdev, false,
1339                                         katom->need_cache_flush_cores_retained);
1340 }
1341
1342 void kbase_backend_complete_wq(struct kbase_device *kbdev,
1343                                                 struct kbase_jd_atom *katom)
1344 {
1345         /*
1346          * If cache flush required due to HW workaround then perform the flush
1347          * now
1348          */
1349         if (katom->need_cache_flush_cores_retained) {
1350                 kbasep_gpu_cacheclean(kbdev, katom);
1351                 katom->need_cache_flush_cores_retained = 0;
1352         }
1353
1354         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10969)            &&
1355             (katom->core_req & BASE_JD_REQ_FS)                        &&
1356             katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT       &&
1357             (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED) &&
1358             !(katom->atom_flags & KBASE_KATOM_FLAGS_RERUN)) {
1359                 dev_dbg(kbdev->dev, "Soft-stopped fragment shader job got a TILE_RANGE_FAULT. Possible HW issue, trying SW workaround\n");
1360                 if (kbasep_10969_workaround_clamp_coordinates(katom)) {
1361                         /* The job had a TILE_RANGE_FAULT after was soft-stopped
1362                          * Due to an HW issue we try to execute the job again.
1363                          */
1364                         dev_dbg(kbdev->dev,
1365                                 "Clamping has been executed, try to rerun the job\n"
1366                         );
1367                         katom->event_code = BASE_JD_EVENT_STOPPED;
1368                         katom->atom_flags |= KBASE_KATOM_FLAGS_RERUN;
1369                 }
1370         }
1371 }
1372
1373 void kbase_gpu_dump_slots(struct kbase_device *kbdev)
1374 {
1375         struct kbasep_js_device_data *js_devdata;
1376         unsigned long flags;
1377         int js;
1378
1379         js_devdata = &kbdev->js_data;
1380
1381         spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1382
1383         dev_info(kbdev->dev, "kbase_gpu_dump_slots:\n");
1384
1385         for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
1386                 int idx;
1387
1388                 for (idx = 0; idx < SLOT_RB_SIZE; idx++) {
1389                         struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev,
1390                                                                         js,
1391                                                                         idx);
1392
1393                         if (katom)
1394                                 dev_info(kbdev->dev,
1395                                 "  js%d idx%d : katom=%p gpu_rb_state=%d\n",
1396                                 js, idx, katom, katom->gpu_rb_state);
1397                         else
1398                                 dev_info(kbdev->dev, "  js%d idx%d : empty\n",
1399                                                                 js, idx);
1400                 }
1401         }
1402
1403         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1404 }
1405
1406
1407