drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_rb.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18
  19 /*
  20  * Register-based HW access backend specific APIs
  21  */
  22
  23 #include <mali_kbase.h>
  24 #include <mali_kbase_hwaccess_jm.h>
  25 #include <mali_kbase_jm.h>
  26 #include <mali_kbase_js.h>
  27 #include <mali_kbase_tlstream.h>
  28 #include <mali_kbase_10969_workaround.h>
  29 #include <backend/gpu/mali_kbase_device_internal.h>
  30 #include <backend/gpu/mali_kbase_jm_internal.h>
  31 #include <backend/gpu/mali_kbase_js_affinity.h>
  32 #include <backend/gpu/mali_kbase_pm_internal.h>
  33
  34 /* Return whether the specified ringbuffer is empty. HW access lock must be
  35  * held */
  36 #define SLOT_RB_EMPTY(rb)   (rb->write_idx == rb->read_idx)
  37 /* Return number of atoms currently in the specified ringbuffer. HW access lock
  38  * must be held */
  39 #define SLOT_RB_ENTRIES(rb) (int)(s8)(rb->write_idx - rb->read_idx)
  40
  41 static void kbase_gpu_release_atom(struct kbase_device *kbdev,
  42                                         struct kbase_jd_atom *katom,
  43                                         ktime_t *end_timestamp);
  44
  45 /**
  46  * kbase_gpu_enqueue_atom - Enqueue an atom in the HW access ringbuffer
  47  * @kbdev: Device pointer
  48  * @katom: Atom to enqueue
  49  *
  50  * Context: Caller must hold the HW access lock
  51  */
  52 static void kbase_gpu_enqueue_atom(struct kbase_device *kbdev,
  53                                         struct kbase_jd_atom *katom)
  54 {
  55         struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[katom->slot_nr];
  56
  57         WARN_ON(SLOT_RB_ENTRIES(rb) >= SLOT_RB_SIZE);
  58
  59         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
  60
  61         rb->entries[rb->write_idx & SLOT_RB_MASK].katom = katom;
  62         rb->write_idx++;
  63
  64         katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED;
  65 }
  66
  67 /**
  68  * kbase_gpu_dequeue_atom - Remove an atom from the HW access ringbuffer, once
  69  * it has been completed
  70  * @kbdev:         Device pointer
  71  * @js:            Job slot to remove atom from
  72  * @end_timestamp: Pointer to timestamp of atom completion. May be NULL, in
  73  *                 which case current time will be used.
  74  *
  75  * Context: Caller must hold the HW access lock
  76  *
  77  * Return: Atom removed from ringbuffer
  78  */
  79 static struct kbase_jd_atom *kbase_gpu_dequeue_atom(struct kbase_device *kbdev,
  80                                                 int js,
  81                                                 ktime_t *end_timestamp)
  82 {
  83         struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
  84         struct kbase_jd_atom *katom;
  85
  86         if (SLOT_RB_EMPTY(rb)) {
  87                 WARN(1, "GPU ringbuffer unexpectedly empty\n");
  88                 return NULL;
  89         }
  90
  91         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
  92
  93         katom = rb->entries[rb->read_idx & SLOT_RB_MASK].katom;
  94
  95         kbase_gpu_release_atom(kbdev, katom, end_timestamp);
  96
  97         rb->read_idx++;
  98
  99         katom->gpu_rb_state = KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB;
 100
 101         kbase_js_debug_log_current_affinities(kbdev);
 102
 103         return katom;
 104 }
 105
 106 struct kbase_jd_atom *kbase_gpu_inspect(struct kbase_device *kbdev, int js,
 107                                         int idx)
 108 {
 109         struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
 110
 111         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 112
 113         if ((SLOT_RB_ENTRIES(rb) - 1) < idx)
 114                 return NULL; /* idx out of range */
 115
 116         return rb->entries[(rb->read_idx + idx) & SLOT_RB_MASK].katom;
 117 }
 118
 119 struct kbase_jd_atom *kbase_backend_inspect_head(struct kbase_device *kbdev,
 120                                         int js)
 121 {
 122         return kbase_gpu_inspect(kbdev, js, 0);
 123 }
 124
 125 struct kbase_jd_atom *kbase_backend_inspect_tail(struct kbase_device *kbdev,
 126                                         int js)
 127 {
 128         struct slot_rb *rb = &kbdev->hwaccess.backend.slot_rb[js];
 129
 130         if (SLOT_RB_EMPTY(rb))
 131                 return NULL;
 132
 133         return rb->entries[(rb->write_idx - 1) & SLOT_RB_MASK].katom;
 134 }
 135
 136 /**
 137  * kbase_gpu_atoms_submitted - Inspect whether a slot has any atoms currently
 138  * on the GPU
 139  * @kbdev:  Device pointer
 140  * @js:     Job slot to inspect
 141  *
 142  * Return: true if there are atoms on the GPU for slot js,
 143  *         false otherwise
 144  */
 145 static bool kbase_gpu_atoms_submitted(struct kbase_device *kbdev, int js)
 146 {
 147         int i;
 148
 149         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 150
 151         for (i = 0; i < SLOT_RB_SIZE; i++) {
 152                 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
 153
 154                 if (!katom)
 155                         return false;
 156                 if (katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED ||
 157                                 katom->gpu_rb_state == KBASE_ATOM_GPU_RB_READY)
 158                         return true;
 159         }
 160
 161         return false;
 162 }
 163
 164 /**
 165  * kbase_gpu_atoms_submitted_any() - Inspect whether there are any atoms
 166  * currently on the GPU
 167  * @kbdev:  Device pointer
 168  *
 169  * Return: true if there are any atoms on the GPU, false otherwise
 170  */
 171 static bool kbase_gpu_atoms_submitted_any(struct kbase_device *kbdev)
 172 {
 173         int js;
 174         int i;
 175
 176         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 177
 178         for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
 179                 for (i = 0; i < SLOT_RB_SIZE; i++) {
 180                         struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
 181
 182                         if (katom && katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED)
 183                                 return true;
 184                 }
 185         }
 186         return false;
 187 }
 188
 189 int kbase_backend_nr_atoms_submitted(struct kbase_device *kbdev, int js)
 190 {
 191         int nr = 0;
 192         int i;
 193
 194         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 195
 196         for (i = 0; i < SLOT_RB_SIZE; i++) {
 197                 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
 198
 199                 if (katom && (katom->gpu_rb_state ==
 200                                                 KBASE_ATOM_GPU_RB_SUBMITTED))
 201                         nr++;
 202         }
 203
 204         return nr;
 205 }
 206
 207 int kbase_backend_nr_atoms_on_slot(struct kbase_device *kbdev, int js)
 208 {
 209         int nr = 0;
 210         int i;
 211
 212         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 213
 214         for (i = 0; i < SLOT_RB_SIZE; i++) {
 215                 if (kbase_gpu_inspect(kbdev, js, i))
 216                         nr++;
 217         }
 218
 219         return nr;
 220 }
 221
 222 static int kbase_gpu_nr_atoms_on_slot_min(struct kbase_device *kbdev, int js,
 223                                 enum kbase_atom_gpu_rb_state min_rb_state)
 224 {
 225         int nr = 0;
 226         int i;
 227
 228         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 229
 230         for (i = 0; i < SLOT_RB_SIZE; i++) {
 231                 struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, i);
 232
 233                 if (katom && (katom->gpu_rb_state >= min_rb_state))
 234                         nr++;
 235         }
 236
 237         return nr;
 238 }
 239
 240 int kbase_backend_slot_free(struct kbase_device *kbdev, int js)
 241 {
 242         if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) !=
 243                                                 KBASE_RESET_GPU_NOT_PENDING) {
 244                 /* The GPU is being reset - so prevent submission */
 245                 return 0;
 246         }
 247
 248         return SLOT_RB_SIZE - kbase_backend_nr_atoms_on_slot(kbdev, js);
 249 }
 250
 251
 252 static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev,
 253                                                 struct kbase_jd_atom *katom);
 254
 255 static bool kbasep_js_job_check_ref_cores(struct kbase_device *kbdev,
 256                                                 int js,
 257                                                 struct kbase_jd_atom *katom)
 258 {
 259         /* The most recently checked affinity. Having this at this scope allows
 260          * us to guarantee that we've checked the affinity in this function
 261          * call.
 262          */
 263         u64 recently_chosen_affinity = 0;
 264         bool chosen_affinity = false;
 265         bool retry;
 266
 267         do {
 268                 retry = false;
 269
 270                 /* NOTE: The following uses a number of FALLTHROUGHs to optimize
 271                  * the calls to this function. Ending of the function is
 272                  * indicated by BREAK OUT */
 273                 switch (katom->coreref_state) {
 274                         /* State when job is first attempted to be run */
 275                 case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
 276                         KBASE_DEBUG_ASSERT(katom->affinity == 0);
 277
 278                         /* Compute affinity */
 279                         if (false == kbase_js_choose_affinity(
 280                                         &recently_chosen_affinity, kbdev, katom,
 281                                                                         js)) {
 282                                 /* No cores are currently available */
 283                                 /* *** BREAK OUT: No state transition *** */
 284                                 break;
 285                         }
 286
 287                         chosen_affinity = true;
 288
 289                         /* Request the cores */
 290                         kbase_pm_request_cores(kbdev,
 291                                         katom->core_req & BASE_JD_REQ_T,
 292                                                 recently_chosen_affinity);
 293
 294                         katom->affinity = recently_chosen_affinity;
 295
 296                         /* Proceed to next state */
 297                         katom->coreref_state =
 298                         KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
 299
 300                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 301
 302                 case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
 303                         {
 304                                 enum kbase_pm_cores_ready cores_ready;
 305
 306                                 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
 307                                         (katom->core_req & BASE_JD_REQ_T));
 308
 309                                 cores_ready = kbase_pm_register_inuse_cores(
 310                                                 kbdev,
 311                                                 katom->core_req & BASE_JD_REQ_T,
 312                                                 katom->affinity);
 313                                 if (cores_ready == KBASE_NEW_AFFINITY) {
 314                                         /* Affinity no longer valid - return to
 315                                          * previous state */
 316                                         kbasep_js_job_check_deref_cores(kbdev,
 317                                                                         katom);
 318                                         KBASE_TRACE_ADD_SLOT_INFO(kbdev,
 319                                         JS_CORE_REF_REGISTER_INUSE_FAILED,
 320                                                         katom->kctx, katom,
 321                                                         katom->jc, js,
 322                                                         (u32) katom->affinity);
 323                                         /* *** BREAK OUT: Return to previous
 324                                          * state, retry *** */
 325                                         retry = true;
 326                                         break;
 327                                 }
 328                                 if (cores_ready == KBASE_CORES_NOT_READY) {
 329                                         /* Stay in this state and return, to
 330                                          * retry at this state later */
 331                                         KBASE_TRACE_ADD_SLOT_INFO(kbdev,
 332                                         JS_CORE_REF_REGISTER_INUSE_FAILED,
 333                                                         katom->kctx, katom,
 334                                                         katom->jc, js,
 335                                                         (u32) katom->affinity);
 336                                         /* *** BREAK OUT: No state transition
 337                                          * *** */
 338                                         break;
 339                                 }
 340                                 /* Proceed to next state */
 341                                 katom->coreref_state =
 342                                 KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
 343                         }
 344
 345                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 346
 347                 case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
 348                         KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
 349                                         (katom->core_req & BASE_JD_REQ_T));
 350
 351                         /* Optimize out choosing the affinity twice in the same
 352                          * function call */
 353                         if (chosen_affinity == false) {
 354                                 /* See if the affinity changed since a previous
 355                                  * call. */
 356                                 if (false == kbase_js_choose_affinity(
 357                                                 &recently_chosen_affinity,
 358                                                         kbdev, katom, js)) {
 359                                         /* No cores are currently available */
 360                                         kbasep_js_job_check_deref_cores(kbdev,
 361                                                                         katom);
 362                                         KBASE_TRACE_ADD_SLOT_INFO(kbdev,
 363                                         JS_CORE_REF_REQUEST_ON_RECHECK_FAILED,
 364                                                 katom->kctx, katom,
 365                                                 katom->jc, js,
 366                                                 (u32) recently_chosen_affinity);
 367                                         /* *** BREAK OUT: Transition to lower
 368                                          * state *** */
 369                                         break;
 370                                 }
 371                                 chosen_affinity = true;
 372                         }
 373
 374                         /* Now see if this requires a different set of cores */
 375                         if (recently_chosen_affinity != katom->affinity) {
 376                                 enum kbase_pm_cores_ready cores_ready;
 377
 378                                 kbase_pm_request_cores(kbdev,
 379                                                 katom->core_req & BASE_JD_REQ_T,
 380                                                 recently_chosen_affinity);
 381
 382                                 /* Register new cores whilst we still hold the
 383                                  * old ones, to minimize power transitions */
 384                                 cores_ready =
 385                                         kbase_pm_register_inuse_cores(kbdev,
 386                                                 katom->core_req & BASE_JD_REQ_T,
 387                                                 recently_chosen_affinity);
 388                                 kbasep_js_job_check_deref_cores(kbdev, katom);
 389
 390                                 /* Fixup the state that was reduced by
 391                                  * deref_cores: */
 392                                 katom->coreref_state =
 393                                 KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
 394                                 katom->affinity = recently_chosen_affinity;
 395                                 if (cores_ready == KBASE_NEW_AFFINITY) {
 396                                         /* Affinity no longer valid - return to
 397                                          * previous state */
 398                                         katom->coreref_state =
 399                                         KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
 400
 401                                         kbasep_js_job_check_deref_cores(kbdev,
 402                                                                         katom);
 403
 404                                         KBASE_TRACE_ADD_SLOT_INFO(kbdev,
 405                                         JS_CORE_REF_REGISTER_INUSE_FAILED,
 406                                                         katom->kctx, katom,
 407                                                         katom->jc, js,
 408                                                         (u32) katom->affinity);
 409                                         /* *** BREAK OUT: Return to previous
 410                                          * state, retry *** */
 411                                         retry = true;
 412                                         break;
 413                                 }
 414                                 /* Now might be waiting for powerup again, with
 415                                  * a new affinity */
 416                                 if (cores_ready == KBASE_CORES_NOT_READY) {
 417                                         /* Return to previous state */
 418                                         katom->coreref_state =
 419                                         KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES;
 420                                         KBASE_TRACE_ADD_SLOT_INFO(kbdev,
 421                                         JS_CORE_REF_REGISTER_ON_RECHECK_FAILED,
 422                                                         katom->kctx, katom,
 423                                                         katom->jc, js,
 424                                                         (u32) katom->affinity);
 425                                         /* *** BREAK OUT: Transition to lower
 426                                          * state *** */
 427                                         break;
 428                                 }
 429                         }
 430                         /* Proceed to next state */
 431                         katom->coreref_state =
 432                         KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS;
 433
 434                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 435                 case KBASE_ATOM_COREREF_STATE_CHECK_AFFINITY_VIOLATIONS:
 436                         KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
 437                                         (katom->core_req & BASE_JD_REQ_T));
 438                         KBASE_DEBUG_ASSERT(katom->affinity ==
 439                                                 recently_chosen_affinity);
 440
 441                         /* Note: this is where the caller must've taken the
 442                          * runpool_irq.lock */
 443
 444                         /* Check for affinity violations - if there are any,
 445                          * then we just ask the caller to requeue and try again
 446                          * later */
 447                         if (kbase_js_affinity_would_violate(kbdev, js,
 448                                         katom->affinity) != false) {
 449                                 /* Return to previous state */
 450                                 katom->coreref_state =
 451                                 KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY;
 452                                 /* *** BREAK OUT: Transition to lower state ***
 453                                  */
 454                                 KBASE_TRACE_ADD_SLOT_INFO(kbdev,
 455                                         JS_CORE_REF_AFFINITY_WOULD_VIOLATE,
 456                                         katom->kctx, katom, katom->jc, js,
 457                                         (u32) katom->affinity);
 458                                 break;
 459                         }
 460
 461                         /* No affinity violations would result, so the cores are
 462                          * ready */
 463                         katom->coreref_state = KBASE_ATOM_COREREF_STATE_READY;
 464                         /* *** BREAK OUT: Cores Ready *** */
 465                         break;
 466
 467                 default:
 468                         KBASE_DEBUG_ASSERT_MSG(false,
 469                                         "Unhandled kbase_atom_coreref_state %d",
 470                                                         katom->coreref_state);
 471                         break;
 472                 }
 473         } while (retry != false);
 474
 475         return (katom->coreref_state == KBASE_ATOM_COREREF_STATE_READY);
 476 }
 477
 478 static void kbasep_js_job_check_deref_cores(struct kbase_device *kbdev,
 479                                                 struct kbase_jd_atom *katom)
 480 {
 481         KBASE_DEBUG_ASSERT(kbdev != NULL);
 482         KBASE_DEBUG_ASSERT(katom != NULL);
 483
 484         switch (katom->coreref_state) {
 485         case KBASE_ATOM_COREREF_STATE_READY:
 486                 /* State where atom was submitted to the HW - just proceed to
 487                  * power-down */
 488                 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
 489                                         (katom->core_req & BASE_JD_REQ_T));
 490
 491                 /* *** FALLTHROUGH *** */
 492
 493         case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
 494                 /* State where cores were registered */
 495                 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
 496                                         (katom->core_req & BASE_JD_REQ_T));
 497                 kbase_pm_release_cores(kbdev, katom->core_req & BASE_JD_REQ_T,
 498                                                         katom->affinity);
 499
 500                 break;
 501
 502         case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
 503                 /* State where cores were requested, but not registered */
 504                 KBASE_DEBUG_ASSERT(katom->affinity != 0 ||
 505                                         (katom->core_req & BASE_JD_REQ_T));
 506                 kbase_pm_unrequest_cores(kbdev, katom->core_req & BASE_JD_REQ_T,
 507                                                         katom->affinity);
 508                 break;
 509
 510         case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
 511                 /* Initial state - nothing required */
 512                 KBASE_DEBUG_ASSERT(katom->affinity == 0);
 513                 break;
 514
 515         default:
 516                 KBASE_DEBUG_ASSERT_MSG(false,
 517                                                 "Unhandled coreref_state: %d",
 518                                                         katom->coreref_state);
 519                 break;
 520         }
 521
 522         katom->affinity = 0;
 523         katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
 524 }
 525
 526 static void kbasep_js_job_check_deref_cores_nokatom(struct kbase_device *kbdev,
 527                 base_jd_core_req core_req, u64 affinity,
 528                 enum kbase_atom_coreref_state coreref_state)
 529 {
 530         KBASE_DEBUG_ASSERT(kbdev != NULL);
 531
 532         switch (coreref_state) {
 533         case KBASE_ATOM_COREREF_STATE_READY:
 534                 /* State where atom was submitted to the HW - just proceed to
 535                  * power-down */
 536                 KBASE_DEBUG_ASSERT(affinity != 0 ||
 537                                         (core_req & BASE_JD_REQ_T));
 538
 539                 /* *** FALLTHROUGH *** */
 540
 541         case KBASE_ATOM_COREREF_STATE_RECHECK_AFFINITY:
 542                 /* State where cores were registered */
 543                 KBASE_DEBUG_ASSERT(affinity != 0 ||
 544                                         (core_req & BASE_JD_REQ_T));
 545                 kbase_pm_release_cores(kbdev, core_req & BASE_JD_REQ_T,
 546                                                         affinity);
 547
 548                 break;
 549
 550         case KBASE_ATOM_COREREF_STATE_WAITING_FOR_REQUESTED_CORES:
 551                 /* State where cores were requested, but not registered */
 552                 KBASE_DEBUG_ASSERT(affinity != 0 ||
 553                                         (core_req & BASE_JD_REQ_T));
 554                 kbase_pm_unrequest_cores(kbdev, core_req & BASE_JD_REQ_T,
 555                                                         affinity);
 556                 break;
 557
 558         case KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED:
 559                 /* Initial state - nothing required */
 560                 KBASE_DEBUG_ASSERT(affinity == 0);
 561                 break;
 562
 563         default:
 564                 KBASE_DEBUG_ASSERT_MSG(false,
 565                                                 "Unhandled coreref_state: %d",
 566                                                         coreref_state);
 567                 break;
 568         }
 569 }
 570
 571 static void kbase_gpu_release_atom(struct kbase_device *kbdev,
 572                                         struct kbase_jd_atom *katom,
 573                                         ktime_t *end_timestamp)
 574 {
 575         switch (katom->gpu_rb_state) {
 576         case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB:
 577                 /* Should be impossible */
 578                 WARN(1, "Attempting to release atom not in ringbuffer\n");
 579                 break;
 580
 581         case KBASE_ATOM_GPU_RB_SUBMITTED:
 582                 /* Inform power management at start/finish of atom so it can
 583                  * update its GPU utilisation metrics. Mark atom as not
 584                  * submitted beforehand. */
 585                 katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY;
 586                 kbase_pm_metrics_update(kbdev, end_timestamp);
 587
 588                 if (katom->core_req & BASE_JD_REQ_PERMON)
 589                         kbase_pm_release_gpu_cycle_counter(kbdev);
 590                 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
 591
 592         case KBASE_ATOM_GPU_RB_READY:
 593                 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
 594
 595         case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_ENTRY:
 596                 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
 597
 598         case KBASE_ATOM_GPU_RB_WAITING_AFFINITY:
 599                 kbase_js_affinity_release_slot_cores(kbdev, katom->slot_nr,
 600                                                         katom->affinity);
 601                 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
 602
 603         case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE:
 604                 break;
 605
 606         case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_EXIT:
 607                 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
 608
 609         case KBASE_ATOM_GPU_RB_WAITING_BLOCKED:
 610                 /* ***FALLTHROUGH: TRANSITION TO LOWER STATE*** */
 611
 612         case KBASE_ATOM_GPU_RB_RETURN_TO_JS:
 613                 break;
 614         }
 615
 616         katom->gpu_rb_state = KBASE_ATOM_GPU_RB_WAITING_BLOCKED;
 617 }
 618
 619 static void kbase_gpu_mark_atom_for_return(struct kbase_device *kbdev,
 620                                                 struct kbase_jd_atom *katom)
 621 {
 622         kbase_gpu_release_atom(kbdev, katom, NULL);
 623         katom->gpu_rb_state = KBASE_ATOM_GPU_RB_RETURN_TO_JS;
 624 }
 625
 626 static inline bool kbase_gpu_rmu_workaround(struct kbase_device *kbdev, int js)
 627 {
 628         struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
 629         bool slot_busy[3];
 630
 631         if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
 632                 return true;
 633         slot_busy[0] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 0,
 634                                         KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
 635         slot_busy[1] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 1,
 636                                         KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
 637         slot_busy[2] = kbase_gpu_nr_atoms_on_slot_min(kbdev, 2,
 638                                         KBASE_ATOM_GPU_RB_WAITING_AFFINITY);
 639
 640         if ((js == 2 && !(slot_busy[0] || slot_busy[1])) ||
 641                 (js != 2 && !slot_busy[2]))
 642                 return true;
 643
 644         /* Don't submit slot 2 atom while GPU has jobs on slots 0/1 */
 645         if (js == 2 && (kbase_gpu_atoms_submitted(kbdev, 0) ||
 646                         kbase_gpu_atoms_submitted(kbdev, 1) ||
 647                         backend->rmu_workaround_flag))
 648                 return false;
 649
 650         /* Don't submit slot 0/1 atom while GPU has jobs on slot 2 */
 651         if (js != 2 && (kbase_gpu_atoms_submitted(kbdev, 2) ||
 652                         !backend->rmu_workaround_flag))
 653                 return false;
 654
 655         backend->rmu_workaround_flag = !backend->rmu_workaround_flag;
 656
 657         return true;
 658 }
 659
 660 static inline bool kbase_gpu_in_protected_mode(struct kbase_device *kbdev)
 661 {
 662         return kbdev->protected_mode;
 663 }
 664
 665 static int kbase_gpu_protected_mode_enter(struct kbase_device *kbdev)
 666 {
 667         int err = -EINVAL;
 668
 669         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 670
 671         WARN_ONCE(!kbdev->protected_ops,
 672                         "Cannot enter protected mode: protected callbacks not specified.\n");
 673
 674         if (kbdev->protected_ops) {
 675                 /* Switch GPU to protected mode */
 676                 err = kbdev->protected_ops->protected_mode_enter(kbdev);
 677
 678                 if (err)
 679                         dev_warn(kbdev->dev, "Failed to enable protected mode: %d\n",
 680                                         err);
 681                 else
 682                         kbdev->protected_mode = true;
 683         }
 684
 685         return err;
 686 }
 687
 688 static int kbase_gpu_protected_mode_reset(struct kbase_device *kbdev)
 689 {
 690         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 691
 692         WARN_ONCE(!kbdev->protected_ops,
 693                         "Cannot exit protected mode: protected callbacks not specified.\n");
 694
 695         if (!kbdev->protected_ops)
 696                 return -EINVAL;
 697
 698         kbdev->protected_mode_transition = true;
 699         kbase_reset_gpu_silent(kbdev);
 700
 701         return 0;
 702 }
 703
 704 static int kbase_jm_exit_protected_mode(struct kbase_device *kbdev,
 705                 struct kbase_jd_atom **katom, int idx, int js)
 706 {
 707         int err = 0;
 708
 709         switch (katom[idx]->exit_protected_state) {
 710         case KBASE_ATOM_EXIT_PROTECTED_CHECK:
 711                 /*
 712                  * If the atom ahead of this one hasn't got to being
 713                  * submitted yet then bail.
 714                  */
 715                 if (idx == 1 &&
 716                         (katom[0]->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED &&
 717                         katom[0]->gpu_rb_state != KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB))
 718                         return -EAGAIN;
 719
 720                 /* If we're not exiting protected mode then we're done here. */
 721                 if (!(kbase_gpu_in_protected_mode(kbdev) &&
 722                                 !kbase_jd_katom_is_protected(katom[idx])))
 723                         return 0;
 724
 725                 /*
 726                  * If there is a transition in progress, or work still
 727                  * on the GPU try again later.
 728                  */
 729                 if (kbdev->protected_mode_transition ||
 730                                 kbase_gpu_atoms_submitted_any(kbdev))
 731                         return -EAGAIN;
 732
 733                 /*
 734                  * Exiting protected mode requires a reset, but first the L2
 735                  * needs to be powered down to ensure it's not active when the
 736                  * reset is issued.
 737                  */
 738                 katom[idx]->exit_protected_state =
 739                                 KBASE_ATOM_EXIT_PROTECTED_IDLE_L2;
 740
 741                 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 742
 743         case KBASE_ATOM_EXIT_PROTECTED_IDLE_L2:
 744                 if (kbase_pm_get_active_cores(kbdev, KBASE_PM_CORE_L2) ||
 745                                 kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_L2)) {
 746                         /*
 747                          * The L2 is still powered, wait for all the users to
 748                          * finish with it before doing the actual reset.
 749                          */
 750                         return -EAGAIN;
 751                 }
 752                 katom[idx]->exit_protected_state =
 753                                 KBASE_ATOM_EXIT_PROTECTED_RESET;
 754
 755                 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 756
 757         case KBASE_ATOM_EXIT_PROTECTED_RESET:
 758                 /* Issue the reset to the GPU */
 759                 err = kbase_gpu_protected_mode_reset(kbdev);
 760                 if (err) {
 761                         /* Failed to exit protected mode, fail atom */
 762                         katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID;
 763                         kbase_gpu_mark_atom_for_return(kbdev, katom[idx]);
 764                         /* Only return if head atom or previous atom
 765                          * already removed - as atoms must be returned
 766                          * in order */
 767                         if (idx == 0 || katom[0]->gpu_rb_state ==
 768                                         KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
 769                                 kbase_gpu_dequeue_atom(kbdev, js, NULL);
 770                                 kbase_jm_return_atom_to_js(kbdev, katom[idx]);
 771                         }
 772
 773                         kbase_vinstr_resume(kbdev->vinstr_ctx);
 774
 775                         return -EINVAL;
 776                 }
 777
 778                 katom[idx]->exit_protected_state =
 779                                 KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT;
 780
 781                 /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 782
 783         case KBASE_ATOM_EXIT_PROTECTED_RESET_WAIT:
 784                 if (kbase_reset_gpu_active(kbdev))
 785                         return -EAGAIN;
 786
 787                 /* protected mode sanity checks */
 788                 KBASE_DEBUG_ASSERT_MSG(
 789                         kbase_jd_katom_is_protected(katom[idx]) == kbase_gpu_in_protected_mode(kbdev),
 790                         "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)",
 791                         kbase_jd_katom_is_protected(katom[idx]), kbase_gpu_in_protected_mode(kbdev));
 792                 KBASE_DEBUG_ASSERT_MSG(
 793                         (kbase_jd_katom_is_protected(katom[idx]) && js == 0) ||
 794                         !kbase_jd_katom_is_protected(katom[idx]),
 795                         "Protected atom on JS%d not supported", js);
 796         }
 797
 798         return 0;
 799 }
 800
 801 void kbase_gpu_slot_update(struct kbase_device *kbdev)
 802 {
 803         int js;
 804
 805         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 806
 807         for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
 808                 struct kbase_jd_atom *katom[2];
 809                 int idx;
 810
 811                 katom[0] = kbase_gpu_inspect(kbdev, js, 0);
 812                 katom[1] = kbase_gpu_inspect(kbdev, js, 1);
 813                 WARN_ON(katom[1] && !katom[0]);
 814
 815                 for (idx = 0; idx < SLOT_RB_SIZE; idx++) {
 816                         bool cores_ready;
 817                         int ret;
 818
 819                         if (!katom[idx])
 820                                 continue;
 821
 822                         switch (katom[idx]->gpu_rb_state) {
 823                         case KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB:
 824                                 /* Should be impossible */
 825                                 WARN(1, "Attempting to update atom not in ringbuffer\n");
 826                                 break;
 827
 828                         case KBASE_ATOM_GPU_RB_WAITING_BLOCKED:
 829                                 if (katom[idx]->atom_flags &
 830                                                 KBASE_KATOM_FLAG_X_DEP_BLOCKED)
 831                                         break;
 832
 833                                 katom[idx]->gpu_rb_state =
 834                                         KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_EXIT;
 835
 836                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 837
 838                         case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_EXIT:
 839                                 /*
 840                                  * Exiting protected mode must be done before
 841                                  * the references on the cores are taken as
 842                                  * a power down the L2 is required which
 843                                  * can't happen after the references for this
 844                                  * atom are taken.
 845                                  */
 846                                 ret = kbase_jm_exit_protected_mode(kbdev,
 847                                                 katom, idx, js);
 848                                 if (ret)
 849                                         break;
 850
 851                                 katom[idx]->gpu_rb_state =
 852                                         KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE;
 853
 854                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 855
 856                         case KBASE_ATOM_GPU_RB_WAITING_FOR_CORE_AVAILABLE:
 857                                 if (katom[idx]->will_fail_event_code) {
 858                                         kbase_gpu_mark_atom_for_return(kbdev,
 859                                                         katom[idx]);
 860                                         /* Set EVENT_DONE so this atom will be
 861                                            completed, not unpulled. */
 862                                         katom[idx]->event_code =
 863                                                 BASE_JD_EVENT_DONE;
 864                                         /* Only return if head atom or previous
 865                                          * atom already removed - as atoms must
 866                                          * be returned in order. */
 867                                         if (idx == 0 || katom[0]->gpu_rb_state ==
 868                                                         KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
 869                                                 kbase_gpu_dequeue_atom(kbdev, js, NULL);
 870                                                 kbase_jm_return_atom_to_js(kbdev, katom[idx]);
 871                                         }
 872                                         break;
 873                                 }
 874
 875
 876                                 cores_ready =
 877                                         kbasep_js_job_check_ref_cores(kbdev, js,
 878                                                                 katom[idx]);
 879
 880                                 if (katom[idx]->event_code ==
 881                                                 BASE_JD_EVENT_PM_EVENT) {
 882                                         katom[idx]->gpu_rb_state =
 883                                                 KBASE_ATOM_GPU_RB_RETURN_TO_JS;
 884                                         break;
 885                                 }
 886
 887                                 if (!cores_ready)
 888                                         break;
 889
 890                                 kbase_js_affinity_retain_slot_cores(kbdev, js,
 891                                                         katom[idx]->affinity);
 892                                 katom[idx]->gpu_rb_state =
 893                                         KBASE_ATOM_GPU_RB_WAITING_AFFINITY;
 894
 895                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 896
 897                         case KBASE_ATOM_GPU_RB_WAITING_AFFINITY:
 898                                 if (!kbase_gpu_rmu_workaround(kbdev, js))
 899                                         break;
 900
 901                                 katom[idx]->gpu_rb_state =
 902                                         KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_ENTRY;
 903
 904                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 905
 906                         case KBASE_ATOM_GPU_RB_WAITING_PROTECTED_MODE_ENTRY:
 907
 908                                 /* Only submit if head atom or previous atom
 909                                  * already submitted */
 910                                 if (idx == 1 &&
 911                                         (katom[0]->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED &&
 912                                         katom[0]->gpu_rb_state != KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB))
 913                                         break;
 914
 915                                 /*
 916                                  * If the GPU is transitioning protected mode
 917                                  * then bail now and we'll be called when the
 918                                  * new state has settled.
 919                                  */
 920                                 if (kbdev->protected_mode_transition)
 921                                         break;
 922
 923                                 if (!kbase_gpu_in_protected_mode(kbdev) && kbase_jd_katom_is_protected(katom[idx])) {
 924                                         int err = 0;
 925
 926                                         /* Not in correct mode, take action */
 927                                         if (kbase_gpu_atoms_submitted_any(kbdev)) {
 928                                                 /*
 929                                                  * We are not in the correct
 930                                                  * GPU mode for this job, and
 931                                                  * we can't switch now because
 932                                                  * there are jobs already
 933                                                  * running.
 934                                                  */
 935                                                 break;
 936                                         }
 937                                         if (kbase_vinstr_try_suspend(kbdev->vinstr_ctx) < 0) {
 938                                                 /*
 939                                                  * We can't switch now because
 940                                                  * the vinstr core state switch
 941                                                  * is not done yet.
 942                                                  */
 943                                                 break;
 944                                         }
 945                                         /* Once reaching this point GPU must be
 946                                          * switched to protected mode or vinstr
 947                                          * re-enabled. */
 948
 949                                         /* No jobs running, so we can switch GPU mode right now */
 950                                         err = kbase_gpu_protected_mode_enter(kbdev);
 951                                         if (err) {
 952                                                 /*
 953                                                  * Failed to switch into protected mode, resume
 954                                                  * vinstr core and fail atom.
 955                                                  */
 956                                                 kbase_vinstr_resume(kbdev->vinstr_ctx);
 957                                                 katom[idx]->event_code = BASE_JD_EVENT_JOB_INVALID;
 958                                                 kbase_gpu_mark_atom_for_return(kbdev, katom[idx]);
 959                                                 /* Only return if head atom or previous atom
 960                                                  * already removed - as atoms must be returned
 961                                                  * in order */
 962                                                 if (idx == 0 || katom[0]->gpu_rb_state ==
 963                                                                 KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
 964                                                         kbase_gpu_dequeue_atom(kbdev, js, NULL);
 965                                                         kbase_jm_return_atom_to_js(kbdev, katom[idx]);
 966                                                 }
 967                                                 break;
 968                                         }
 969                                 }
 970
 971                                 /* Protected mode sanity checks */
 972                                 KBASE_DEBUG_ASSERT_MSG(
 973                                         kbase_jd_katom_is_protected(katom[idx]) == kbase_gpu_in_protected_mode(kbdev),
 974                                         "Protected mode of atom (%d) doesn't match protected mode of GPU (%d)",
 975                                         kbase_jd_katom_is_protected(katom[idx]), kbase_gpu_in_protected_mode(kbdev));
 976                                 katom[idx]->gpu_rb_state =
 977                                         KBASE_ATOM_GPU_RB_READY;
 978
 979                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
 980
 981                         case KBASE_ATOM_GPU_RB_READY:
 982
 983                                 /* Only submit if head atom or previous atom
 984                                  * already submitted */
 985                                 if (idx == 1 &&
 986                                         (katom[0]->gpu_rb_state !=
 987                                                 KBASE_ATOM_GPU_RB_SUBMITTED &&
 988                                         katom[0]->gpu_rb_state !=
 989                                         KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB))
 990                                         break;
 991
 992                                 /* Check if this job needs the cycle counter
 993                                  * enabled before submission */
 994                                 if (katom[idx]->core_req & BASE_JD_REQ_PERMON)
 995                                         kbase_pm_request_gpu_cycle_counter_l2_is_on(
 996                                                                         kbdev);
 997
 998                                 kbase_job_hw_submit(kbdev, katom[idx], js);
 999                                 katom[idx]->gpu_rb_state =
1000                                                 KBASE_ATOM_GPU_RB_SUBMITTED;
1001
1002                                 /* Inform power management at start/finish of
1003                                  * atom so it can update its GPU utilisation
1004                                  * metrics. */
1005                                 kbase_pm_metrics_update(kbdev,
1006                                                 &katom[idx]->start_timestamp);
1007
1008                         /* ***FALLTHROUGH: TRANSITION TO HIGHER STATE*** */
1009
1010                         case KBASE_ATOM_GPU_RB_SUBMITTED:
1011                                 /* Atom submitted to HW, nothing else to do */
1012                                 break;
1013
1014                         case KBASE_ATOM_GPU_RB_RETURN_TO_JS:
1015                                 /* Only return if head atom or previous atom
1016                                  * already removed - as atoms must be returned
1017                                  * in order */
1018                                 if (idx == 0 || katom[0]->gpu_rb_state ==
1019                                         KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB) {
1020                                         kbase_gpu_dequeue_atom(kbdev, js, NULL);
1021                                         kbase_jm_return_atom_to_js(kbdev,
1022                                                                 katom[idx]);
1023                                 }
1024                                 break;
1025                         }
1026                 }
1027         }
1028
1029         /* Warn if PRLAM-8987 affinity restrictions are violated */
1030         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8987))
1031                 WARN_ON((kbase_gpu_atoms_submitted(kbdev, 0) ||
1032                         kbase_gpu_atoms_submitted(kbdev, 1)) &&
1033                         kbase_gpu_atoms_submitted(kbdev, 2));
1034 }
1035
1036
1037 void kbase_backend_run_atom(struct kbase_device *kbdev,
1038                                 struct kbase_jd_atom *katom)
1039 {
1040         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1041
1042         kbase_gpu_enqueue_atom(kbdev, katom);
1043         kbase_gpu_slot_update(kbdev);
1044 }
1045
1046 bool kbase_gpu_irq_evict(struct kbase_device *kbdev, int js)
1047 {
1048         struct kbase_jd_atom *katom;
1049         struct kbase_jd_atom *next_katom;
1050
1051         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1052
1053         katom = kbase_gpu_inspect(kbdev, js, 0);
1054         next_katom = kbase_gpu_inspect(kbdev, js, 1);
1055
1056         if (next_katom && katom->kctx == next_katom->kctx &&
1057                 next_katom->gpu_rb_state == KBASE_ATOM_GPU_RB_SUBMITTED &&
1058                 (kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO), NULL)
1059                                                                         != 0 ||
1060                 kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI), NULL)
1061                                                                         != 0)) {
1062                 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT),
1063                                 JS_COMMAND_NOP, NULL);
1064                 next_katom->gpu_rb_state = KBASE_ATOM_GPU_RB_READY;
1065                 return true;
1066         }
1067
1068         return false;
1069 }
1070
1071 void kbase_gpu_complete_hw(struct kbase_device *kbdev, int js,
1072                                 u32 completion_code,
1073                                 u64 job_tail,
1074                                 ktime_t *end_timestamp)
1075 {
1076         struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, js, 0);
1077         struct kbase_context *kctx = katom->kctx;
1078
1079         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1080
1081         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_6787) &&
1082                         completion_code != BASE_JD_EVENT_DONE &&
1083                         !(completion_code & BASE_JD_SW_EVENT)) {
1084                 katom->need_cache_flush_cores_retained = katom->affinity;
1085                 kbase_pm_request_cores(kbdev, false, katom->affinity);
1086         } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10676)) {
1087                 if (kbdev->gpu_props.num_core_groups > 1 &&
1088                         !(katom->affinity &
1089                         kbdev->gpu_props.props.coherency_info.group[0].core_mask
1090                                                                         ) &&
1091                         (katom->affinity &
1092                         kbdev->gpu_props.props.coherency_info.group[1].core_mask
1093                                                                         )) {
1094                         dev_info(kbdev->dev, "JD: Flushing cache due to PRLAM-10676\n");
1095                         katom->need_cache_flush_cores_retained =
1096                                                                 katom->affinity;
1097                         kbase_pm_request_cores(kbdev, false,
1098                                                         katom->affinity);
1099                 }
1100         }
1101
1102         katom = kbase_gpu_dequeue_atom(kbdev, js, end_timestamp);
1103         kbase_timeline_job_slot_done(kbdev, katom->kctx, katom, js, 0);
1104         kbase_tlstream_tl_nret_atom_lpu(
1105                         katom,
1106                         &kbdev->gpu_props.props.raw_props.js_features[
1107                                 katom->slot_nr]);
1108         kbase_tlstream_tl_nret_atom_as(katom, &kbdev->as[kctx->as_nr]);
1109         kbase_tlstream_tl_nret_ctx_lpu(
1110                         kctx,
1111                         &kbdev->gpu_props.props.raw_props.js_features[
1112                                 katom->slot_nr]);
1113
1114         if (completion_code == BASE_JD_EVENT_STOPPED) {
1115                 struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js,
1116                                                                         0);
1117
1118                 /*
1119                  * Dequeue next atom from ringbuffers on same slot if required.
1120                  * This atom will already have been removed from the NEXT
1121                  * registers by kbase_gpu_soft_hard_stop_slot(), to ensure that
1122                  * the atoms on this slot are returned in the correct order.
1123                  */
1124                 if (next_katom && katom->kctx == next_katom->kctx) {
1125                         kbase_gpu_dequeue_atom(kbdev, js, end_timestamp);
1126                         kbase_jm_return_atom_to_js(kbdev, next_katom);
1127                 }
1128         } else if (completion_code != BASE_JD_EVENT_DONE) {
1129                 struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1130                 int i;
1131
1132 #if KBASE_TRACE_DUMP_ON_JOB_SLOT_ERROR != 0
1133                 KBASE_TRACE_DUMP(kbdev);
1134 #endif
1135                 kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
1136
1137                 /*
1138                  * Remove all atoms on the same context from ringbuffers. This
1139                  * will not remove atoms that are already on the GPU, as these
1140                  * are guaranteed not to have fail dependencies on the failed
1141                  * atom.
1142                  */
1143                 for (i = 0; i < kbdev->gpu_props.num_job_slots; i++) {
1144                         struct kbase_jd_atom *katom_idx0 =
1145                                                 kbase_gpu_inspect(kbdev, i, 0);
1146                         struct kbase_jd_atom *katom_idx1 =
1147                                                 kbase_gpu_inspect(kbdev, i, 1);
1148
1149                         if (katom_idx0 && katom_idx0->kctx == katom->kctx &&
1150                                 katom_idx0->gpu_rb_state !=
1151                                 KBASE_ATOM_GPU_RB_SUBMITTED) {
1152                                 /* Dequeue katom_idx0 from ringbuffer */
1153                                 kbase_gpu_dequeue_atom(kbdev, i, end_timestamp);
1154
1155                                 if (katom_idx1 &&
1156                                         katom_idx1->kctx == katom->kctx &&
1157                                         katom_idx0->gpu_rb_state !=
1158                                                 KBASE_ATOM_GPU_RB_SUBMITTED) {
1159                                         /* Dequeue katom_idx1 from ringbuffer */
1160                                         kbase_gpu_dequeue_atom(kbdev, i,
1161                                                         end_timestamp);
1162
1163                                         katom_idx1->event_code =
1164                                                         BASE_JD_EVENT_STOPPED;
1165                                         kbase_jm_return_atom_to_js(kbdev,
1166                                                                 katom_idx1);
1167                                 }
1168                                 katom_idx0->event_code = BASE_JD_EVENT_STOPPED;
1169                                 kbase_jm_return_atom_to_js(kbdev, katom_idx0);
1170
1171                         } else if (katom_idx1 &&
1172                                         katom_idx1->kctx == katom->kctx &&
1173                                         katom_idx1->gpu_rb_state !=
1174                                                 KBASE_ATOM_GPU_RB_SUBMITTED) {
1175                                 /* Can not dequeue this atom yet - will be
1176                                  * dequeued when atom at idx0 completes */
1177                                 katom_idx1->event_code = BASE_JD_EVENT_STOPPED;
1178                                 kbase_gpu_mark_atom_for_return(kbdev,
1179                                                                 katom_idx1);
1180                         }
1181                 }
1182         }
1183
1184         KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_JOB_DONE, kctx, katom, katom->jc,
1185                                         js, completion_code);
1186
1187         if (job_tail != 0 && job_tail != katom->jc) {
1188                 bool was_updated = (job_tail != katom->jc);
1189
1190                 /* Some of the job has been executed, so we update the job chain
1191                  * address to where we should resume from */
1192                 katom->jc = job_tail;
1193                 if (was_updated)
1194                         KBASE_TRACE_ADD_SLOT(kbdev, JM_UPDATE_HEAD, katom->kctx,
1195                                                 katom, job_tail, js);
1196         }
1197
1198         /* Only update the event code for jobs that weren't cancelled */
1199         if (katom->event_code != BASE_JD_EVENT_JOB_CANCELLED)
1200                 katom->event_code = (base_jd_event_code)completion_code;
1201
1202         kbase_device_trace_register_access(kctx, REG_WRITE,
1203                                                 JOB_CONTROL_REG(JOB_IRQ_CLEAR),
1204                                                 1 << js);
1205
1206         /* Complete the job, and start new ones
1207          *
1208          * Also defer remaining work onto the workqueue:
1209          * - Re-queue Soft-stopped jobs
1210          * - For any other jobs, queue the job back into the dependency system
1211          * - Schedule out the parent context if necessary, and schedule a new
1212          *   one in.
1213          */
1214 #ifdef CONFIG_GPU_TRACEPOINTS
1215         {
1216                 /* The atom in the HEAD */
1217                 struct kbase_jd_atom *next_katom = kbase_gpu_inspect(kbdev, js,
1218                                                                         0);
1219
1220                 if (next_katom && next_katom->gpu_rb_state ==
1221                                                 KBASE_ATOM_GPU_RB_SUBMITTED) {
1222                         char js_string[16];
1223
1224                         trace_gpu_sched_switch(kbasep_make_job_slot_string(js,
1225                                                                 js_string),
1226                                                 ktime_to_ns(*end_timestamp),
1227                                                 (u32)next_katom->kctx->id, 0,
1228                                                 next_katom->work_id);
1229                         kbdev->hwaccess.backend.slot_rb[js].last_context =
1230                                                         next_katom->kctx;
1231                 } else {
1232                         char js_string[16];
1233
1234                         trace_gpu_sched_switch(kbasep_make_job_slot_string(js,
1235                                                                 js_string),
1236                                                 ktime_to_ns(ktime_get()), 0, 0,
1237                                                 0);
1238                         kbdev->hwaccess.backend.slot_rb[js].last_context = 0;
1239                 }
1240         }
1241 #endif
1242
1243         if (completion_code == BASE_JD_EVENT_STOPPED)
1244                 kbase_jm_return_atom_to_js(kbdev, katom);
1245         else
1246                 kbase_jm_complete(kbdev, katom, end_timestamp);
1247
1248         /* Job completion may have unblocked other atoms. Try to update all job
1249          * slots */
1250         kbase_gpu_slot_update(kbdev);
1251 }
1252
1253 void kbase_backend_reset(struct kbase_device *kbdev, ktime_t *end_timestamp)
1254 {
1255         int js;
1256
1257         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1258
1259         for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
1260                 int idx;
1261
1262                 for (idx = 0; idx < 2; idx++) {
1263                         struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev,
1264                                                                         js, 0);
1265                         bool keep_in_jm_rb = false;
1266
1267                         if (!katom)
1268                                 continue;
1269
1270                         if (katom->gpu_rb_state < KBASE_ATOM_GPU_RB_SUBMITTED)
1271                                 keep_in_jm_rb = true;
1272
1273                         kbase_gpu_release_atom(kbdev, katom, NULL);
1274
1275                         /*
1276                          * If the atom wasn't on HW when the reset was issued
1277                          * then leave it in the RB and next time we're kicked
1278                          * it will be processed again from the starting state.
1279                          */
1280                         if (keep_in_jm_rb) {
1281                                 katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
1282                                 katom->exit_protected_state = KBASE_ATOM_EXIT_PROTECTED_CHECK;
1283                                 continue;
1284                         }
1285
1286                         /*
1287                          * The atom was on the HW when the reset was issued
1288                          * all we can do is fail the atom.
1289                          */
1290                         kbase_gpu_dequeue_atom(kbdev, js, NULL);
1291                         katom->event_code = BASE_JD_EVENT_JOB_CANCELLED;
1292                         kbase_jm_complete(kbdev, katom, end_timestamp);
1293                 }
1294         }
1295 }
1296
1297 static inline void kbase_gpu_stop_atom(struct kbase_device *kbdev,
1298                                         int js,
1299                                         struct kbase_jd_atom *katom,
1300                                         u32 action)
1301 {
1302         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1303         u32 hw_action = action & JS_COMMAND_MASK;
1304
1305         kbase_job_check_enter_disjoint(kbdev, action, katom->core_req, katom);
1306         kbasep_job_slot_soft_or_hard_stop_do_action(kbdev, js, hw_action,
1307                                                         katom->core_req, katom);
1308         kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
1309 }
1310
1311 static inline void kbase_gpu_remove_atom(struct kbase_device *kbdev,
1312                                                 struct kbase_jd_atom *katom,
1313                                                 u32 action,
1314                                                 bool disjoint)
1315 {
1316         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1317
1318         katom->event_code = BASE_JD_EVENT_REMOVED_FROM_NEXT;
1319         kbase_gpu_mark_atom_for_return(kbdev, katom);
1320         kbasep_js_clear_submit_allowed(js_devdata, katom->kctx);
1321
1322         if (disjoint)
1323                 kbase_job_check_enter_disjoint(kbdev, action, katom->core_req,
1324                                                                         katom);
1325 }
1326
1327 static int should_stop_x_dep_slot(struct kbase_jd_atom *katom)
1328 {
1329         if (katom->x_post_dep) {
1330                 struct kbase_jd_atom *dep_atom = katom->x_post_dep;
1331
1332                 if (dep_atom->gpu_rb_state !=
1333                                         KBASE_ATOM_GPU_RB_NOT_IN_SLOT_RB &&
1334                         dep_atom->gpu_rb_state !=
1335                                         KBASE_ATOM_GPU_RB_RETURN_TO_JS)
1336                         return dep_atom->slot_nr;
1337         }
1338         return -1;
1339 }
1340
1341 static void kbase_job_evicted(struct kbase_jd_atom *katom)
1342 {
1343         kbase_timeline_job_slot_done(katom->kctx->kbdev, katom->kctx, katom,
1344                         katom->slot_nr, KBASE_JS_ATOM_DONE_EVICTED_FROM_NEXT);
1345 }
1346
1347 bool kbase_backend_soft_hard_stop_slot(struct kbase_device *kbdev,
1348                                         struct kbase_context *kctx,
1349                                         int js,
1350                                         struct kbase_jd_atom *katom,
1351                                         u32 action)
1352 {
1353         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
1354
1355         struct kbase_jd_atom *katom_idx0;
1356         struct kbase_jd_atom *katom_idx1;
1357
1358         bool katom_idx0_valid, katom_idx1_valid;
1359
1360         bool ret = false;
1361
1362         int stop_x_dep_idx0 = -1, stop_x_dep_idx1 = -1;
1363
1364         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
1365
1366         katom_idx0 = kbase_gpu_inspect(kbdev, js, 0);
1367         katom_idx1 = kbase_gpu_inspect(kbdev, js, 1);
1368
1369         if (katom) {
1370                 katom_idx0_valid = (katom_idx0 == katom);
1371                 /* If idx0 is to be removed and idx1 is on the same context,
1372                  * then idx1 must also be removed otherwise the atoms might be
1373                  * returned out of order */
1374                 if (katom_idx1)
1375                         katom_idx1_valid = (katom_idx1 == katom) ||
1376                                                 (katom_idx0_valid &&
1377                                                         (katom_idx0->kctx ==
1378                                                         katom_idx1->kctx));
1379                 else
1380                         katom_idx1_valid = false;
1381         } else {
1382                 katom_idx0_valid = (katom_idx0 &&
1383                                         (!kctx || katom_idx0->kctx == kctx));
1384                 katom_idx1_valid = (katom_idx1 &&
1385                                         (!kctx || katom_idx1->kctx == kctx));
1386         }
1387
1388         if (katom_idx0_valid)
1389                 stop_x_dep_idx0 = should_stop_x_dep_slot(katom_idx0);
1390         if (katom_idx1_valid)
1391                 stop_x_dep_idx1 = should_stop_x_dep_slot(katom_idx1);
1392
1393         if (katom_idx0_valid) {
1394                 if (katom_idx0->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) {
1395                         /* Simple case - just dequeue and return */
1396                         kbase_gpu_dequeue_atom(kbdev, js, NULL);
1397                         if (katom_idx1_valid) {
1398                                 kbase_gpu_dequeue_atom(kbdev, js, NULL);
1399                                 katom_idx1->event_code =
1400                                                 BASE_JD_EVENT_REMOVED_FROM_NEXT;
1401                                 kbase_jm_return_atom_to_js(kbdev, katom_idx1);
1402                                 kbasep_js_clear_submit_allowed(js_devdata,
1403                                                         katom_idx1->kctx);
1404                         }
1405
1406                         katom_idx0->event_code =
1407                                                 BASE_JD_EVENT_REMOVED_FROM_NEXT;
1408                         kbase_jm_return_atom_to_js(kbdev, katom_idx0);
1409                         kbasep_js_clear_submit_allowed(js_devdata,
1410                                                         katom_idx0->kctx);
1411                 } else {
1412                         /* katom_idx0 is on GPU */
1413                         if (katom_idx1 && katom_idx1->gpu_rb_state ==
1414                                                 KBASE_ATOM_GPU_RB_SUBMITTED) {
1415                                 /* katom_idx0 and katom_idx1 are on GPU */
1416
1417                                 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1418                                                 JS_COMMAND_NEXT), NULL) == 0) {
1419                                         /* idx0 has already completed - stop
1420                                          * idx1 if needed*/
1421                                         if (katom_idx1_valid) {
1422                                                 kbase_gpu_stop_atom(kbdev, js,
1423                                                                 katom_idx1,
1424                                                                 action);
1425                                                 ret = true;
1426                                         }
1427                                 } else {
1428                                         /* idx1 is in NEXT registers - attempt
1429                                          * to remove */
1430                                         kbase_reg_write(kbdev,
1431                                                         JOB_SLOT_REG(js,
1432                                                         JS_COMMAND_NEXT),
1433                                                         JS_COMMAND_NOP, NULL);
1434
1435                                         if (kbase_reg_read(kbdev,
1436                                                         JOB_SLOT_REG(js,
1437                                                         JS_HEAD_NEXT_LO), NULL)
1438                                                                         != 0 ||
1439                                                 kbase_reg_read(kbdev,
1440                                                         JOB_SLOT_REG(js,
1441                                                         JS_HEAD_NEXT_HI), NULL)
1442                                                                         != 0) {
1443                                                 /* idx1 removed successfully,
1444                                                  * will be handled in IRQ */
1445                                                 kbase_job_evicted(katom_idx1);
1446                                                 kbase_gpu_remove_atom(kbdev,
1447                                                                 katom_idx1,
1448                                                                 action, true);
1449                                                 stop_x_dep_idx1 =
1450                                         should_stop_x_dep_slot(katom_idx1);
1451
1452                                                 /* stop idx0 if still on GPU */
1453                                                 kbase_gpu_stop_atom(kbdev, js,
1454                                                                 katom_idx0,
1455                                                                 action);
1456                                                 ret = true;
1457                                         } else if (katom_idx1_valid) {
1458                                                 /* idx0 has already completed,
1459                                                  * stop idx1 if needed */
1460                                                 kbase_gpu_stop_atom(kbdev, js,
1461                                                                 katom_idx1,
1462                                                                 action);
1463                                                 ret = true;
1464                                         }
1465                                 }
1466                         } else if (katom_idx1_valid) {
1467                                 /* idx1 not on GPU but must be dequeued*/
1468
1469                                 /* idx1 will be handled in IRQ */
1470                                 kbase_gpu_remove_atom(kbdev, katom_idx1, action,
1471                                                                         false);
1472                                 /* stop idx0 */
1473                                 /* This will be repeated for anything removed
1474                                  * from the next registers, since their normal
1475                                  * flow was also interrupted, and this function
1476                                  * might not enter disjoint state e.g. if we
1477                                  * don't actually do a hard stop on the head
1478                                  * atom */
1479                                 kbase_gpu_stop_atom(kbdev, js, katom_idx0,
1480                                                                         action);
1481                                 ret = true;
1482                         } else {
1483                                 /* no atom in idx1 */
1484                                 /* just stop idx0 */
1485                                 kbase_gpu_stop_atom(kbdev, js, katom_idx0,
1486                                                                         action);
1487                                 ret = true;
1488                         }
1489                 }
1490         } else if (katom_idx1_valid) {
1491                 if (katom_idx1->gpu_rb_state != KBASE_ATOM_GPU_RB_SUBMITTED) {
1492                         /* Mark for return */
1493                         /* idx1 will be returned once idx0 completes */
1494                         kbase_gpu_remove_atom(kbdev, katom_idx1, action,
1495                                                                         false);
1496                 } else {
1497                         /* idx1 is on GPU */
1498                         if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1499                                                 JS_COMMAND_NEXT), NULL) == 0) {
1500                                 /* idx0 has already completed - stop idx1 */
1501                                 kbase_gpu_stop_atom(kbdev, js, katom_idx1,
1502                                                                         action);
1503                                 ret = true;
1504                         } else {
1505                                 /* idx1 is in NEXT registers - attempt to
1506                                  * remove */
1507                                 kbase_reg_write(kbdev, JOB_SLOT_REG(js,
1508                                                         JS_COMMAND_NEXT),
1509                                                         JS_COMMAND_NOP, NULL);
1510
1511                                 if (kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1512                                                 JS_HEAD_NEXT_LO), NULL) != 0 ||
1513                                     kbase_reg_read(kbdev, JOB_SLOT_REG(js,
1514                                                 JS_HEAD_NEXT_HI), NULL) != 0) {
1515                                         /* idx1 removed successfully, will be
1516                                          * handled in IRQ once idx0 completes */
1517                                         kbase_job_evicted(katom_idx1);
1518                                         kbase_gpu_remove_atom(kbdev, katom_idx1,
1519                                                                         action,
1520                                                                         false);
1521                                 } else {
1522                                         /* idx0 has already completed - stop
1523                                          * idx1 */
1524                                         kbase_gpu_stop_atom(kbdev, js,
1525                                                                 katom_idx1,
1526                                                                 action);
1527                                         ret = true;
1528                                 }
1529                         }
1530                 }
1531         }
1532
1533
1534         if (stop_x_dep_idx0 != -1)
1535                 kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx0,
1536                                                                 NULL, action);
1537
1538         if (stop_x_dep_idx1 != -1)
1539                 kbase_backend_soft_hard_stop_slot(kbdev, kctx, stop_x_dep_idx1,
1540                                                                 NULL, action);
1541
1542         return ret;
1543 }
1544
1545 void kbase_gpu_cacheclean(struct kbase_device *kbdev,
1546                                         struct kbase_jd_atom *katom)
1547 {
1548         /* Limit the number of loops to avoid a hang if the interrupt is missed
1549          */
1550         u32 max_loops = KBASE_CLEAN_CACHE_MAX_LOOPS;
1551
1552         mutex_lock(&kbdev->cacheclean_lock);
1553
1554         /* use GPU_COMMAND completion solution */
1555         /* clean & invalidate the caches */
1556         KBASE_TRACE_ADD(kbdev, CORE_GPU_CLEAN_INV_CACHES, NULL, NULL, 0u, 0);
1557         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1558                                         GPU_COMMAND_CLEAN_INV_CACHES, NULL);
1559
1560         /* wait for cache flush to complete before continuing */
1561         while (--max_loops &&
1562                 (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) &
1563                                                 CLEAN_CACHES_COMPLETED) == 0)
1564                 ;
1565
1566         /* clear the CLEAN_CACHES_COMPLETED irq */
1567         KBASE_TRACE_ADD(kbdev, CORE_GPU_IRQ_CLEAR, NULL, NULL, 0u,
1568                                                         CLEAN_CACHES_COMPLETED);
1569         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR),
1570                                                 CLEAN_CACHES_COMPLETED, NULL);
1571         KBASE_DEBUG_ASSERT_MSG(kbdev->hwcnt.backend.state !=
1572                                                 KBASE_INSTR_STATE_CLEANING,
1573             "Instrumentation code was cleaning caches, but Job Management code cleared their IRQ - Instrumentation code will now hang.");
1574
1575         mutex_unlock(&kbdev->cacheclean_lock);
1576
1577         kbase_pm_unrequest_cores(kbdev, false,
1578                                         katom->need_cache_flush_cores_retained);
1579 }
1580
1581 void kbase_backend_complete_wq(struct kbase_device *kbdev,
1582                                                 struct kbase_jd_atom *katom)
1583 {
1584         /*
1585          * If cache flush required due to HW workaround then perform the flush
1586          * now
1587          */
1588         if (katom->need_cache_flush_cores_retained) {
1589                 kbase_gpu_cacheclean(kbdev, katom);
1590                 katom->need_cache_flush_cores_retained = 0;
1591         }
1592
1593         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10969)            &&
1594             (katom->core_req & BASE_JD_REQ_FS)                        &&
1595             katom->event_code == BASE_JD_EVENT_TILE_RANGE_FAULT       &&
1596             (katom->atom_flags & KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED) &&
1597             !(katom->atom_flags & KBASE_KATOM_FLAGS_RERUN)) {
1598                 dev_dbg(kbdev->dev, "Soft-stopped fragment shader job got a TILE_RANGE_FAULT. Possible HW issue, trying SW workaround\n");
1599                 if (kbasep_10969_workaround_clamp_coordinates(katom)) {
1600                         /* The job had a TILE_RANGE_FAULT after was soft-stopped
1601                          * Due to an HW issue we try to execute the job again.
1602                          */
1603                         dev_dbg(kbdev->dev,
1604                                 "Clamping has been executed, try to rerun the job\n"
1605                         );
1606                         katom->event_code = BASE_JD_EVENT_STOPPED;
1607                         katom->atom_flags |= KBASE_KATOM_FLAGS_RERUN;
1608                 }
1609         }
1610
1611         /* Clear the coreref_state now - while check_deref_cores() may not have
1612          * been called yet, the caller will have taken a copy of this field. If
1613          * this is not done, then if the atom is re-scheduled (following a soft
1614          * stop) then the core reference would not be retaken. */
1615         katom->coreref_state = KBASE_ATOM_COREREF_STATE_NO_CORES_REQUESTED;
1616         katom->affinity = 0;
1617 }
1618
1619 void kbase_backend_complete_wq_post_sched(struct kbase_device *kbdev,
1620                 base_jd_core_req core_req, u64 affinity,
1621                 enum kbase_atom_coreref_state coreref_state)
1622 {
1623         kbasep_js_job_check_deref_cores_nokatom(kbdev, core_req, affinity,
1624                         coreref_state);
1625
1626         if (!kbdev->pm.active_count) {
1627                 mutex_lock(&kbdev->js_data.runpool_mutex);
1628                 mutex_lock(&kbdev->pm.lock);
1629                 kbase_pm_update_active(kbdev);
1630                 mutex_unlock(&kbdev->pm.lock);
1631                 mutex_unlock(&kbdev->js_data.runpool_mutex);
1632         }
1633 }
1634
1635 void kbase_gpu_dump_slots(struct kbase_device *kbdev)
1636 {
1637         struct kbasep_js_device_data *js_devdata;
1638         unsigned long flags;
1639         int js;
1640
1641         js_devdata = &kbdev->js_data;
1642
1643         spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1644
1645         dev_info(kbdev->dev, "kbase_gpu_dump_slots:\n");
1646
1647         for (js = 0; js < kbdev->gpu_props.num_job_slots; js++) {
1648                 int idx;
1649
1650                 for (idx = 0; idx < SLOT_RB_SIZE; idx++) {
1651                         struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev,
1652                                                                         js,
1653                                                                         idx);
1654
1655                         if (katom)
1656                                 dev_info(kbdev->dev,
1657                                 "  js%d idx%d : katom=%p gpu_rb_state=%d\n",
1658                                 js, idx, katom, katom->gpu_rb_state);
1659                         else
1660                                 dev_info(kbdev->dev, "  js%d idx%d : empty\n",
1661                                                                 js, idx);
1662                 }
1663         }
1664
1665         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1666 }
1667
1668
1669