drivers/gpu/arm/midgard_for_linux/backend/gpu/mali_kbase_jm_hw.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18
  19
  20 /*
  21  * Base kernel job manager APIs
  22  */
  23
  24 #include <mali_kbase.h>
  25 #include <mali_kbase_config.h>
  26 #include <mali_midg_regmap.h>
  27 #if defined(CONFIG_MALI_GATOR_SUPPORT)
  28 #include <mali_kbase_gator.h>
  29 #endif
  30 #if defined(CONFIG_MALI_MIPE_ENABLED)
  31 #include <mali_kbase_tlstream.h>
  32 #endif
  33 #include <mali_kbase_hw.h>
  34 #include <mali_kbase_config_defaults.h>
  35 #include <mali_kbase_hwaccess_jm.h>
  36 #include <backend/gpu/mali_kbase_device_internal.h>
  37 #include <backend/gpu/mali_kbase_irq_internal.h>
  38 #include <backend/gpu/mali_kbase_js_affinity.h>
  39 #include <backend/gpu/mali_kbase_jm_internal.h>
  40
  41 #define beenthere(kctx, f, a...) \
  42                         dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a)
  43
  44 #if KBASE_GPU_RESET_EN
  45 static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev);
  46 static void kbasep_reset_timeout_worker(struct work_struct *data);
  47 static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer);
  48 #endif /* KBASE_GPU_RESET_EN */
  49
  50 static inline int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js,
  51                                                 struct kbase_context *kctx)
  52 {
  53         return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), kctx);
  54 }
  55
  56 void kbase_job_hw_submit(struct kbase_device *kbdev,
  57                                 struct kbase_jd_atom *katom,
  58                                 int js)
  59 {
  60         struct kbase_context *kctx;
  61         u32 cfg;
  62         u64 jc_head = katom->jc;
  63
  64         KBASE_DEBUG_ASSERT(kbdev);
  65         KBASE_DEBUG_ASSERT(katom);
  66
  67         kctx = katom->kctx;
  68
  69         /* Command register must be available */
  70         KBASE_DEBUG_ASSERT(kbasep_jm_is_js_free(kbdev, js, kctx));
  71         /* Affinity is not violating */
  72         kbase_js_debug_log_current_affinities(kbdev);
  73         KBASE_DEBUG_ASSERT(!kbase_js_affinity_would_violate(kbdev, js,
  74                                                         katom->affinity));
  75
  76         kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO),
  77                                                 jc_head & 0xFFFFFFFF, kctx);
  78         kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI),
  79                                                 jc_head >> 32, kctx);
  80
  81         kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_LO),
  82                                         katom->affinity & 0xFFFFFFFF, kctx);
  83         kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_HI),
  84                                         katom->affinity >> 32, kctx);
  85
  86         /* start MMU, medium priority, cache clean/flush on end, clean/flush on
  87          * start */
  88         cfg = kctx->as_nr;
  89
  90 #ifndef CONFIG_MALI_COH_GPU
  91         cfg |= JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE;
  92         cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE;
  93 #endif
  94
  95         cfg |= JS_CONFIG_START_MMU;
  96         cfg |= JS_CONFIG_THREAD_PRI(8);
  97
  98         if (kbase_hw_has_feature(kbdev,
  99                                 BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
 100                 if (!kbdev->hwaccess.backend.slot_rb[js].job_chain_flag) {
 101                         cfg |= JS_CONFIG_JOB_CHAIN_FLAG;
 102                         katom->atom_flags |= KBASE_KATOM_FLAGS_JOBCHAIN;
 103                         kbdev->hwaccess.backend.slot_rb[js].job_chain_flag =
 104                                                                 true;
 105                 } else {
 106                         katom->atom_flags &= ~KBASE_KATOM_FLAGS_JOBCHAIN;
 107                         kbdev->hwaccess.backend.slot_rb[js].job_chain_flag =
 108                                                                 false;
 109                 }
 110         }
 111
 112         kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_CONFIG_NEXT), cfg, kctx);
 113
 114
 115         /* Write an approximate start timestamp.
 116          * It's approximate because there might be a job in the HEAD register.
 117          * In such cases, we'll try to make a better approximation in the IRQ
 118          * handler (up to the KBASE_JS_IRQ_THROTTLE_TIME_US). */
 119         katom->start_timestamp = ktime_get();
 120
 121         /* GO ! */
 122         dev_dbg(kbdev->dev, "JS: Submitting atom %p from ctx %p to js[%d] with head=0x%llx, affinity=0x%llx",
 123                                 katom, kctx, js, jc_head, katom->affinity);
 124
 125         KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js,
 126                                                         (u32) katom->affinity);
 127
 128 #if defined(CONFIG_MALI_GATOR_SUPPORT)
 129         kbase_trace_mali_job_slots_event(
 130                                 GATOR_MAKE_EVENT(GATOR_JOB_SLOT_START, js),
 131                                 kctx, kbase_jd_atom_id(kctx, katom));
 132 #endif
 133 #if defined(CONFIG_MALI_MIPE_ENABLED)
 134         kbase_tlstream_tl_attrib_atom_config(katom, jc_head,
 135                         katom->affinity, cfg);
 136         kbase_tlstream_tl_ret_ctx_lpu(
 137                 kctx,
 138                 &kbdev->gpu_props.props.raw_props.js_features[
 139                         katom->slot_nr]);
 140         kbase_tlstream_tl_ret_atom_as(katom, &kbdev->as[kctx->as_nr]);
 141         kbase_tlstream_tl_ret_atom_lpu(
 142                         katom,
 143                         &kbdev->gpu_props.props.raw_props.js_features[js],
 144                         "ctx_nr,atom_nr");
 145 #endif
 146 #ifdef CONFIG_GPU_TRACEPOINTS
 147         if (!kbase_backend_nr_atoms_submitted(kbdev, js)) {
 148                 /* If this is the only job on the slot, trace it as starting */
 149                 char js_string[16];
 150
 151                 trace_gpu_sched_switch(
 152                                 kbasep_make_job_slot_string(js, js_string),
 153                                 ktime_to_ns(katom->start_timestamp),
 154                                 (u32)katom->kctx->id, 0, katom->work_id);
 155                 kbdev->hwaccess.backend.slot_rb[js].last_context = katom->kctx;
 156         }
 157 #endif
 158         kbase_timeline_job_slot_submit(kbdev, kctx, katom, js);
 159
 160         kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT),
 161                                                 JS_COMMAND_START, katom->kctx);
 162 }
 163
 164 /**
 165  * kbasep_job_slot_update_head_start_timestamp - Update timestamp
 166  * @kbdev: kbase device
 167  * @js: job slot
 168  * @end_timestamp: timestamp
 169  *
 170  * Update the start_timestamp of the job currently in the HEAD, based on the
 171  * fact that we got an IRQ for the previous set of completed jobs.
 172  *
 173  * The estimate also takes into account the %KBASE_JS_IRQ_THROTTLE_TIME_US and
 174  * the time the job was submitted, to work out the best estimate (which might
 175  * still result in an over-estimate to the calculated time spent)
 176  */
 177 static void kbasep_job_slot_update_head_start_timestamp(
 178                                                 struct kbase_device *kbdev,
 179                                                 int js,
 180                                                 ktime_t end_timestamp)
 181 {
 182         if (kbase_backend_nr_atoms_on_slot(kbdev, js) > 0) {
 183                 struct kbase_jd_atom *katom;
 184                 ktime_t new_timestamp;
 185                 ktime_t timestamp_diff;
 186                 /* The atom in the HEAD */
 187                 katom = kbase_gpu_inspect(kbdev, js, 0);
 188
 189                 KBASE_DEBUG_ASSERT(katom != NULL);
 190
 191                 /* Account for any IRQ Throttle time - makes an overestimate of
 192                  * the time spent by the job */
 193                 new_timestamp = ktime_sub_ns(end_timestamp,
 194                                         KBASE_JS_IRQ_THROTTLE_TIME_US * 1000);
 195                 timestamp_diff = ktime_sub(new_timestamp,
 196                                                         katom->start_timestamp);
 197                 if (ktime_to_ns(timestamp_diff) >= 0) {
 198                         /* Only update the timestamp if it's a better estimate
 199                          * than what's currently stored. This is because our
 200                          * estimate that accounts for the throttle time may be
 201                          * too much of an overestimate */
 202                         katom->start_timestamp = new_timestamp;
 203                 }
 204         }
 205 }
 206
 207 void kbase_job_done(struct kbase_device *kbdev, u32 done)
 208 {
 209         unsigned long flags;
 210         int i;
 211         u32 count = 0;
 212         ktime_t end_timestamp = ktime_get();
 213         struct kbasep_js_device_data *js_devdata;
 214
 215         KBASE_DEBUG_ASSERT(kbdev);
 216         js_devdata = &kbdev->js_data;
 217
 218         KBASE_TRACE_ADD(kbdev, JM_IRQ, NULL, NULL, 0, done);
 219
 220         memset(&kbdev->slot_submit_count_irq[0], 0,
 221                                         sizeof(kbdev->slot_submit_count_irq));
 222
 223         /* write irq throttle register, this will prevent irqs from occurring
 224          * until the given number of gpu clock cycles have passed */
 225         {
 226                 int irq_throttle_cycles =
 227                                 atomic_read(&kbdev->irq_throttle_cycles);
 228
 229                 kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_THROTTLE),
 230                                                 irq_throttle_cycles, NULL);
 231         }
 232
 233         spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
 234
 235         while (done) {
 236                 u32 failed = done >> 16;
 237
 238                 /* treat failed slots as finished slots */
 239                 u32 finished = (done & 0xFFFF) | failed;
 240
 241                 /* Note: This is inherently unfair, as we always check
 242                  * for lower numbered interrupts before the higher
 243                  * numbered ones.*/
 244                 i = ffs(finished) - 1;
 245                 KBASE_DEBUG_ASSERT(i >= 0);
 246
 247                 do {
 248                         int nr_done;
 249                         u32 active;
 250                         u32 completion_code = BASE_JD_EVENT_DONE;/* assume OK */
 251                         u64 job_tail = 0;
 252
 253                         if (failed & (1u << i)) {
 254                                 /* read out the job slot status code if the job
 255                                  * slot reported failure */
 256                                 completion_code = kbase_reg_read(kbdev,
 257                                         JOB_SLOT_REG(i, JS_STATUS), NULL);
 258
 259                                 switch (completion_code) {
 260                                 case BASE_JD_EVENT_STOPPED:
 261 #if defined(CONFIG_MALI_GATOR_SUPPORT)
 262                                         kbase_trace_mali_job_slots_event(
 263                                                 GATOR_MAKE_EVENT(
 264                                                 GATOR_JOB_SLOT_SOFT_STOPPED, i),
 265                                                                 NULL, 0);
 266 #endif
 267 #if defined(CONFIG_MALI_MIPE_ENABLED)
 268                                         kbase_tlstream_aux_job_softstop(i);
 269 #endif
 270                                         /* Soft-stopped job - read the value of
 271                                          * JS<n>_TAIL so that the job chain can
 272                                          * be resumed */
 273                                         job_tail = (u64)kbase_reg_read(kbdev,
 274                                                 JOB_SLOT_REG(i, JS_TAIL_LO),
 275                                                                         NULL) |
 276                                                 ((u64)kbase_reg_read(kbdev,
 277                                                 JOB_SLOT_REG(i, JS_TAIL_HI),
 278                                                                 NULL) << 32);
 279                                         break;
 280                                 case BASE_JD_EVENT_NOT_STARTED:
 281                                         /* PRLAM-10673 can cause a TERMINATED
 282                                          * job to come back as NOT_STARTED, but
 283                                          * the error interrupt helps us detect
 284                                          * it */
 285                                         completion_code =
 286                                                 BASE_JD_EVENT_TERMINATED;
 287                                         /* fall through */
 288                                 default:
 289                                         dev_warn(kbdev->dev, "error detected from slot %d, job status 0x%08x (%s)",
 290                                                         i, completion_code,
 291                                                         kbase_exception_name
 292                                                         (kbdev,
 293                                                         completion_code));
 294                                 }
 295
 296                                 kbase_gpu_irq_evict(kbdev, i);
 297                         }
 298
 299                         kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR),
 300                                         done & ((1 << i) | (1 << (i + 16))),
 301                                         NULL);
 302                         active = kbase_reg_read(kbdev,
 303                                         JOB_CONTROL_REG(JOB_IRQ_JS_STATE),
 304                                         NULL);
 305
 306                         if (((active >> i) & 1) == 0 &&
 307                                         (((done >> (i + 16)) & 1) == 0)) {
 308                                 /* There is a potential race we must work
 309                                  * around:
 310                                  *
 311                                  *  1. A job slot has a job in both current and
 312                                  *     next registers
 313                                  *  2. The job in current completes
 314                                  *     successfully, the IRQ handler reads
 315                                  *     RAWSTAT and calls this function with the
 316                                  *     relevant bit set in "done"
 317                                  *  3. The job in the next registers becomes the
 318                                  *     current job on the GPU
 319                                  *  4. Sometime before the JOB_IRQ_CLEAR line
 320                                  *     above the job on the GPU _fails_
 321                                  *  5. The IRQ_CLEAR clears the done bit but not
 322                                  *     the failed bit. This atomically sets
 323                                  *     JOB_IRQ_JS_STATE. However since both jobs
 324                                  *     have now completed the relevant bits for
 325                                  *     the slot are set to 0.
 326                                  *
 327                                  * If we now did nothing then we'd incorrectly
 328                                  * assume that _both_ jobs had completed
 329                                  * successfully (since we haven't yet observed
 330                                  * the fail bit being set in RAWSTAT).
 331                                  *
 332                                  * So at this point if there are no active jobs
 333                                  * left we check to see if RAWSTAT has a failure
 334                                  * bit set for the job slot. If it does we know
 335                                  * that there has been a new failure that we
 336                                  * didn't previously know about, so we make sure
 337                                  * that we record this in active (but we wait
 338                                  * for the next loop to deal with it).
 339                                  *
 340                                  * If we were handling a job failure (i.e. done
 341                                  * has the relevant high bit set) then we know
 342                                  * that the value read back from
 343                                  * JOB_IRQ_JS_STATE is the correct number of
 344                                  * remaining jobs because the failed job will
 345                                  * have prevented any futher jobs from starting
 346                                  * execution.
 347                                  */
 348                                 u32 rawstat = kbase_reg_read(kbdev,
 349                                         JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL);
 350
 351                                 if ((rawstat >> (i + 16)) & 1) {
 352                                         /* There is a failed job that we've
 353                                          * missed - add it back to active */
 354                                         active |= (1u << i);
 355                                 }
 356                         }
 357
 358                         dev_dbg(kbdev->dev, "Job ended with status 0x%08X\n",
 359                                                         completion_code);
 360
 361                         nr_done = kbase_backend_nr_atoms_submitted(kbdev, i);
 362                         nr_done -= (active >> i) & 1;
 363                         nr_done -= (active >> (i + 16)) & 1;
 364
 365                         if (nr_done <= 0) {
 366                                 dev_warn(kbdev->dev, "Spurious interrupt on slot %d",
 367                                                                         i);
 368
 369                                 goto spurious;
 370                         }
 371
 372                         count += nr_done;
 373
 374                         while (nr_done) {
 375                                 if (nr_done == 1) {
 376                                         kbase_gpu_complete_hw(kbdev, i,
 377                                                                 completion_code,
 378                                                                 job_tail,
 379                                                                 &end_timestamp);
 380                                         kbase_jm_try_kick_all(kbdev);
 381                                 } else {
 382                                         /* More than one job has completed.
 383                                          * Since this is not the last job being
 384                                          * reported this time it must have
 385                                          * passed. This is because the hardware
 386                                          * will not allow further jobs in a job
 387                                          * slot to complete until the failed job
 388                                          * is cleared from the IRQ status.
 389                                          */
 390                                         kbase_gpu_complete_hw(kbdev, i,
 391                                                         BASE_JD_EVENT_DONE,
 392                                                         0,
 393                                                         &end_timestamp);
 394                                 }
 395                                 nr_done--;
 396                         }
 397  spurious:
 398                         done = kbase_reg_read(kbdev,
 399                                         JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL);
 400
 401                         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10883)) {
 402                                 /* Workaround for missing interrupt caused by
 403                                  * PRLAM-10883 */
 404                                 if (((active >> i) & 1) && (0 ==
 405                                                 kbase_reg_read(kbdev,
 406                                                         JOB_SLOT_REG(i,
 407                                                         JS_STATUS), NULL))) {
 408                                         /* Force job slot to be processed again
 409                                          */
 410                                         done |= (1u << i);
 411                                 }
 412                         }
 413
 414                         failed = done >> 16;
 415                         finished = (done & 0xFFFF) | failed;
 416                         if (done)
 417                                 end_timestamp = ktime_get();
 418                 } while (finished & (1 << i));
 419
 420                 kbasep_job_slot_update_head_start_timestamp(kbdev, i,
 421                                                                 end_timestamp);
 422         }
 423
 424         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
 425 #if KBASE_GPU_RESET_EN
 426         if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
 427                                                 KBASE_RESET_GPU_COMMITTED) {
 428                 /* If we're trying to reset the GPU then we might be able to do
 429                  * it early (without waiting for a timeout) because some jobs
 430                  * have completed
 431                  */
 432                 kbasep_try_reset_gpu_early(kbdev);
 433         }
 434 #endif /* KBASE_GPU_RESET_EN */
 435         KBASE_TRACE_ADD(kbdev, JM_IRQ_END, NULL, NULL, 0, count);
 436 }
 437 KBASE_EXPORT_TEST_API(kbase_job_done);
 438
 439 static bool kbasep_soft_stop_allowed(struct kbase_device *kbdev,
 440                                                                 u16 core_reqs)
 441 {
 442         bool soft_stops_allowed = true;
 443
 444         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) {
 445                 if ((core_reqs & BASE_JD_REQ_T) != 0)
 446                         soft_stops_allowed = false;
 447         }
 448         return soft_stops_allowed;
 449 }
 450
 451 static bool kbasep_hard_stop_allowed(struct kbase_device *kbdev,
 452                                                                 u16 core_reqs)
 453 {
 454         bool hard_stops_allowed = true;
 455
 456         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8394)) {
 457                 if ((core_reqs & BASE_JD_REQ_T) != 0)
 458                         hard_stops_allowed = false;
 459         }
 460         return hard_stops_allowed;
 461 }
 462
 463 void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev,
 464                                         int js,
 465                                         u32 action,
 466                                         u16 core_reqs,
 467                                         struct kbase_jd_atom *target_katom)
 468 {
 469         struct kbase_context *kctx = target_katom->kctx;
 470 #if KBASE_TRACE_ENABLE
 471         u32 status_reg_before;
 472         u64 job_in_head_before;
 473         u32 status_reg_after;
 474
 475         KBASE_DEBUG_ASSERT(!(action & (~JS_COMMAND_MASK)));
 476
 477         /* Check the head pointer */
 478         job_in_head_before = ((u64) kbase_reg_read(kbdev,
 479                                         JOB_SLOT_REG(js, JS_HEAD_LO), NULL))
 480                         | (((u64) kbase_reg_read(kbdev,
 481                                         JOB_SLOT_REG(js, JS_HEAD_HI), NULL))
 482                                                                         << 32);
 483         status_reg_before = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS),
 484                                                                         NULL);
 485 #endif
 486
 487         if (action == JS_COMMAND_SOFT_STOP) {
 488                 bool soft_stop_allowed = kbasep_soft_stop_allowed(kbdev,
 489                                                                 core_reqs);
 490
 491                 if (!soft_stop_allowed) {
 492 #ifdef CONFIG_MALI_DEBUG
 493                         dev_dbg(kbdev->dev, "Attempt made to soft-stop a job that cannot be soft-stopped. core_reqs = 0x%X",
 494                                                 (unsigned int)core_reqs);
 495 #endif                          /* CONFIG_MALI_DEBUG */
 496                         return;
 497                 }
 498
 499                 /* We are about to issue a soft stop, so mark the atom as having
 500                  * been soft stopped */
 501                 target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED;
 502         }
 503
 504         if (action == JS_COMMAND_HARD_STOP) {
 505                 bool hard_stop_allowed = kbasep_hard_stop_allowed(kbdev,
 506                                                                 core_reqs);
 507
 508                 if (!hard_stop_allowed) {
 509                         /* Jobs can be hard-stopped for the following reasons:
 510                          *  * CFS decides the job has been running too long (and
 511                          *    soft-stop has not occurred). In this case the GPU
 512                          *    will be reset by CFS if the job remains on the
 513                          *    GPU.
 514                          *
 515                          *  * The context is destroyed, kbase_jd_zap_context
 516                          *    will attempt to hard-stop the job. However it also
 517                          *    has a watchdog which will cause the GPU to be
 518                          *    reset if the job remains on the GPU.
 519                          *
 520                          *  * An (unhandled) MMU fault occurred. As long as
 521                          *    BASE_HW_ISSUE_8245 is defined then the GPU will be
 522                          *    reset.
 523                          *
 524                          * All three cases result in the GPU being reset if the
 525                          * hard-stop fails, so it is safe to just return and
 526                          * ignore the hard-stop request.
 527                          */
 528                         dev_warn(kbdev->dev, "Attempt made to hard-stop a job that cannot be hard-stopped. core_reqs = 0x%X",
 529                                                 (unsigned int)core_reqs);
 530                         return;
 531                 }
 532                 target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_HARD_STOPPED;
 533         }
 534
 535         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316) &&
 536                                         action == JS_COMMAND_SOFT_STOP) {
 537                 int i;
 538
 539                 for (i = 0; i < kbase_backend_nr_atoms_submitted(kbdev, js);
 540                                                                         i++) {
 541                         struct kbase_jd_atom *katom;
 542
 543                         katom = kbase_gpu_inspect(kbdev, js, i);
 544
 545                         KBASE_DEBUG_ASSERT(katom);
 546
 547                         /* For HW_ISSUE_8316, only 'bad' jobs attacking the
 548                          * system can cause this issue: normally, all memory
 549                          * should be allocated in multiples of 4 pages, and
 550                          * growable memory should be changed size in multiples
 551                          * of 4 pages.
 552                          *
 553                          * Whilst such 'bad' jobs can be cleared by a GPU reset,
 554                          * the locking up of a uTLB entry caused by the bad job
 555                          * could also stall other ASs, meaning that other ASs'
 556                          * jobs don't complete in the 'grace' period before the
 557                          * reset. We don't want to lose other ASs' jobs when
 558                          * they would normally complete fine, so we must 'poke'
 559                          * the MMU regularly to help other ASs complete */
 560                         kbase_as_poking_timer_retain_atom(kbdev, katom->kctx,
 561                                                                         katom);
 562                 }
 563         }
 564
 565         if (kbase_hw_has_feature(kbdev,
 566                                 BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
 567                 if (action == JS_COMMAND_SOFT_STOP)
 568                         action = (target_katom->atom_flags &
 569                                                 KBASE_KATOM_FLAGS_JOBCHAIN) ?
 570                                         JS_COMMAND_SOFT_STOP_1 :
 571                                         JS_COMMAND_SOFT_STOP_0;
 572                 else
 573                         action = (target_katom->atom_flags &
 574                                                 KBASE_KATOM_FLAGS_JOBCHAIN) ?
 575                                         JS_COMMAND_HARD_STOP_1 :
 576                                         JS_COMMAND_HARD_STOP_0;
 577         }
 578
 579         kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND), action, kctx);
 580
 581 #if KBASE_TRACE_ENABLE
 582         status_reg_after = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS),
 583                                                                         NULL);
 584         if (status_reg_after == BASE_JD_EVENT_ACTIVE) {
 585                 struct kbase_jd_atom *head;
 586                 struct kbase_context *head_kctx;
 587
 588                 head = kbase_gpu_inspect(kbdev, js, 0);
 589                 head_kctx = head->kctx;
 590
 591                 if (status_reg_before == BASE_JD_EVENT_ACTIVE)
 592                         KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, head_kctx,
 593                                                 head, job_in_head_before, js);
 594                 else
 595                         KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL,
 596                                                 0, js);
 597
 598                 switch (action) {
 599                 case JS_COMMAND_SOFT_STOP:
 600                         KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, head_kctx,
 601                                                         head, head->jc, js);
 602                         break;
 603                 case JS_COMMAND_SOFT_STOP_0:
 604                         KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx,
 605                                                         head, head->jc, js);
 606                         break;
 607                 case JS_COMMAND_SOFT_STOP_1:
 608                         KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx,
 609                                                         head, head->jc, js);
 610                         break;
 611                 case JS_COMMAND_HARD_STOP:
 612                         KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, head_kctx,
 613                                                         head, head->jc, js);
 614                         break;
 615                 case JS_COMMAND_HARD_STOP_0:
 616                         KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, head_kctx,
 617                                                         head, head->jc, js);
 618                         break;
 619                 case JS_COMMAND_HARD_STOP_1:
 620                         KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, head_kctx,
 621                                                         head, head->jc, js);
 622                         break;
 623                 default:
 624                         BUG();
 625                         break;
 626                 }
 627         } else {
 628                 if (status_reg_before == BASE_JD_EVENT_ACTIVE)
 629                         KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL,
 630                                                         job_in_head_before, js);
 631                 else
 632                         KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL,
 633                                                         0, js);
 634
 635                 switch (action) {
 636                 case JS_COMMAND_SOFT_STOP:
 637                         KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0,
 638                                                         js);
 639                         break;
 640                 case JS_COMMAND_SOFT_STOP_0:
 641                         KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL,
 642                                                         0, js);
 643                         break;
 644                 case JS_COMMAND_SOFT_STOP_1:
 645                         KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL,
 646                                                         0, js);
 647                         break;
 648                 case JS_COMMAND_HARD_STOP:
 649                         KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0,
 650                                                         js);
 651                         break;
 652                 case JS_COMMAND_HARD_STOP_0:
 653                         KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL,
 654                                                         0, js);
 655                         break;
 656                 case JS_COMMAND_HARD_STOP_1:
 657                         KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL,
 658                                                         0, js);
 659                         break;
 660                 default:
 661                         BUG();
 662                         break;
 663                 }
 664         }
 665 #endif
 666 }
 667
 668 void kbase_backend_jm_kill_jobs_from_kctx(struct kbase_context *kctx)
 669 {
 670         unsigned long flags;
 671         struct kbase_device *kbdev;
 672         struct kbasep_js_device_data *js_devdata;
 673         int i;
 674
 675         KBASE_DEBUG_ASSERT(kctx != NULL);
 676         kbdev = kctx->kbdev;
 677         KBASE_DEBUG_ASSERT(kbdev != NULL);
 678         js_devdata = &kbdev->js_data;
 679
 680         /* Cancel any remaining running jobs for this kctx  */
 681         mutex_lock(&kctx->jctx.lock);
 682         spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
 683
 684         /* Invalidate all jobs in context, to prevent re-submitting */
 685         for (i = 0; i < BASE_JD_ATOM_COUNT; i++) {
 686                 if (!work_pending(&kctx->jctx.atoms[i].work))
 687                         kctx->jctx.atoms[i].event_code =
 688                                                 BASE_JD_EVENT_JOB_CANCELLED;
 689         }
 690
 691         for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
 692                 kbase_job_slot_hardstop(kctx, i, NULL);
 693
 694         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
 695         mutex_unlock(&kctx->jctx.lock);
 696 }
 697
 698 void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx,
 699                                 struct kbase_jd_atom *target_katom)
 700 {
 701         struct kbase_device *kbdev;
 702         struct kbasep_js_device_data *js_devdata;
 703         int js = target_katom->slot_nr;
 704         int priority = target_katom->sched_priority;
 705         int i;
 706
 707         KBASE_DEBUG_ASSERT(kctx != NULL);
 708         kbdev = kctx->kbdev;
 709         KBASE_DEBUG_ASSERT(kbdev != NULL);
 710         js_devdata = &kbdev->js_data;
 711
 712         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 713
 714         for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) {
 715                 struct kbase_jd_atom *katom;
 716
 717                 katom = kbase_gpu_inspect(kbdev, js, i);
 718                 if (!katom)
 719                         continue;
 720
 721                 if (katom->kctx != kctx)
 722                         continue;
 723
 724                 if (katom->sched_priority > priority)
 725                         kbase_job_slot_softstop(kbdev, js, katom);
 726         }
 727 }
 728
 729 struct zap_reset_data {
 730         /* The stages are:
 731          * 1. The timer has never been called
 732          * 2. The zap has timed out, all slots are soft-stopped - the GPU reset
 733          *    will happen. The GPU has been reset when
 734          *    kbdev->hwaccess.backend.reset_waitq is signalled
 735          *
 736          * (-1 - The timer has been cancelled)
 737          */
 738         int stage;
 739         struct kbase_device *kbdev;
 740         struct hrtimer timer;
 741         spinlock_t lock; /* protects updates to stage member */
 742 };
 743
 744 static enum hrtimer_restart zap_timeout_callback(struct hrtimer *timer)
 745 {
 746         struct zap_reset_data *reset_data = container_of(timer,
 747                                                 struct zap_reset_data, timer);
 748         struct kbase_device *kbdev = reset_data->kbdev;
 749         unsigned long flags;
 750
 751         spin_lock_irqsave(&reset_data->lock, flags);
 752
 753         if (reset_data->stage == -1)
 754                 goto out;
 755
 756 #if KBASE_GPU_RESET_EN
 757         if (kbase_prepare_to_reset_gpu(kbdev)) {
 758                 dev_err(kbdev->dev, "Issueing GPU soft-reset because jobs failed to be killed (within %d ms) as part of context termination (e.g. process exit)\n",
 759                                                                 ZAP_TIMEOUT);
 760                 kbase_reset_gpu(kbdev);
 761         }
 762 #endif /* KBASE_GPU_RESET_EN */
 763         reset_data->stage = 2;
 764
 765  out:
 766         spin_unlock_irqrestore(&reset_data->lock, flags);
 767
 768         return HRTIMER_NORESTART;
 769 }
 770
 771 void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx)
 772 {
 773         struct kbase_device *kbdev = kctx->kbdev;
 774         struct zap_reset_data reset_data;
 775         unsigned long flags;
 776
 777         hrtimer_init_on_stack(&reset_data.timer, CLOCK_MONOTONIC,
 778                                                         HRTIMER_MODE_REL);
 779         reset_data.timer.function = zap_timeout_callback;
 780
 781         spin_lock_init(&reset_data.lock);
 782
 783         reset_data.kbdev = kbdev;
 784         reset_data.stage = 1;
 785
 786         hrtimer_start(&reset_data.timer, HR_TIMER_DELAY_MSEC(ZAP_TIMEOUT),
 787                                                         HRTIMER_MODE_REL);
 788
 789         /* Wait for all jobs to finish, and for the context to be not-scheduled
 790          * (due to kbase_job_zap_context(), we also guarentee it's not in the JS
 791          * policy queue either */
 792         wait_event(kctx->jctx.zero_jobs_wait, kctx->jctx.job_nr == 0);
 793         wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait,
 794                         kctx->jctx.sched_info.ctx.is_scheduled == false);
 795
 796         spin_lock_irqsave(&reset_data.lock, flags);
 797         if (reset_data.stage == 1) {
 798                 /* The timer hasn't run yet - so cancel it */
 799                 reset_data.stage = -1;
 800         }
 801         spin_unlock_irqrestore(&reset_data.lock, flags);
 802
 803         hrtimer_cancel(&reset_data.timer);
 804
 805         if (reset_data.stage == 2) {
 806                 /* The reset has already started.
 807                  * Wait for the reset to complete
 808                  */
 809                 wait_event(kbdev->hwaccess.backend.reset_wait,
 810                                 atomic_read(&kbdev->hwaccess.backend.reset_gpu)
 811                                                 == KBASE_RESET_GPU_NOT_PENDING);
 812         }
 813         destroy_hrtimer_on_stack(&reset_data.timer);
 814
 815         dev_dbg(kbdev->dev, "Zap: Finished Context %p", kctx);
 816
 817         /* Ensure that the signallers of the waitqs have finished */
 818         mutex_lock(&kctx->jctx.lock);
 819         mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
 820         mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
 821         mutex_unlock(&kctx->jctx.lock);
 822 }
 823
 824 u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev)
 825 {
 826         u32 flush_id = 0;
 827
 828
 829         return flush_id;
 830 }
 831
 832 int kbase_job_slot_init(struct kbase_device *kbdev)
 833 {
 834 #if KBASE_GPU_RESET_EN
 835         kbdev->hwaccess.backend.reset_workq = alloc_workqueue(
 836                                                 "Mali reset workqueue", 0, 1);
 837         if (NULL == kbdev->hwaccess.backend.reset_workq)
 838                 return -EINVAL;
 839
 840         KBASE_DEBUG_ASSERT(0 ==
 841                 object_is_on_stack(&kbdev->hwaccess.backend.reset_work));
 842         INIT_WORK(&kbdev->hwaccess.backend.reset_work,
 843                                                 kbasep_reset_timeout_worker);
 844
 845         hrtimer_init(&kbdev->hwaccess.backend.reset_timer, CLOCK_MONOTONIC,
 846                                                         HRTIMER_MODE_REL);
 847         kbdev->hwaccess.backend.reset_timer.function =
 848                                                 kbasep_reset_timer_callback;
 849 #endif
 850
 851         return 0;
 852 }
 853 KBASE_EXPORT_TEST_API(kbase_job_slot_init);
 854
 855 void kbase_job_slot_halt(struct kbase_device *kbdev)
 856 {
 857         CSTD_UNUSED(kbdev);
 858 }
 859
 860 void kbase_job_slot_term(struct kbase_device *kbdev)
 861 {
 862 #if KBASE_GPU_RESET_EN
 863         destroy_workqueue(kbdev->hwaccess.backend.reset_workq);
 864 #endif
 865 }
 866 KBASE_EXPORT_TEST_API(kbase_job_slot_term);
 867
 868 #if KBASE_GPU_RESET_EN
 869 /**
 870  * kbasep_check_for_afbc_on_slot() - Check whether AFBC is in use on this slot
 871  * @kbdev: kbase device pointer
 872  * @kctx:  context to check against
 873  * @js:    slot to check
 874  * @target_katom: An atom to check, or NULL if all atoms from @kctx on
 875  *                slot @js should be checked
 876  *
 877  * This checks are based upon parameters that would normally be passed to
 878  * kbase_job_slot_hardstop().
 879  *
 880  * In the event of @target_katom being NULL, this will check the last jobs that
 881  * are likely to be running on the slot to see if a) they belong to kctx, and
 882  * so would be stopped, and b) whether they have AFBC
 883  *
 884  * In that case, It's guaranteed that a job currently executing on the HW with
 885  * AFBC will be detected. However, this is a conservative check because it also
 886  * detects jobs that have just completed too.
 887  *
 888  * Return: true when hard-stop _might_ stop an afbc atom, else false.
 889  */
 890 static bool kbasep_check_for_afbc_on_slot(struct kbase_device *kbdev,
 891                 struct kbase_context *kctx, int js,
 892                 struct kbase_jd_atom *target_katom)
 893 {
 894         bool ret = false;
 895         int i;
 896
 897         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 898
 899         /* When we have an atom the decision can be made straight away. */
 900         if (target_katom)
 901                 return !!(target_katom->core_req & BASE_JD_REQ_FS_AFBC);
 902
 903         /* Otherwise, we must chweck the hardware to see if it has atoms from
 904          * this context with AFBC. */
 905         for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) {
 906                 struct kbase_jd_atom *katom;
 907
 908                 katom = kbase_gpu_inspect(kbdev, js, i);
 909                 if (!katom)
 910                         continue;
 911
 912                 /* Ignore atoms from other contexts, they won't be stopped when
 913                  * we use this for checking if we should hard-stop them */
 914                 if (katom->kctx != kctx)
 915                         continue;
 916
 917                 /* An atom on this slot and this context: check for AFBC */
 918                 if (katom->core_req & BASE_JD_REQ_FS_AFBC) {
 919                         ret = true;
 920                         break;
 921                 }
 922         }
 923
 924         return ret;
 925 }
 926 #endif /* KBASE_GPU_RESET_EN */
 927
 928 /**
 929  * kbase_job_slot_softstop_swflags - Soft-stop a job with flags
 930  * @kbdev:         The kbase device
 931  * @js:            The job slot to soft-stop
 932  * @target_katom:  The job that should be soft-stopped (or NULL for any job)
 933  * @sw_flags:      Flags to pass in about the soft-stop
 934  *
 935  * Context:
 936  *   The job slot lock must be held when calling this function.
 937  *   The job slot must not already be in the process of being soft-stopped.
 938  *
 939  * Soft-stop the specified job slot, with extra information about the stop
 940  *
 941  * Where possible any job in the next register is evicted before the soft-stop.
 942  */
 943 void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js,
 944                         struct kbase_jd_atom *target_katom, u32 sw_flags)
 945 {
 946         KBASE_DEBUG_ASSERT(!(sw_flags & JS_COMMAND_MASK));
 947         kbase_backend_soft_hard_stop_slot(kbdev, NULL, js, target_katom,
 948                         JS_COMMAND_SOFT_STOP | sw_flags);
 949 }
 950
 951 /**
 952  * kbase_job_slot_softstop - Soft-stop the specified job slot
 953  * @kbdev:         The kbase device
 954  * @js:            The job slot to soft-stop
 955  * @target_katom:  The job that should be soft-stopped (or NULL for any job)
 956  * Context:
 957  *   The job slot lock must be held when calling this function.
 958  *   The job slot must not already be in the process of being soft-stopped.
 959  *
 960  * Where possible any job in the next register is evicted before the soft-stop.
 961  */
 962 void kbase_job_slot_softstop(struct kbase_device *kbdev, int js,
 963                                 struct kbase_jd_atom *target_katom)
 964 {
 965         kbase_job_slot_softstop_swflags(kbdev, js, target_katom, 0u);
 966 }
 967
 968 /**
 969  * kbase_job_slot_hardstop - Hard-stop the specified job slot
 970  * @kctx:         The kbase context that contains the job(s) that should
 971  *                be hard-stopped
 972  * @js:           The job slot to hard-stop
 973  * @target_katom: The job that should be hard-stopped (or NULL for all
 974  *                jobs from the context)
 975  * Context:
 976  *   The job slot lock must be held when calling this function.
 977  */
 978 void kbase_job_slot_hardstop(struct kbase_context *kctx, int js,
 979                                 struct kbase_jd_atom *target_katom)
 980 {
 981         struct kbase_device *kbdev = kctx->kbdev;
 982         bool stopped;
 983 #if KBASE_GPU_RESET_EN
 984         /* We make the check for AFBC before evicting/stopping atoms.  Note
 985          * that no other thread can modify the slots whilst we have the
 986          * runpool_irq lock. */
 987         int needs_workaround_for_afbc =
 988                         kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3542)
 989                         && kbasep_check_for_afbc_on_slot(kbdev, kctx, js,
 990                                          target_katom);
 991 #endif
 992
 993         stopped = kbase_backend_soft_hard_stop_slot(kbdev, kctx, js,
 994                                                         target_katom,
 995                                                         JS_COMMAND_HARD_STOP);
 996 #if KBASE_GPU_RESET_EN
 997         if (stopped && (kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_8401) ||
 998                         kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_9510) ||
 999                         needs_workaround_for_afbc)) {
1000                 /* MIDBASE-2916 if a fragment job with AFBC encoding is
1001                  * hardstopped, ensure to do a soft reset also in order to
1002                  * clear the GPU status.
1003                  * Workaround for HW issue 8401 has an issue,so after
1004                  * hard-stopping just reset the GPU. This will ensure that the
1005                  * jobs leave the GPU.*/
1006                 if (kbase_prepare_to_reset_gpu_locked(kbdev)) {
1007                         dev_err(kbdev->dev, "Issueing GPU soft-reset after hard stopping due to hardware issue");
1008                         kbase_reset_gpu_locked(kbdev);
1009                 }
1010         }
1011 #endif
1012 }
1013
1014 /**
1015  * kbase_job_check_enter_disjoint - potentiall enter disjoint mode
1016  * @kbdev: kbase device
1017  * @action: the event which has occurred
1018  * @core_reqs: core requirements of the atom
1019  * @target_katom: the atom which is being affected
1020  *
1021  * For a certain soft/hard-stop action, work out whether to enter disjoint
1022  * state.
1023  *
1024  * This does not register multiple disjoint events if the atom has already
1025  * started a disjoint period
1026  *
1027  * @core_reqs can be supplied as 0 if the atom had not started on the hardware
1028  * (and so a 'real' soft/hard-stop was not required, but it still interrupted
1029  * flow, perhaps on another context)
1030  *
1031  * kbase_job_check_leave_disjoint() should be used to end the disjoint
1032  * state when the soft/hard-stop action is complete
1033  */
1034 void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action,
1035                 u16 core_reqs, struct kbase_jd_atom *target_katom)
1036 {
1037         u32 hw_action = action & JS_COMMAND_MASK;
1038
1039         /* For hard-stop, don't enter if hard-stop not allowed */
1040         if (hw_action == JS_COMMAND_HARD_STOP &&
1041                         !kbasep_hard_stop_allowed(kbdev, core_reqs))
1042                 return;
1043
1044         /* For soft-stop, don't enter if soft-stop not allowed, or isn't
1045          * causing disjoint */
1046         if (hw_action == JS_COMMAND_SOFT_STOP &&
1047                         !(kbasep_soft_stop_allowed(kbdev, core_reqs) &&
1048                           (action & JS_COMMAND_SW_CAUSES_DISJOINT)))
1049                 return;
1050
1051         /* Nothing to do if already logged disjoint state on this atom */
1052         if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT)
1053                 return;
1054
1055         target_katom->atom_flags |= KBASE_KATOM_FLAG_IN_DISJOINT;
1056         kbase_disjoint_state_up(kbdev);
1057 }
1058
1059 /**
1060  * kbase_job_check_enter_disjoint - potentially leave disjoint state
1061  * @kbdev: kbase device
1062  * @target_katom: atom which is finishing
1063  *
1064  * Work out whether to leave disjoint state when finishing an atom that was
1065  * originated by kbase_job_check_enter_disjoint().
1066  */
1067 void kbase_job_check_leave_disjoint(struct kbase_device *kbdev,
1068                 struct kbase_jd_atom *target_katom)
1069 {
1070         if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) {
1071                 target_katom->atom_flags &= ~KBASE_KATOM_FLAG_IN_DISJOINT;
1072                 kbase_disjoint_state_down(kbdev);
1073         }
1074 }
1075
1076
1077 #if KBASE_GPU_RESET_EN
1078 static void kbase_debug_dump_registers(struct kbase_device *kbdev)
1079 {
1080         int i;
1081
1082         dev_err(kbdev->dev, "Register state:");
1083         dev_err(kbdev->dev, "  GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x",
1084                 kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL),
1085                 kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS), NULL));
1086         dev_err(kbdev->dev, "  JOB_IRQ_RAWSTAT=0x%08x JOB_IRQ_JS_STATE=0x%08x JOB_IRQ_THROTTLE=0x%08x",
1087                 kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL),
1088                 kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_JS_STATE), NULL),
1089                 kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_THROTTLE), NULL));
1090         for (i = 0; i < 3; i++) {
1091                 dev_err(kbdev->dev, "  JS%d_STATUS=0x%08x      JS%d_HEAD_LO=0x%08x",
1092                         i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_STATUS),
1093                                         NULL),
1094                         i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_HEAD_LO),
1095                                         NULL));
1096         }
1097         dev_err(kbdev->dev, "  MMU_IRQ_RAWSTAT=0x%08x GPU_FAULTSTATUS=0x%08x",
1098                 kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_RAWSTAT), NULL),
1099                 kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS), NULL));
1100         dev_err(kbdev->dev, "  GPU_IRQ_MASK=0x%08x    JOB_IRQ_MASK=0x%08x     MMU_IRQ_MASK=0x%08x",
1101                 kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL),
1102                 kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), NULL),
1103                 kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL));
1104         dev_err(kbdev->dev, "  PWR_OVERRIDE0=0x%08x   PWR_OVERRIDE1=0x%08x",
1105                 kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE0), NULL),
1106                 kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE1), NULL));
1107         dev_err(kbdev->dev, "  SHADER_CONFIG=0x%08x   L2_MMU_CONFIG=0x%08x",
1108                 kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), NULL),
1109                 kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), NULL));
1110 }
1111
1112 static void kbasep_save_hwcnt_setup(struct kbase_device *kbdev,
1113                                 struct kbase_context *kctx,
1114                                 struct kbase_uk_hwcnt_setup *hwcnt_setup)
1115 {
1116         hwcnt_setup->dump_buffer =
1117                 kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO), kctx) &
1118                                                                 0xffffffff;
1119         hwcnt_setup->dump_buffer |= (u64)
1120                 kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI), kctx) <<
1121                                                                         32;
1122         hwcnt_setup->jm_bm =
1123                 kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN), kctx);
1124         hwcnt_setup->shader_bm =
1125                 kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN), kctx);
1126         hwcnt_setup->tiler_bm =
1127                 kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN), kctx);
1128         hwcnt_setup->mmu_l2_bm =
1129                 kbase_reg_read(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN), kctx);
1130 }
1131
1132 static void kbasep_reset_timeout_worker(struct work_struct *data)
1133 {
1134         unsigned long flags, mmu_flags;
1135         struct kbase_device *kbdev;
1136         int i;
1137         ktime_t end_timestamp = ktime_get();
1138         struct kbasep_js_device_data *js_devdata;
1139         struct kbase_uk_hwcnt_setup hwcnt_setup = { {0} };
1140         enum kbase_instr_state bckp_state;
1141         bool try_schedule = false;
1142         bool restore_hwc = false;
1143
1144         KBASE_DEBUG_ASSERT(data);
1145
1146         kbdev = container_of(data, struct kbase_device,
1147                                                 hwaccess.backend.reset_work);
1148
1149         KBASE_DEBUG_ASSERT(kbdev);
1150         js_devdata = &kbdev->js_data;
1151
1152         KBASE_TRACE_ADD(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0);
1153
1154         /* Make sure the timer has completed - this cannot be done from
1155          * interrupt context, so this cannot be done within
1156          * kbasep_try_reset_gpu_early. */
1157         hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer);
1158
1159         if (kbase_pm_context_active_handle_suspend(kbdev,
1160                                 KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
1161                 /* This would re-activate the GPU. Since it's already idle,
1162                  * there's no need to reset it */
1163                 atomic_set(&kbdev->hwaccess.backend.reset_gpu,
1164                                                 KBASE_RESET_GPU_NOT_PENDING);
1165                 kbase_disjoint_state_down(kbdev);
1166                 wake_up(&kbdev->hwaccess.backend.reset_wait);
1167                 return;
1168         }
1169
1170         KBASE_DEBUG_ASSERT(kbdev->irq_reset_flush == false);
1171
1172         spin_lock_irqsave(&kbdev->mmu_mask_change, mmu_flags);
1173         /* We're about to flush out the IRQs and their bottom half's */
1174         kbdev->irq_reset_flush = true;
1175
1176         /* Disable IRQ to avoid IRQ handlers to kick in after releasing the
1177          * spinlock; this also clears any outstanding interrupts */
1178         spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
1179         kbase_pm_disable_interrupts(kbdev);
1180         spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
1181
1182         spin_unlock_irqrestore(&kbdev->mmu_mask_change, mmu_flags);
1183
1184         /* Ensure that any IRQ handlers have finished
1185          * Must be done without any locks IRQ handlers will take */
1186         kbase_synchronize_irqs(kbdev);
1187
1188         /* Flush out any in-flight work items */
1189         kbase_flush_mmu_wqs(kbdev);
1190
1191         /* The flush has completed so reset the active indicator */
1192         kbdev->irq_reset_flush = false;
1193
1194         mutex_lock(&kbdev->pm.lock);
1195         /* We hold the pm lock, so there ought to be a current policy */
1196         KBASE_DEBUG_ASSERT(kbdev->pm.backend.pm_current_policy);
1197
1198         /* All slot have been soft-stopped and we've waited
1199          * SOFT_STOP_RESET_TIMEOUT for the slots to clear, at this point we
1200          * assume that anything that is still left on the GPU is stuck there and
1201          * we'll kill it when we reset the GPU */
1202
1203         dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)",
1204                                                                 RESET_TIMEOUT);
1205
1206         spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
1207
1208         if (kbdev->hwcnt.backend.state == KBASE_INSTR_STATE_RESETTING) {
1209                 /* the same interrupt handler preempted itself */
1210                 /* GPU is being reset */
1211                 spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
1212                 wait_event(kbdev->hwcnt.backend.wait,
1213                                         kbdev->hwcnt.backend.triggered != 0);
1214                 spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
1215         }
1216         /* Save the HW counters setup */
1217         if (kbdev->hwcnt.kctx != NULL) {
1218                 struct kbase_context *kctx = kbdev->hwcnt.kctx;
1219
1220                 if (kctx->jctx.sched_info.ctx.is_scheduled) {
1221                         kbasep_save_hwcnt_setup(kbdev, kctx, &hwcnt_setup);
1222
1223                         restore_hwc = true;
1224                 }
1225         }
1226
1227         /* Output the state of some interesting registers to help in the
1228          * debugging of GPU resets */
1229         kbase_debug_dump_registers(kbdev);
1230
1231         bckp_state = kbdev->hwcnt.backend.state;
1232         kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_RESETTING;
1233         kbdev->hwcnt.backend.triggered = 0;
1234
1235         spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
1236
1237         /* Reset the GPU */
1238         kbase_pm_init_hw(kbdev, 0);
1239
1240         /* Complete any jobs that were still on the GPU */
1241         spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1242         kbase_backend_reset(kbdev, &end_timestamp);
1243         kbase_pm_metrics_update(kbdev, NULL);
1244         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1245
1246         mutex_unlock(&kbdev->pm.lock);
1247
1248         mutex_lock(&js_devdata->runpool_mutex);
1249
1250         /* Reprogram the GPU's MMU */
1251         for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
1252                 struct kbase_as *as = &kbdev->as[i];
1253
1254                 mutex_lock(&as->transaction_mutex);
1255                 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1256
1257                 if (js_devdata->runpool_irq.per_as_data[i].kctx)
1258                         kbase_mmu_update(
1259                                 js_devdata->runpool_irq.per_as_data[i].kctx);
1260                 else
1261                         kbase_mmu_disable_as(kbdev, i);
1262
1263                 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1264                 mutex_unlock(&as->transaction_mutex);
1265         }
1266
1267         kbase_pm_enable_interrupts(kbdev);
1268
1269         atomic_set(&kbdev->hwaccess.backend.reset_gpu,
1270                                                 KBASE_RESET_GPU_NOT_PENDING);
1271
1272         kbase_disjoint_state_down(kbdev);
1273
1274         wake_up(&kbdev->hwaccess.backend.reset_wait);
1275         dev_err(kbdev->dev, "Reset complete");
1276
1277         if (js_devdata->nr_contexts_pullable > 0 && !kbdev->poweroff_pending)
1278                 try_schedule = true;
1279
1280         mutex_unlock(&js_devdata->runpool_mutex);
1281
1282         spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
1283         /* Restore the HW counters setup */
1284         if (restore_hwc) {
1285                 struct kbase_context *kctx = kbdev->hwcnt.kctx;
1286
1287                 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG),
1288                                 (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) |
1289                                 PRFCNT_CONFIG_MODE_OFF, kctx);
1290                 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_LO),
1291                                 hwcnt_setup.dump_buffer & 0xFFFFFFFF, kctx);
1292                 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_BASE_HI),
1293                                 hwcnt_setup.dump_buffer >> 32,        kctx);
1294                 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_JM_EN),
1295                                 hwcnt_setup.jm_bm,                    kctx);
1296                 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_SHADER_EN),
1297                                 hwcnt_setup.shader_bm,                kctx);
1298                 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_MMU_L2_EN),
1299                                 hwcnt_setup.mmu_l2_bm,                kctx);
1300
1301                 /* Due to PRLAM-8186 we need to disable the Tiler before we
1302                  * enable the HW counter dump. */
1303                 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
1304                         kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN),
1305                                                 0, kctx);
1306                 else
1307                         kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN),
1308                                                 hwcnt_setup.tiler_bm, kctx);
1309
1310                 kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_CONFIG),
1311                                 (kctx->as_nr << PRFCNT_CONFIG_AS_SHIFT) |
1312                                 PRFCNT_CONFIG_MODE_MANUAL, kctx);
1313
1314                 /* If HW has PRLAM-8186 we can now re-enable the tiler HW
1315                  * counters dump */
1316                 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8186))
1317                         kbase_reg_write(kbdev, GPU_CONTROL_REG(PRFCNT_TILER_EN),
1318                                                 hwcnt_setup.tiler_bm, kctx);
1319         }
1320         kbdev->hwcnt.backend.state = bckp_state;
1321         switch (kbdev->hwcnt.backend.state) {
1322         /* Cases for waking kbasep_cache_clean_worker worker */
1323         case KBASE_INSTR_STATE_CLEANED:
1324                 /* Cache-clean IRQ occurred, but we reset:
1325                  * Wakeup incase the waiter saw RESETTING */
1326         case KBASE_INSTR_STATE_REQUEST_CLEAN:
1327                 /* After a clean was requested, but before the regs were
1328                  * written:
1329                  * Wakeup incase the waiter saw RESETTING */
1330                 wake_up(&kbdev->hwcnt.backend.cache_clean_wait);
1331                 break;
1332         case KBASE_INSTR_STATE_CLEANING:
1333                 /* Either:
1334                  * 1) We've not got the Cache-clean IRQ yet: it was lost, or:
1335                  * 2) We got it whilst resetting: it was voluntarily lost
1336                  *
1337                  * So, move to the next state and wakeup: */
1338                 kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_CLEANED;
1339                 wake_up(&kbdev->hwcnt.backend.cache_clean_wait);
1340                 break;
1341
1342         /* Cases for waking anyone else */
1343         case KBASE_INSTR_STATE_DUMPING:
1344                 /* If dumping, abort the dump, because we may've lost the IRQ */
1345                 kbdev->hwcnt.backend.state = KBASE_INSTR_STATE_IDLE;
1346                 kbdev->hwcnt.backend.triggered = 1;
1347                 wake_up(&kbdev->hwcnt.backend.wait);
1348                 break;
1349         case KBASE_INSTR_STATE_DISABLED:
1350         case KBASE_INSTR_STATE_IDLE:
1351         case KBASE_INSTR_STATE_FAULT:
1352                 /* Every other reason: wakeup in that state */
1353                 kbdev->hwcnt.backend.triggered = 1;
1354                 wake_up(&kbdev->hwcnt.backend.wait);
1355                 break;
1356
1357         /* Unhandled cases */
1358         case KBASE_INSTR_STATE_RESETTING:
1359         default:
1360                 BUG();
1361                 break;
1362         }
1363         spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
1364
1365         /* Resume the vinstr core */
1366         kbase_vinstr_hwc_resume(kbdev->vinstr_ctx);
1367
1368         /* Note: counter dumping may now resume */
1369
1370         mutex_lock(&kbdev->pm.lock);
1371
1372         /* Find out what cores are required now */
1373         kbase_pm_update_cores_state(kbdev);
1374
1375         /* Synchronously request and wait for those cores, because if
1376          * instrumentation is enabled it would need them immediately. */
1377         kbase_pm_check_transitions_sync(kbdev);
1378
1379         mutex_unlock(&kbdev->pm.lock);
1380
1381         /* Try submitting some jobs to restart processing */
1382         if (try_schedule) {
1383                 KBASE_TRACE_ADD(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u,
1384                                                                         0);
1385                 kbase_js_sched_all(kbdev);
1386         }
1387
1388         kbase_pm_context_idle(kbdev);
1389         KBASE_TRACE_ADD(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0);
1390 }
1391
1392 static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer)
1393 {
1394         struct kbase_device *kbdev = container_of(timer, struct kbase_device,
1395                                                 hwaccess.backend.reset_timer);
1396
1397         KBASE_DEBUG_ASSERT(kbdev);
1398
1399         /* Reset still pending? */
1400         if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu,
1401                         KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) ==
1402                                                 KBASE_RESET_GPU_COMMITTED)
1403                 queue_work(kbdev->hwaccess.backend.reset_workq,
1404                                         &kbdev->hwaccess.backend.reset_work);
1405
1406         return HRTIMER_NORESTART;
1407 }
1408
1409 /*
1410  * If all jobs are evicted from the GPU then we can reset the GPU
1411  * immediately instead of waiting for the timeout to elapse
1412  */
1413
1414 static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev)
1415 {
1416         int i;
1417         int pending_jobs = 0;
1418
1419         KBASE_DEBUG_ASSERT(kbdev);
1420
1421         /* Count the number of jobs */
1422         for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
1423                 pending_jobs += kbase_backend_nr_atoms_submitted(kbdev, i);
1424
1425         if (pending_jobs > 0) {
1426                 /* There are still jobs on the GPU - wait */
1427                 return;
1428         }
1429
1430         /* To prevent getting incorrect registers when dumping failed job,
1431          * skip early reset.
1432          */
1433         if (kbdev->job_fault_debug != false)
1434                 return;
1435
1436         /* Check that the reset has been committed to (i.e. kbase_reset_gpu has
1437          * been called), and that no other thread beat this thread to starting
1438          * the reset */
1439         if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu,
1440                         KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) !=
1441                                                 KBASE_RESET_GPU_COMMITTED) {
1442                 /* Reset has already occurred */
1443                 return;
1444         }
1445
1446         queue_work(kbdev->hwaccess.backend.reset_workq,
1447                                         &kbdev->hwaccess.backend.reset_work);
1448 }
1449
1450 static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev)
1451 {
1452         unsigned long flags;
1453         struct kbasep_js_device_data *js_devdata;
1454
1455         js_devdata = &kbdev->js_data;
1456         spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1457         kbasep_try_reset_gpu_early_locked(kbdev);
1458         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1459 }
1460
1461 /**
1462  * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU
1463  * @kbdev: kbase device
1464  *
1465  * This function just soft-stops all the slots to ensure that as many jobs as
1466  * possible are saved.
1467  *
1468  * Return:
1469  *   The function returns a boolean which should be interpreted as follows:
1470  *   true - Prepared for reset, kbase_reset_gpu should be called.
1471  *   false - Another thread is performing a reset, kbase_reset_gpu should
1472  *   not be called.
1473  */
1474 bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev)
1475 {
1476         int i;
1477
1478         KBASE_DEBUG_ASSERT(kbdev);
1479
1480         if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu,
1481                                                 KBASE_RESET_GPU_NOT_PENDING,
1482                                                 KBASE_RESET_GPU_PREPARED) !=
1483                                                 KBASE_RESET_GPU_NOT_PENDING) {
1484                 /* Some other thread is already resetting the GPU */
1485                 return false;
1486         }
1487
1488         kbase_disjoint_state_up(kbdev);
1489
1490         for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
1491                 kbase_job_slot_softstop(kbdev, i, NULL);
1492
1493         return true;
1494 }
1495
1496 bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev)
1497 {
1498         unsigned long flags;
1499         bool ret;
1500         struct kbasep_js_device_data *js_devdata;
1501
1502         js_devdata = &kbdev->js_data;
1503         spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1504         ret = kbase_prepare_to_reset_gpu_locked(kbdev);
1505         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1506
1507         return ret;
1508 }
1509 KBASE_EXPORT_TEST_API(kbase_prepare_to_reset_gpu);
1510
1511 /*
1512  * This function should be called after kbase_prepare_to_reset_gpu if it
1513  * returns true. It should never be called without a corresponding call to
1514  * kbase_prepare_to_reset_gpu.
1515  *
1516  * After this function is called (or not called if kbase_prepare_to_reset_gpu
1517  * returned false), the caller should wait for
1518  * kbdev->hwaccess.backend.reset_waitq to be signalled to know when the reset
1519  * has completed.
1520  */
1521 void kbase_reset_gpu(struct kbase_device *kbdev)
1522 {
1523         KBASE_DEBUG_ASSERT(kbdev);
1524
1525         /* Note this is an assert/atomic_set because it is a software issue for
1526          * a race to be occuring here */
1527         KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
1528                                                 KBASE_RESET_GPU_PREPARED);
1529         atomic_set(&kbdev->hwaccess.backend.reset_gpu,
1530                                                 KBASE_RESET_GPU_COMMITTED);
1531
1532         dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n",
1533                         kbdev->reset_timeout_ms);
1534
1535         hrtimer_start(&kbdev->hwaccess.backend.reset_timer,
1536                         HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms),
1537                         HRTIMER_MODE_REL);
1538
1539         /* Try resetting early */
1540         kbasep_try_reset_gpu_early(kbdev);
1541 }
1542 KBASE_EXPORT_TEST_API(kbase_reset_gpu);
1543
1544 void kbase_reset_gpu_locked(struct kbase_device *kbdev)
1545 {
1546         KBASE_DEBUG_ASSERT(kbdev);
1547
1548         /* Note this is an assert/atomic_set because it is a software issue for
1549          * a race to be occuring here */
1550         KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
1551                                                 KBASE_RESET_GPU_PREPARED);
1552         atomic_set(&kbdev->hwaccess.backend.reset_gpu,
1553                                                 KBASE_RESET_GPU_COMMITTED);
1554
1555         dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n",
1556                         kbdev->reset_timeout_ms);
1557         hrtimer_start(&kbdev->hwaccess.backend.reset_timer,
1558                         HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms),
1559                         HRTIMER_MODE_REL);
1560
1561         /* Try resetting early */
1562         kbasep_try_reset_gpu_early_locked(kbdev);
1563 }
1564 #endif /* KBASE_GPU_RESET_EN */