drivers/gpu/arm/midgard/backend/gpu/mali_kbase_jm_hw.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18 /*
  19  * Base kernel job manager APIs
  20  */
  21
  22 #include <mali_kbase.h>
  23 #include <mali_kbase_config.h>
  24 #include <mali_midg_regmap.h>
  25 #if defined(CONFIG_MALI_GATOR_SUPPORT)
  26 #include <mali_kbase_gator.h>
  27 #endif
  28 #include <mali_kbase_tlstream.h>
  29 #include <mali_kbase_vinstr.h>
  30 #include <mali_kbase_hw.h>
  31 #include <mali_kbase_hwaccess_jm.h>
  32 #include <backend/gpu/mali_kbase_device_internal.h>
  33 #include <backend/gpu/mali_kbase_irq_internal.h>
  34 #include <backend/gpu/mali_kbase_js_affinity.h>
  35 #include <backend/gpu/mali_kbase_jm_internal.h>
  36
  37 #define beenthere(kctx, f, a...) \
  38                         dev_dbg(kctx->kbdev->dev, "%s:" f, __func__, ##a)
  39
  40 #if KBASE_GPU_RESET_EN
  41 static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev);
  42 static void kbasep_reset_timeout_worker(struct work_struct *data);
  43 static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer);
  44 #endif /* KBASE_GPU_RESET_EN */
  45
  46 static inline int kbasep_jm_is_js_free(struct kbase_device *kbdev, int js,
  47                                                 struct kbase_context *kctx)
  48 {
  49         return !kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT), kctx);
  50 }
  51
  52 void kbase_job_hw_submit(struct kbase_device *kbdev,
  53                                 struct kbase_jd_atom *katom,
  54                                 int js)
  55 {
  56         struct kbase_context *kctx;
  57         u32 cfg;
  58         u64 jc_head = katom->jc;
  59
  60         KBASE_DEBUG_ASSERT(kbdev);
  61         KBASE_DEBUG_ASSERT(katom);
  62
  63         kctx = katom->kctx;
  64
  65         /* Command register must be available */
  66         KBASE_DEBUG_ASSERT(kbasep_jm_is_js_free(kbdev, js, kctx));
  67         /* Affinity is not violating */
  68         kbase_js_debug_log_current_affinities(kbdev);
  69         KBASE_DEBUG_ASSERT(!kbase_js_affinity_would_violate(kbdev, js,
  70                                                         katom->affinity));
  71
  72         kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_LO),
  73                                                 jc_head & 0xFFFFFFFF, kctx);
  74         kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_HEAD_NEXT_HI),
  75                                                 jc_head >> 32, kctx);
  76
  77         kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_LO),
  78                                         katom->affinity & 0xFFFFFFFF, kctx);
  79         kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_AFFINITY_NEXT_HI),
  80                                         katom->affinity >> 32, kctx);
  81
  82         /* start MMU, medium priority, cache clean/flush on end, clean/flush on
  83          * start */
  84         cfg = kctx->as_nr;
  85
  86         if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION))
  87                 cfg |= JS_CONFIG_ENABLE_FLUSH_REDUCTION;
  88
  89 #ifndef CONFIG_MALI_COH_GPU
  90         if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_START))
  91                 cfg |= JS_CONFIG_START_FLUSH_NO_ACTION;
  92         else
  93                 cfg |= JS_CONFIG_START_FLUSH_CLEAN_INVALIDATE;
  94
  95         if (0 != (katom->core_req & BASE_JD_REQ_SKIP_CACHE_END))
  96                 cfg |= JS_CONFIG_END_FLUSH_NO_ACTION;
  97         else
  98                 cfg |= JS_CONFIG_END_FLUSH_CLEAN_INVALIDATE;
  99 #endif /* CONFIG_MALI_COH_GPU */
 100
 101         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10649) ||
 102                 !kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3982))
 103                 cfg |= JS_CONFIG_START_MMU;
 104
 105         cfg |= JS_CONFIG_THREAD_PRI(8);
 106
 107         if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE) &&
 108                 (katom->atom_flags & KBASE_KATOM_FLAG_PROTECTED))
 109                 cfg |= JS_CONFIG_DISABLE_DESCRIPTOR_WR_BK;
 110
 111         if (kbase_hw_has_feature(kbdev,
 112                                 BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
 113                 if (!kbdev->hwaccess.backend.slot_rb[js].job_chain_flag) {
 114                         cfg |= JS_CONFIG_JOB_CHAIN_FLAG;
 115                         katom->atom_flags |= KBASE_KATOM_FLAGS_JOBCHAIN;
 116                         kbdev->hwaccess.backend.slot_rb[js].job_chain_flag =
 117                                                                 true;
 118                 } else {
 119                         katom->atom_flags &= ~KBASE_KATOM_FLAGS_JOBCHAIN;
 120                         kbdev->hwaccess.backend.slot_rb[js].job_chain_flag =
 121                                                                 false;
 122                 }
 123         }
 124
 125         kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_CONFIG_NEXT), cfg, kctx);
 126
 127         if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION))
 128                 kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_FLUSH_ID_NEXT),
 129                                 katom->flush_id, kctx);
 130
 131         /* Write an approximate start timestamp.
 132          * It's approximate because there might be a job in the HEAD register.
 133          * In such cases, we'll try to make a better approximation in the IRQ
 134          * handler (up to the KBASE_JS_IRQ_THROTTLE_TIME_US). */
 135         katom->start_timestamp = ktime_get();
 136
 137         /* GO ! */
 138         dev_dbg(kbdev->dev, "JS: Submitting atom %p from ctx %p to js[%d] with head=0x%llx, affinity=0x%llx",
 139                                 katom, kctx, js, jc_head, katom->affinity);
 140
 141         KBASE_TRACE_ADD_SLOT_INFO(kbdev, JM_SUBMIT, kctx, katom, jc_head, js,
 142                                                         (u32) katom->affinity);
 143
 144 #if defined(CONFIG_MALI_GATOR_SUPPORT)
 145         kbase_trace_mali_job_slots_event(
 146                                 GATOR_MAKE_EVENT(GATOR_JOB_SLOT_START, js),
 147                                 kctx, kbase_jd_atom_id(kctx, katom));
 148 #endif
 149         kbase_tlstream_tl_attrib_atom_config(katom, jc_head,
 150                         katom->affinity, cfg);
 151         kbase_tlstream_tl_ret_ctx_lpu(
 152                 kctx,
 153                 &kbdev->gpu_props.props.raw_props.js_features[
 154                         katom->slot_nr]);
 155         kbase_tlstream_tl_ret_atom_as(katom, &kbdev->as[kctx->as_nr]);
 156         kbase_tlstream_tl_ret_atom_lpu(
 157                         katom,
 158                         &kbdev->gpu_props.props.raw_props.js_features[js],
 159                         "ctx_nr,atom_nr");
 160 #ifdef CONFIG_GPU_TRACEPOINTS
 161         if (!kbase_backend_nr_atoms_submitted(kbdev, js)) {
 162                 /* If this is the only job on the slot, trace it as starting */
 163                 char js_string[16];
 164
 165                 trace_gpu_sched_switch(
 166                                 kbasep_make_job_slot_string(js, js_string),
 167                                 ktime_to_ns(katom->start_timestamp),
 168                                 (u32)katom->kctx->id, 0, katom->work_id);
 169                 kbdev->hwaccess.backend.slot_rb[js].last_context = katom->kctx;
 170         }
 171 #endif
 172         kbase_timeline_job_slot_submit(kbdev, kctx, katom, js);
 173
 174         kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND_NEXT),
 175                                                 JS_COMMAND_START, katom->kctx);
 176 }
 177
 178 /**
 179  * kbasep_job_slot_update_head_start_timestamp - Update timestamp
 180  * @kbdev: kbase device
 181  * @js: job slot
 182  * @end_timestamp: timestamp
 183  *
 184  * Update the start_timestamp of the job currently in the HEAD, based on the
 185  * fact that we got an IRQ for the previous set of completed jobs.
 186  *
 187  * The estimate also takes into account the %KBASE_JS_IRQ_THROTTLE_TIME_US and
 188  * the time the job was submitted, to work out the best estimate (which might
 189  * still result in an over-estimate to the calculated time spent)
 190  */
 191 static void kbasep_job_slot_update_head_start_timestamp(
 192                                                 struct kbase_device *kbdev,
 193                                                 int js,
 194                                                 ktime_t end_timestamp)
 195 {
 196         if (kbase_backend_nr_atoms_on_slot(kbdev, js) > 0) {
 197                 struct kbase_jd_atom *katom;
 198                 ktime_t new_timestamp;
 199                 ktime_t timestamp_diff;
 200                 /* The atom in the HEAD */
 201                 katom = kbase_gpu_inspect(kbdev, js, 0);
 202
 203                 KBASE_DEBUG_ASSERT(katom != NULL);
 204
 205                 /* Account for any IRQ Throttle time - makes an overestimate of
 206                  * the time spent by the job */
 207                 new_timestamp = ktime_sub_ns(end_timestamp,
 208                                         KBASE_JS_IRQ_THROTTLE_TIME_US * 1000);
 209                 timestamp_diff = ktime_sub(new_timestamp,
 210                                                         katom->start_timestamp);
 211                 if (ktime_to_ns(timestamp_diff) >= 0) {
 212                         /* Only update the timestamp if it's a better estimate
 213                          * than what's currently stored. This is because our
 214                          * estimate that accounts for the throttle time may be
 215                          * too much of an overestimate */
 216                         katom->start_timestamp = new_timestamp;
 217                 }
 218         }
 219 }
 220
 221 /**
 222  * kbasep_trace_tl_nret_atom_lpu - Call nret_atom_lpu timeline tracepoint
 223  * @kbdev: kbase device
 224  * @i: job slot
 225  *
 226  * Get kbase atom by calling kbase_gpu_inspect for given job slot.
 227  * Then use obtained katom and name of slot associated with the given
 228  * job slot number in tracepoint call to the instrumentation module
 229  * informing that given atom is no longer executed on given lpu (job slot).
 230  */
 231 static void kbasep_trace_tl_nret_atom_lpu(struct kbase_device *kbdev, int i)
 232 {
 233         struct kbase_jd_atom *katom = kbase_gpu_inspect(kbdev, i, 0);
 234
 235         kbase_tlstream_tl_nret_atom_lpu(katom,
 236                 &kbdev->gpu_props.props.raw_props.js_features[i]);
 237 }
 238
 239 void kbase_job_done(struct kbase_device *kbdev, u32 done)
 240 {
 241         unsigned long flags;
 242         int i;
 243         u32 count = 0;
 244         ktime_t end_timestamp = ktime_get();
 245         struct kbasep_js_device_data *js_devdata;
 246
 247         KBASE_DEBUG_ASSERT(kbdev);
 248         js_devdata = &kbdev->js_data;
 249
 250         KBASE_TRACE_ADD(kbdev, JM_IRQ, NULL, NULL, 0, done);
 251
 252         memset(&kbdev->slot_submit_count_irq[0], 0,
 253                                         sizeof(kbdev->slot_submit_count_irq));
 254
 255         /* write irq throttle register, this will prevent irqs from occurring
 256          * until the given number of gpu clock cycles have passed */
 257         {
 258                 int irq_throttle_cycles =
 259                                 atomic_read(&kbdev->irq_throttle_cycles);
 260
 261                 kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_THROTTLE),
 262                                                 irq_throttle_cycles, NULL);
 263         }
 264
 265         spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
 266
 267         while (done) {
 268                 u32 failed = done >> 16;
 269
 270                 /* treat failed slots as finished slots */
 271                 u32 finished = (done & 0xFFFF) | failed;
 272
 273                 /* Note: This is inherently unfair, as we always check
 274                  * for lower numbered interrupts before the higher
 275                  * numbered ones.*/
 276                 i = ffs(finished) - 1;
 277                 KBASE_DEBUG_ASSERT(i >= 0);
 278
 279                 do {
 280                         int nr_done;
 281                         u32 active;
 282                         u32 completion_code = BASE_JD_EVENT_DONE;/* assume OK */
 283                         u64 job_tail = 0;
 284
 285                         if (failed & (1u << i)) {
 286                                 /* read out the job slot status code if the job
 287                                  * slot reported failure */
 288                                 completion_code = kbase_reg_read(kbdev,
 289                                         JOB_SLOT_REG(i, JS_STATUS), NULL);
 290
 291                                 switch (completion_code) {
 292                                 case BASE_JD_EVENT_STOPPED:
 293 #if defined(CONFIG_MALI_GATOR_SUPPORT)
 294                                         kbase_trace_mali_job_slots_event(
 295                                                 GATOR_MAKE_EVENT(
 296                                                 GATOR_JOB_SLOT_SOFT_STOPPED, i),
 297                                                                 NULL, 0);
 298 #endif
 299
 300                                         kbase_tlstream_aux_job_softstop(i);
 301
 302                                         kbasep_trace_tl_nret_atom_lpu(
 303                                                 kbdev, i);
 304
 305                                         /* Soft-stopped job - read the value of
 306                                          * JS<n>_TAIL so that the job chain can
 307                                          * be resumed */
 308                                         job_tail = (u64)kbase_reg_read(kbdev,
 309                                                 JOB_SLOT_REG(i, JS_TAIL_LO),
 310                                                                         NULL) |
 311                                                 ((u64)kbase_reg_read(kbdev,
 312                                                 JOB_SLOT_REG(i, JS_TAIL_HI),
 313                                                                 NULL) << 32);
 314                                         break;
 315                                 case BASE_JD_EVENT_NOT_STARTED:
 316                                         /* PRLAM-10673 can cause a TERMINATED
 317                                          * job to come back as NOT_STARTED, but
 318                                          * the error interrupt helps us detect
 319                                          * it */
 320                                         completion_code =
 321                                                 BASE_JD_EVENT_TERMINATED;
 322                                         /* fall through */
 323                                 default:
 324                                         dev_warn(kbdev->dev, "error detected from slot %d, job status 0x%08x (%s)",
 325                                                         i, completion_code,
 326                                                         kbase_exception_name
 327                                                         (kbdev,
 328                                                         completion_code));
 329                                 }
 330
 331                                 kbase_gpu_irq_evict(kbdev, i);
 332                         }
 333
 334                         kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR),
 335                                         done & ((1 << i) | (1 << (i + 16))),
 336                                         NULL);
 337                         active = kbase_reg_read(kbdev,
 338                                         JOB_CONTROL_REG(JOB_IRQ_JS_STATE),
 339                                         NULL);
 340
 341                         if (((active >> i) & 1) == 0 &&
 342                                         (((done >> (i + 16)) & 1) == 0)) {
 343                                 /* There is a potential race we must work
 344                                  * around:
 345                                  *
 346                                  *  1. A job slot has a job in both current and
 347                                  *     next registers
 348                                  *  2. The job in current completes
 349                                  *     successfully, the IRQ handler reads
 350                                  *     RAWSTAT and calls this function with the
 351                                  *     relevant bit set in "done"
 352                                  *  3. The job in the next registers becomes the
 353                                  *     current job on the GPU
 354                                  *  4. Sometime before the JOB_IRQ_CLEAR line
 355                                  *     above the job on the GPU _fails_
 356                                  *  5. The IRQ_CLEAR clears the done bit but not
 357                                  *     the failed bit. This atomically sets
 358                                  *     JOB_IRQ_JS_STATE. However since both jobs
 359                                  *     have now completed the relevant bits for
 360                                  *     the slot are set to 0.
 361                                  *
 362                                  * If we now did nothing then we'd incorrectly
 363                                  * assume that _both_ jobs had completed
 364                                  * successfully (since we haven't yet observed
 365                                  * the fail bit being set in RAWSTAT).
 366                                  *
 367                                  * So at this point if there are no active jobs
 368                                  * left we check to see if RAWSTAT has a failure
 369                                  * bit set for the job slot. If it does we know
 370                                  * that there has been a new failure that we
 371                                  * didn't previously know about, so we make sure
 372                                  * that we record this in active (but we wait
 373                                  * for the next loop to deal with it).
 374                                  *
 375                                  * If we were handling a job failure (i.e. done
 376                                  * has the relevant high bit set) then we know
 377                                  * that the value read back from
 378                                  * JOB_IRQ_JS_STATE is the correct number of
 379                                  * remaining jobs because the failed job will
 380                                  * have prevented any futher jobs from starting
 381                                  * execution.
 382                                  */
 383                                 u32 rawstat = kbase_reg_read(kbdev,
 384                                         JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL);
 385
 386                                 if ((rawstat >> (i + 16)) & 1) {
 387                                         /* There is a failed job that we've
 388                                          * missed - add it back to active */
 389                                         active |= (1u << i);
 390                                 }
 391                         }
 392
 393                         dev_dbg(kbdev->dev, "Job ended with status 0x%08X\n",
 394                                                         completion_code);
 395
 396                         nr_done = kbase_backend_nr_atoms_submitted(kbdev, i);
 397                         nr_done -= (active >> i) & 1;
 398                         nr_done -= (active >> (i + 16)) & 1;
 399
 400                         if (nr_done <= 0) {
 401                                 dev_warn(kbdev->dev, "Spurious interrupt on slot %d",
 402                                                                         i);
 403
 404                                 goto spurious;
 405                         }
 406
 407                         count += nr_done;
 408
 409                         while (nr_done) {
 410                                 if (nr_done == 1) {
 411                                         kbase_gpu_complete_hw(kbdev, i,
 412                                                                 completion_code,
 413                                                                 job_tail,
 414                                                                 &end_timestamp);
 415                                         kbase_jm_try_kick_all(kbdev);
 416                                 } else {
 417                                         /* More than one job has completed.
 418                                          * Since this is not the last job being
 419                                          * reported this time it must have
 420                                          * passed. This is because the hardware
 421                                          * will not allow further jobs in a job
 422                                          * slot to complete until the failed job
 423                                          * is cleared from the IRQ status.
 424                                          */
 425                                         kbase_gpu_complete_hw(kbdev, i,
 426                                                         BASE_JD_EVENT_DONE,
 427                                                         0,
 428                                                         &end_timestamp);
 429                                 }
 430                                 nr_done--;
 431                         }
 432  spurious:
 433                         done = kbase_reg_read(kbdev,
 434                                         JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL);
 435
 436                         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10883)) {
 437                                 /* Workaround for missing interrupt caused by
 438                                  * PRLAM-10883 */
 439                                 if (((active >> i) & 1) && (0 ==
 440                                                 kbase_reg_read(kbdev,
 441                                                         JOB_SLOT_REG(i,
 442                                                         JS_STATUS), NULL))) {
 443                                         /* Force job slot to be processed again
 444                                          */
 445                                         done |= (1u << i);
 446                                 }
 447                         }
 448
 449                         failed = done >> 16;
 450                         finished = (done & 0xFFFF) | failed;
 451                         if (done)
 452                                 end_timestamp = ktime_get();
 453                 } while (finished & (1 << i));
 454
 455                 kbasep_job_slot_update_head_start_timestamp(kbdev, i,
 456                                                                 end_timestamp);
 457         }
 458
 459         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
 460 #if KBASE_GPU_RESET_EN
 461         if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
 462                                                 KBASE_RESET_GPU_COMMITTED) {
 463                 /* If we're trying to reset the GPU then we might be able to do
 464                  * it early (without waiting for a timeout) because some jobs
 465                  * have completed
 466                  */
 467                 kbasep_try_reset_gpu_early(kbdev);
 468         }
 469 #endif /* KBASE_GPU_RESET_EN */
 470         KBASE_TRACE_ADD(kbdev, JM_IRQ_END, NULL, NULL, 0, count);
 471 }
 472 KBASE_EXPORT_TEST_API(kbase_job_done);
 473
 474 static bool kbasep_soft_stop_allowed(struct kbase_device *kbdev,
 475                                         struct kbase_jd_atom *katom)
 476 {
 477         bool soft_stops_allowed = true;
 478
 479         if (kbase_jd_katom_is_protected(katom)) {
 480                 soft_stops_allowed = false;
 481         } else if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8408)) {
 482                 if ((katom->core_req & BASE_JD_REQ_T) != 0)
 483                         soft_stops_allowed = false;
 484         }
 485         return soft_stops_allowed;
 486 }
 487
 488 static bool kbasep_hard_stop_allowed(struct kbase_device *kbdev,
 489                                                 base_jd_core_req core_reqs)
 490 {
 491         bool hard_stops_allowed = true;
 492
 493         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8394)) {
 494                 if ((core_reqs & BASE_JD_REQ_T) != 0)
 495                         hard_stops_allowed = false;
 496         }
 497         return hard_stops_allowed;
 498 }
 499
 500 void kbasep_job_slot_soft_or_hard_stop_do_action(struct kbase_device *kbdev,
 501                                         int js,
 502                                         u32 action,
 503                                         base_jd_core_req core_reqs,
 504                                         struct kbase_jd_atom *target_katom)
 505 {
 506         struct kbase_context *kctx = target_katom->kctx;
 507 #if KBASE_TRACE_ENABLE
 508         u32 status_reg_before;
 509         u64 job_in_head_before;
 510         u32 status_reg_after;
 511
 512         KBASE_DEBUG_ASSERT(!(action & (~JS_COMMAND_MASK)));
 513
 514         /* Check the head pointer */
 515         job_in_head_before = ((u64) kbase_reg_read(kbdev,
 516                                         JOB_SLOT_REG(js, JS_HEAD_LO), NULL))
 517                         | (((u64) kbase_reg_read(kbdev,
 518                                         JOB_SLOT_REG(js, JS_HEAD_HI), NULL))
 519                                                                         << 32);
 520         status_reg_before = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS),
 521                                                                         NULL);
 522 #endif
 523
 524         if (action == JS_COMMAND_SOFT_STOP) {
 525                 bool soft_stop_allowed = kbasep_soft_stop_allowed(kbdev,
 526                                                                 target_katom);
 527
 528                 if (!soft_stop_allowed) {
 529 #ifdef CONFIG_MALI_DEBUG
 530                         dev_dbg(kbdev->dev,
 531                                         "Attempt made to soft-stop a job that cannot be soft-stopped. core_reqs = 0x%X",
 532                                         (unsigned int)core_reqs);
 533 #endif                          /* CONFIG_MALI_DEBUG */
 534                         return;
 535                 }
 536
 537                 /* We are about to issue a soft stop, so mark the atom as having
 538                  * been soft stopped */
 539                 target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_SOFT_STOPPPED;
 540
 541                 /* Mark the point where we issue the soft-stop command */
 542                 kbase_tlstream_aux_issue_job_softstop(target_katom);
 543
 544                 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8316)) {
 545                         int i;
 546
 547                         for (i = 0;
 548                              i < kbase_backend_nr_atoms_submitted(kbdev, js);
 549                              i++) {
 550                                 struct kbase_jd_atom *katom;
 551
 552                                 katom = kbase_gpu_inspect(kbdev, js, i);
 553
 554                                 KBASE_DEBUG_ASSERT(katom);
 555
 556                                 /* For HW_ISSUE_8316, only 'bad' jobs attacking
 557                                  * the system can cause this issue: normally,
 558                                  * all memory should be allocated in multiples
 559                                  * of 4 pages, and growable memory should be
 560                                  * changed size in multiples of 4 pages.
 561                                  *
 562                                  * Whilst such 'bad' jobs can be cleared by a
 563                                  * GPU reset, the locking up of a uTLB entry
 564                                  * caused by the bad job could also stall other
 565                                  * ASs, meaning that other ASs' jobs don't
 566                                  * complete in the 'grace' period before the
 567                                  * reset. We don't want to lose other ASs' jobs
 568                                  * when they would normally complete fine, so we
 569                                  * must 'poke' the MMU regularly to help other
 570                                  * ASs complete */
 571                                 kbase_as_poking_timer_retain_atom(
 572                                                 kbdev, katom->kctx, katom);
 573                         }
 574                 }
 575
 576                 if (kbase_hw_has_feature(
 577                                 kbdev,
 578                                 BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
 579                         action = (target_katom->atom_flags &
 580                                         KBASE_KATOM_FLAGS_JOBCHAIN) ?
 581                                 JS_COMMAND_SOFT_STOP_1 :
 582                                 JS_COMMAND_SOFT_STOP_0;
 583                 }
 584         } else if (action == JS_COMMAND_HARD_STOP) {
 585                 bool hard_stop_allowed = kbasep_hard_stop_allowed(kbdev,
 586                                                                 core_reqs);
 587
 588                 if (!hard_stop_allowed) {
 589                         /* Jobs can be hard-stopped for the following reasons:
 590                          *  * CFS decides the job has been running too long (and
 591                          *    soft-stop has not occurred). In this case the GPU
 592                          *    will be reset by CFS if the job remains on the
 593                          *    GPU.
 594                          *
 595                          *  * The context is destroyed, kbase_jd_zap_context
 596                          *    will attempt to hard-stop the job. However it also
 597                          *    has a watchdog which will cause the GPU to be
 598                          *    reset if the job remains on the GPU.
 599                          *
 600                          *  * An (unhandled) MMU fault occurred. As long as
 601                          *    BASE_HW_ISSUE_8245 is defined then the GPU will be
 602                          *    reset.
 603                          *
 604                          * All three cases result in the GPU being reset if the
 605                          * hard-stop fails, so it is safe to just return and
 606                          * ignore the hard-stop request.
 607                          */
 608                         dev_warn(kbdev->dev,
 609                                         "Attempt made to hard-stop a job that cannot be hard-stopped. core_reqs = 0x%X",
 610                                         (unsigned int)core_reqs);
 611                         return;
 612                 }
 613                 target_katom->atom_flags |= KBASE_KATOM_FLAG_BEEN_HARD_STOPPED;
 614
 615                 if (kbase_hw_has_feature(
 616                                 kbdev,
 617                                 BASE_HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) {
 618                         action = (target_katom->atom_flags &
 619                                         KBASE_KATOM_FLAGS_JOBCHAIN) ?
 620                                 JS_COMMAND_HARD_STOP_1 :
 621                                 JS_COMMAND_HARD_STOP_0;
 622                 }
 623         }
 624
 625         kbase_reg_write(kbdev, JOB_SLOT_REG(js, JS_COMMAND), action, kctx);
 626
 627 #if KBASE_TRACE_ENABLE
 628         status_reg_after = kbase_reg_read(kbdev, JOB_SLOT_REG(js, JS_STATUS),
 629                                                                         NULL);
 630         if (status_reg_after == BASE_JD_EVENT_ACTIVE) {
 631                 struct kbase_jd_atom *head;
 632                 struct kbase_context *head_kctx;
 633
 634                 head = kbase_gpu_inspect(kbdev, js, 0);
 635                 head_kctx = head->kctx;
 636
 637                 if (status_reg_before == BASE_JD_EVENT_ACTIVE)
 638                         KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, head_kctx,
 639                                                 head, job_in_head_before, js);
 640                 else
 641                         KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL,
 642                                                 0, js);
 643
 644                 switch (action) {
 645                 case JS_COMMAND_SOFT_STOP:
 646                         KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, head_kctx,
 647                                                         head, head->jc, js);
 648                         break;
 649                 case JS_COMMAND_SOFT_STOP_0:
 650                         KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, head_kctx,
 651                                                         head, head->jc, js);
 652                         break;
 653                 case JS_COMMAND_SOFT_STOP_1:
 654                         KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, head_kctx,
 655                                                         head, head->jc, js);
 656                         break;
 657                 case JS_COMMAND_HARD_STOP:
 658                         KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, head_kctx,
 659                                                         head, head->jc, js);
 660                         break;
 661                 case JS_COMMAND_HARD_STOP_0:
 662                         KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, head_kctx,
 663                                                         head, head->jc, js);
 664                         break;
 665                 case JS_COMMAND_HARD_STOP_1:
 666                         KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, head_kctx,
 667                                                         head, head->jc, js);
 668                         break;
 669                 default:
 670                         BUG();
 671                         break;
 672                 }
 673         } else {
 674                 if (status_reg_before == BASE_JD_EVENT_ACTIVE)
 675                         KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL,
 676                                                         job_in_head_before, js);
 677                 else
 678                         KBASE_TRACE_ADD_SLOT(kbdev, JM_CHECK_HEAD, NULL, NULL,
 679                                                         0, js);
 680
 681                 switch (action) {
 682                 case JS_COMMAND_SOFT_STOP:
 683                         KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP, NULL, NULL, 0,
 684                                                         js);
 685                         break;
 686                 case JS_COMMAND_SOFT_STOP_0:
 687                         KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_0, NULL, NULL,
 688                                                         0, js);
 689                         break;
 690                 case JS_COMMAND_SOFT_STOP_1:
 691                         KBASE_TRACE_ADD_SLOT(kbdev, JM_SOFTSTOP_1, NULL, NULL,
 692                                                         0, js);
 693                         break;
 694                 case JS_COMMAND_HARD_STOP:
 695                         KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP, NULL, NULL, 0,
 696                                                         js);
 697                         break;
 698                 case JS_COMMAND_HARD_STOP_0:
 699                         KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_0, NULL, NULL,
 700                                                         0, js);
 701                         break;
 702                 case JS_COMMAND_HARD_STOP_1:
 703                         KBASE_TRACE_ADD_SLOT(kbdev, JM_HARDSTOP_1, NULL, NULL,
 704                                                         0, js);
 705                         break;
 706                 default:
 707                         BUG();
 708                         break;
 709                 }
 710         }
 711 #endif
 712 }
 713
 714 void kbase_backend_jm_kill_jobs_from_kctx(struct kbase_context *kctx)
 715 {
 716         unsigned long flags;
 717         struct kbase_device *kbdev;
 718         struct kbasep_js_device_data *js_devdata;
 719         int i;
 720
 721         KBASE_DEBUG_ASSERT(kctx != NULL);
 722         kbdev = kctx->kbdev;
 723         KBASE_DEBUG_ASSERT(kbdev != NULL);
 724         js_devdata = &kbdev->js_data;
 725
 726         /* Cancel any remaining running jobs for this kctx  */
 727         mutex_lock(&kctx->jctx.lock);
 728         spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
 729
 730         /* Invalidate all jobs in context, to prevent re-submitting */
 731         for (i = 0; i < BASE_JD_ATOM_COUNT; i++) {
 732                 if (!work_pending(&kctx->jctx.atoms[i].work))
 733                         kctx->jctx.atoms[i].event_code =
 734                                                 BASE_JD_EVENT_JOB_CANCELLED;
 735         }
 736
 737         for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
 738                 kbase_job_slot_hardstop(kctx, i, NULL);
 739
 740         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
 741         mutex_unlock(&kctx->jctx.lock);
 742 }
 743
 744 void kbase_job_slot_ctx_priority_check_locked(struct kbase_context *kctx,
 745                                 struct kbase_jd_atom *target_katom)
 746 {
 747         struct kbase_device *kbdev;
 748         int js = target_katom->slot_nr;
 749         int priority = target_katom->sched_priority;
 750         int i;
 751
 752         KBASE_DEBUG_ASSERT(kctx != NULL);
 753         kbdev = kctx->kbdev;
 754         KBASE_DEBUG_ASSERT(kbdev != NULL);
 755
 756         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 757
 758         for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) {
 759                 struct kbase_jd_atom *katom;
 760
 761                 katom = kbase_gpu_inspect(kbdev, js, i);
 762                 if (!katom)
 763                         continue;
 764
 765                 if (katom->kctx != kctx)
 766                         continue;
 767
 768                 if (katom->sched_priority > priority)
 769                         kbase_job_slot_softstop(kbdev, js, katom);
 770         }
 771 }
 772
 773 struct zap_reset_data {
 774         /* The stages are:
 775          * 1. The timer has never been called
 776          * 2. The zap has timed out, all slots are soft-stopped - the GPU reset
 777          *    will happen. The GPU has been reset when
 778          *    kbdev->hwaccess.backend.reset_waitq is signalled
 779          *
 780          * (-1 - The timer has been cancelled)
 781          */
 782         int stage;
 783         struct kbase_device *kbdev;
 784         struct hrtimer timer;
 785         spinlock_t lock; /* protects updates to stage member */
 786 };
 787
 788 static enum hrtimer_restart zap_timeout_callback(struct hrtimer *timer)
 789 {
 790         struct zap_reset_data *reset_data = container_of(timer,
 791                                                 struct zap_reset_data, timer);
 792         struct kbase_device *kbdev = reset_data->kbdev;
 793         unsigned long flags;
 794
 795         spin_lock_irqsave(&reset_data->lock, flags);
 796
 797         if (reset_data->stage == -1)
 798                 goto out;
 799
 800 #if KBASE_GPU_RESET_EN
 801         if (kbase_prepare_to_reset_gpu(kbdev)) {
 802                 dev_err(kbdev->dev, "Issueing GPU soft-reset because jobs failed to be killed (within %d ms) as part of context termination (e.g. process exit)\n",
 803                                                                 ZAP_TIMEOUT);
 804                 kbase_reset_gpu(kbdev);
 805         }
 806 #endif /* KBASE_GPU_RESET_EN */
 807         reset_data->stage = 2;
 808
 809  out:
 810         spin_unlock_irqrestore(&reset_data->lock, flags);
 811
 812         return HRTIMER_NORESTART;
 813 }
 814
 815 void kbase_jm_wait_for_zero_jobs(struct kbase_context *kctx)
 816 {
 817         struct kbase_device *kbdev = kctx->kbdev;
 818         struct zap_reset_data reset_data;
 819         unsigned long flags;
 820
 821         hrtimer_init_on_stack(&reset_data.timer, CLOCK_MONOTONIC,
 822                                                         HRTIMER_MODE_REL);
 823         reset_data.timer.function = zap_timeout_callback;
 824
 825         spin_lock_init(&reset_data.lock);
 826
 827         reset_data.kbdev = kbdev;
 828         reset_data.stage = 1;
 829
 830         hrtimer_start(&reset_data.timer, HR_TIMER_DELAY_MSEC(ZAP_TIMEOUT),
 831                                                         HRTIMER_MODE_REL);
 832
 833         /* Wait for all jobs to finish, and for the context to be not-scheduled
 834          * (due to kbase_job_zap_context(), we also guarentee it's not in the JS
 835          * policy queue either */
 836         wait_event(kctx->jctx.zero_jobs_wait, kctx->jctx.job_nr == 0);
 837         wait_event(kctx->jctx.sched_info.ctx.is_scheduled_wait,
 838                         kctx->jctx.sched_info.ctx.is_scheduled == false);
 839
 840         spin_lock_irqsave(&reset_data.lock, flags);
 841         if (reset_data.stage == 1) {
 842                 /* The timer hasn't run yet - so cancel it */
 843                 reset_data.stage = -1;
 844         }
 845         spin_unlock_irqrestore(&reset_data.lock, flags);
 846
 847         hrtimer_cancel(&reset_data.timer);
 848
 849         if (reset_data.stage == 2) {
 850                 /* The reset has already started.
 851                  * Wait for the reset to complete
 852                  */
 853                 wait_event(kbdev->hwaccess.backend.reset_wait,
 854                                 atomic_read(&kbdev->hwaccess.backend.reset_gpu)
 855                                                 == KBASE_RESET_GPU_NOT_PENDING);
 856         }
 857         destroy_hrtimer_on_stack(&reset_data.timer);
 858
 859         dev_dbg(kbdev->dev, "Zap: Finished Context %p", kctx);
 860
 861         /* Ensure that the signallers of the waitqs have finished */
 862         mutex_lock(&kctx->jctx.lock);
 863         mutex_lock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
 864         mutex_unlock(&kctx->jctx.sched_info.ctx.jsctx_mutex);
 865         mutex_unlock(&kctx->jctx.lock);
 866 }
 867
 868 u32 kbase_backend_get_current_flush_id(struct kbase_device *kbdev)
 869 {
 870         u32 flush_id = 0;
 871
 872         if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_FLUSH_REDUCTION)) {
 873                 mutex_lock(&kbdev->pm.lock);
 874                 if (kbdev->pm.backend.gpu_powered)
 875                         flush_id = kbase_reg_read(kbdev,
 876                                         GPU_CONTROL_REG(LATEST_FLUSH), NULL);
 877                 mutex_unlock(&kbdev->pm.lock);
 878         }
 879
 880         return flush_id;
 881 }
 882
 883 int kbase_job_slot_init(struct kbase_device *kbdev)
 884 {
 885 #if KBASE_GPU_RESET_EN
 886         kbdev->hwaccess.backend.reset_workq = alloc_workqueue(
 887                                                 "Mali reset workqueue", 0, 1);
 888         if (NULL == kbdev->hwaccess.backend.reset_workq)
 889                 return -EINVAL;
 890
 891         KBASE_DEBUG_ASSERT(0 ==
 892                 object_is_on_stack(&kbdev->hwaccess.backend.reset_work));
 893         INIT_WORK(&kbdev->hwaccess.backend.reset_work,
 894                                                 kbasep_reset_timeout_worker);
 895
 896         hrtimer_init(&kbdev->hwaccess.backend.reset_timer, CLOCK_MONOTONIC,
 897                                                         HRTIMER_MODE_REL);
 898         kbdev->hwaccess.backend.reset_timer.function =
 899                                                 kbasep_reset_timer_callback;
 900 #endif
 901
 902         return 0;
 903 }
 904 KBASE_EXPORT_TEST_API(kbase_job_slot_init);
 905
 906 void kbase_job_slot_halt(struct kbase_device *kbdev)
 907 {
 908         CSTD_UNUSED(kbdev);
 909 }
 910
 911 void kbase_job_slot_term(struct kbase_device *kbdev)
 912 {
 913 #if KBASE_GPU_RESET_EN
 914         destroy_workqueue(kbdev->hwaccess.backend.reset_workq);
 915 #endif
 916 }
 917 KBASE_EXPORT_TEST_API(kbase_job_slot_term);
 918
 919 #if KBASE_GPU_RESET_EN
 920 /**
 921  * kbasep_check_for_afbc_on_slot() - Check whether AFBC is in use on this slot
 922  * @kbdev: kbase device pointer
 923  * @kctx:  context to check against
 924  * @js:    slot to check
 925  * @target_katom: An atom to check, or NULL if all atoms from @kctx on
 926  *                slot @js should be checked
 927  *
 928  * This checks are based upon parameters that would normally be passed to
 929  * kbase_job_slot_hardstop().
 930  *
 931  * In the event of @target_katom being NULL, this will check the last jobs that
 932  * are likely to be running on the slot to see if a) they belong to kctx, and
 933  * so would be stopped, and b) whether they have AFBC
 934  *
 935  * In that case, It's guaranteed that a job currently executing on the HW with
 936  * AFBC will be detected. However, this is a conservative check because it also
 937  * detects jobs that have just completed too.
 938  *
 939  * Return: true when hard-stop _might_ stop an afbc atom, else false.
 940  */
 941 static bool kbasep_check_for_afbc_on_slot(struct kbase_device *kbdev,
 942                 struct kbase_context *kctx, int js,
 943                 struct kbase_jd_atom *target_katom)
 944 {
 945         bool ret = false;
 946         int i;
 947
 948         lockdep_assert_held(&kbdev->js_data.runpool_irq.lock);
 949
 950         /* When we have an atom the decision can be made straight away. */
 951         if (target_katom)
 952                 return !!(target_katom->core_req & BASE_JD_REQ_FS_AFBC);
 953
 954         /* Otherwise, we must chweck the hardware to see if it has atoms from
 955          * this context with AFBC. */
 956         for (i = 0; i < kbase_backend_nr_atoms_on_slot(kbdev, js); i++) {
 957                 struct kbase_jd_atom *katom;
 958
 959                 katom = kbase_gpu_inspect(kbdev, js, i);
 960                 if (!katom)
 961                         continue;
 962
 963                 /* Ignore atoms from other contexts, they won't be stopped when
 964                  * we use this for checking if we should hard-stop them */
 965                 if (katom->kctx != kctx)
 966                         continue;
 967
 968                 /* An atom on this slot and this context: check for AFBC */
 969                 if (katom->core_req & BASE_JD_REQ_FS_AFBC) {
 970                         ret = true;
 971                         break;
 972                 }
 973         }
 974
 975         return ret;
 976 }
 977 #endif /* KBASE_GPU_RESET_EN */
 978
 979 /**
 980  * kbase_job_slot_softstop_swflags - Soft-stop a job with flags
 981  * @kbdev:         The kbase device
 982  * @js:            The job slot to soft-stop
 983  * @target_katom:  The job that should be soft-stopped (or NULL for any job)
 984  * @sw_flags:      Flags to pass in about the soft-stop
 985  *
 986  * Context:
 987  *   The job slot lock must be held when calling this function.
 988  *   The job slot must not already be in the process of being soft-stopped.
 989  *
 990  * Soft-stop the specified job slot, with extra information about the stop
 991  *
 992  * Where possible any job in the next register is evicted before the soft-stop.
 993  */
 994 void kbase_job_slot_softstop_swflags(struct kbase_device *kbdev, int js,
 995                         struct kbase_jd_atom *target_katom, u32 sw_flags)
 996 {
 997         KBASE_DEBUG_ASSERT(!(sw_flags & JS_COMMAND_MASK));
 998         kbase_backend_soft_hard_stop_slot(kbdev, NULL, js, target_katom,
 999                         JS_COMMAND_SOFT_STOP | sw_flags);
1000 }
1001
1002 /**
1003  * kbase_job_slot_softstop - Soft-stop the specified job slot
1004  * @kbdev:         The kbase device
1005  * @js:            The job slot to soft-stop
1006  * @target_katom:  The job that should be soft-stopped (or NULL for any job)
1007  * Context:
1008  *   The job slot lock must be held when calling this function.
1009  *   The job slot must not already be in the process of being soft-stopped.
1010  *
1011  * Where possible any job in the next register is evicted before the soft-stop.
1012  */
1013 void kbase_job_slot_softstop(struct kbase_device *kbdev, int js,
1014                                 struct kbase_jd_atom *target_katom)
1015 {
1016         kbase_job_slot_softstop_swflags(kbdev, js, target_katom, 0u);
1017 }
1018
1019 /**
1020  * kbase_job_slot_hardstop - Hard-stop the specified job slot
1021  * @kctx:         The kbase context that contains the job(s) that should
1022  *                be hard-stopped
1023  * @js:           The job slot to hard-stop
1024  * @target_katom: The job that should be hard-stopped (or NULL for all
1025  *                jobs from the context)
1026  * Context:
1027  *   The job slot lock must be held when calling this function.
1028  */
1029 void kbase_job_slot_hardstop(struct kbase_context *kctx, int js,
1030                                 struct kbase_jd_atom *target_katom)
1031 {
1032         struct kbase_device *kbdev = kctx->kbdev;
1033         bool stopped;
1034 #if KBASE_GPU_RESET_EN
1035         /* We make the check for AFBC before evicting/stopping atoms.  Note
1036          * that no other thread can modify the slots whilst we have the
1037          * runpool_irq lock. */
1038         int needs_workaround_for_afbc =
1039                         kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3542)
1040                         && kbasep_check_for_afbc_on_slot(kbdev, kctx, js,
1041                                          target_katom);
1042 #endif
1043
1044         stopped = kbase_backend_soft_hard_stop_slot(kbdev, kctx, js,
1045                                                         target_katom,
1046                                                         JS_COMMAND_HARD_STOP);
1047 #if KBASE_GPU_RESET_EN
1048         if (stopped && (kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_8401) ||
1049                         kbase_hw_has_issue(kctx->kbdev, BASE_HW_ISSUE_9510) ||
1050                         needs_workaround_for_afbc)) {
1051                 /* MIDBASE-2916 if a fragment job with AFBC encoding is
1052                  * hardstopped, ensure to do a soft reset also in order to
1053                  * clear the GPU status.
1054                  * Workaround for HW issue 8401 has an issue,so after
1055                  * hard-stopping just reset the GPU. This will ensure that the
1056                  * jobs leave the GPU.*/
1057                 if (kbase_prepare_to_reset_gpu_locked(kbdev)) {
1058                         dev_err(kbdev->dev, "Issueing GPU soft-reset after hard stopping due to hardware issue");
1059                         kbase_reset_gpu_locked(kbdev);
1060                 }
1061         }
1062 #endif
1063 }
1064
1065 /**
1066  * kbase_job_check_enter_disjoint - potentiall enter disjoint mode
1067  * @kbdev: kbase device
1068  * @action: the event which has occurred
1069  * @core_reqs: core requirements of the atom
1070  * @target_katom: the atom which is being affected
1071  *
1072  * For a certain soft/hard-stop action, work out whether to enter disjoint
1073  * state.
1074  *
1075  * This does not register multiple disjoint events if the atom has already
1076  * started a disjoint period
1077  *
1078  * @core_reqs can be supplied as 0 if the atom had not started on the hardware
1079  * (and so a 'real' soft/hard-stop was not required, but it still interrupted
1080  * flow, perhaps on another context)
1081  *
1082  * kbase_job_check_leave_disjoint() should be used to end the disjoint
1083  * state when the soft/hard-stop action is complete
1084  */
1085 void kbase_job_check_enter_disjoint(struct kbase_device *kbdev, u32 action,
1086                 base_jd_core_req core_reqs, struct kbase_jd_atom *target_katom)
1087 {
1088         u32 hw_action = action & JS_COMMAND_MASK;
1089
1090         /* For hard-stop, don't enter if hard-stop not allowed */
1091         if (hw_action == JS_COMMAND_HARD_STOP &&
1092                         !kbasep_hard_stop_allowed(kbdev, core_reqs))
1093                 return;
1094
1095         /* For soft-stop, don't enter if soft-stop not allowed, or isn't
1096          * causing disjoint */
1097         if (hw_action == JS_COMMAND_SOFT_STOP &&
1098                         !(kbasep_soft_stop_allowed(kbdev, target_katom) &&
1099                           (action & JS_COMMAND_SW_CAUSES_DISJOINT)))
1100                 return;
1101
1102         /* Nothing to do if already logged disjoint state on this atom */
1103         if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT)
1104                 return;
1105
1106         target_katom->atom_flags |= KBASE_KATOM_FLAG_IN_DISJOINT;
1107         kbase_disjoint_state_up(kbdev);
1108 }
1109
1110 /**
1111  * kbase_job_check_enter_disjoint - potentially leave disjoint state
1112  * @kbdev: kbase device
1113  * @target_katom: atom which is finishing
1114  *
1115  * Work out whether to leave disjoint state when finishing an atom that was
1116  * originated by kbase_job_check_enter_disjoint().
1117  */
1118 void kbase_job_check_leave_disjoint(struct kbase_device *kbdev,
1119                 struct kbase_jd_atom *target_katom)
1120 {
1121         if (target_katom->atom_flags & KBASE_KATOM_FLAG_IN_DISJOINT) {
1122                 target_katom->atom_flags &= ~KBASE_KATOM_FLAG_IN_DISJOINT;
1123                 kbase_disjoint_state_down(kbdev);
1124         }
1125 }
1126
1127
1128 #if KBASE_GPU_RESET_EN
1129 static void kbase_debug_dump_registers(struct kbase_device *kbdev)
1130 {
1131         int i;
1132
1133         dev_err(kbdev->dev, "Register state:");
1134         dev_err(kbdev->dev, "  GPU_IRQ_RAWSTAT=0x%08x GPU_STATUS=0x%08x",
1135                 kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL),
1136                 kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_STATUS), NULL));
1137         dev_err(kbdev->dev, "  JOB_IRQ_RAWSTAT=0x%08x JOB_IRQ_JS_STATE=0x%08x JOB_IRQ_THROTTLE=0x%08x",
1138                 kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_RAWSTAT), NULL),
1139                 kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_JS_STATE), NULL),
1140                 kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_THROTTLE), NULL));
1141         for (i = 0; i < 3; i++) {
1142                 dev_err(kbdev->dev, "  JS%d_STATUS=0x%08x      JS%d_HEAD_LO=0x%08x",
1143                         i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_STATUS),
1144                                         NULL),
1145                         i, kbase_reg_read(kbdev, JOB_SLOT_REG(i, JS_HEAD_LO),
1146                                         NULL));
1147         }
1148         dev_err(kbdev->dev, "  MMU_IRQ_RAWSTAT=0x%08x GPU_FAULTSTATUS=0x%08x",
1149                 kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_RAWSTAT), NULL),
1150                 kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_FAULTSTATUS), NULL));
1151         dev_err(kbdev->dev, "  GPU_IRQ_MASK=0x%08x    JOB_IRQ_MASK=0x%08x     MMU_IRQ_MASK=0x%08x",
1152                 kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), NULL),
1153                 kbase_reg_read(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), NULL),
1154                 kbase_reg_read(kbdev, MMU_REG(MMU_IRQ_MASK), NULL));
1155         dev_err(kbdev->dev, "  PWR_OVERRIDE0=0x%08x   PWR_OVERRIDE1=0x%08x",
1156                 kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE0), NULL),
1157                 kbase_reg_read(kbdev, GPU_CONTROL_REG(PWR_OVERRIDE1), NULL));
1158         dev_err(kbdev->dev, "  SHADER_CONFIG=0x%08x   L2_MMU_CONFIG=0x%08x",
1159                 kbase_reg_read(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), NULL),
1160                 kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), NULL));
1161 }
1162
1163 static void kbasep_reset_timeout_worker(struct work_struct *data)
1164 {
1165         unsigned long flags, mmu_flags;
1166         struct kbase_device *kbdev;
1167         int i;
1168         ktime_t end_timestamp = ktime_get();
1169         struct kbasep_js_device_data *js_devdata;
1170         bool try_schedule = false;
1171         bool silent = false;
1172
1173         KBASE_DEBUG_ASSERT(data);
1174
1175         kbdev = container_of(data, struct kbase_device,
1176                                                 hwaccess.backend.reset_work);
1177
1178         KBASE_DEBUG_ASSERT(kbdev);
1179         js_devdata = &kbdev->js_data;
1180
1181         if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
1182                         KBASE_RESET_GPU_SILENT)
1183                 silent = true;
1184
1185         KBASE_TRACE_ADD(kbdev, JM_BEGIN_RESET_WORKER, NULL, NULL, 0u, 0);
1186
1187         /* Suspend vinstr.
1188          * This call will block until vinstr is suspended. */
1189         kbase_vinstr_suspend(kbdev->vinstr_ctx);
1190
1191         /* Make sure the timer has completed - this cannot be done from
1192          * interrupt context, so this cannot be done within
1193          * kbasep_try_reset_gpu_early. */
1194         hrtimer_cancel(&kbdev->hwaccess.backend.reset_timer);
1195
1196         if (kbase_pm_context_active_handle_suspend(kbdev,
1197                                 KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE)) {
1198                 /* This would re-activate the GPU. Since it's already idle,
1199                  * there's no need to reset it */
1200                 atomic_set(&kbdev->hwaccess.backend.reset_gpu,
1201                                                 KBASE_RESET_GPU_NOT_PENDING);
1202                 kbase_disjoint_state_down(kbdev);
1203                 wake_up(&kbdev->hwaccess.backend.reset_wait);
1204                 return;
1205         }
1206
1207         KBASE_DEBUG_ASSERT(kbdev->irq_reset_flush == false);
1208
1209         spin_lock_irqsave(&kbdev->mmu_mask_change, mmu_flags);
1210         /* We're about to flush out the IRQs and their bottom half's */
1211         kbdev->irq_reset_flush = true;
1212
1213         /* Disable IRQ to avoid IRQ handlers to kick in after releasing the
1214          * spinlock; this also clears any outstanding interrupts */
1215         spin_lock_irqsave(&kbdev->hwcnt.lock, flags);
1216         kbase_pm_disable_interrupts(kbdev);
1217         spin_unlock_irqrestore(&kbdev->hwcnt.lock, flags);
1218
1219         spin_unlock_irqrestore(&kbdev->mmu_mask_change, mmu_flags);
1220
1221         /* Ensure that any IRQ handlers have finished
1222          * Must be done without any locks IRQ handlers will take */
1223         kbase_synchronize_irqs(kbdev);
1224
1225         /* Flush out any in-flight work items */
1226         kbase_flush_mmu_wqs(kbdev);
1227
1228         /* The flush has completed so reset the active indicator */
1229         kbdev->irq_reset_flush = false;
1230
1231         mutex_lock(&kbdev->pm.lock);
1232         /* We hold the pm lock, so there ought to be a current policy */
1233         KBASE_DEBUG_ASSERT(kbdev->pm.backend.pm_current_policy);
1234
1235         /* All slot have been soft-stopped and we've waited
1236          * SOFT_STOP_RESET_TIMEOUT for the slots to clear, at this point we
1237          * assume that anything that is still left on the GPU is stuck there and
1238          * we'll kill it when we reset the GPU */
1239
1240         if (!silent)
1241                 dev_err(kbdev->dev, "Resetting GPU (allowing up to %d ms)",
1242                                                                 RESET_TIMEOUT);
1243
1244         /* Output the state of some interesting registers to help in the
1245          * debugging of GPU resets */
1246         if (!silent)
1247                 kbase_debug_dump_registers(kbdev);
1248
1249         /* Reset the GPU */
1250         kbase_pm_init_hw(kbdev, 0);
1251
1252         /* Complete any jobs that were still on the GPU */
1253         spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1254         kbase_backend_reset(kbdev, &end_timestamp);
1255         kbase_pm_metrics_update(kbdev, NULL);
1256         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1257
1258         mutex_unlock(&kbdev->pm.lock);
1259
1260         mutex_lock(&js_devdata->runpool_mutex);
1261
1262         /* Reprogram the GPU's MMU */
1263         for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
1264                 struct kbase_as *as = &kbdev->as[i];
1265
1266                 mutex_lock(&as->transaction_mutex);
1267                 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1268
1269                 if (js_devdata->runpool_irq.per_as_data[i].kctx)
1270                         kbase_mmu_update(
1271                                 js_devdata->runpool_irq.per_as_data[i].kctx);
1272                 else
1273                         kbase_mmu_disable_as(kbdev, i);
1274
1275                 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1276                 mutex_unlock(&as->transaction_mutex);
1277         }
1278
1279         kbase_pm_enable_interrupts(kbdev);
1280
1281         atomic_set(&kbdev->hwaccess.backend.reset_gpu,
1282                                                 KBASE_RESET_GPU_NOT_PENDING);
1283
1284         kbase_disjoint_state_down(kbdev);
1285
1286         wake_up(&kbdev->hwaccess.backend.reset_wait);
1287         if (!silent)
1288                 dev_err(kbdev->dev, "Reset complete");
1289
1290         if (js_devdata->nr_contexts_pullable > 0 && !kbdev->poweroff_pending)
1291                 try_schedule = true;
1292
1293         mutex_unlock(&js_devdata->runpool_mutex);
1294
1295         mutex_lock(&kbdev->pm.lock);
1296
1297         /* Find out what cores are required now */
1298         kbase_pm_update_cores_state(kbdev);
1299
1300         /* Synchronously request and wait for those cores, because if
1301          * instrumentation is enabled it would need them immediately. */
1302         kbase_pm_check_transitions_sync(kbdev);
1303
1304         mutex_unlock(&kbdev->pm.lock);
1305
1306         /* Try submitting some jobs to restart processing */
1307         if (try_schedule) {
1308                 KBASE_TRACE_ADD(kbdev, JM_SUBMIT_AFTER_RESET, NULL, NULL, 0u,
1309                                                                         0);
1310                 kbase_js_sched_all(kbdev);
1311         }
1312
1313         kbase_pm_context_idle(kbdev);
1314
1315         /* Release vinstr */
1316         kbase_vinstr_resume(kbdev->vinstr_ctx);
1317
1318         KBASE_TRACE_ADD(kbdev, JM_END_RESET_WORKER, NULL, NULL, 0u, 0);
1319 }
1320
1321 static enum hrtimer_restart kbasep_reset_timer_callback(struct hrtimer *timer)
1322 {
1323         struct kbase_device *kbdev = container_of(timer, struct kbase_device,
1324                                                 hwaccess.backend.reset_timer);
1325
1326         KBASE_DEBUG_ASSERT(kbdev);
1327
1328         /* Reset still pending? */
1329         if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu,
1330                         KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) ==
1331                                                 KBASE_RESET_GPU_COMMITTED)
1332                 queue_work(kbdev->hwaccess.backend.reset_workq,
1333                                         &kbdev->hwaccess.backend.reset_work);
1334
1335         return HRTIMER_NORESTART;
1336 }
1337
1338 /*
1339  * If all jobs are evicted from the GPU then we can reset the GPU
1340  * immediately instead of waiting for the timeout to elapse
1341  */
1342
1343 static void kbasep_try_reset_gpu_early_locked(struct kbase_device *kbdev)
1344 {
1345         int i;
1346         int pending_jobs = 0;
1347
1348         KBASE_DEBUG_ASSERT(kbdev);
1349
1350         /* Count the number of jobs */
1351         for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
1352                 pending_jobs += kbase_backend_nr_atoms_submitted(kbdev, i);
1353
1354         if (pending_jobs > 0) {
1355                 /* There are still jobs on the GPU - wait */
1356                 return;
1357         }
1358
1359         /* To prevent getting incorrect registers when dumping failed job,
1360          * skip early reset.
1361          */
1362         if (kbdev->job_fault_debug != false)
1363                 return;
1364
1365         /* Check that the reset has been committed to (i.e. kbase_reset_gpu has
1366          * been called), and that no other thread beat this thread to starting
1367          * the reset */
1368         if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu,
1369                         KBASE_RESET_GPU_COMMITTED, KBASE_RESET_GPU_HAPPENING) !=
1370                                                 KBASE_RESET_GPU_COMMITTED) {
1371                 /* Reset has already occurred */
1372                 return;
1373         }
1374
1375         queue_work(kbdev->hwaccess.backend.reset_workq,
1376                                         &kbdev->hwaccess.backend.reset_work);
1377 }
1378
1379 static void kbasep_try_reset_gpu_early(struct kbase_device *kbdev)
1380 {
1381         unsigned long flags;
1382         struct kbasep_js_device_data *js_devdata;
1383
1384         js_devdata = &kbdev->js_data;
1385         spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1386         kbasep_try_reset_gpu_early_locked(kbdev);
1387         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1388 }
1389
1390 /**
1391  * kbase_prepare_to_reset_gpu_locked - Prepare for resetting the GPU
1392  * @kbdev: kbase device
1393  *
1394  * This function just soft-stops all the slots to ensure that as many jobs as
1395  * possible are saved.
1396  *
1397  * Return:
1398  *   The function returns a boolean which should be interpreted as follows:
1399  *   true - Prepared for reset, kbase_reset_gpu_locked should be called.
1400  *   false - Another thread is performing a reset, kbase_reset_gpu should
1401  *   not be called.
1402  */
1403 bool kbase_prepare_to_reset_gpu_locked(struct kbase_device *kbdev)
1404 {
1405         int i;
1406
1407         KBASE_DEBUG_ASSERT(kbdev);
1408
1409         if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu,
1410                                                 KBASE_RESET_GPU_NOT_PENDING,
1411                                                 KBASE_RESET_GPU_PREPARED) !=
1412                                                 KBASE_RESET_GPU_NOT_PENDING) {
1413                 /* Some other thread is already resetting the GPU */
1414                 return false;
1415         }
1416
1417         kbase_disjoint_state_up(kbdev);
1418
1419         for (i = 0; i < kbdev->gpu_props.num_job_slots; i++)
1420                 kbase_job_slot_softstop(kbdev, i, NULL);
1421
1422         return true;
1423 }
1424
1425 bool kbase_prepare_to_reset_gpu(struct kbase_device *kbdev)
1426 {
1427         unsigned long flags;
1428         bool ret;
1429         struct kbasep_js_device_data *js_devdata;
1430
1431         js_devdata = &kbdev->js_data;
1432         spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
1433         ret = kbase_prepare_to_reset_gpu_locked(kbdev);
1434         spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
1435
1436         return ret;
1437 }
1438 KBASE_EXPORT_TEST_API(kbase_prepare_to_reset_gpu);
1439
1440 /*
1441  * This function should be called after kbase_prepare_to_reset_gpu if it
1442  * returns true. It should never be called without a corresponding call to
1443  * kbase_prepare_to_reset_gpu.
1444  *
1445  * After this function is called (or not called if kbase_prepare_to_reset_gpu
1446  * returned false), the caller should wait for
1447  * kbdev->hwaccess.backend.reset_waitq to be signalled to know when the reset
1448  * has completed.
1449  */
1450 void kbase_reset_gpu(struct kbase_device *kbdev)
1451 {
1452         KBASE_DEBUG_ASSERT(kbdev);
1453
1454         /* Note this is an assert/atomic_set because it is a software issue for
1455          * a race to be occuring here */
1456         KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
1457                                                 KBASE_RESET_GPU_PREPARED);
1458         atomic_set(&kbdev->hwaccess.backend.reset_gpu,
1459                                                 KBASE_RESET_GPU_COMMITTED);
1460
1461         dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n",
1462                         kbdev->reset_timeout_ms);
1463
1464         hrtimer_start(&kbdev->hwaccess.backend.reset_timer,
1465                         HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms),
1466                         HRTIMER_MODE_REL);
1467
1468         /* Try resetting early */
1469         kbasep_try_reset_gpu_early(kbdev);
1470 }
1471 KBASE_EXPORT_TEST_API(kbase_reset_gpu);
1472
1473 void kbase_reset_gpu_locked(struct kbase_device *kbdev)
1474 {
1475         KBASE_DEBUG_ASSERT(kbdev);
1476
1477         /* Note this is an assert/atomic_set because it is a software issue for
1478          * a race to be occuring here */
1479         KBASE_DEBUG_ASSERT(atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
1480                                                 KBASE_RESET_GPU_PREPARED);
1481         atomic_set(&kbdev->hwaccess.backend.reset_gpu,
1482                                                 KBASE_RESET_GPU_COMMITTED);
1483
1484         dev_err(kbdev->dev, "Preparing to soft-reset GPU: Waiting (upto %d ms) for all jobs to complete soft-stop\n",
1485                         kbdev->reset_timeout_ms);
1486         hrtimer_start(&kbdev->hwaccess.backend.reset_timer,
1487                         HR_TIMER_DELAY_MSEC(kbdev->reset_timeout_ms),
1488                         HRTIMER_MODE_REL);
1489
1490         /* Try resetting early */
1491         kbasep_try_reset_gpu_early_locked(kbdev);
1492 }
1493
1494 void kbase_reset_gpu_silent(struct kbase_device *kbdev)
1495 {
1496         if (atomic_cmpxchg(&kbdev->hwaccess.backend.reset_gpu,
1497                                                 KBASE_RESET_GPU_NOT_PENDING,
1498                                                 KBASE_RESET_GPU_SILENT) !=
1499                                                 KBASE_RESET_GPU_NOT_PENDING) {
1500                 /* Some other thread is already resetting the GPU */
1501                 return;
1502         }
1503
1504         kbase_disjoint_state_up(kbdev);
1505
1506         queue_work(kbdev->hwaccess.backend.reset_workq,
1507                         &kbdev->hwaccess.backend.reset_work);
1508 }
1509
1510 bool kbase_reset_gpu_active(struct kbase_device *kbdev)
1511 {
1512         if (atomic_read(&kbdev->hwaccess.backend.reset_gpu) ==
1513                         KBASE_RESET_GPU_NOT_PENDING)
1514                 return false;
1515
1516         return true;
1517 }
1518 #endif /* KBASE_GPU_RESET_EN */