drivers/gpu/arm/midgard/backend/gpu/mali_kbase_js_backend.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2014-2016 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18
  19 /*
  20  * Register-based HW access backend specific job scheduler APIs
  21  */
  22
  23 #include <mali_kbase.h>
  24 #include <mali_kbase_hwaccess_jm.h>
  25 #include <backend/gpu/mali_kbase_jm_internal.h>
  26 #include <backend/gpu/mali_kbase_js_internal.h>
  27
  28 /*
  29  * Define for when dumping is enabled.
  30  * This should not be based on the instrumentation level as whether dumping is
  31  * enabled for a particular level is down to the integrator. However this is
  32  * being used for now as otherwise the cinstr headers would be needed.
  33  */
  34 #define CINSTR_DUMPING_ENABLED (2 == MALI_INSTRUMENTATION_LEVEL)
  35
  36 /*
  37  * Hold the runpool_mutex for this
  38  */
  39 static inline bool timer_callback_should_run(struct kbase_device *kbdev)
  40 {
  41         struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
  42         s8 nr_running_ctxs;
  43
  44         lockdep_assert_held(&kbdev->js_data.runpool_mutex);
  45
  46         /* Timer must stop if we are suspending */
  47         if (backend->suspend_timer)
  48                 return false;
  49
  50         /* nr_contexts_pullable is updated with the runpool_mutex. However, the
  51          * locking in the caller gives us a barrier that ensures
  52          * nr_contexts_pullable is up-to-date for reading */
  53         nr_running_ctxs = atomic_read(&kbdev->js_data.nr_contexts_runnable);
  54
  55 #ifdef CONFIG_MALI_DEBUG
  56         if (kbdev->js_data.softstop_always) {
  57                 /* Debug support for allowing soft-stop on a single context */
  58                 return true;
  59         }
  60 #endif                          /* CONFIG_MALI_DEBUG */
  61
  62         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_9435)) {
  63                 /* Timeouts would have to be 4x longer (due to micro-
  64                  * architectural design) to support OpenCL conformance tests, so
  65                  * only run the timer when there's:
  66                  * - 2 or more CL contexts
  67                  * - 1 or more GLES contexts
  68                  *
  69                  * NOTE: We will treat a context that has both Compute and Non-
  70                  * Compute jobs will be treated as an OpenCL context (hence, we
  71                  * don't check KBASEP_JS_CTX_ATTR_NON_COMPUTE).
  72                  */
  73                 {
  74                         s8 nr_compute_ctxs =
  75                                 kbasep_js_ctx_attr_count_on_runpool(kbdev,
  76                                                 KBASEP_JS_CTX_ATTR_COMPUTE);
  77                         s8 nr_noncompute_ctxs = nr_running_ctxs -
  78                                                         nr_compute_ctxs;
  79
  80                         return (bool) (nr_compute_ctxs >= 2 ||
  81                                                         nr_noncompute_ctxs > 0);
  82                 }
  83         } else {
  84                 /* Run the timer callback whenever you have at least 1 context
  85                  */
  86                 return (bool) (nr_running_ctxs > 0);
  87         }
  88 }
  89
  90 static enum hrtimer_restart timer_callback(struct hrtimer *timer)
  91 {
  92         unsigned long flags;
  93         struct kbase_device *kbdev;
  94         struct kbasep_js_device_data *js_devdata;
  95         struct kbase_backend_data *backend;
  96         int s;
  97         bool reset_needed = false;
  98
  99         KBASE_DEBUG_ASSERT(timer != NULL);
 100
 101         backend = container_of(timer, struct kbase_backend_data,
 102                                                         scheduling_timer);
 103         kbdev = container_of(backend, struct kbase_device, hwaccess.backend);
 104         js_devdata = &kbdev->js_data;
 105
 106         /* Loop through the slots */
 107         spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 108         for (s = 0; s < kbdev->gpu_props.num_job_slots; s++) {
 109                 struct kbase_jd_atom *atom = NULL;
 110
 111                 if (kbase_backend_nr_atoms_on_slot(kbdev, s) > 0) {
 112                         atom = kbase_gpu_inspect(kbdev, s, 0);
 113                         KBASE_DEBUG_ASSERT(atom != NULL);
 114                 }
 115
 116                 if (atom != NULL) {
 117                         /* The current version of the model doesn't support
 118                          * Soft-Stop */
 119                         if (!kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_5736)) {
 120                                 u32 ticks = atom->sched_info.cfs.ticks++;
 121
 122 #if !CINSTR_DUMPING_ENABLED
 123                                 u32 soft_stop_ticks, hard_stop_ticks,
 124                                                                 gpu_reset_ticks;
 125                                 if (atom->core_req & BASE_JD_REQ_ONLY_COMPUTE) {
 126                                         soft_stop_ticks =
 127                                                 js_devdata->soft_stop_ticks_cl;
 128                                         hard_stop_ticks =
 129                                                 js_devdata->hard_stop_ticks_cl;
 130                                         gpu_reset_ticks =
 131                                                 js_devdata->gpu_reset_ticks_cl;
 132                                 } else {
 133                                         soft_stop_ticks =
 134                                                 js_devdata->soft_stop_ticks;
 135                                         hard_stop_ticks =
 136                                                 js_devdata->hard_stop_ticks_ss;
 137                                         gpu_reset_ticks =
 138                                                 js_devdata->gpu_reset_ticks_ss;
 139                                 }
 140
 141                                 /* If timeouts have been changed then ensure
 142                                  * that atom tick count is not greater than the
 143                                  * new soft_stop timeout. This ensures that
 144                                  * atoms do not miss any of the timeouts due to
 145                                  * races between this worker and the thread
 146                                  * changing the timeouts. */
 147                                 if (backend->timeouts_updated &&
 148                                                 ticks > soft_stop_ticks)
 149                                         ticks = atom->sched_info.cfs.ticks =
 150                                                         soft_stop_ticks;
 151
 152                                 /* Job is Soft-Stoppable */
 153                                 if (ticks == soft_stop_ticks) {
 154                                         int disjoint_threshold =
 155                 KBASE_DISJOINT_STATE_INTERLEAVED_CONTEXT_COUNT_THRESHOLD;
 156                                         u32 softstop_flags = 0u;
 157                                         /* Job has been scheduled for at least
 158                                          * js_devdata->soft_stop_ticks ticks.
 159                                          * Soft stop the slot so we can run
 160                                          * other jobs.
 161                                          */
 162                                         dev_dbg(kbdev->dev, "Soft-stop");
 163 #if !KBASE_DISABLE_SCHEDULING_SOFT_STOPS
 164                                         /* nr_user_contexts_running is updated
 165                                          * with the runpool_mutex, but we can't
 166                                          * take that here.
 167                                          *
 168                                          * However, if it's about to be
 169                                          * increased then the new context can't
 170                                          * run any jobs until they take the
 171                                          * hwaccess_lock, so it's OK to observe
 172                                          * the older value.
 173                                          *
 174                                          * Similarly, if it's about to be
 175                                          * decreased, the last job from another
 176                                          * context has already finished, so it's
 177                                          * not too bad that we observe the older
 178                                          * value and register a disjoint event
 179                                          * when we try soft-stopping */
 180                                         if (js_devdata->nr_user_contexts_running
 181                                                         >= disjoint_threshold)
 182                                                 softstop_flags |=
 183                                                 JS_COMMAND_SW_CAUSES_DISJOINT;
 184
 185                                         kbase_job_slot_softstop_swflags(kbdev,
 186                                                 s, atom, softstop_flags);
 187 #endif
 188                                 } else if (ticks == hard_stop_ticks) {
 189                                         /* Job has been scheduled for at least
 190                                          * js_devdata->hard_stop_ticks_ss ticks.
 191                                          * It should have been soft-stopped by
 192                                          * now. Hard stop the slot.
 193                                          */
 194 #if !KBASE_DISABLE_SCHEDULING_HARD_STOPS
 195                                         int ms =
 196                                                 js_devdata->scheduling_period_ns
 197                                                                 / 1000000u;
 198                                         dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)",
 199                                                         (unsigned long)ticks,
 200                                                         (unsigned long)ms);
 201                                         kbase_job_slot_hardstop(atom->kctx, s,
 202                                                                         atom);
 203 #endif
 204                                 } else if (ticks == gpu_reset_ticks) {
 205                                         /* Job has been scheduled for at least
 206                                          * js_devdata->gpu_reset_ticks_ss ticks.
 207                                          * It should have left the GPU by now.
 208                                          * Signal that the GPU needs to be
 209                                          * reset.
 210                                          */
 211                                         reset_needed = true;
 212                                 }
 213 #else                           /* !CINSTR_DUMPING_ENABLED */
 214                                 /* NOTE: During CINSTR_DUMPING_ENABLED, we use
 215                                  * the alternate timeouts, which makes the hard-
 216                                  * stop and GPU reset timeout much longer. We
 217                                  * also ensure that we don't soft-stop at all.
 218                                  */
 219                                 if (ticks == js_devdata->soft_stop_ticks) {
 220                                         /* Job has been scheduled for at least
 221                                          * js_devdata->soft_stop_ticks. We do
 222                                          * not soft-stop during
 223                                          * CINSTR_DUMPING_ENABLED, however.
 224                                          */
 225                                         dev_dbg(kbdev->dev, "Soft-stop");
 226                                 } else if (ticks ==
 227                                         js_devdata->hard_stop_ticks_dumping) {
 228                                         /* Job has been scheduled for at least
 229                                          * js_devdata->hard_stop_ticks_dumping
 230                                          * ticks. Hard stop the slot.
 231                                          */
 232 #if !KBASE_DISABLE_SCHEDULING_HARD_STOPS
 233                                         int ms =
 234                                                 js_devdata->scheduling_period_ns
 235                                                                 / 1000000u;
 236                                         dev_warn(kbdev->dev, "JS: Job Hard-Stopped (took more than %lu ticks at %lu ms/tick)",
 237                                                         (unsigned long)ticks,
 238                                                         (unsigned long)ms);
 239                                         kbase_job_slot_hardstop(atom->kctx, s,
 240                                                                         atom);
 241 #endif
 242                                 } else if (ticks ==
 243                                         js_devdata->gpu_reset_ticks_dumping) {
 244                                         /* Job has been scheduled for at least
 245                                          * js_devdata->gpu_reset_ticks_dumping
 246                                          * ticks. It should have left the GPU by
 247                                          * now. Signal that the GPU needs to be
 248                                          * reset.
 249                                          */
 250                                         reset_needed = true;
 251                                 }
 252 #endif                          /* !CINSTR_DUMPING_ENABLED */
 253                         }
 254                 }
 255         }
 256 #if KBASE_GPU_RESET_EN
 257         if (reset_needed) {
 258                 dev_err(kbdev->dev, "JS: Job has been on the GPU for too long (JS_RESET_TICKS_SS/DUMPING timeout hit). Issueing GPU soft-reset to resolve.");
 259
 260                 if (kbase_prepare_to_reset_gpu_locked(kbdev))
 261                         kbase_reset_gpu_locked(kbdev);
 262         }
 263 #endif /* KBASE_GPU_RESET_EN */
 264         /* the timer is re-issued if there is contexts in the run-pool */
 265
 266         if (backend->timer_running)
 267                 hrtimer_start(&backend->scheduling_timer,
 268                         HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns),
 269                         HRTIMER_MODE_REL);
 270
 271         backend->timeouts_updated = false;
 272
 273         spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 274
 275         return HRTIMER_NORESTART;
 276 }
 277
 278 void kbase_backend_ctx_count_changed(struct kbase_device *kbdev)
 279 {
 280         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
 281         struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
 282         unsigned long flags;
 283
 284         lockdep_assert_held(&js_devdata->runpool_mutex);
 285
 286         if (!timer_callback_should_run(kbdev)) {
 287                 /* Take spinlock to force synchronisation with timer */
 288                 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 289                 backend->timer_running = false;
 290                 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 291                 /* From now on, return value of timer_callback_should_run() will
 292                  * also cause the timer to not requeue itself. Its return value
 293                  * cannot change, because it depends on variables updated with
 294                  * the runpool_mutex held, which the caller of this must also
 295                  * hold */
 296                 hrtimer_cancel(&backend->scheduling_timer);
 297         }
 298
 299         if (timer_callback_should_run(kbdev) && !backend->timer_running) {
 300                 /* Take spinlock to force synchronisation with timer */
 301                 spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 302                 backend->timer_running = true;
 303                 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 304                 hrtimer_start(&backend->scheduling_timer,
 305                         HR_TIMER_DELAY_NSEC(js_devdata->scheduling_period_ns),
 306                                                         HRTIMER_MODE_REL);
 307
 308                 KBASE_TRACE_ADD(kbdev, JS_POLICY_TIMER_START, NULL, NULL, 0u,
 309                                                                         0u);
 310         }
 311 }
 312
 313 int kbase_backend_timer_init(struct kbase_device *kbdev)
 314 {
 315         struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
 316
 317         hrtimer_init(&backend->scheduling_timer, CLOCK_MONOTONIC,
 318                                                         HRTIMER_MODE_REL);
 319         backend->scheduling_timer.function = timer_callback;
 320
 321         backend->timer_running = false;
 322
 323         return 0;
 324 }
 325
 326 void kbase_backend_timer_term(struct kbase_device *kbdev)
 327 {
 328         struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
 329
 330         hrtimer_cancel(&backend->scheduling_timer);
 331 }
 332
 333 void kbase_backend_timer_suspend(struct kbase_device *kbdev)
 334 {
 335         struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
 336
 337         backend->suspend_timer = true;
 338
 339         kbase_backend_ctx_count_changed(kbdev);
 340 }
 341
 342 void kbase_backend_timer_resume(struct kbase_device *kbdev)
 343 {
 344         struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
 345
 346         backend->suspend_timer = false;
 347
 348         kbase_backend_ctx_count_changed(kbdev);
 349 }
 350
 351 void kbase_backend_timeouts_changed(struct kbase_device *kbdev)
 352 {
 353         struct kbase_backend_data *backend = &kbdev->hwaccess.backend;
 354
 355         backend->timeouts_updated = true;
 356 }
 357