drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_backend.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18
  19 /*
  20  * GPU backend implementation of base kernel power management APIs
  21  */
  22
  23 #include <mali_kbase.h>
  24 #include <mali_midg_regmap.h>
  25 #include <mali_kbase_config_defaults.h>
  26 #ifdef CONFIG_MALI_PLATFORM_DEVICETREE
  27 #include <linux/pm_runtime.h>
  28 #endif /* CONFIG_MALI_PLATFORM_DEVICETREE */
  29
  30 #include <mali_kbase_pm.h>
  31 #include <mali_kbase_hwaccess_jm.h>
  32 #include <backend/gpu/mali_kbase_js_internal.h>
  33 #include <backend/gpu/mali_kbase_pm_internal.h>
  34
  35 static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data);
  36
  37 void kbase_pm_register_access_enable(struct kbase_device *kbdev)
  38 {
  39         struct kbase_pm_callback_conf *callbacks;
  40
  41         callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS;
  42
  43         if (callbacks)
  44                 callbacks->power_on_callback(kbdev);
  45
  46         kbdev->pm.backend.gpu_powered = true;
  47 }
  48
  49 void kbase_pm_register_access_disable(struct kbase_device *kbdev)
  50 {
  51         struct kbase_pm_callback_conf *callbacks;
  52
  53         callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS;
  54
  55         if (callbacks)
  56                 callbacks->power_off_callback(kbdev);
  57
  58         kbdev->pm.backend.gpu_powered = false;
  59 }
  60
  61 int kbase_hwaccess_pm_init(struct kbase_device *kbdev)
  62 {
  63         int ret = 0;
  64         struct kbase_pm_callback_conf *callbacks;
  65
  66         KBASE_DEBUG_ASSERT(kbdev != NULL);
  67
  68         mutex_init(&kbdev->pm.lock);
  69
  70         kbdev->pm.backend.gpu_poweroff_wait_wq = alloc_workqueue("kbase_pm_poweroff_wait",
  71                         WQ_HIGHPRI | WQ_UNBOUND, 1);
  72         if (!kbdev->pm.backend.gpu_poweroff_wait_wq)
  73                 return -ENOMEM;
  74
  75         INIT_WORK(&kbdev->pm.backend.gpu_poweroff_wait_work,
  76                         kbase_pm_gpu_poweroff_wait_wq);
  77
  78         kbdev->pm.backend.gpu_powered = false;
  79         kbdev->pm.suspending = false;
  80 #ifdef CONFIG_MALI_DEBUG
  81         kbdev->pm.backend.driver_ready_for_irqs = false;
  82 #endif /* CONFIG_MALI_DEBUG */
  83         kbdev->pm.backend.gpu_in_desired_state = true;
  84         init_waitqueue_head(&kbdev->pm.backend.gpu_in_desired_state_wait);
  85
  86         callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS;
  87         if (callbacks) {
  88                 kbdev->pm.backend.callback_power_on =
  89                                         callbacks->power_on_callback;
  90                 kbdev->pm.backend.callback_power_off =
  91                                         callbacks->power_off_callback;
  92                 kbdev->pm.backend.callback_power_suspend =
  93                                         callbacks->power_suspend_callback;
  94                 kbdev->pm.backend.callback_power_resume =
  95                                         callbacks->power_resume_callback;
  96                 kbdev->pm.callback_power_runtime_init =
  97                                         callbacks->power_runtime_init_callback;
  98                 kbdev->pm.callback_power_runtime_term =
  99                                         callbacks->power_runtime_term_callback;
 100                 kbdev->pm.backend.callback_power_runtime_on =
 101                                         callbacks->power_runtime_on_callback;
 102                 kbdev->pm.backend.callback_power_runtime_off =
 103                                         callbacks->power_runtime_off_callback;
 104                 kbdev->pm.backend.callback_power_runtime_idle =
 105                                         callbacks->power_runtime_idle_callback;
 106         } else {
 107                 kbdev->pm.backend.callback_power_on = NULL;
 108                 kbdev->pm.backend.callback_power_off = NULL;
 109                 kbdev->pm.backend.callback_power_suspend = NULL;
 110                 kbdev->pm.backend.callback_power_resume = NULL;
 111                 kbdev->pm.callback_power_runtime_init = NULL;
 112                 kbdev->pm.callback_power_runtime_term = NULL;
 113                 kbdev->pm.backend.callback_power_runtime_on = NULL;
 114                 kbdev->pm.backend.callback_power_runtime_off = NULL;
 115                 kbdev->pm.backend.callback_power_runtime_idle = NULL;
 116         }
 117
 118         /* Initialise the metrics subsystem */
 119         ret = kbasep_pm_metrics_init(kbdev);
 120         if (ret)
 121                 return ret;
 122
 123         init_waitqueue_head(&kbdev->pm.backend.l2_powered_wait);
 124         kbdev->pm.backend.l2_powered = 0;
 125
 126         init_waitqueue_head(&kbdev->pm.backend.reset_done_wait);
 127         kbdev->pm.backend.reset_done = false;
 128
 129         init_waitqueue_head(&kbdev->pm.zero_active_count_wait);
 130         kbdev->pm.active_count = 0;
 131
 132         spin_lock_init(&kbdev->pm.backend.gpu_cycle_counter_requests_lock);
 133         spin_lock_init(&kbdev->pm.backend.gpu_powered_lock);
 134
 135         init_waitqueue_head(&kbdev->pm.backend.poweroff_wait);
 136
 137         if (kbase_pm_ca_init(kbdev) != 0)
 138                 goto workq_fail;
 139
 140         if (kbase_pm_policy_init(kbdev) != 0)
 141                 goto pm_policy_fail;
 142
 143         return 0;
 144
 145 pm_policy_fail:
 146         kbase_pm_ca_term(kbdev);
 147 workq_fail:
 148         kbasep_pm_metrics_term(kbdev);
 149         return -EINVAL;
 150 }
 151
 152 void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume)
 153 {
 154         lockdep_assert_held(&kbdev->pm.lock);
 155
 156         /* Turn clocks and interrupts on - no-op if we haven't done a previous
 157          * kbase_pm_clock_off() */
 158         kbase_pm_clock_on(kbdev, is_resume);
 159
 160         /* Update core status as required by the policy */
 161         KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 162                                 SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_START);
 163         kbase_pm_update_cores_state(kbdev);
 164         KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 165                                 SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_END);
 166
 167         /* NOTE: We don't wait to reach the desired state, since running atoms
 168          * will wait for that state to be reached anyway */
 169 }
 170
 171 static void kbase_pm_gpu_poweroff_wait_wq(struct work_struct *data)
 172 {
 173         struct kbase_device *kbdev = container_of(data, struct kbase_device,
 174                         pm.backend.gpu_poweroff_wait_work);
 175         struct kbase_pm_device_data *pm = &kbdev->pm;
 176         struct kbase_pm_backend_data *backend = &pm->backend;
 177         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
 178         unsigned long flags;
 179
 180         /* Wait for power transitions to complete. We do this with no locks held
 181          * so that we don't deadlock with any pending workqueues */
 182         KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 183                                 SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_START);
 184         kbase_pm_check_transitions_sync(kbdev);
 185         KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 186                                 SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_END);
 187
 188         mutex_lock(&js_devdata->runpool_mutex);
 189         mutex_lock(&kbdev->pm.lock);
 190
 191         if (!backend->poweron_required) {
 192                 WARN_ON(kbdev->l2_available_bitmap ||
 193                                 kbdev->shader_available_bitmap ||
 194                                 kbdev->tiler_available_bitmap);
 195
 196                 /* Consume any change-state events */
 197                 kbase_timeline_pm_check_handle_event(kbdev,
 198                                         KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
 199
 200                 /* Disable interrupts and turn the clock off */
 201                 if (!kbase_pm_clock_off(kbdev, backend->poweroff_is_suspend)) {
 202                         /*
 203                          * Page/bus faults are pending, must drop locks to
 204                          * process.  Interrupts are disabled so no more faults
 205                          * should be generated at this point.
 206                          */
 207                         mutex_unlock(&kbdev->pm.lock);
 208                         mutex_unlock(&js_devdata->runpool_mutex);
 209                         kbase_flush_mmu_wqs(kbdev);
 210                         mutex_lock(&js_devdata->runpool_mutex);
 211                         mutex_lock(&kbdev->pm.lock);
 212
 213                         /* Turn off clock now that fault have been handled. We
 214                          * dropped locks so poweron_required may have changed -
 215                          * power back on if this is the case.*/
 216                         if (backend->poweron_required)
 217                                 kbase_pm_clock_on(kbdev, false);
 218                         else
 219                                 WARN_ON(!kbase_pm_clock_off(kbdev,
 220                                                 backend->poweroff_is_suspend));
 221                 }
 222         }
 223
 224         spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 225         backend->poweroff_wait_in_progress = false;
 226         if (backend->poweron_required) {
 227                 backend->poweron_required = false;
 228                 kbase_pm_update_cores_state_nolock(kbdev);
 229         }
 230         spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 231
 232         mutex_unlock(&kbdev->pm.lock);
 233         mutex_unlock(&js_devdata->runpool_mutex);
 234
 235         wake_up(&kbdev->pm.backend.poweroff_wait);
 236 }
 237
 238 void kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend)
 239 {
 240         unsigned long flags;
 241
 242         lockdep_assert_held(&kbdev->pm.lock);
 243
 244         spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 245         if (!kbdev->pm.backend.poweroff_wait_in_progress) {
 246                 /* Force all cores off */
 247                 kbdev->pm.backend.desired_shader_state = 0;
 248                 kbdev->pm.backend.desired_tiler_state = 0;
 249
 250                 /* Force all cores to be unavailable, in the situation where
 251                  * transitions are in progress for some cores but not others,
 252                  * and kbase_pm_check_transitions_nolock can not immediately
 253                  * power off the cores */
 254                 kbdev->shader_available_bitmap = 0;
 255                 kbdev->tiler_available_bitmap = 0;
 256                 kbdev->l2_available_bitmap = 0;
 257
 258                 kbdev->pm.backend.poweroff_wait_in_progress = true;
 259                 kbdev->pm.backend.poweroff_is_suspend = is_suspend;
 260
 261                 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 262                 /*Kick off wq here. Callers will have to wait*/
 263                 queue_work(kbdev->pm.backend.gpu_poweroff_wait_wq,
 264                                 &kbdev->pm.backend.gpu_poweroff_wait_work);
 265         } else {
 266                 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 267         }
 268 }
 269
 270 static bool is_poweroff_in_progress(struct kbase_device *kbdev)
 271 {
 272         bool ret;
 273         unsigned long flags;
 274
 275         spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 276         ret = (kbdev->pm.backend.poweroff_wait_in_progress == false);
 277         spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 278
 279         return ret;
 280 }
 281
 282 void kbase_pm_wait_for_poweroff_complete(struct kbase_device *kbdev)
 283 {
 284         wait_event_killable(kbdev->pm.backend.poweroff_wait,
 285                         is_poweroff_in_progress(kbdev));
 286 }
 287
 288 int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev,
 289                 unsigned int flags)
 290 {
 291         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
 292         unsigned long irq_flags;
 293         int ret;
 294
 295         KBASE_DEBUG_ASSERT(kbdev != NULL);
 296
 297         mutex_lock(&js_devdata->runpool_mutex);
 298         mutex_lock(&kbdev->pm.lock);
 299
 300         /* A suspend won't happen during startup/insmod */
 301         KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
 302
 303         /* Power up the GPU, don't enable IRQs as we are not ready to receive
 304          * them. */
 305         ret = kbase_pm_init_hw(kbdev, flags);
 306         if (ret) {
 307                 mutex_unlock(&kbdev->pm.lock);
 308                 mutex_unlock(&js_devdata->runpool_mutex);
 309                 return ret;
 310         }
 311
 312         kbasep_pm_read_present_cores(kbdev);
 313
 314         kbdev->pm.debug_core_mask_all = kbdev->pm.debug_core_mask[0] =
 315                         kbdev->pm.debug_core_mask[1] =
 316                         kbdev->pm.debug_core_mask[2] =
 317                         kbdev->gpu_props.props.raw_props.shader_present;
 318
 319         /* Pretend the GPU is active to prevent a power policy turning the GPU
 320          * cores off */
 321         kbdev->pm.active_count = 1;
 322
 323         spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock,
 324                                                                 irq_flags);
 325         /* Ensure cycle counter is off */
 326         kbdev->pm.backend.gpu_cycle_counter_requests = 0;
 327         spin_unlock_irqrestore(
 328                         &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
 329                                                                 irq_flags);
 330
 331         /* We are ready to receive IRQ's now as power policy is set up, so
 332          * enable them now. */
 333 #ifdef CONFIG_MALI_DEBUG
 334         spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, irq_flags);
 335         kbdev->pm.backend.driver_ready_for_irqs = true;
 336         spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, irq_flags);
 337 #endif
 338         kbase_pm_enable_interrupts(kbdev);
 339
 340         /* Turn on the GPU and any cores needed by the policy */
 341         kbase_pm_do_poweron(kbdev, false);
 342         mutex_unlock(&kbdev->pm.lock);
 343         mutex_unlock(&js_devdata->runpool_mutex);
 344
 345         /* Idle the GPU and/or cores, if the policy wants it to */
 346         kbase_pm_context_idle(kbdev);
 347
 348         return 0;
 349 }
 350
 351 void kbase_hwaccess_pm_halt(struct kbase_device *kbdev)
 352 {
 353         KBASE_DEBUG_ASSERT(kbdev != NULL);
 354
 355         mutex_lock(&kbdev->pm.lock);
 356         kbase_pm_cancel_deferred_poweroff(kbdev);
 357         kbase_pm_do_poweroff(kbdev, false);
 358         mutex_unlock(&kbdev->pm.lock);
 359 }
 360
 361 KBASE_EXPORT_TEST_API(kbase_hwaccess_pm_halt);
 362
 363 void kbase_hwaccess_pm_term(struct kbase_device *kbdev)
 364 {
 365         KBASE_DEBUG_ASSERT(kbdev != NULL);
 366         KBASE_DEBUG_ASSERT(kbdev->pm.active_count == 0);
 367         KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests == 0);
 368
 369         /* Free any resources the policy allocated */
 370         kbase_pm_policy_term(kbdev);
 371         kbase_pm_ca_term(kbdev);
 372
 373         /* Shut down the metrics subsystem */
 374         kbasep_pm_metrics_term(kbdev);
 375
 376         destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wait_wq);
 377 }
 378
 379 void kbase_pm_power_changed(struct kbase_device *kbdev)
 380 {
 381         bool cores_are_available;
 382         unsigned long flags;
 383
 384         KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 385                                 SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_START);
 386         spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 387         cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
 388         KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 389                                 SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_END);
 390
 391         if (cores_are_available) {
 392                 /* Log timelining information that a change in state has
 393                  * completed */
 394                 kbase_timeline_pm_handle_event(kbdev,
 395                                 KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
 396
 397                 kbase_backend_slot_update(kbdev);
 398         }
 399         spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 400 }
 401
 402 void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev,
 403                 u64 new_core_mask_js0, u64 new_core_mask_js1,
 404                 u64 new_core_mask_js2)
 405 {
 406         kbdev->pm.debug_core_mask[0] = new_core_mask_js0;
 407         kbdev->pm.debug_core_mask[1] = new_core_mask_js1;
 408         kbdev->pm.debug_core_mask[2] = new_core_mask_js2;
 409         kbdev->pm.debug_core_mask_all = new_core_mask_js0 | new_core_mask_js1 |
 410                         new_core_mask_js2;
 411
 412         kbase_pm_update_cores_state_nolock(kbdev);
 413 }
 414
 415 void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev)
 416 {
 417         kbase_pm_update_active(kbdev);
 418 }
 419
 420 void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev)
 421 {
 422         kbase_pm_update_active(kbdev);
 423 }
 424
 425 void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev)
 426 {
 427         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
 428
 429         /* Force power off the GPU and all cores (regardless of policy), only
 430          * after the PM active count reaches zero (otherwise, we risk turning it
 431          * off prematurely) */
 432         mutex_lock(&js_devdata->runpool_mutex);
 433         mutex_lock(&kbdev->pm.lock);
 434
 435         kbase_pm_cancel_deferred_poweroff(kbdev);
 436         kbase_pm_do_poweroff(kbdev, true);
 437
 438         kbase_backend_timer_suspend(kbdev);
 439
 440         mutex_unlock(&kbdev->pm.lock);
 441         mutex_unlock(&js_devdata->runpool_mutex);
 442
 443         kbase_pm_wait_for_poweroff_complete(kbdev);
 444 }
 445
 446 void kbase_hwaccess_pm_resume(struct kbase_device *kbdev)
 447 {
 448         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
 449
 450         mutex_lock(&js_devdata->runpool_mutex);
 451         mutex_lock(&kbdev->pm.lock);
 452
 453         kbdev->pm.suspending = false;
 454         kbase_pm_do_poweron(kbdev, true);
 455
 456         kbase_backend_timer_resume(kbdev);
 457
 458         mutex_unlock(&kbdev->pm.lock);
 459         mutex_unlock(&js_devdata->runpool_mutex);
 460 }