drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_backend.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18
  19 /*
  20  * GPU backend implementation of base kernel power management APIs
  21  */
  22
  23 /* #define ENABLE_DEBUG_LOG */
  24 #include "../../platform/rk/custom_log.h"
  25
  26 #include <mali_kbase.h>
  27 #include <mali_midg_regmap.h>
  28 #include <mali_kbase_config_defaults.h>
  29 #ifdef CONFIG_MALI_PLATFORM_DEVICETREE
  30 #include <linux/pm_runtime.h>
  31 #endif /* CONFIG_MALI_PLATFORM_DEVICETREE */
  32
  33 #include <mali_kbase_pm.h>
  34 #include <backend/gpu/mali_kbase_jm_internal.h>
  35 #include <backend/gpu/mali_kbase_js_internal.h>
  36 #include <backend/gpu/mali_kbase_pm_internal.h>
  37
  38 static int rk_slowdown_clk_gpu_before_poweroff_cores(struct kbase_device *kbdev)
  39 {
  40         int ret = 0;
  41         const unsigned long freq = 200 * 1000 * 1000;
  42
  43         mutex_lock(&kbdev->mutex_for_clk);
  44         ret = clk_set_rate(kbdev->clock, freq);
  45         if (ret)
  46                 E("Failed to set clock to %lu.", freq);
  47         kbdev->is_power_off = true;
  48         mutex_unlock(&kbdev->mutex_for_clk);
  49
  50         return ret;
  51 }
  52
  53 static int rk_restore_clk_gpu(struct kbase_device *kbdev)
  54 {
  55         int ret = 0;
  56
  57         mutex_lock(&kbdev->mutex_for_clk);
  58         if (kbdev->freq != 0)
  59                 ret = clk_set_rate(kbdev->clock, kbdev->freq);
  60         if (ret)
  61                 E("Failed to set clock to %lu.", kbdev->freq);
  62         kbdev->is_power_off = false;
  63         mutex_unlock(&kbdev->mutex_for_clk);
  64
  65         return ret;
  66 }
  67
  68 void kbase_pm_register_access_enable(struct kbase_device *kbdev)
  69 {
  70         struct kbase_pm_callback_conf *callbacks;
  71
  72         callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS;
  73
  74         if (callbacks)
  75                 callbacks->power_on_callback(kbdev);
  76
  77         kbdev->pm.backend.gpu_powered = true;
  78 }
  79
  80 void kbase_pm_register_access_disable(struct kbase_device *kbdev)
  81 {
  82         struct kbase_pm_callback_conf *callbacks;
  83
  84         callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS;
  85
  86         if (callbacks)
  87                 callbacks->power_off_callback(kbdev);
  88
  89         kbdev->pm.backend.gpu_powered = false;
  90 }
  91
  92 int kbase_hwaccess_pm_init(struct kbase_device *kbdev)
  93 {
  94         int ret = 0;
  95         struct kbase_pm_callback_conf *callbacks;
  96
  97         KBASE_DEBUG_ASSERT(kbdev != NULL);
  98
  99         mutex_init(&kbdev->pm.lock);
 100
 101         kbdev->pm.backend.gpu_powered = false;
 102         kbdev->pm.suspending = false;
 103 #ifdef CONFIG_MALI_DEBUG
 104         kbdev->pm.backend.driver_ready_for_irqs = false;
 105 #endif /* CONFIG_MALI_DEBUG */
 106         kbdev->pm.backend.gpu_in_desired_state = true;
 107         init_waitqueue_head(&kbdev->pm.backend.gpu_in_desired_state_wait);
 108
 109         callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS;
 110         if (callbacks) {
 111                 kbdev->pm.backend.callback_power_on =
 112                                         callbacks->power_on_callback;
 113                 kbdev->pm.backend.callback_power_off =
 114                                         callbacks->power_off_callback;
 115                 kbdev->pm.backend.callback_power_suspend =
 116                                         callbacks->power_suspend_callback;
 117                 kbdev->pm.backend.callback_power_resume =
 118                                         callbacks->power_resume_callback;
 119                 kbdev->pm.callback_power_runtime_init =
 120                                         callbacks->power_runtime_init_callback;
 121                 kbdev->pm.callback_power_runtime_term =
 122                                         callbacks->power_runtime_term_callback;
 123                 kbdev->pm.backend.callback_power_runtime_on =
 124                                         callbacks->power_runtime_on_callback;
 125                 kbdev->pm.backend.callback_power_runtime_off =
 126                                         callbacks->power_runtime_off_callback;
 127                 kbdev->pm.backend.callback_power_runtime_idle =
 128                                         callbacks->power_runtime_idle_callback;
 129         } else {
 130                 kbdev->pm.backend.callback_power_on = NULL;
 131                 kbdev->pm.backend.callback_power_off = NULL;
 132                 kbdev->pm.backend.callback_power_suspend = NULL;
 133                 kbdev->pm.backend.callback_power_resume = NULL;
 134                 kbdev->pm.callback_power_runtime_init = NULL;
 135                 kbdev->pm.callback_power_runtime_term = NULL;
 136                 kbdev->pm.backend.callback_power_runtime_on = NULL;
 137                 kbdev->pm.backend.callback_power_runtime_off = NULL;
 138                 kbdev->pm.backend.callback_power_runtime_idle = NULL;
 139         }
 140
 141         /* Initialise the metrics subsystem */
 142         ret = kbasep_pm_metrics_init(kbdev);
 143         if (ret)
 144                 return ret;
 145
 146         init_waitqueue_head(&kbdev->pm.backend.l2_powered_wait);
 147         kbdev->pm.backend.l2_powered = 0;
 148
 149         init_waitqueue_head(&kbdev->pm.backend.reset_done_wait);
 150         kbdev->pm.backend.reset_done = false;
 151
 152         init_waitqueue_head(&kbdev->pm.zero_active_count_wait);
 153         kbdev->pm.active_count = 0;
 154
 155         spin_lock_init(&kbdev->pm.power_change_lock);
 156         spin_lock_init(&kbdev->pm.backend.gpu_cycle_counter_requests_lock);
 157         spin_lock_init(&kbdev->pm.backend.gpu_powered_lock);
 158
 159         if (kbase_pm_ca_init(kbdev) != 0)
 160                 goto workq_fail;
 161
 162         if (kbase_pm_policy_init(kbdev) != 0)
 163                 goto pm_policy_fail;
 164
 165         return 0;
 166
 167 pm_policy_fail:
 168         kbase_pm_ca_term(kbdev);
 169 workq_fail:
 170         kbasep_pm_metrics_term(kbdev);
 171         return -EINVAL;
 172 }
 173
 174 void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume)
 175 {
 176         lockdep_assert_held(&kbdev->pm.lock);
 177
 178         /* Turn clocks and interrupts on - no-op if we haven't done a previous
 179          * kbase_pm_clock_off() */
 180         kbase_pm_clock_on(kbdev, is_resume);
 181
 182         /* Update core status as required by the policy */
 183         KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 184                                 SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_START);
 185         kbase_pm_update_cores_state(kbdev);
 186         KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 187                                 SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_END);
 188
 189         /* NOTE: We don't wait to reach the desired state, since running atoms
 190          * will wait for that state to be reached anyway */
 191
 192         D("to restore clk_gpu.");
 193         rk_restore_clk_gpu(kbdev);
 194 }
 195
 196 bool kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend)
 197 {
 198         unsigned long flags;
 199         bool cores_are_available;
 200
 201         lockdep_assert_held(&kbdev->pm.lock);
 202
 203         D("to slowdown clk_gpu before poweroff pm_cores.");
 204         rk_slowdown_clk_gpu_before_poweroff_cores(kbdev);
 205
 206         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 207
 208         /* Force all cores off */
 209         kbdev->pm.backend.desired_shader_state = 0;
 210         kbdev->pm.backend.desired_tiler_state = 0;
 211
 212         /* Force all cores to be unavailable, in the situation where
 213          * transitions are in progress for some cores but not others,
 214          * and kbase_pm_check_transitions_nolock can not immediately
 215          * power off the cores */
 216         kbdev->shader_available_bitmap = 0;
 217         kbdev->tiler_available_bitmap = 0;
 218         kbdev->l2_available_bitmap = 0;
 219
 220         KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 221                                 SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_START);
 222         cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
 223         KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 224                                 SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_END);
 225         /* Don't need 'cores_are_available', because we don't return anything */
 226         CSTD_UNUSED(cores_are_available);
 227
 228         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 229
 230         /* NOTE: We won't wait to reach the core's desired state, even if we're
 231          * powering off the GPU itself too. It's safe to cut the power whilst
 232          * they're transitioning to off, because the cores should be idle and
 233          * all cache flushes should already have occurred */
 234
 235         /* Consume any change-state events */
 236         kbase_timeline_pm_check_handle_event(kbdev,
 237                                 KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
 238         /* Disable interrupts and turn the clock off */
 239         return kbase_pm_clock_off(kbdev, is_suspend);
 240 }
 241
 242 int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev,
 243                 unsigned int flags)
 244 {
 245         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
 246         unsigned long irq_flags;
 247         int ret;
 248
 249         KBASE_DEBUG_ASSERT(kbdev != NULL);
 250
 251         mutex_lock(&js_devdata->runpool_mutex);
 252         mutex_lock(&kbdev->pm.lock);
 253
 254         /* A suspend won't happen during startup/insmod */
 255         KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
 256
 257         /* Power up the GPU, don't enable IRQs as we are not ready to receive
 258          * them. */
 259         ret = kbase_pm_init_hw(kbdev, flags);
 260         if (ret) {
 261                 mutex_unlock(&kbdev->pm.lock);
 262                 mutex_unlock(&js_devdata->runpool_mutex);
 263                 return ret;
 264         }
 265
 266         kbasep_pm_read_present_cores(kbdev);
 267
 268         kbdev->pm.debug_core_mask_all = kbdev->pm.debug_core_mask[0] =
 269                         kbdev->pm.debug_core_mask[1] =
 270                         kbdev->pm.debug_core_mask[2] =
 271                         kbdev->gpu_props.props.raw_props.shader_present;
 272
 273         /* Pretend the GPU is active to prevent a power policy turning the GPU
 274          * cores off */
 275         kbdev->pm.active_count = 1;
 276
 277         spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock,
 278                                                                 irq_flags);
 279         /* Ensure cycle counter is off */
 280         kbdev->pm.backend.gpu_cycle_counter_requests = 0;
 281         spin_unlock_irqrestore(
 282                         &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
 283                                                                 irq_flags);
 284
 285         /* We are ready to receive IRQ's now as power policy is set up, so
 286          * enable them now. */
 287 #ifdef CONFIG_MALI_DEBUG
 288         spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, irq_flags);
 289         kbdev->pm.backend.driver_ready_for_irqs = true;
 290         spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, irq_flags);
 291 #endif
 292         kbase_pm_enable_interrupts(kbdev);
 293
 294         /* Turn on the GPU and any cores needed by the policy */
 295         kbase_pm_do_poweron(kbdev, false);
 296         mutex_unlock(&kbdev->pm.lock);
 297         mutex_unlock(&js_devdata->runpool_mutex);
 298
 299         /* Idle the GPU and/or cores, if the policy wants it to */
 300         kbase_pm_context_idle(kbdev);
 301
 302         return 0;
 303 }
 304
 305 void kbase_hwaccess_pm_halt(struct kbase_device *kbdev)
 306 {
 307         KBASE_DEBUG_ASSERT(kbdev != NULL);
 308
 309         mutex_lock(&kbdev->pm.lock);
 310         kbase_pm_cancel_deferred_poweroff(kbdev);
 311         if (!kbase_pm_do_poweroff(kbdev, false)) {
 312                 /* Page/bus faults are pending, must drop pm.lock to process.
 313                  * Interrupts are disabled so no more faults should be
 314                  * generated at this point */
 315                 mutex_unlock(&kbdev->pm.lock);
 316                 kbase_flush_mmu_wqs(kbdev);
 317                 mutex_lock(&kbdev->pm.lock);
 318                 WARN_ON(!kbase_pm_do_poweroff(kbdev, false));
 319         }
 320         mutex_unlock(&kbdev->pm.lock);
 321 }
 322
 323 KBASE_EXPORT_TEST_API(kbase_hwaccess_pm_halt);
 324
 325 void kbase_hwaccess_pm_term(struct kbase_device *kbdev)
 326 {
 327         KBASE_DEBUG_ASSERT(kbdev != NULL);
 328         KBASE_DEBUG_ASSERT(kbdev->pm.active_count == 0);
 329         KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests == 0);
 330
 331         /* Free any resources the policy allocated */
 332         kbase_pm_policy_term(kbdev);
 333         kbase_pm_ca_term(kbdev);
 334
 335         /* Shut down the metrics subsystem */
 336         kbasep_pm_metrics_term(kbdev);
 337 }
 338
 339 void kbase_pm_power_changed(struct kbase_device *kbdev)
 340 {
 341         bool cores_are_available;
 342         unsigned long flags;
 343
 344         KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 345                                 SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_START);
 346         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 347         cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
 348         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 349         KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 350                                 SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_END);
 351
 352         if (cores_are_available) {
 353                 /* Log timelining information that a change in state has
 354                  * completed */
 355                 kbase_timeline_pm_handle_event(kbdev,
 356                                 KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
 357
 358                 spin_lock_irqsave(&kbdev->js_data.runpool_irq.lock, flags);
 359                 kbase_gpu_slot_update(kbdev);
 360                 spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock, flags);
 361         }
 362 }
 363
 364 void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev,
 365                 u64 new_core_mask_js0, u64 new_core_mask_js1,
 366                 u64 new_core_mask_js2)
 367 {
 368         kbdev->pm.debug_core_mask[0] = new_core_mask_js0;
 369         kbdev->pm.debug_core_mask[1] = new_core_mask_js1;
 370         kbdev->pm.debug_core_mask[2] = new_core_mask_js2;
 371         kbdev->pm.debug_core_mask_all = new_core_mask_js0 | new_core_mask_js1 |
 372                         new_core_mask_js2;
 373
 374         kbase_pm_update_cores_state_nolock(kbdev);
 375 }
 376
 377 void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev)
 378 {
 379         kbase_pm_update_active(kbdev);
 380 }
 381
 382 void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev)
 383 {
 384         kbase_pm_update_active(kbdev);
 385 }
 386
 387 void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev)
 388 {
 389         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
 390
 391         /* Force power off the GPU and all cores (regardless of policy), only
 392          * after the PM active count reaches zero (otherwise, we risk turning it
 393          * off prematurely) */
 394         mutex_lock(&js_devdata->runpool_mutex);
 395         mutex_lock(&kbdev->pm.lock);
 396         kbase_pm_cancel_deferred_poweroff(kbdev);
 397         if (!kbase_pm_do_poweroff(kbdev, true)) {
 398                 /* Page/bus faults are pending, must drop pm.lock to process.
 399                  * Interrupts are disabled so no more faults should be
 400                  * generated at this point */
 401                 mutex_unlock(&kbdev->pm.lock);
 402                 kbase_flush_mmu_wqs(kbdev);
 403                 mutex_lock(&kbdev->pm.lock);
 404                 WARN_ON(!kbase_pm_do_poweroff(kbdev, false));
 405         }
 406
 407         kbase_backend_timer_suspend(kbdev);
 408
 409         mutex_unlock(&kbdev->pm.lock);
 410         mutex_unlock(&js_devdata->runpool_mutex);
 411 }
 412
 413 void kbase_hwaccess_pm_resume(struct kbase_device *kbdev)
 414 {
 415         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
 416
 417         mutex_lock(&js_devdata->runpool_mutex);
 418         mutex_lock(&kbdev->pm.lock);
 419
 420         kbdev->pm.suspending = false;
 421         kbase_pm_do_poweron(kbdev, true);
 422
 423         kbase_backend_timer_resume(kbdev);
 424
 425         mutex_unlock(&kbdev->pm.lock);
 426         mutex_unlock(&js_devdata->runpool_mutex);
 427 }