drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_backend.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18
  19 /**
  20  * @file mali_kbase_pm_hwaccess.c
  21  * GPU backend implementation of base kernel power management APIs
  22  */
  23
  24 #include <mali_kbase.h>
  25 #include <mali_midg_regmap.h>
  26 #include <mali_kbase_config_defaults.h>
  27 #ifdef CONFIG_MALI_PLATFORM_DEVICETREE
  28 #include <linux/pm_runtime.h>
  29 #endif /* CONFIG_MALI_PLATFORM_DEVICETREE */
  30
  31 #include <mali_kbase_pm.h>
  32 #include <backend/gpu/mali_kbase_jm_internal.h>
  33 #include <backend/gpu/mali_kbase_pm_internal.h>
  34
  35 void kbase_pm_register_access_enable(struct kbase_device *kbdev)
  36 {
  37         struct kbase_pm_callback_conf *callbacks;
  38
  39 #ifdef CONFIG_MALI_PLATFORM_DEVICETREE
  40         pm_runtime_enable(kbdev->dev);
  41 #endif /* CONFIG_MALI_PLATFORM_DEVICETREE */
  42         callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS;
  43
  44         if (callbacks)
  45                 callbacks->power_on_callback(kbdev);
  46
  47         kbdev->pm.backend.gpu_powered = true;
  48 }
  49
  50 void kbase_pm_register_access_disable(struct kbase_device *kbdev)
  51 {
  52         struct kbase_pm_callback_conf *callbacks;
  53
  54         callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS;
  55
  56         if (callbacks)
  57                 callbacks->power_off_callback(kbdev);
  58
  59         kbdev->pm.backend.gpu_powered = false;
  60 #ifdef CONFIG_MALI_PLATFORM_DEVICETREE
  61         pm_runtime_disable(kbdev->dev);
  62 #endif
  63 }
  64
  65 int kbase_hwaccess_pm_init(struct kbase_device *kbdev)
  66 {
  67         int ret = 0;
  68         struct kbase_pm_callback_conf *callbacks;
  69
  70         KBASE_DEBUG_ASSERT(kbdev != NULL);
  71
  72         mutex_init(&kbdev->pm.lock);
  73
  74         kbdev->pm.backend.gpu_powered = false;
  75         kbdev->pm.suspending = false;
  76 #ifdef CONFIG_MALI_DEBUG
  77         kbdev->pm.backend.driver_ready_for_irqs = false;
  78 #endif /* CONFIG_MALI_DEBUG */
  79         kbdev->pm.backend.gpu_in_desired_state = true;
  80         init_waitqueue_head(&kbdev->pm.backend.gpu_in_desired_state_wait);
  81
  82         callbacks = (struct kbase_pm_callback_conf *)POWER_MANAGEMENT_CALLBACKS;
  83         if (callbacks) {
  84                 kbdev->pm.backend.callback_power_on =
  85                                         callbacks->power_on_callback;
  86                 kbdev->pm.backend.callback_power_off =
  87                                         callbacks->power_off_callback;
  88                 kbdev->pm.backend.callback_power_suspend =
  89                                         callbacks->power_suspend_callback;
  90                 kbdev->pm.backend.callback_power_resume =
  91                                         callbacks->power_resume_callback;
  92                 kbdev->pm.callback_power_runtime_init =
  93                                         callbacks->power_runtime_init_callback;
  94                 kbdev->pm.callback_power_runtime_term =
  95                                         callbacks->power_runtime_term_callback;
  96                 kbdev->pm.backend.callback_power_runtime_on =
  97                                         callbacks->power_runtime_on_callback;
  98                 kbdev->pm.backend.callback_power_runtime_off =
  99                                         callbacks->power_runtime_off_callback;
 100         } else {
 101                 kbdev->pm.backend.callback_power_on = NULL;
 102                 kbdev->pm.backend.callback_power_off = NULL;
 103                 kbdev->pm.backend.callback_power_suspend = NULL;
 104                 kbdev->pm.backend.callback_power_resume = NULL;
 105                 kbdev->pm.callback_power_runtime_init = NULL;
 106                 kbdev->pm.callback_power_runtime_term = NULL;
 107                 kbdev->pm.backend.callback_power_runtime_on = NULL;
 108                 kbdev->pm.backend.callback_power_runtime_off = NULL;
 109         }
 110
 111         /* Initialise the metrics subsystem */
 112         ret = kbasep_pm_metrics_init(kbdev);
 113         if (ret)
 114                 return ret;
 115
 116         init_waitqueue_head(&kbdev->pm.backend.l2_powered_wait);
 117         kbdev->pm.backend.l2_powered = 0;
 118
 119         init_waitqueue_head(&kbdev->pm.backend.reset_done_wait);
 120         kbdev->pm.backend.reset_done = false;
 121
 122         init_waitqueue_head(&kbdev->pm.zero_active_count_wait);
 123         kbdev->pm.active_count = 0;
 124
 125         spin_lock_init(&kbdev->pm.power_change_lock);
 126         spin_lock_init(&kbdev->pm.backend.gpu_cycle_counter_requests_lock);
 127         spin_lock_init(&kbdev->pm.backend.gpu_powered_lock);
 128
 129         if (kbase_pm_ca_init(kbdev) != 0)
 130                 goto workq_fail;
 131
 132         if (kbase_pm_policy_init(kbdev) != 0)
 133                 goto pm_policy_fail;
 134
 135         return 0;
 136
 137 pm_policy_fail:
 138         kbase_pm_ca_term(kbdev);
 139 workq_fail:
 140         kbasep_pm_metrics_term(kbdev);
 141         return -EINVAL;
 142 }
 143
 144 void kbase_pm_do_poweron(struct kbase_device *kbdev, bool is_resume)
 145 {
 146         lockdep_assert_held(&kbdev->pm.lock);
 147
 148         /* Turn clocks and interrupts on - no-op if we haven't done a previous
 149          * kbase_pm_clock_off() */
 150         kbase_pm_clock_on(kbdev, is_resume);
 151
 152         /* Update core status as required by the policy */
 153         KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 154                                 SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_START);
 155         kbase_pm_update_cores_state(kbdev);
 156         KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 157                                 SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_END);
 158
 159         /* NOTE: We don't wait to reach the desired state, since running atoms
 160          * will wait for that state to be reached anyway */
 161 }
 162
 163 bool kbase_pm_do_poweroff(struct kbase_device *kbdev, bool is_suspend)
 164 {
 165         unsigned long flags;
 166         bool cores_are_available;
 167
 168         lockdep_assert_held(&kbdev->pm.lock);
 169
 170         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 171
 172         /* Force all cores off */
 173         kbdev->pm.backend.desired_shader_state = 0;
 174
 175         /* Force all cores to be unavailable, in the situation where
 176          * transitions are in progress for some cores but not others,
 177          * and kbase_pm_check_transitions_nolock can not immediately
 178          * power off the cores */
 179         kbdev->shader_available_bitmap = 0;
 180         kbdev->tiler_available_bitmap = 0;
 181         kbdev->l2_available_bitmap = 0;
 182
 183         KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 184                                 SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_START);
 185         cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
 186         KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 187                                 SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_END);
 188         /* Don't need 'cores_are_available', because we don't return anything */
 189         CSTD_UNUSED(cores_are_available);
 190
 191         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 192
 193         /* NOTE: We won't wait to reach the core's desired state, even if we're
 194          * powering off the GPU itself too. It's safe to cut the power whilst
 195          * they're transitioning to off, because the cores should be idle and
 196          * all cache flushes should already have occurred */
 197
 198         /* Consume any change-state events */
 199         kbase_timeline_pm_check_handle_event(kbdev,
 200                                 KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
 201         /* Disable interrupts and turn the clock off */
 202         return kbase_pm_clock_off(kbdev, is_suspend);
 203 }
 204
 205 int kbase_hwaccess_pm_powerup(struct kbase_device *kbdev,
 206                 unsigned int flags)
 207 {
 208         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
 209         unsigned long irq_flags;
 210         int ret;
 211
 212         KBASE_DEBUG_ASSERT(kbdev != NULL);
 213
 214         mutex_lock(&js_devdata->runpool_mutex);
 215         mutex_lock(&kbdev->pm.lock);
 216
 217         /* A suspend won't happen during startup/insmod */
 218         KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
 219
 220         /* Power up the GPU, don't enable IRQs as we are not ready to receive
 221          * them. */
 222         ret = kbase_pm_init_hw(kbdev, flags);
 223         if (ret) {
 224                 mutex_unlock(&kbdev->pm.lock);
 225                 mutex_unlock(&js_devdata->runpool_mutex);
 226                 return ret;
 227         }
 228
 229         kbasep_pm_read_present_cores(kbdev);
 230
 231         kbdev->pm.debug_core_mask = kbdev->shader_present_bitmap;
 232
 233         /* Pretend the GPU is active to prevent a power policy turning the GPU
 234          * cores off */
 235         kbdev->pm.active_count = 1;
 236
 237         spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock,
 238                                                                 irq_flags);
 239         /* Ensure cycle counter is off */
 240         kbdev->pm.backend.gpu_cycle_counter_requests = 0;
 241         spin_unlock_irqrestore(
 242                         &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
 243                                                                 irq_flags);
 244
 245         /* We are ready to receive IRQ's now as power policy is set up, so
 246          * enable them now. */
 247 #ifdef CONFIG_MALI_DEBUG
 248         spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, irq_flags);
 249         kbdev->pm.backend.driver_ready_for_irqs = true;
 250         spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, irq_flags);
 251 #endif
 252         kbase_pm_enable_interrupts(kbdev);
 253
 254         /* Turn on the GPU and any cores needed by the policy */
 255         kbase_pm_do_poweron(kbdev, false);
 256         mutex_unlock(&kbdev->pm.lock);
 257         mutex_unlock(&js_devdata->runpool_mutex);
 258
 259         /* Idle the GPU and/or cores, if the policy wants it to */
 260         kbase_pm_context_idle(kbdev);
 261
 262         return 0;
 263 }
 264
 265 void kbase_hwaccess_pm_halt(struct kbase_device *kbdev)
 266 {
 267         KBASE_DEBUG_ASSERT(kbdev != NULL);
 268
 269         mutex_lock(&kbdev->pm.lock);
 270         kbase_pm_cancel_deferred_poweroff(kbdev);
 271         if (!kbase_pm_do_poweroff(kbdev, false)) {
 272                 /* Page/bus faults are pending, must drop pm.lock to process.
 273                  * Interrupts are disabled so no more faults should be
 274                  * generated at this point */
 275                 mutex_unlock(&kbdev->pm.lock);
 276                 kbase_flush_mmu_wqs(kbdev);
 277                 mutex_lock(&kbdev->pm.lock);
 278                 WARN_ON(!kbase_pm_do_poweroff(kbdev, false));
 279         }
 280         mutex_unlock(&kbdev->pm.lock);
 281 }
 282
 283 KBASE_EXPORT_TEST_API(kbase_hwaccess_pm_halt);
 284
 285 void kbase_hwaccess_pm_term(struct kbase_device *kbdev)
 286 {
 287         KBASE_DEBUG_ASSERT(kbdev != NULL);
 288         KBASE_DEBUG_ASSERT(kbdev->pm.active_count == 0);
 289         KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests == 0);
 290
 291         /* Free any resources the policy allocated */
 292         kbase_pm_policy_term(kbdev);
 293         kbase_pm_ca_term(kbdev);
 294
 295         /* Shut down the metrics subsystem */
 296         kbasep_pm_metrics_term(kbdev);
 297 }
 298
 299 void kbase_pm_power_changed(struct kbase_device *kbdev)
 300 {
 301         bool cores_are_available;
 302         unsigned long flags;
 303
 304         KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 305                                 SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_START);
 306         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 307         cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
 308         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 309         KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 310                                 SW_FLOW_PM_CHECKTRANS_GPU_INTERRUPT_END);
 311
 312         if (cores_are_available) {
 313                 /* Log timelining information that a change in state has
 314                  * completed */
 315                 kbase_timeline_pm_handle_event(kbdev,
 316                                 KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
 317
 318                 spin_lock_irqsave(&kbdev->js_data.runpool_irq.lock, flags);
 319                 kbase_gpu_slot_update(kbdev);
 320                 spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock, flags);
 321         }
 322 }
 323
 324 void kbase_pm_set_debug_core_mask(struct kbase_device *kbdev, u64 new_core_mask)
 325 {
 326         kbdev->pm.debug_core_mask = new_core_mask;
 327
 328         kbase_pm_update_cores_state_nolock(kbdev);
 329 }
 330
 331 void kbase_hwaccess_pm_gpu_active(struct kbase_device *kbdev)
 332 {
 333         kbase_pm_update_active(kbdev);
 334 }
 335
 336 void kbase_hwaccess_pm_gpu_idle(struct kbase_device *kbdev)
 337 {
 338         kbase_pm_update_active(kbdev);
 339 }
 340
 341 void kbase_hwaccess_pm_suspend(struct kbase_device *kbdev)
 342 {
 343         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
 344
 345         /* Force power off the GPU and all cores (regardless of policy), only
 346          * after the PM active count reaches zero (otherwise, we risk turning it
 347          * off prematurely) */
 348         mutex_lock(&js_devdata->runpool_mutex);
 349         mutex_lock(&kbdev->pm.lock);
 350         kbase_pm_cancel_deferred_poweroff(kbdev);
 351         if (!kbase_pm_do_poweroff(kbdev, true)) {
 352                 /* Page/bus faults are pending, must drop pm.lock to process.
 353                  * Interrupts are disabled so no more faults should be
 354                  * generated at this point */
 355                 mutex_unlock(&kbdev->pm.lock);
 356                 kbase_flush_mmu_wqs(kbdev);
 357                 mutex_lock(&kbdev->pm.lock);
 358                 WARN_ON(!kbase_pm_do_poweroff(kbdev, false));
 359         }
 360
 361         mutex_unlock(&kbdev->pm.lock);
 362         mutex_unlock(&js_devdata->runpool_mutex);
 363 }
 364
 365 void kbase_hwaccess_pm_resume(struct kbase_device *kbdev)
 366 {
 367         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
 368
 369         mutex_lock(&js_devdata->runpool_mutex);
 370         mutex_lock(&kbdev->pm.lock);
 371         kbdev->pm.suspending = false;
 372         kbase_pm_do_poweron(kbdev, true);
 373         mutex_unlock(&kbdev->pm.lock);
 374         mutex_unlock(&js_devdata->runpool_mutex);
 375 }