3 * (C) COPYRIGHT ARM Limited. All rights reserved.
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
21 * @file mali_kbase_pm.c
22 * Base kernel power management APIs
25 #include <mali_kbase.h>
26 #include <mali_midg_regmap.h>
28 #include <mali_kbase_pm.h>
30 void kbase_pm_register_access_enable(kbase_device *kbdev)
32 kbase_pm_callback_conf *callbacks;
34 callbacks = (kbase_pm_callback_conf *) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS);
37 callbacks->power_on_callback(kbdev);
40 void kbase_pm_register_access_disable(kbase_device *kbdev)
42 kbase_pm_callback_conf *callbacks;
44 callbacks = (kbase_pm_callback_conf *) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS);
47 callbacks->power_off_callback(kbdev);
50 mali_error kbase_pm_init(kbase_device *kbdev)
52 mali_error ret = MALI_ERROR_NONE;
53 kbase_pm_callback_conf *callbacks;
55 KBASE_DEBUG_ASSERT(kbdev != NULL);
57 mutex_init(&kbdev->pm.lock);
59 kbdev->pm.gpu_powered = MALI_FALSE;
60 kbdev->pm.suspending = MALI_FALSE;
61 #ifdef CONFIG_MALI_DEBUG
62 kbdev->pm.driver_ready_for_irqs = MALI_FALSE;
63 #endif /* CONFIG_MALI_DEBUG */
64 kbdev->pm.gpu_in_desired_state = MALI_TRUE;
65 init_waitqueue_head(&kbdev->pm.gpu_in_desired_state_wait);
67 callbacks = (kbase_pm_callback_conf *) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_POWER_MANAGEMENT_CALLBACKS);
69 kbdev->pm.callback_power_on = callbacks->power_on_callback;
70 kbdev->pm.callback_power_off = callbacks->power_off_callback;
71 kbdev->pm.callback_power_suspend =
72 callbacks->power_suspend_callback;
73 kbdev->pm.callback_power_resume =
74 callbacks->power_resume_callback;
75 kbdev->pm.callback_power_runtime_init = callbacks->power_runtime_init_callback;
76 kbdev->pm.callback_power_runtime_term = callbacks->power_runtime_term_callback;
77 kbdev->pm.callback_power_runtime_on = callbacks->power_runtime_on_callback;
78 kbdev->pm.callback_power_runtime_off = callbacks->power_runtime_off_callback;
80 kbdev->pm.callback_power_on = NULL;
81 kbdev->pm.callback_power_off = NULL;
82 kbdev->pm.callback_power_suspend = NULL;
83 kbdev->pm.callback_power_resume = NULL;
84 kbdev->pm.callback_power_runtime_init = NULL;
85 kbdev->pm.callback_power_runtime_term = NULL;
86 kbdev->pm.callback_power_runtime_on = NULL;
87 kbdev->pm.callback_power_runtime_off = NULL;
90 kbdev->pm.platform_dvfs_frequency = (u32) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_POWER_MANAGEMENT_DVFS_FREQ);
92 /* Initialise the metrics subsystem */
93 ret = kbasep_pm_metrics_init(kbdev);
94 if (MALI_ERROR_NONE != ret)
97 init_waitqueue_head(&kbdev->pm.l2_powered_wait);
98 kbdev->pm.l2_powered = 0;
100 init_waitqueue_head(&kbdev->pm.reset_done_wait);
101 kbdev->pm.reset_done = MALI_FALSE;
103 init_waitqueue_head(&kbdev->pm.zero_active_count_wait);
104 kbdev->pm.active_count = 0;
106 spin_lock_init(&kbdev->pm.power_change_lock);
107 spin_lock_init(&kbdev->pm.gpu_cycle_counter_requests_lock);
108 spin_lock_init(&kbdev->pm.gpu_powered_lock);
110 if (MALI_ERROR_NONE != kbase_pm_ca_init(kbdev))
113 if (MALI_ERROR_NONE != kbase_pm_policy_init(kbdev))
116 return MALI_ERROR_NONE;
119 kbase_pm_ca_term(kbdev);
121 kbasep_pm_metrics_term(kbdev);
122 return MALI_ERROR_FUNCTION_FAILED;
125 KBASE_EXPORT_TEST_API(kbase_pm_init)
127 void kbase_pm_do_poweron(kbase_device *kbdev, mali_bool is_resume)
129 lockdep_assert_held(&kbdev->pm.lock);
131 /* Turn clocks and interrupts on - no-op if we haven't done a previous
132 * kbase_pm_clock_off() */
133 kbase_pm_clock_on(kbdev, is_resume);
135 /* Update core status as required by the policy */
136 KBASE_TIMELINE_PM_CHECKTRANS(kbdev, SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_START);
137 kbase_pm_update_cores_state(kbdev);
138 KBASE_TIMELINE_PM_CHECKTRANS(kbdev, SW_FLOW_PM_CHECKTRANS_PM_DO_POWERON_END);
140 /* NOTE: We don't wait to reach the desired state, since running atoms
141 * will wait for that state to be reached anyway */
144 void kbase_pm_do_poweroff(kbase_device *kbdev, mali_bool is_suspend)
147 mali_bool cores_are_available;
149 lockdep_assert_held(&kbdev->pm.lock);
151 spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
153 /* Force all cores off */
154 kbdev->pm.desired_shader_state = 0;
156 /* Force all cores to be unavailable, in the situation where
157 * transitions are in progress for some cores but not others,
158 * and kbase_pm_check_transitions_nolock can not immediately
159 * power off the cores */
160 kbdev->shader_available_bitmap = 0;
161 kbdev->tiler_available_bitmap = 0;
162 kbdev->l2_available_bitmap = 0;
164 KBASE_TIMELINE_PM_CHECKTRANS(kbdev, SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_START);
165 cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
166 KBASE_TIMELINE_PM_CHECKTRANS(kbdev, SW_FLOW_PM_CHECKTRANS_PM_DO_POWEROFF_END);
167 /* Don't need 'cores_are_available', because we don't return anything */
168 CSTD_UNUSED(cores_are_available);
170 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
172 /* NOTE: We won't wait to reach the core's desired state, even if we're
173 * powering off the GPU itself too. It's safe to cut the power whilst
174 * they're transitioning to off, because the cores should be idle and all
175 * cache flushes should already have occurred */
177 /* Consume any change-state events */
178 kbase_timeline_pm_check_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
179 /* Disable interrupts and turn the clock off */
180 kbase_pm_clock_off(kbdev, is_suspend);
183 mali_error kbase_pm_powerup(kbase_device *kbdev)
188 KBASE_DEBUG_ASSERT(kbdev != NULL);
190 mutex_lock(&kbdev->pm.lock);
192 /* A suspend won't happen during startup/insmod */
193 KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
195 /* Power up the GPU, don't enable IRQs as we are not ready to receive them. */
196 ret = kbase_pm_init_hw(kbdev, MALI_FALSE );
197 if (ret != MALI_ERROR_NONE) {
198 mutex_unlock(&kbdev->pm.lock);
202 kbasep_pm_read_present_cores(kbdev);
204 kbdev->pm.debug_core_mask = kbdev->shader_present_bitmap;
206 /* Pretend the GPU is active to prevent a power policy turning the GPU cores off */
207 kbdev->pm.active_count = 1;
209 spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
210 /* Ensure cycle counter is off */
211 kbdev->pm.gpu_cycle_counter_requests = 0;
212 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_STOP, NULL);
213 spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
215 /* We are ready to receive IRQ's now as power policy is set up, so enable them now. */
216 #ifdef CONFIG_MALI_DEBUG
217 spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags);
218 kbdev->pm.driver_ready_for_irqs = MALI_TRUE;
219 spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
221 kbase_pm_enable_interrupts(kbdev);
223 /* Turn on the GPU and any cores needed by the policy */
224 kbase_pm_do_poweron(kbdev, MALI_FALSE);
225 mutex_unlock(&kbdev->pm.lock);
227 /* Idle the GPU and/or cores, if the policy wants it to */
228 kbase_pm_context_idle(kbdev);
230 return MALI_ERROR_NONE;
233 KBASE_EXPORT_TEST_API(kbase_pm_powerup)
235 void kbase_pm_context_active(kbase_device *kbdev)
237 (void)kbase_pm_context_active_handle_suspend(kbdev, KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE);
240 int kbase_pm_context_active_handle_suspend(kbase_device *kbdev, kbase_pm_suspend_handler suspend_handler)
245 KBASE_DEBUG_ASSERT(kbdev != NULL);
247 /* Trace timeline information about how long it took to handle the decision
248 * to powerup. Sometimes the event might be missed due to reading the count
249 * outside of mutex, but this is necessary to get the trace timing
251 old_count = kbdev->pm.active_count;
253 kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE);
255 mutex_lock(&kbdev->pm.lock);
256 if (kbase_pm_is_suspending(kbdev))
258 switch (suspend_handler) {
259 case KBASE_PM_SUSPEND_HANDLER_DONT_REACTIVATE:
260 if (kbdev->pm.active_count != 0 )
263 case KBASE_PM_SUSPEND_HANDLER_DONT_INCREASE:
264 mutex_unlock(&kbdev->pm.lock);
266 kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE);
269 case KBASE_PM_SUSPEND_HANDLER_NOT_POSSIBLE:
272 KBASE_DEBUG_ASSERT_MSG(MALI_FALSE,"unreachable");
276 c = ++kbdev->pm.active_count;
278 KBASE_TRACE_ADD_REFCOUNT(kbdev, PM_CONTEXT_ACTIVE, NULL, NULL, 0u, c);
280 /* Trace the event being handled */
282 kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_ACTIVE);
285 /* First context active: Power on the GPU and any cores requested by
287 kbase_pm_update_active(kbdev);
289 kbasep_pm_record_gpu_active(kbdev);
292 mutex_unlock(&kbdev->pm.lock);
297 KBASE_EXPORT_TEST_API(kbase_pm_context_active)
299 void kbase_pm_context_idle(kbase_device *kbdev)
304 KBASE_DEBUG_ASSERT(kbdev != NULL);
306 /* Trace timeline information about how long it took to handle the decision
307 * to powerdown. Sometimes the event might be missed due to reading the
308 * count outside of mutex, but this is necessary to get the trace timing
310 old_count = kbdev->pm.active_count;
312 kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_IDLE);
314 mutex_lock(&kbdev->pm.lock);
316 c = --kbdev->pm.active_count;
318 KBASE_TRACE_ADD_REFCOUNT(kbdev, PM_CONTEXT_IDLE, NULL, NULL, 0u, c);
320 KBASE_DEBUG_ASSERT(c >= 0);
322 /* Trace the event being handled */
324 kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_IDLE);
327 /* Last context has gone idle */
328 kbase_pm_update_active(kbdev);
330 kbasep_pm_record_gpu_idle(kbdev);
332 /* Wake up anyone waiting for this to become 0 (e.g. suspend). The
333 * waiters must synchronize with us by locking the pm.lock after
335 wake_up(&kbdev->pm.zero_active_count_wait);
338 mutex_unlock(&kbdev->pm.lock);
341 KBASE_EXPORT_TEST_API(kbase_pm_context_idle)
343 void kbase_pm_halt(kbase_device *kbdev)
345 KBASE_DEBUG_ASSERT(kbdev != NULL);
347 mutex_lock(&kbdev->pm.lock);
348 kbase_pm_cancel_deferred_poweroff(kbdev);
349 kbase_pm_do_poweroff(kbdev, MALI_FALSE);
350 mutex_unlock(&kbdev->pm.lock);
353 KBASE_EXPORT_TEST_API(kbase_pm_halt)
355 void kbase_pm_term(kbase_device *kbdev)
357 KBASE_DEBUG_ASSERT(kbdev != NULL);
358 KBASE_DEBUG_ASSERT(kbdev->pm.active_count == 0);
359 KBASE_DEBUG_ASSERT(kbdev->pm.gpu_cycle_counter_requests == 0);
361 /* Free any resources the policy allocated */
362 kbase_pm_policy_term(kbdev);
363 kbase_pm_ca_term(kbdev);
365 /* Shut down the metrics subsystem */
366 kbasep_pm_metrics_term(kbdev);
369 KBASE_EXPORT_TEST_API(kbase_pm_term)
371 void kbase_pm_suspend(struct kbase_device *kbdev)
373 int nr_keep_gpu_powered_ctxs;
374 KBASE_DEBUG_ASSERT(kbdev);
376 mutex_lock(&kbdev->pm.lock);
377 KBASE_DEBUG_ASSERT(!kbase_pm_is_suspending(kbdev));
378 kbdev->pm.suspending = MALI_TRUE;
379 mutex_unlock(&kbdev->pm.lock);
381 /* From now on, the active count will drop towards zero. Sometimes, it'll
382 * go up briefly before going down again. However, once it reaches zero it
383 * will stay there - guaranteeing that we've idled all pm references */
385 /* Suspend job scheduler and associated components, so that it releases all
386 * the PM active count references */
387 kbasep_js_suspend(kbdev);
389 /* Suspend any counter collection that might be happening */
390 kbase_instr_hwcnt_suspend(kbdev);
392 /* Cancel the keep_gpu_powered calls */
393 for (nr_keep_gpu_powered_ctxs = atomic_read(&kbdev->keep_gpu_powered_count);
394 nr_keep_gpu_powered_ctxs > 0 ;
395 --nr_keep_gpu_powered_ctxs ) {
396 kbase_pm_context_idle(kbdev);
399 /* Wait for the active count to reach zero. This is not the same as
400 * waiting for a power down, since not all policies power down when this
402 wait_event(kbdev->pm.zero_active_count_wait, kbdev->pm.active_count == 0);
404 /* NOTE: We synchronize with anything that was just finishing a
405 * kbase_pm_context_idle() call by locking the pm.lock below */
407 /* Force power off the GPU and all cores (regardless of policy), only after
408 * the PM active count reaches zero (otherwise, we risk turning it off
410 mutex_lock(&kbdev->pm.lock);
411 kbase_pm_cancel_deferred_poweroff(kbdev);
412 kbase_pm_do_poweroff(kbdev, MALI_TRUE);
413 mutex_unlock(&kbdev->pm.lock);
416 void kbase_pm_resume(struct kbase_device *kbdev)
418 int nr_keep_gpu_powered_ctxs;
420 /* MUST happen before any pm_context_active calls occur */
421 mutex_lock(&kbdev->pm.lock);
422 kbdev->pm.suspending = MALI_FALSE;
423 kbase_pm_do_poweron(kbdev, MALI_TRUE);
424 mutex_unlock(&kbdev->pm.lock);
426 /* Initial active call, to power on the GPU/cores if needed */
427 kbase_pm_context_active(kbdev);
429 /* Restore the keep_gpu_powered calls */
430 for (nr_keep_gpu_powered_ctxs = atomic_read(&kbdev->keep_gpu_powered_count);
431 nr_keep_gpu_powered_ctxs > 0 ;
432 --nr_keep_gpu_powered_ctxs ) {
433 kbase_pm_context_active(kbdev);
436 /* Re-enable instrumentation, if it was previously disabled */
437 kbase_instr_hwcnt_resume(kbdev);
439 /* Resume any blocked atoms (which may cause contexts to be scheduled in
440 * and dependent atoms to run) */
441 kbase_resume_suspended_soft_jobs(kbdev);
443 /* Resume the Job Scheduler and associated components, and start running
445 kbasep_js_resume(kbdev);
447 /* Matching idle call, to power off the GPU/cores if we didn't actually
448 * need it and the policy doesn't want it on */
449 kbase_pm_context_idle(kbdev);