3 * (C) COPYRIGHT ARM Limited. All rights reserved.
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
21 * @file mali_kbase_pm_driver.c
22 * Base kernel Power Management hardware control
25 #include <kbase/src/common/mali_kbase.h>
26 #include <kbase/src/common/mali_midg_regmap.h>
27 #include <kbase/src/common/mali_kbase_gator.h>
28 #include <kbase/src/common/mali_kbase_pm.h>
31 #define MOCKABLE(function) function##_original
33 #define MOCKABLE(function) function
34 #endif /* MALI_MOCK_TEST */
36 /** Actions that can be performed on a core.
38 * This enumeration is private to the file. Its values are set to allow @ref core_type_to_reg function,
39 * which decodes this enumeration, to be simpler and more efficient.
41 typedef enum kbasep_pm_action {
43 ACTION_READY = (SHADER_READY_LO - SHADER_PRESENT_LO),
44 ACTION_PWRON = (SHADER_PWRON_LO - SHADER_PRESENT_LO),
45 ACTION_PWROFF = (SHADER_PWROFF_LO - SHADER_PRESENT_LO),
46 ACTION_PWRTRANS = (SHADER_PWRTRANS_LO - SHADER_PRESENT_LO),
47 ACTION_PWRACTIVE = (SHADER_PWRACTIVE_LO - SHADER_PRESENT_LO)
50 /** Decode a core type and action to a register.
52 * Given a core type (defined by @ref kbase_pm_core_type) and an action (defined by @ref kbasep_pm_action) this
53 * function will return the register offset that will perform the action on the core type. The register returned is
54 * the \c _LO register and an offset must be applied to use the \c _HI register.
56 * @param core_type The type of core
57 * @param action The type of action
59 * @return The register offset of the \c _LO register that performs an action of type \c action on a core of type \c
62 static u32 core_type_to_reg(kbase_pm_core_type core_type, kbasep_pm_action action)
64 return core_type + action;
67 /** Invokes an action on a core set
69 * This function performs the action given by \c action on a set of cores of a type given by \c core_type. It is a
70 * static function used by @ref kbase_pm_transition_core_type
72 * @param kbdev The kbase device structure of the device
73 * @param core_type The type of core that the action should be performed on
74 * @param cores A bit mask of cores to perform the action on (low 32 bits)
75 * @param action The action to perform on the cores
77 STATIC void kbase_pm_invoke(kbase_device *kbdev, kbase_pm_core_type core_type, u64 cores, kbasep_pm_action action)
80 u32 lo = cores & 0xFFFFFFFF;
81 u32 hi = (cores >> 32) & 0xFFFFFFFF;
83 lockdep_assert_held(&kbdev->pm.power_change_lock);
85 reg = core_type_to_reg(core_type, action);
87 KBASE_DEBUG_ASSERT(reg);
88 #ifdef CONFIG_MALI_GATOR_SUPPORT
90 if (action == ACTION_PWRON)
91 kbase_trace_mali_pm_power_on(core_type, cores);
92 else if (action == ACTION_PWROFF)
93 kbase_trace_mali_pm_power_off(core_type, cores);
95 #endif /* CONFIG_MALI_GATOR_SUPPORT */
98 if (action == ACTION_PWRON)
100 case KBASE_PM_CORE_SHADER:
101 KBASE_TRACE_ADD(kbdev, PM_PWRON, NULL, NULL, 0u, lo);
103 case KBASE_PM_CORE_TILER:
104 KBASE_TRACE_ADD(kbdev, PM_PWRON_TILER, NULL, NULL, 0u, lo);
106 case KBASE_PM_CORE_L2:
107 KBASE_TRACE_ADD(kbdev, PM_PWRON_L2, NULL, NULL, 0u, lo);
113 else if (action == ACTION_PWROFF)
115 case KBASE_PM_CORE_SHADER:
116 KBASE_TRACE_ADD(kbdev, PM_PWROFF, NULL, NULL, 0u, lo);
118 case KBASE_PM_CORE_TILER:
119 KBASE_TRACE_ADD(kbdev, PM_PWROFF_TILER, NULL, NULL, 0u, lo);
121 case KBASE_PM_CORE_L2:
122 KBASE_TRACE_ADD(kbdev, PM_PWROFF_L2, NULL, NULL, 0u, lo);
131 kbase_reg_write(kbdev, GPU_CONTROL_REG(reg), lo, NULL);
134 kbase_reg_write(kbdev, GPU_CONTROL_REG(reg + 4), hi, NULL);
137 /** Get information about a core set
139 * This function gets information (chosen by \c action) about a set of cores of a type given by \c core_type. It is a
140 * static function used by @ref kbase_pm_get_present_cores, @ref kbase_pm_get_active_cores, @ref
141 * kbase_pm_get_trans_cores and @ref kbase_pm_get_ready_cores.
143 * @param kbdev The kbase device structure of the device
144 * @param core_type The type of core that the should be queried
145 * @param action The property of the cores to query
147 * @return A bit mask specifying the state of the cores
149 static u64 kbase_pm_get_state(kbase_device *kbdev, kbase_pm_core_type core_type, kbasep_pm_action action)
154 reg = core_type_to_reg(core_type, action);
156 KBASE_DEBUG_ASSERT(reg);
158 lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg), NULL);
159 hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg + 4), NULL);
161 return (((u64) hi) << 32) | ((u64) lo);
164 void kbasep_pm_read_present_cores(kbase_device *kbdev)
166 kbdev->shader_present_bitmap = kbase_pm_get_state(kbdev, KBASE_PM_CORE_SHADER, ACTION_PRESENT);
167 kbdev->tiler_present_bitmap = kbase_pm_get_state(kbdev, KBASE_PM_CORE_TILER, ACTION_PRESENT);
168 kbdev->l2_present_bitmap = kbase_pm_get_state(kbdev, KBASE_PM_CORE_L2, ACTION_PRESENT);
169 kbdev->l3_present_bitmap = kbase_pm_get_state(kbdev, KBASE_PM_CORE_L3, ACTION_PRESENT);
171 kbdev->shader_inuse_bitmap = 0;
172 kbdev->shader_needed_bitmap = 0;
173 kbdev->shader_available_bitmap = 0;
174 kbdev->tiler_available_bitmap = 0;
175 kbdev->l2_users_count = 0;
176 kbdev->l2_available_bitmap = 0;
177 kbdev->tiler_needed_cnt = 0;
178 kbdev->tiler_inuse_cnt = 0;
180 memset(kbdev->shader_needed_cnt, 0, sizeof(kbdev->shader_needed_cnt));
183 KBASE_EXPORT_TEST_API(kbasep_pm_read_present_cores)
185 /** Get the cores that are present
187 u64 kbase_pm_get_present_cores(kbase_device *kbdev, kbase_pm_core_type type)
189 KBASE_DEBUG_ASSERT(kbdev != NULL);
192 case KBASE_PM_CORE_L3:
193 return kbdev->l3_present_bitmap;
195 case KBASE_PM_CORE_L2:
196 return kbdev->l2_present_bitmap;
198 case KBASE_PM_CORE_SHADER:
199 return kbdev->shader_present_bitmap;
201 case KBASE_PM_CORE_TILER:
202 return kbdev->tiler_present_bitmap;
205 KBASE_DEBUG_ASSERT(0);
209 KBASE_EXPORT_TEST_API(kbase_pm_get_present_cores)
211 /** Get the cores that are "active" (busy processing work)
213 u64 kbase_pm_get_active_cores(kbase_device *kbdev, kbase_pm_core_type type)
215 return kbase_pm_get_state(kbdev, type, ACTION_PWRACTIVE);
218 KBASE_EXPORT_TEST_API(kbase_pm_get_active_cores)
220 /** Get the cores that are transitioning between power states
222 u64 kbase_pm_get_trans_cores(kbase_device *kbdev, kbase_pm_core_type type)
224 return kbase_pm_get_state(kbdev, type, ACTION_PWRTRANS);
227 KBASE_EXPORT_TEST_API(kbase_pm_get_trans_cores)
228 /** Get the cores that are powered on
230 u64 kbase_pm_get_ready_cores(kbase_device *kbdev, kbase_pm_core_type type)
233 result = kbase_pm_get_state(kbdev, type, ACTION_READY);
236 case KBASE_PM_CORE_SHADER:
237 KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED, NULL, NULL, 0u, (u32) result);
239 case KBASE_PM_CORE_TILER:
240 KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, NULL, 0u, (u32) result);
242 case KBASE_PM_CORE_L2:
243 KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, NULL, 0u, (u32) result);
246 /* NB: L3 not currently traced */
253 KBASE_EXPORT_TEST_API(kbase_pm_get_ready_cores)
255 /** Perform power transitions for a particular core type.
257 * This function will perform any available power transitions to make the actual hardware state closer to the desired
258 * state. If a core is currently transitioning then changes to the power state of that call cannot be made until the
259 * transition has finished. Cores which are not present in the hardware are ignored if they are specified in the
260 * desired_state bitmask, however the return value will always be 0 in this case.
262 * @param kbdev The kbase device
263 * @param type The core type to perform transitions for
264 * @param desired_state A bit mask of the desired state of the cores
265 * @param in_use A bit mask of the cores that are currently running jobs.
266 * These cores have to be kept powered up because there are jobs
267 * running (or about to run) on them.
268 * @param[out] available Receives a bit mask of the cores that the job scheduler can use to submit jobs to.
269 * May be NULL if this is not needed.
270 * @param[in,out] powering_on Bit mask to update with cores that are transitioning to a power-on state.
272 * @return MALI_TRUE if the desired state has been reached, MALI_FALSE otherwise
274 STATIC mali_bool kbase_pm_transition_core_type(kbase_device *kbdev, kbase_pm_core_type type, u64 desired_state,
275 u64 in_use, u64 * const available, u64 *powering_on)
282 u64 powering_on_trans;
283 u64 desired_state_in_use;
285 lockdep_assert_held(&kbdev->pm.power_change_lock);
287 /* Get current state */
288 present = kbase_pm_get_present_cores(kbdev, type);
289 trans = kbase_pm_get_trans_cores(kbdev, type);
290 ready = kbase_pm_get_ready_cores(kbdev, type);
292 powering_on_trans = trans & *powering_on;
293 *powering_on = powering_on_trans;
295 if (available != NULL)
296 *available = (ready | powering_on_trans) & desired_state;
298 /* Update desired state to include the in-use cores. These have to be kept powered up because there are jobs
299 * running or about to run on these cores
301 desired_state_in_use = desired_state | in_use;
303 /* Update state of whether l2 caches are powered */
304 if (type == KBASE_PM_CORE_L2) {
305 if ((ready == present) && (desired_state_in_use == ready) && (trans == 0)) {
306 /* All are ready, none will be turned off, and none are transitioning */
307 kbdev->pm.l2_powered = 1;
308 if (kbdev->l2_users_count > 0) {
309 /* Notify any registered l2 cache users (optimized out when no users waiting) */
310 wake_up(&kbdev->pm.l2_powered_wait);
313 kbdev->pm.l2_powered = 0;
317 if (desired_state_in_use == ready && (trans == 0))
320 /* Restrict the cores to those that are actually present */
321 powerup = desired_state_in_use & present;
322 powerdown = (~desired_state_in_use) & present;
324 /* Restrict to cores that are not already in the desired state */
328 /* Don't transition any cores that are already transitioning, except for
329 * Mali cores that support the following case:
331 * If the SHADER_PWRON or TILER_PWRON registers are written to turn on
332 * a core that is currently transitioning to power off, then this is
333 * remembered and the shader core is automatically powered up again once
334 * the original transition completes. Once the automatic power on is
335 * complete any job scheduled on the shader core should start.
339 if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS))
340 if (KBASE_PM_CORE_SHADER == type || KBASE_PM_CORE_TILER == type)
341 trans = powering_on_trans; /* for exception cases, only mask off cores in power on transitions */
345 /* Perform transitions if any */
346 kbase_pm_invoke(kbdev, type, powerup, ACTION_PWRON);
347 kbase_pm_invoke(kbdev, type, powerdown, ACTION_PWROFF);
349 /* Recalculate cores transitioning on, and re-evaluate our state */
350 powering_on_trans |= powerup;
351 *powering_on = powering_on_trans;
352 if (available != NULL)
353 *available = (ready | powering_on_trans) & desired_state;
358 KBASE_EXPORT_TEST_API(kbase_pm_transition_core_type)
360 /** Determine which caches should be on for a particular core state.
362 * This function takes a bit mask of the present caches and the cores (or caches) that are attached to the caches that
363 * will be powered. It then computes which caches should be turned on to allow the cores requested to be powered up.
365 * @param present The bit mask of present caches
366 * @param cores_powered A bit mask of cores (or L2 caches) that are desired to be powered
368 * @return A bit mask of the caches that should be turned on
370 STATIC u64 get_desired_cache_status(u64 present, u64 cores_powered)
375 /* Find out which is the highest set bit */
376 u64 bit = fls64(present) - 1;
377 u64 bit_mask = 1ull << bit;
378 /* Create a mask which has all bits from 'bit' upwards set */
380 u64 mask = ~(bit_mask - 1);
382 /* If there are any cores powered at this bit or above (that haven't previously been processed) then we need
384 if (cores_powered & mask)
387 /* Remove bits from cores_powered and present */
388 cores_powered &= ~mask;
389 present &= ~bit_mask;
395 KBASE_EXPORT_TEST_API(get_desired_cache_status)
397 mali_bool MOCKABLE(kbase_pm_check_transitions_nolock) (struct kbase_device *kbdev)
399 mali_bool cores_are_available = MALI_FALSE;
400 mali_bool in_desired_state = MALI_TRUE;
401 u64 desired_l2_state;
402 u64 desired_l3_state;
404 u64 tiler_available_bitmap;
405 u64 shader_available_bitmap;
406 u64 shader_ready_bitmap;
407 u64 shader_transitioning_bitmap;
408 u64 l2_available_bitmap;
410 KBASE_DEBUG_ASSERT(NULL != kbdev);
411 lockdep_assert_held(&kbdev->pm.power_change_lock);
413 spin_lock(&kbdev->pm.gpu_powered_lock);
414 if (kbdev->pm.gpu_powered == MALI_FALSE) {
415 spin_unlock(&kbdev->pm.gpu_powered_lock);
416 if (kbdev->pm.desired_shader_state == 0 && kbdev->pm.desired_tiler_state == 0)
421 /* Trace that a change-state is being requested, and that it took
422 * (effectively) no time to start it. This is useful for counting how many
423 * state changes occurred, in a way that's backwards-compatible with
424 * processing the trace data */
425 kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE);
426 kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE);
428 /* If any cores are already powered then, we must keep the caches on */
429 cores_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER);
431 cores_powered |= kbdev->pm.desired_shader_state;
433 /* If there are l2 cache users registered, keep all l2s powered even if all other cores are off. */
434 if (kbdev->l2_users_count > 0)
435 cores_powered |= kbdev->l2_present_bitmap;
437 desired_l2_state = get_desired_cache_status(kbdev->l2_present_bitmap, cores_powered);
439 /* If any l2 cache is on, then enable l2 #0, for use by job manager */
440 if (0 != desired_l2_state) {
441 desired_l2_state |= 1;
442 /* Also enable tiler if l2 cache is powered */
443 kbdev->pm.desired_tiler_state = kbdev->tiler_present_bitmap;
445 kbdev->pm.desired_tiler_state = 0;
448 desired_l3_state = get_desired_cache_status(kbdev->l3_present_bitmap, desired_l2_state);
450 in_desired_state &= kbase_pm_transition_core_type(kbdev, KBASE_PM_CORE_L3, desired_l3_state, 0, NULL, &kbdev->pm.powering_on_l3_state);
451 in_desired_state &= kbase_pm_transition_core_type(kbdev, KBASE_PM_CORE_L2, desired_l2_state, 0, &l2_available_bitmap, &kbdev->pm.powering_on_l2_state);
453 if( kbdev->l2_available_bitmap != l2_available_bitmap)
455 KBASE_TIMELINE_POWER_L2(kbdev,l2_available_bitmap);
458 kbdev->l2_available_bitmap = l2_available_bitmap;
460 if (in_desired_state) {
462 in_desired_state &= kbase_pm_transition_core_type(kbdev, KBASE_PM_CORE_TILER, kbdev->pm.desired_tiler_state, 0, &tiler_available_bitmap, &kbdev->pm.powering_on_tiler_state);
463 in_desired_state &= kbase_pm_transition_core_type(kbdev, KBASE_PM_CORE_SHADER, kbdev->pm.desired_shader_state, kbdev->shader_inuse_bitmap, &shader_available_bitmap, &kbdev->pm.powering_on_shader_state);
465 if (kbdev->shader_available_bitmap != shader_available_bitmap) {
466 KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, NULL, 0u, (u32) shader_available_bitmap);
467 KBASE_TIMELINE_POWER_SHADER(kbdev, shader_available_bitmap);
470 kbdev->shader_available_bitmap = shader_available_bitmap;
472 if (kbdev->tiler_available_bitmap != tiler_available_bitmap) {
473 KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, NULL, 0u, (u32) tiler_available_bitmap);
474 KBASE_TIMELINE_POWER_TILER(kbdev, tiler_available_bitmap);
477 kbdev->tiler_available_bitmap = tiler_available_bitmap;
479 } else if ((l2_available_bitmap & kbdev->tiler_present_bitmap) != kbdev->tiler_present_bitmap) {
480 tiler_available_bitmap = 0;
482 if (kbdev->tiler_available_bitmap != tiler_available_bitmap) {
483 KBASE_TIMELINE_POWER_TILER(kbdev, tiler_available_bitmap);
486 kbdev->tiler_available_bitmap = tiler_available_bitmap;
489 /* State updated for slow-path waiters */
490 kbdev->pm.gpu_in_desired_state = in_desired_state;
492 shader_ready_bitmap = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER);
493 shader_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_SHADER);
495 /* Determine whether the cores are now available (even if the set of
496 * available cores is empty). Note that they can be available even if we've
497 * not finished transitioning to the desired state */
498 if ((kbdev->shader_available_bitmap & kbdev->pm.desired_shader_state) == kbdev->pm.desired_shader_state
499 && (kbdev->tiler_available_bitmap & kbdev->pm.desired_tiler_state) == kbdev->pm.desired_tiler_state) {
500 cores_are_available = MALI_TRUE;
502 KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE, NULL, NULL, 0u, (u32)(kbdev->shader_available_bitmap & kbdev->pm.desired_shader_state));
503 KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE_TILER, NULL, NULL, 0u, (u32)(kbdev->tiler_available_bitmap & kbdev->pm.desired_tiler_state));
505 /* Log timelining information about handling events that power up
506 * cores, to match up either with immediate submission either because
507 * cores already available, or from PM IRQ */
508 if (!in_desired_state)
509 kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
512 if (in_desired_state) {
513 KBASE_DEBUG_ASSERT(cores_are_available);
515 #ifdef CONFIG_MALI_GATOR_SUPPORT
516 kbase_trace_mali_pm_status(KBASE_PM_CORE_L3, kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L3));
517 kbase_trace_mali_pm_status(KBASE_PM_CORE_L2, kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2));
518 kbase_trace_mali_pm_status(KBASE_PM_CORE_SHADER, kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER));
519 kbase_trace_mali_pm_status(KBASE_PM_CORE_TILER, kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_TILER));
520 #endif /* CONFIG_MALI_GATOR_SUPPORT */
522 KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, NULL, kbdev->pm.gpu_in_desired_state, (u32)kbdev->pm.desired_shader_state);
523 KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED_TILER, NULL, NULL, 0u, (u32)kbdev->pm.desired_tiler_state);
525 /* Log timelining information for synchronous waiters */
526 kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
527 /* Wake slow-path waiters. Job scheduler does not use this. */
528 KBASE_TRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, NULL, 0u, 0);
529 wake_up(&kbdev->pm.gpu_in_desired_state_wait);
532 spin_unlock(&kbdev->pm.gpu_powered_lock);
534 /* kbase_pm_ca_update_core_status can cause one-level recursion into
535 * this function, so it must only be called once all changes to kbdev
536 * have been committed, and after the gpu_powered_lock has been
538 if (kbdev->shader_ready_bitmap != shader_ready_bitmap ||
539 kbdev->shader_transitioning_bitmap != shader_transitioning_bitmap) {
540 kbdev->shader_ready_bitmap = shader_ready_bitmap;
541 kbdev->shader_transitioning_bitmap = shader_transitioning_bitmap;
543 kbase_pm_ca_update_core_status(kbdev, shader_ready_bitmap, shader_transitioning_bitmap);
546 /* The core availability policy is not allowed to keep core group 0 off */
547 if (!((shader_ready_bitmap | shader_transitioning_bitmap) & kbdev->gpu_props.props.coherency_info.group[0].core_mask) &&
548 !(kbase_pm_ca_get_core_mask(kbdev) & kbdev->gpu_props.props.coherency_info.group[0].core_mask))
551 /* The core availability policy is allowed to keep core group 1 off,
552 * but all jobs specifically targeting CG1 must fail */
553 if (!((shader_ready_bitmap | shader_transitioning_bitmap) & kbdev->gpu_props.props.coherency_info.group[1].core_mask) &&
554 !(kbase_pm_ca_get_core_mask(kbdev) & kbdev->gpu_props.props.coherency_info.group[1].core_mask))
555 kbdev->pm.cg1_disabled = MALI_TRUE;
557 kbdev->pm.cg1_disabled = MALI_FALSE;
559 return cores_are_available;
561 KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_nolock)
563 void kbase_pm_check_transitions_sync(struct kbase_device *kbdev)
566 mali_bool cores_are_available;
567 /* Force the transition to be checked and reported - the cores may be
568 * 'available' (for job submission) but not fully powered up. */
569 spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
570 cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
571 /* Don't need 'cores_are_available', because we don't return anything */
572 CSTD_UNUSED(cores_are_available);
573 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
576 wait_event(kbdev->pm.gpu_in_desired_state_wait, kbdev->pm.gpu_in_desired_state);
578 /* Log timelining information that a change in state has completed */
579 kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
581 KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_sync)
583 void kbase_pm_enable_interrupts(kbase_device *kbdev)
587 KBASE_DEBUG_ASSERT(NULL != kbdev);
589 * Clear all interrupts,
590 * and unmask them all.
592 spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
593 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL, NULL);
594 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), GPU_IRQ_REG_ALL, NULL);
595 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
597 kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF, NULL);
598 kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0xFFFFFFFF, NULL);
600 kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL);
601 kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFFFFFF, NULL);
604 KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts)
606 void kbase_pm_disable_interrupts(kbase_device *kbdev)
610 KBASE_DEBUG_ASSERT(NULL != kbdev);
612 * Mask all interrupts,
613 * and clear them all.
615 spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
616 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), 0, NULL);
617 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL, NULL);
618 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
620 kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0, NULL);
621 kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF, NULL);
623 kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL);
624 kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL);
627 KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts)
631 * 0x0000: PMU TAG (RO) (0xCAFECAFE)
632 * 0x0004: PMU VERSION ID (RO) (0x00000000)
633 * 0x0008: CLOCK ENABLE (RW) (31:1 SBZ, 0 CLOCK STATE)
635 void kbase_pm_clock_on(kbase_device *kbdev, mali_bool is_resume)
637 mali_bool reset_required = is_resume;
639 KBASE_DEBUG_ASSERT(NULL != kbdev);
640 lockdep_assert_held(&kbdev->pm.lock);
642 if (kbdev->pm.gpu_powered) {
643 /* Already turned on */
644 KBASE_DEBUG_ASSERT(!is_resume);
648 KBASE_TRACE_ADD(kbdev, PM_GPU_ON, NULL, NULL, 0u, 0u);
650 if (is_resume && kbdev->pm.callback_power_resume) {
651 kbdev->pm.callback_power_resume(kbdev);
652 } else if (kbdev->pm.callback_power_on) {
653 if (kbdev->pm.callback_power_on(kbdev))
654 reset_required = MALI_TRUE;
657 if (reset_required) {
658 /* GPU state was lost, reset GPU to ensure it is in a
659 * consistent state */
660 kbase_pm_init_hw(kbdev, MALI_TRUE);
663 spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags);
664 kbdev->pm.gpu_powered = MALI_TRUE;
665 spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
667 /* Lastly, enable the interrupts */
668 kbase_pm_enable_interrupts(kbdev);
671 KBASE_EXPORT_TEST_API(kbase_pm_clock_on)
673 void kbase_pm_clock_off(kbase_device *kbdev, mali_bool is_suspend)
676 KBASE_DEBUG_ASSERT(NULL != kbdev);
677 lockdep_assert_held(&kbdev->pm.lock);
679 /* ASSERT that the cores should now be unavailable. No lock needed. */
680 KBASE_DEBUG_ASSERT(kbdev->shader_available_bitmap == 0u);
682 if (!kbdev->pm.gpu_powered) {
683 /* Already turned off */
684 if (is_suspend && kbdev->pm.callback_power_suspend)
685 kbdev->pm.callback_power_suspend(kbdev);
689 KBASE_TRACE_ADD(kbdev, PM_GPU_OFF, NULL, NULL, 0u, 0u);
691 /* Disable interrupts. This also clears any outstanding interrupts */
692 kbase_pm_disable_interrupts(kbdev);
693 /* Ensure that any IRQ handlers have finished */
694 kbase_synchronize_irqs(kbdev);
696 /* The GPU power may be turned off from this point */
697 spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags);
698 kbdev->pm.gpu_powered = MALI_FALSE;
699 spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
701 if (is_suspend && kbdev->pm.callback_power_suspend)
702 kbdev->pm.callback_power_suspend(kbdev);
703 else if (kbdev->pm.callback_power_off)
704 kbdev->pm.callback_power_off(kbdev);
707 KBASE_EXPORT_TEST_API(kbase_pm_clock_off)
709 struct kbasep_reset_timeout_data {
710 struct hrtimer timer;
715 void kbase_pm_reset_done(kbase_device *kbdev)
717 KBASE_DEBUG_ASSERT(kbdev != NULL);
718 kbdev->pm.reset_done = MALI_TRUE;
719 wake_up(&kbdev->pm.reset_done_wait);
723 * Wait for the RESET_COMPLETED IRQ to occur, then reset the waiting state.
725 STATIC void kbase_pm_wait_for_reset(kbase_device *kbdev)
727 lockdep_assert_held(&kbdev->pm.lock);
729 wait_event(kbdev->pm.reset_done_wait, (kbdev->pm.reset_done));
730 kbdev->pm.reset_done = MALI_FALSE;
733 KBASE_EXPORT_TEST_API(kbase_pm_reset_done)
735 static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer)
737 struct kbasep_reset_timeout_data *rtdata = container_of(timer, struct kbasep_reset_timeout_data, timer);
739 rtdata->timed_out = 1;
741 /* Set the wait queue to wake up kbase_pm_init_hw even though the reset hasn't completed */
742 kbase_pm_reset_done(rtdata->kbdev);
744 return HRTIMER_NORESTART;
747 static void kbase_pm_hw_issues(kbase_device *kbdev)
752 /* Needed due to MIDBASE-1494: LS_PAUSEBUFFER_DISABLE. See PRLAM-8443. */
753 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8443))
754 value |= SC_LS_PAUSEBUFFER_DISABLE;
756 /* Needed due to MIDBASE-2054: SDC_DISABLE_OQ_DISCARD. See PRLAM-10327. */
757 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10327))
758 value |= SC_SDC_DISABLE_OQ_DISCARD;
760 /* Enable alternative hardware counter selection if configured. */
761 if (kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_ALTERNATIVE_HWC))
762 value |= SC_ALT_COUNTERS;
764 /* Use software control of forward pixel kill when needed. See MIDEUR-174. */
765 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_2121))
766 value |= SC_OVERRIDE_FWD_PIXEL_KILL;
768 /* Needed due to MIDBASE-2795. ENABLE_TEXGRD_FLAGS. See PRLAM-10797. */
769 if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10797))
770 value |= SC_ENABLE_TEXGRD_FLAGS;
773 kbase_reg_write(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), value, NULL);
775 /* Limit the GPU bus bandwidth if the platform needs this. */
776 value = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), NULL);
778 /* Limit read ID width for AXI */
779 config_value = (u32) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_ARID_LIMIT);
780 value &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_READS);
781 value |= (config_value & 0x3) << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT;
783 /* Limit write ID width for AXI */
784 config_value = (u32) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_AWID_LIMIT);
785 value &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES);
786 value |= (config_value & 0x3) << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT;
788 kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), value, NULL);
791 mali_error kbase_pm_init_hw(kbase_device *kbdev, mali_bool enable_irqs )
794 struct kbasep_reset_timeout_data rtdata;
796 KBASE_DEBUG_ASSERT(NULL != kbdev);
797 lockdep_assert_held(&kbdev->pm.lock);
799 /* Ensure the clock is on before attempting to access the hardware */
800 if (!kbdev->pm.gpu_powered) {
801 if (kbdev->pm.callback_power_on)
802 kbdev->pm.callback_power_on(kbdev);
804 spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags);
805 kbdev->pm.gpu_powered = MALI_TRUE;
806 spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
809 /* Ensure interrupts are off to begin with, this also clears any outstanding interrupts */
810 kbase_pm_disable_interrupts(kbdev);
812 /* Prepare for the soft-reset */
813 kbdev->pm.reset_done = MALI_FALSE;
815 /* The cores should be made unavailable due to the reset */
816 spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
817 if (kbdev->shader_available_bitmap != 0u)
818 KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, NULL, 0u, (u32)0u);
819 if (kbdev->tiler_available_bitmap != 0u)
820 KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, NULL, 0u, (u32)0u);
821 kbdev->shader_available_bitmap = 0u;
822 kbdev->tiler_available_bitmap = 0u;
823 kbdev->l2_available_bitmap = 0u;
824 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
826 /* Soft reset the GPU */
827 KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0);
828 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_SOFT_RESET, NULL);
830 /* Unmask the reset complete interrupt only */
831 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), RESET_COMPLETED, NULL);
833 /* Initialize a structure for tracking the status of the reset */
834 rtdata.kbdev = kbdev;
835 rtdata.timed_out = 0;
837 /* Create a timer to use as a timeout on the reset */
838 hrtimer_init_on_stack(&rtdata.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
839 rtdata.timer.function = kbasep_reset_timeout;
841 hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), HRTIMER_MODE_REL);
843 /* Wait for the RESET_COMPLETED interrupt to be raised */
844 kbase_pm_wait_for_reset(kbdev);
846 if (rtdata.timed_out == 0) {
847 /* GPU has been reset */
848 hrtimer_cancel(&rtdata.timer);
849 destroy_hrtimer_on_stack(&rtdata.timer);
853 /* No interrupt has been received - check if the RAWSTAT register says the reset has completed */
854 if (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) & RESET_COMPLETED) {
855 /* The interrupt is set in the RAWSTAT; this suggests that the interrupts are not getting to the CPU */
856 KBASE_DEBUG_PRINT_WARN(KBASE_PM, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n");
857 /* If interrupts aren't working we can't continue. */
858 destroy_hrtimer_on_stack(&rtdata.timer);
862 /* The GPU doesn't seem to be responding to the reset so try a hard reset */
863 KBASE_DEBUG_PRINT_ERROR(KBASE_PM, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n", RESET_TIMEOUT);
864 KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0);
865 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET, NULL);
867 /* Restart the timer to wait for the hard reset to complete */
868 rtdata.timed_out = 0;
870 hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), HRTIMER_MODE_REL);
872 /* Wait for the RESET_COMPLETED interrupt to be raised */
873 kbase_pm_wait_for_reset(kbdev);
875 if (rtdata.timed_out == 0) {
876 /* GPU has been reset */
877 hrtimer_cancel(&rtdata.timer);
878 destroy_hrtimer_on_stack(&rtdata.timer);
882 destroy_hrtimer_on_stack(&rtdata.timer);
884 KBASE_DEBUG_PRINT_ERROR(KBASE_PM, "Failed to hard-reset the GPU (timed out after %d ms)\n", RESET_TIMEOUT);
886 /* The GPU still hasn't reset, give up */
887 return MALI_ERROR_FUNCTION_FAILED;
890 /* Re-enable interrupts if requested*/
893 kbase_pm_enable_interrupts(kbdev);
895 /* If cycle counter was in use-re enable it */
896 spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
898 if (kbdev->pm.gpu_cycle_counter_requests)
899 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_START, NULL);
901 spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
903 kbase_pm_hw_issues(kbdev);
905 return MALI_ERROR_NONE;
908 KBASE_EXPORT_TEST_API(kbase_pm_init_hw)
910 void kbase_pm_request_gpu_cycle_counter(kbase_device *kbdev)
913 KBASE_DEBUG_ASSERT(kbdev != NULL);
915 KBASE_DEBUG_ASSERT(kbdev->pm.gpu_powered);
917 spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
919 KBASE_DEBUG_ASSERT(kbdev->pm.gpu_cycle_counter_requests < INT_MAX);
921 ++kbdev->pm.gpu_cycle_counter_requests;
923 if (1 == kbdev->pm.gpu_cycle_counter_requests)
924 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_START, NULL);
926 spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
929 KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter)
931 void kbase_pm_release_gpu_cycle_counter(kbase_device *kbdev)
934 KBASE_DEBUG_ASSERT(kbdev != NULL);
936 spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
938 KBASE_DEBUG_ASSERT(kbdev->pm.gpu_cycle_counter_requests > 0);
940 --kbdev->pm.gpu_cycle_counter_requests;
942 if (0 == kbdev->pm.gpu_cycle_counter_requests)
943 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_STOP, NULL);
945 spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
948 KBASE_EXPORT_TEST_API(kbase_pm_release_gpu_cycle_counter)