3 * (C) COPYRIGHT ARM Limited. All rights reserved.
5 * This program is free software and is provided to you under the terms of the
6 * GNU General Public License version 2 as published by the Free Software
7 * Foundation, and any use by you of this program is subject to the terms
10 * A copy of the licence is included with the program, and can also be obtained
11 * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
12 * Boston, MA 02110-1301, USA.
19 * @file mali_kbase_pm_policy.c
20 * Power policy API implementations
23 #include <kbase/src/common/mali_kbase.h>
24 #include <kbase/src/common/mali_midg_regmap.h>
25 #include <kbase/src/common/mali_kbase_gator.h>
26 #include <kbase/src/common/mali_kbase_pm.h>
28 extern const kbase_pm_policy kbase_pm_always_on_policy_ops;
29 extern const kbase_pm_policy kbase_pm_coarse_demand_policy_ops;
30 extern const kbase_pm_policy kbase_pm_demand_policy_ops;
32 #if MALI_CUSTOMER_RELEASE == 0
33 extern const kbase_pm_policy kbase_pm_fast_start_policy_ops;
34 extern const kbase_pm_policy kbase_pm_demand_always_powered_policy_ops;
37 static const kbase_pm_policy *const policy_list[] = {
38 #ifdef CONFIG_MALI_NO_MALI
39 &kbase_pm_always_on_policy_ops,
40 &kbase_pm_demand_policy_ops,
41 &kbase_pm_coarse_demand_policy_ops,
42 #if MALI_CUSTOMER_RELEASE == 0
43 &kbase_pm_demand_always_powered_policy_ops,
44 &kbase_pm_fast_start_policy_ops,
46 #else /* CONFIG_MALI_NO_MALI */
47 &kbase_pm_demand_policy_ops,
48 &kbase_pm_always_on_policy_ops,
49 &kbase_pm_coarse_demand_policy_ops,
50 #if MALI_CUSTOMER_RELEASE == 0
51 &kbase_pm_demand_always_powered_policy_ops,
52 &kbase_pm_fast_start_policy_ops,
54 #endif /* CONFIG_MALI_NO_MALI */
57 /** The number of policies available in the system.
58 * This is derived from the number of functions listed in policy_get_functions.
60 #define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list))
63 /* Function IDs for looking up Timeline Trace codes in kbase_pm_change_state_trace_code */
66 KBASE_PM_FUNC_ID_REQUEST_CORES_START,
67 KBASE_PM_FUNC_ID_REQUEST_CORES_END,
68 KBASE_PM_FUNC_ID_RELEASE_CORES_START,
69 KBASE_PM_FUNC_ID_RELEASE_CORES_END,
70 /* Note: kbase_pm_unrequest_cores() is on the slow path, and we neither
71 * expect to hit it nor tend to hit it very much anyway. We can detect
72 * whether we need more instrumentation by a difference between
73 * PM_CHECKTRANS events and PM_SEND/HANDLE_EVENT. */
75 /* Must be the last */
76 KBASE_PM_FUNC_ID_COUNT
80 /* State changes during request/unrequest/release-ing cores */
83 KBASE_PM_CHANGE_STATE_SHADER = (1u << 0),
84 KBASE_PM_CHANGE_STATE_TILER = (1u << 1),
86 /* These two must be last */
87 KBASE_PM_CHANGE_STATE_MASK = (KBASE_PM_CHANGE_STATE_TILER|KBASE_PM_CHANGE_STATE_SHADER),
88 KBASE_PM_CHANGE_STATE_COUNT = KBASE_PM_CHANGE_STATE_MASK + 1
90 typedef u32 kbase_pm_change_state;
93 #ifdef CONFIG_MALI_TRACE_TIMELINE
94 /* Timeline Trace code lookups for each function */
95 static u32 kbase_pm_change_state_trace_code[KBASE_PM_FUNC_ID_COUNT][KBASE_PM_CHANGE_STATE_COUNT] =
97 /* kbase_pm_request_cores */
98 [KBASE_PM_FUNC_ID_REQUEST_CORES_START][0] = 0,
99 [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] =
100 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_START,
101 [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_TILER] =
102 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_START,
103 [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER|KBASE_PM_CHANGE_STATE_TILER] =
104 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_START,
106 [KBASE_PM_FUNC_ID_REQUEST_CORES_END][0] = 0,
107 [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] =
108 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_END,
109 [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_TILER] =
110 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_END,
111 [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER|KBASE_PM_CHANGE_STATE_TILER] =
112 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_END,
114 /* kbase_pm_release_cores */
115 [KBASE_PM_FUNC_ID_RELEASE_CORES_START][0] = 0,
116 [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] =
117 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_START,
118 [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_TILER] =
119 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_START,
120 [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER|KBASE_PM_CHANGE_STATE_TILER] =
121 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_START,
123 [KBASE_PM_FUNC_ID_RELEASE_CORES_END][0] = 0,
124 [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] =
125 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_END,
126 [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_TILER] =
127 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_END,
128 [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER|KBASE_PM_CHANGE_STATE_TILER] =
129 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_END
132 STATIC INLINE void kbase_timeline_pm_cores_func(kbase_device *kbdev,
133 kbase_pm_func_id func_id,
134 kbase_pm_change_state state)
137 KBASE_DEBUG_ASSERT(func_id >= 0 && func_id < KBASE_PM_FUNC_ID_COUNT);
138 KBASE_DEBUG_ASSERT(state != 0 && (state & KBASE_PM_CHANGE_STATE_MASK) == state);
140 trace_code = kbase_pm_change_state_trace_code[func_id][state];
141 KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code);
144 #else /* CONFIG_MALI_TRACE_TIMELINE */
145 STATIC INLINE void kbase_timeline_pm_cores_func(kbase_device *kbdev,
146 kbase_pm_func_id func_id,
147 kbase_pm_change_state state)
151 #endif /* CONFIG_MALI_TRACE_TIMELINE */
153 static enum hrtimer_restart kbasep_pm_do_gpu_poweroff_callback(struct hrtimer *timer)
157 kbdev = container_of(timer, kbase_device, pm.gpu_poweroff_timer);
159 /* It is safe for this call to do nothing if the work item is already queued.
160 * The worker function will read the must up-to-date state of kbdev->pm.gpu_poweroff_pending
163 * If a state change occurs while the worker function is processing, this
164 * call will succeed as a work item can be requeued once it has started
167 if (kbdev->pm.gpu_poweroff_pending)
168 queue_work(kbdev->pm.gpu_poweroff_wq, &kbdev->pm.gpu_poweroff_work);
170 if (kbdev->pm.shader_poweroff_pending) {
173 spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
175 if (kbdev->pm.shader_poweroff_pending) {
176 kbdev->pm.shader_poweroff_pending_time--;
178 KBASE_DEBUG_ASSERT(kbdev->pm.shader_poweroff_pending_time >= 0);
180 if (kbdev->pm.shader_poweroff_pending_time == 0) {
181 u64 prev_shader_state = kbdev->pm.desired_shader_state;
183 kbdev->pm.desired_shader_state &= ~kbdev->pm.shader_poweroff_pending;
184 kbdev->pm.shader_poweroff_pending = 0;
186 if (prev_shader_state != kbdev->pm.desired_shader_state ||
187 kbdev->pm.ca_in_transition != MALI_FALSE) {
188 mali_bool cores_are_available;
190 KBASE_TIMELINE_PM_CHECKTRANS(kbdev, SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_START);
191 cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
192 KBASE_TIMELINE_PM_CHECKTRANS(kbdev, SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_END);
194 /* Don't need 'cores_are_available', because we don't return anything */
195 CSTD_UNUSED(cores_are_available);
200 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
203 hrtimer_add_expires(timer, kbdev->pm.gpu_poweroff_time);
204 return HRTIMER_RESTART;
207 static void kbasep_pm_do_gpu_poweroff_wq(struct work_struct *data)
211 mali_bool do_poweroff = MALI_FALSE;
213 kbdev = container_of(data, kbase_device, pm.gpu_poweroff_work);
215 mutex_lock(&kbdev->pm.lock);
217 if (kbdev->pm.gpu_poweroff_pending == 0) {
218 mutex_unlock(&kbdev->pm.lock);
222 kbdev->pm.gpu_poweroff_pending--;
224 if (kbdev->pm.gpu_poweroff_pending > 0) {
225 mutex_unlock(&kbdev->pm.lock);
229 KBASE_DEBUG_ASSERT(kbdev->pm.gpu_poweroff_pending == 0);
231 spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
233 /* Only power off the GPU if a request is still pending */
234 if (kbdev->pm.pm_current_policy->get_core_active(kbdev) == MALI_FALSE)
235 do_poweroff = MALI_TRUE;
237 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
239 if (do_poweroff != MALI_FALSE) {
240 kbdev->pm.poweroff_timer_running = MALI_FALSE;
241 /* Power off the GPU */
242 kbase_pm_do_poweroff(kbdev, MALI_FALSE);
243 hrtimer_cancel(&kbdev->pm.gpu_poweroff_timer);
246 mutex_unlock(&kbdev->pm.lock);
249 mali_error kbase_pm_policy_init(kbase_device *kbdev)
251 KBASE_DEBUG_ASSERT(kbdev != NULL);
253 kbdev->pm.gpu_poweroff_wq = alloc_workqueue("kbase_pm_do_poweroff", WQ_HIGHPRI | WQ_UNBOUND, 1);
254 if (NULL == kbdev->pm.gpu_poweroff_wq)
255 return MALI_ERROR_OUT_OF_MEMORY;
256 INIT_WORK(&kbdev->pm.gpu_poweroff_work, kbasep_pm_do_gpu_poweroff_wq);
258 hrtimer_init(&kbdev->pm.gpu_poweroff_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
259 kbdev->pm.gpu_poweroff_timer.function = kbasep_pm_do_gpu_poweroff_callback;
261 kbdev->pm.pm_current_policy = policy_list[0];
263 kbdev->pm.pm_current_policy->init(kbdev);
265 kbdev->pm.gpu_poweroff_time = HR_TIMER_DELAY_NSEC(kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_PM_GPU_POWEROFF_TICK_NS));
267 kbdev->pm.poweroff_shader_ticks = kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_PM_POWEROFF_TICK_SHADER);
268 kbdev->pm.poweroff_gpu_ticks = kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_PM_POWEROFF_TICK_GPU);
270 return MALI_ERROR_NONE;
273 void kbase_pm_policy_term(kbase_device *kbdev)
275 kbdev->pm.pm_current_policy->term(kbdev);
278 void kbase_pm_cancel_deferred_poweroff(kbase_device *kbdev)
282 lockdep_assert_held(&kbdev->pm.lock);
284 hrtimer_cancel(&kbdev->pm.gpu_poweroff_timer);
286 /* If wq is already running but is held off by pm.lock, make sure it has no effect */
287 kbdev->pm.gpu_poweroff_pending = 0;
289 spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
291 kbdev->pm.shader_poweroff_pending = 0;
292 kbdev->pm.shader_poweroff_pending_time = 0;
294 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
297 void kbase_pm_update_active(kbase_device *kbdev)
302 lockdep_assert_held(&kbdev->pm.lock);
304 /* pm_current_policy will never be NULL while pm.lock is held */
305 KBASE_DEBUG_ASSERT(kbdev->pm.pm_current_policy);
307 spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
309 active = kbdev->pm.pm_current_policy->get_core_active(kbdev);
311 if (active != MALI_FALSE) {
312 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
314 if (kbdev->pm.gpu_poweroff_pending) {
315 /* Cancel any pending power off request */
316 kbdev->pm.gpu_poweroff_pending = 0;
318 /* If a request was pending then the GPU was still powered, so no need to continue */
322 if (!kbdev->pm.poweroff_timer_running && !kbdev->pm.gpu_powered) {
323 kbdev->pm.poweroff_timer_running = MALI_TRUE;
324 hrtimer_start(&kbdev->pm.gpu_poweroff_timer, kbdev->pm.gpu_poweroff_time, HRTIMER_MODE_REL);
327 /* Power on the GPU and any cores requested by the policy */
328 kbase_pm_do_poweron(kbdev, MALI_FALSE);
330 /* It is an error for the power policy to power off the GPU
331 * when there are contexts active */
332 KBASE_DEBUG_ASSERT(kbdev->pm.active_count == 0);
334 if (kbdev->pm.shader_poweroff_pending) {
335 kbdev->pm.shader_poweroff_pending = 0;
336 kbdev->pm.shader_poweroff_pending_time = 0;
339 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
342 /* Request power off */
343 if (kbdev->pm.gpu_powered) {
344 kbdev->pm.gpu_poweroff_pending = kbdev->pm.poweroff_gpu_ticks;
345 if (!kbdev->pm.poweroff_timer_running) {
346 /* Start timer if not running (eg if power policy has been changed from always_on
347 * to something else). This will ensure the GPU is actually powered off */
348 kbdev->pm.poweroff_timer_running = MALI_TRUE;
349 hrtimer_start(&kbdev->pm.gpu_poweroff_timer, kbdev->pm.gpu_poweroff_time, HRTIMER_MODE_REL);
355 void kbase_pm_update_cores_state_nolock(kbase_device *kbdev)
358 mali_bool cores_are_available;
360 lockdep_assert_held(&kbdev->pm.power_change_lock);
362 if (kbdev->pm.pm_current_policy == NULL)
365 desired_bitmap = kbdev->pm.pm_current_policy->get_core_mask(kbdev);
366 desired_bitmap &= kbase_pm_ca_get_core_mask(kbdev);
368 /* Enable core 0 if tiler required, regardless of core availability */
369 if (kbdev->tiler_needed_cnt > 0 || kbdev->tiler_inuse_cnt > 0)
372 if (kbdev->pm.desired_shader_state != desired_bitmap)
373 KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, NULL, 0u, (u32)desired_bitmap);
375 /* Are any cores being powered on? */
376 if (~kbdev->pm.desired_shader_state & desired_bitmap ||
377 kbdev->pm.ca_in_transition != MALI_FALSE) {
378 kbdev->pm.desired_shader_state = desired_bitmap;
380 /* If any cores are being powered on, transition immediately */
381 cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
383 /* Ensure timer does not power off wanted cores */
384 if (kbdev->pm.shader_poweroff_pending != 0) {
385 kbdev->pm.shader_poweroff_pending &= ~kbdev->pm.desired_shader_state;
386 if (kbdev->pm.shader_poweroff_pending == 0)
387 kbdev->pm.shader_poweroff_pending_time = 0;
389 } else if (kbdev->pm.desired_shader_state & ~desired_bitmap) {
390 /* Start timer to power off cores */
391 kbdev->pm.shader_poweroff_pending |= (kbdev->pm.desired_shader_state & ~desired_bitmap);
392 kbdev->pm.shader_poweroff_pending_time = kbdev->pm.poweroff_shader_ticks;
393 } else if (kbdev->pm.active_count == 0 && desired_bitmap != 0 && kbdev->pm.poweroff_timer_running) {
394 /* If power policy is keeping cores on despite there being no active contexts
395 * then disable poweroff timer as it isn't required */
396 kbdev->pm.poweroff_timer_running = MALI_FALSE;
397 hrtimer_cancel(&kbdev->pm.gpu_poweroff_timer);
400 /* Don't need 'cores_are_available', because we don't return anything */
401 CSTD_UNUSED(cores_are_available);
404 void kbase_pm_update_cores_state(kbase_device *kbdev)
408 spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
410 kbase_pm_update_cores_state_nolock(kbdev);
412 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
415 int kbase_pm_list_policies(const kbase_pm_policy * const **list)
425 KBASE_EXPORT_TEST_API(kbase_pm_list_policies)
427 const kbase_pm_policy *kbase_pm_get_policy(kbase_device *kbdev)
429 KBASE_DEBUG_ASSERT(kbdev != NULL);
431 return kbdev->pm.pm_current_policy;
434 KBASE_EXPORT_TEST_API(kbase_pm_get_policy)
436 void kbase_pm_set_policy(kbase_device *kbdev, const kbase_pm_policy *new_policy)
438 const kbase_pm_policy *old_policy;
441 KBASE_DEBUG_ASSERT(kbdev != NULL);
442 KBASE_DEBUG_ASSERT(new_policy != NULL);
444 KBASE_TRACE_ADD(kbdev, PM_SET_POLICY, NULL, NULL, 0u, new_policy->id);
446 /* During a policy change we pretend the GPU is active */
447 /* A suspend won't happen here, because we're in a syscall from a userspace thread */
448 kbase_pm_context_active(kbdev);
450 mutex_lock(&kbdev->pm.lock);
452 /* Remove the policy to prevent IRQ handlers from working on it */
453 spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
454 old_policy = kbdev->pm.pm_current_policy;
455 kbdev->pm.pm_current_policy = NULL;
456 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
458 KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u, old_policy->id);
459 if (old_policy->term)
460 old_policy->term(kbdev);
462 KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u, new_policy->id);
463 if (new_policy->init)
464 new_policy->init(kbdev);
466 spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
467 kbdev->pm.pm_current_policy = new_policy;
468 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
470 /* If any core power state changes were previously attempted, but couldn't
471 * be made because the policy was changing (current_policy was NULL), then
472 * re-try them here. */
473 kbase_pm_update_active(kbdev);
474 kbase_pm_update_cores_state(kbdev);
476 mutex_unlock(&kbdev->pm.lock);
478 /* Now the policy change is finished, we release our fake context active reference */
479 kbase_pm_context_idle(kbdev);
482 KBASE_EXPORT_TEST_API(kbase_pm_set_policy)
484 /** Check whether a state change has finished, and trace it as completed */
485 STATIC void kbase_pm_trace_check_and_finish_state_change(kbase_device *kbdev)
487 if ((kbdev->shader_available_bitmap & kbdev->pm.desired_shader_state) == kbdev->pm.desired_shader_state
488 && (kbdev->tiler_available_bitmap & kbdev->pm.desired_tiler_state) == kbdev->pm.desired_tiler_state)
489 kbase_timeline_pm_check_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
492 void kbase_pm_request_cores(kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores)
497 kbase_pm_change_state change_gpu_state = 0u;
499 KBASE_DEBUG_ASSERT(kbdev != NULL);
501 spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
503 cores = shader_cores;
505 int bitnum = fls64(cores) - 1;
506 u64 bit = 1ULL << bitnum;
508 /* It should be almost impossible for this to overflow. It would require 2^32 atoms
509 * to request a particular core, which would require 2^24 contexts to submit. This
510 * would require an amount of memory that is impossible on a 32-bit system and
511 * extremely unlikely on a 64-bit system. */
512 int cnt = ++kbdev->shader_needed_cnt[bitnum];
515 kbdev->shader_needed_bitmap |= bit;
516 change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
522 if (tiler_required != MALI_FALSE) {
523 ++kbdev->tiler_needed_cnt;
525 KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt != 0);
527 /* For tiler jobs, we must make sure that core 0 is not turned off if it's already on.
528 * However, it's safe for core 0 to be left off and turned on later whilst a tiler job
529 * is running. Hence, we don't need to update the cores state immediately. Also,
530 * attempts to turn off cores will always check the tiler_needed/inuse state first anyway.
532 * Finally, kbase_js_choose_affinity() ensures core 0 is always requested for tiler jobs
533 * anyway. Hence when there's only a tiler job in the system, this will still cause
534 * kbase_pm_update_cores_state_nolock() to be called.
536 * Note that we still need to keep track of tiler_needed/inuse_cnt, to ensure that
537 * kbase_pm_update_cores_state_nolock() can override the core availability policy and
538 * force core 0 to be powered when a tiler job is in the system. */
541 if (change_gpu_state) {
542 KBASE_TRACE_ADD(kbdev, PM_REQUEST_CHANGE_SHADER_NEEDED, NULL, NULL, 0u, (u32) kbdev->shader_needed_bitmap);
544 kbase_timeline_pm_cores_func(kbdev, KBASE_PM_FUNC_ID_REQUEST_CORES_START, change_gpu_state);
545 kbase_pm_update_cores_state_nolock(kbdev);
546 kbase_timeline_pm_cores_func(kbdev, KBASE_PM_FUNC_ID_REQUEST_CORES_END, change_gpu_state);
549 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
552 KBASE_EXPORT_TEST_API(kbase_pm_request_cores)
554 void kbase_pm_unrequest_cores(kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores)
558 kbase_pm_change_state change_gpu_state = 0u;
560 KBASE_DEBUG_ASSERT(kbdev != NULL);
562 spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
564 while (shader_cores) {
565 int bitnum = fls64(shader_cores) - 1;
566 u64 bit = 1ULL << bitnum;
569 KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);
571 cnt = --kbdev->shader_needed_cnt[bitnum];
574 kbdev->shader_needed_bitmap &= ~bit;
576 change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
579 shader_cores &= ~bit;
582 if (tiler_required != MALI_FALSE) {
583 KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);
585 --kbdev->tiler_needed_cnt;
587 /* Whilst tiler jobs must not allow core 0 to be turned off, we don't need to make an
588 * extra call to kbase_pm_update_cores_state_nolock() to ensure core 0 is turned off
589 * when the last tiler job unrequests cores: kbase_js_choose_affinity() ensures core 0
590 * was originally requested for tiler jobs. Hence when there's only a tiler job in the
591 * system, this will still cause kbase_pm_update_cores_state_nolock() to be called. */
594 if (change_gpu_state) {
595 KBASE_TRACE_ADD(kbdev, PM_UNREQUEST_CHANGE_SHADER_NEEDED, NULL, NULL, 0u, (u32) kbdev->shader_needed_bitmap);
597 kbase_pm_update_cores_state_nolock(kbdev);
599 /* Trace that any state change effectively completes immediately -
600 * no-one will wait on the state change */
601 kbase_pm_trace_check_and_finish_state_change(kbdev);
604 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
607 KBASE_EXPORT_TEST_API(kbase_pm_unrequest_cores)
609 kbase_pm_cores_ready kbase_pm_register_inuse_cores(kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores)
612 u64 prev_shader_needed; /* Just for tracing */
613 u64 prev_shader_inuse; /* Just for tracing */
615 spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
617 prev_shader_needed = kbdev->shader_needed_bitmap;
618 prev_shader_inuse = kbdev->shader_inuse_bitmap;
620 /* If desired_shader_state does not contain the requested cores, then power
621 * management is not attempting to powering those cores (most likely
622 * due to core availability policy) and a new job affinity must be
624 if ((kbdev->pm.desired_shader_state & shader_cores) != shader_cores) {
625 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
627 return KBASE_NEW_AFFINITY;
630 if ((kbdev->shader_available_bitmap & shader_cores) != shader_cores ||
631 (tiler_required != MALI_FALSE && !kbdev->tiler_available_bitmap)) {
632 /* Trace ongoing core transition */
633 kbase_timeline_pm_l2_transition_start(kbdev);
634 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
635 return KBASE_CORES_NOT_READY;
638 /* If we started to trace a state change, then trace it has being finished
639 * by now, at the very latest */
640 kbase_pm_trace_check_and_finish_state_change(kbdev);
641 /* Trace core transition done */
642 kbase_timeline_pm_l2_transition_done(kbdev);
644 while (shader_cores) {
645 int bitnum = fls64(shader_cores) - 1;
646 u64 bit = 1ULL << bitnum;
649 KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);
651 cnt = --kbdev->shader_needed_cnt[bitnum];
654 kbdev->shader_needed_bitmap &= ~bit;
656 /* shader_inuse_cnt should not overflow because there can only be a
657 * very limited number of jobs on the h/w at one time */
659 kbdev->shader_inuse_cnt[bitnum]++;
660 kbdev->shader_inuse_bitmap |= bit;
662 shader_cores &= ~bit;
665 if (tiler_required != MALI_FALSE) {
666 KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);
668 --kbdev->tiler_needed_cnt;
670 kbdev->tiler_inuse_cnt++;
672 KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt != 0);
675 if (prev_shader_needed != kbdev->shader_needed_bitmap)
676 KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_NEEDED, NULL, NULL, 0u, (u32) kbdev->shader_needed_bitmap);
678 if (prev_shader_inuse != kbdev->shader_inuse_bitmap)
679 KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_INUSE, NULL, NULL, 0u, (u32) kbdev->shader_inuse_bitmap);
681 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
683 return KBASE_CORES_READY;
686 KBASE_EXPORT_TEST_API(kbase_pm_register_inuse_cores)
688 void kbase_pm_release_cores(kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores)
691 kbase_pm_change_state change_gpu_state = 0u;
693 KBASE_DEBUG_ASSERT(kbdev != NULL);
695 spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
697 while (shader_cores) {
698 int bitnum = fls64(shader_cores) - 1;
699 u64 bit = 1ULL << bitnum;
702 KBASE_DEBUG_ASSERT(kbdev->shader_inuse_cnt[bitnum] > 0);
704 cnt = --kbdev->shader_inuse_cnt[bitnum];
707 kbdev->shader_inuse_bitmap &= ~bit;
708 change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
711 shader_cores &= ~bit;
714 if (tiler_required != MALI_FALSE) {
715 KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt > 0);
717 --kbdev->tiler_inuse_cnt;
719 /* Whilst tiler jobs must not allow core 0 to be turned off, we don't need to make an
720 * extra call to kbase_pm_update_cores_state_nolock() to ensure core 0 is turned off
721 * when the last tiler job finishes: kbase_js_choose_affinity() ensures core 0 was
722 * originally requested for tiler jobs. Hence when there's only a tiler job in the
723 * system, this will still cause kbase_pm_update_cores_state_nolock() to be called */
726 if (change_gpu_state) {
727 KBASE_TRACE_ADD(kbdev, PM_RELEASE_CHANGE_SHADER_INUSE, NULL, NULL, 0u, (u32) kbdev->shader_inuse_bitmap);
729 kbase_timeline_pm_cores_func(kbdev, KBASE_PM_FUNC_ID_RELEASE_CORES_START, change_gpu_state);
730 kbase_pm_update_cores_state_nolock(kbdev);
731 kbase_timeline_pm_cores_func(kbdev, KBASE_PM_FUNC_ID_RELEASE_CORES_END, change_gpu_state);
733 /* Trace that any state change completed immediately */
734 kbase_pm_trace_check_and_finish_state_change(kbdev);
737 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
740 KBASE_EXPORT_TEST_API(kbase_pm_release_cores)
742 void kbase_pm_request_cores_sync(struct kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores)
744 kbase_pm_request_cores(kbdev, tiler_required, shader_cores);
746 kbase_pm_check_transitions_sync(kbdev);
749 KBASE_EXPORT_TEST_API(kbase_pm_request_cores_sync)
751 void kbase_pm_request_l2_caches(kbase_device *kbdev)
754 u32 prior_l2_users_count;
755 spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
757 prior_l2_users_count = kbdev->l2_users_count++;
759 KBASE_DEBUG_ASSERT(kbdev->l2_users_count != 0);
761 if (!prior_l2_users_count)
762 kbase_pm_update_cores_state_nolock(kbdev);
764 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
765 wait_event(kbdev->pm.l2_powered_wait, kbdev->pm.l2_powered == 1);
767 /* Trace that any state change completed immediately */
768 kbase_pm_trace_check_and_finish_state_change(kbdev);
771 KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches)
773 void kbase_pm_release_l2_caches(kbase_device *kbdev)
776 spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
778 KBASE_DEBUG_ASSERT(kbdev->l2_users_count > 0);
780 --kbdev->l2_users_count;
782 if (!kbdev->l2_users_count) {
783 kbase_pm_update_cores_state_nolock(kbdev);
784 /* Trace that any state change completed immediately */
785 kbase_pm_trace_check_and_finish_state_change(kbdev);
788 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
791 KBASE_EXPORT_TEST_API(kbase_pm_release_l2_caches)