drivers/gpu/arm/midgard/mali_kbase_pm_policy.c

   1 /*
   2  *
   3  * (C) COPYRIGHT ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18 /**
  19  * @file mali_kbase_pm_policy.c
  20  * Power policy API implementations
  21  */
  22
  23 #include <mali_kbase.h>
  24 #include <mali_midg_regmap.h>
  25 #include <mali_kbase_gator.h>
  26 #include <mali_kbase_pm.h>
  27
  28 extern const kbase_pm_policy kbase_pm_always_on_policy_ops;
  29 extern const kbase_pm_policy kbase_pm_coarse_demand_policy_ops;
  30 extern const kbase_pm_policy kbase_pm_demand_policy_ops;
  31
  32 #if MALI_CUSTOMER_RELEASE == 0
  33 extern const kbase_pm_policy kbase_pm_fast_start_policy_ops;
  34 extern const kbase_pm_policy kbase_pm_demand_always_powered_policy_ops;
  35 #endif
  36
  37 static const kbase_pm_policy *const policy_list[] = {
  38 #ifdef CONFIG_MALI_NO_MALI
  39         &kbase_pm_always_on_policy_ops,
  40         &kbase_pm_demand_policy_ops,
  41         &kbase_pm_coarse_demand_policy_ops,
  42 #if MALI_CUSTOMER_RELEASE == 0
  43         &kbase_pm_demand_always_powered_policy_ops,
  44         &kbase_pm_fast_start_policy_ops,
  45 #endif
  46 #else                           /* CONFIG_MALI_NO_MALI */
  47         &kbase_pm_demand_policy_ops,
  48         &kbase_pm_always_on_policy_ops,
  49         &kbase_pm_coarse_demand_policy_ops,
  50 #if MALI_CUSTOMER_RELEASE == 0
  51         &kbase_pm_demand_always_powered_policy_ops,
  52         &kbase_pm_fast_start_policy_ops,
  53 #endif
  54 #endif                          /* CONFIG_MALI_NO_MALI */
  55 };
  56
  57 /** The number of policies available in the system.
  58  * This is derived from the number of functions listed in policy_get_functions.
  59  */
  60 #define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list))
  61
  62
  63 /* Function IDs for looking up Timeline Trace codes in kbase_pm_change_state_trace_code */
  64 typedef enum
  65 {
  66         KBASE_PM_FUNC_ID_REQUEST_CORES_START,
  67         KBASE_PM_FUNC_ID_REQUEST_CORES_END,
  68         KBASE_PM_FUNC_ID_RELEASE_CORES_START,
  69         KBASE_PM_FUNC_ID_RELEASE_CORES_END,
  70         /* Note: kbase_pm_unrequest_cores() is on the slow path, and we neither
  71          * expect to hit it nor tend to hit it very much anyway. We can detect
  72          * whether we need more instrumentation by a difference between
  73          * PM_CHECKTRANS events and PM_SEND/HANDLE_EVENT. */
  74
  75         /* Must be the last */
  76         KBASE_PM_FUNC_ID_COUNT
  77 } kbase_pm_func_id;
  78
  79
  80 /* State changes during request/unrequest/release-ing cores */
  81 enum
  82 {
  83         KBASE_PM_CHANGE_STATE_SHADER = (1u << 0),
  84         KBASE_PM_CHANGE_STATE_TILER  = (1u << 1),
  85
  86         /* These two must be last */
  87         KBASE_PM_CHANGE_STATE_MASK = (KBASE_PM_CHANGE_STATE_TILER|KBASE_PM_CHANGE_STATE_SHADER),
  88         KBASE_PM_CHANGE_STATE_COUNT = KBASE_PM_CHANGE_STATE_MASK + 1
  89 };
  90 typedef u32 kbase_pm_change_state;
  91
  92
  93 #ifdef CONFIG_MALI_TRACE_TIMELINE
  94 /* Timeline Trace code lookups for each function */
  95 static u32 kbase_pm_change_state_trace_code[KBASE_PM_FUNC_ID_COUNT][KBASE_PM_CHANGE_STATE_COUNT] =
  96 {
  97         /* kbase_pm_request_cores */
  98         [KBASE_PM_FUNC_ID_REQUEST_CORES_START][0] = 0,
  99         [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] =
 100                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_START,
 101         [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_TILER] =
 102                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_START,
 103         [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER|KBASE_PM_CHANGE_STATE_TILER] =
 104                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_START,
 105
 106         [KBASE_PM_FUNC_ID_REQUEST_CORES_END][0] = 0,
 107         [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] =
 108                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_END,
 109         [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_TILER] =
 110                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_END,
 111         [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER|KBASE_PM_CHANGE_STATE_TILER] =
 112                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_END,
 113
 114         /* kbase_pm_release_cores */
 115         [KBASE_PM_FUNC_ID_RELEASE_CORES_START][0] = 0,
 116         [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] =
 117                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_START,
 118         [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_TILER] =
 119                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_START,
 120         [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER|KBASE_PM_CHANGE_STATE_TILER] =
 121                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_START,
 122
 123         [KBASE_PM_FUNC_ID_RELEASE_CORES_END][0] = 0,
 124         [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] =
 125                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_END,
 126         [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_TILER] =
 127                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_END,
 128         [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER|KBASE_PM_CHANGE_STATE_TILER] =
 129                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_END
 130 };
 131
 132 STATIC INLINE void kbase_timeline_pm_cores_func(kbase_device *kbdev,
 133                                                 kbase_pm_func_id func_id,
 134                                                 kbase_pm_change_state state)
 135 {
 136         int trace_code;
 137         KBASE_DEBUG_ASSERT(func_id >= 0 && func_id < KBASE_PM_FUNC_ID_COUNT);
 138         KBASE_DEBUG_ASSERT(state != 0 && (state & KBASE_PM_CHANGE_STATE_MASK) == state);
 139
 140         trace_code = kbase_pm_change_state_trace_code[func_id][state];
 141         KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code);
 142 }
 143
 144 #else /* CONFIG_MALI_TRACE_TIMELINE */
 145 STATIC INLINE void kbase_timeline_pm_cores_func(kbase_device *kbdev,
 146                                                 kbase_pm_func_id func_id,
 147                                                 kbase_pm_change_state state)
 148 {
 149 }
 150
 151 #endif /* CONFIG_MALI_TRACE_TIMELINE */
 152
 153 static enum hrtimer_restart kbasep_pm_do_gpu_poweroff_callback(struct hrtimer *timer)
 154 {
 155         kbase_device *kbdev;
 156
 157         kbdev = container_of(timer, kbase_device, pm.gpu_poweroff_timer);
 158
 159         /* It is safe for this call to do nothing if the work item is already queued.
 160          * The worker function will read the must up-to-date state of kbdev->pm.gpu_poweroff_pending
 161          * under lock.
 162          *
 163          * If a state change occurs while the worker function is processing, this
 164          * call will succeed as a work item can be requeued once it has started
 165          * processing.
 166          */
 167         if (kbdev->pm.gpu_poweroff_pending)
 168                 queue_work(kbdev->pm.gpu_poweroff_wq, &kbdev->pm.gpu_poweroff_work);
 169
 170         if (kbdev->pm.shader_poweroff_pending) {
 171                 unsigned long flags;
 172
 173                 spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 174
 175                 if (kbdev->pm.shader_poweroff_pending) {
 176                         kbdev->pm.shader_poweroff_pending_time--;
 177
 178                         KBASE_DEBUG_ASSERT(kbdev->pm.shader_poweroff_pending_time >= 0);
 179
 180                         if (kbdev->pm.shader_poweroff_pending_time == 0) {
 181                                 u64 prev_shader_state = kbdev->pm.desired_shader_state;
 182
 183                                 kbdev->pm.desired_shader_state &= ~kbdev->pm.shader_poweroff_pending;
 184                                 kbdev->pm.shader_poweroff_pending = 0;
 185
 186                                 if (prev_shader_state != kbdev->pm.desired_shader_state ||
 187                                     kbdev->pm.ca_in_transition != MALI_FALSE) {
 188                                         mali_bool cores_are_available;
 189
 190                                         KBASE_TIMELINE_PM_CHECKTRANS(kbdev, SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_START);
 191                                         cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
 192                                         KBASE_TIMELINE_PM_CHECKTRANS(kbdev, SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_END);
 193
 194                                         /* Don't need 'cores_are_available', because we don't return anything */
 195                                         CSTD_UNUSED(cores_are_available);
 196                                 }
 197                         }
 198                 }
 199
 200                 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 201         }
 202
 203         hrtimer_add_expires(timer, kbdev->pm.gpu_poweroff_time);
 204         return HRTIMER_RESTART;
 205 }
 206
 207 static void kbasep_pm_do_gpu_poweroff_wq(struct work_struct *data)
 208 {
 209         unsigned long flags;
 210         kbase_device *kbdev;
 211         mali_bool do_poweroff = MALI_FALSE;
 212
 213         kbdev = container_of(data, kbase_device, pm.gpu_poweroff_work);
 214
 215         mutex_lock(&kbdev->pm.lock);
 216
 217         if (kbdev->pm.gpu_poweroff_pending == 0) {
 218                 mutex_unlock(&kbdev->pm.lock);
 219                 return;
 220         }
 221
 222         kbdev->pm.gpu_poweroff_pending--;
 223
 224         if (kbdev->pm.gpu_poweroff_pending > 0) {
 225                 mutex_unlock(&kbdev->pm.lock);
 226                 return;
 227         }
 228
 229         KBASE_DEBUG_ASSERT(kbdev->pm.gpu_poweroff_pending == 0);
 230
 231         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 232
 233         /* Only power off the GPU if a request is still pending */
 234         if (kbdev->pm.pm_current_policy->get_core_active(kbdev) == MALI_FALSE)
 235                 do_poweroff = MALI_TRUE;
 236
 237         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 238
 239         if (do_poweroff != MALI_FALSE) {
 240                 kbdev->pm.poweroff_timer_running = MALI_FALSE;
 241                 /* Power off the GPU */
 242                 kbase_pm_do_poweroff(kbdev, MALI_FALSE);
 243                 hrtimer_cancel(&kbdev->pm.gpu_poweroff_timer);
 244         }
 245
 246         mutex_unlock(&kbdev->pm.lock);
 247 }
 248
 249 mali_error kbase_pm_policy_init(kbase_device *kbdev)
 250 {
 251         KBASE_DEBUG_ASSERT(kbdev != NULL);
 252
 253         kbdev->pm.gpu_poweroff_wq = alloc_workqueue("kbase_pm_do_poweroff", WQ_HIGHPRI | WQ_UNBOUND, 1);
 254         if (NULL == kbdev->pm.gpu_poweroff_wq)
 255                 return MALI_ERROR_OUT_OF_MEMORY;
 256         INIT_WORK(&kbdev->pm.gpu_poweroff_work, kbasep_pm_do_gpu_poweroff_wq);
 257
 258         hrtimer_init(&kbdev->pm.gpu_poweroff_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 259         kbdev->pm.gpu_poweroff_timer.function = kbasep_pm_do_gpu_poweroff_callback;
 260
 261         kbdev->pm.pm_current_policy = policy_list[0];
 262
 263         kbdev->pm.pm_current_policy->init(kbdev);
 264
 265         kbdev->pm.gpu_poweroff_time = HR_TIMER_DELAY_NSEC(kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_PM_GPU_POWEROFF_TICK_NS));
 266
 267         kbdev->pm.poweroff_shader_ticks = kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_PM_POWEROFF_TICK_SHADER);
 268         kbdev->pm.poweroff_gpu_ticks = kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_PM_POWEROFF_TICK_GPU);
 269
 270         return MALI_ERROR_NONE;
 271 }
 272
 273 void kbase_pm_policy_term(kbase_device *kbdev)
 274 {
 275         kbdev->pm.pm_current_policy->term(kbdev);
 276 }
 277
 278 void kbase_pm_cancel_deferred_poweroff(kbase_device *kbdev)
 279 {
 280         unsigned long flags;
 281
 282         lockdep_assert_held(&kbdev->pm.lock);
 283
 284         hrtimer_cancel(&kbdev->pm.gpu_poweroff_timer);
 285
 286         /* If wq is already running but is held off by pm.lock, make sure it has no effect */
 287         kbdev->pm.gpu_poweroff_pending = 0;
 288
 289         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 290
 291         kbdev->pm.shader_poweroff_pending = 0;
 292         kbdev->pm.shader_poweroff_pending_time = 0;
 293
 294         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 295 }
 296
 297 void kbase_pm_update_active(kbase_device *kbdev)
 298 {
 299         unsigned long flags;
 300         mali_bool active;
 301
 302         lockdep_assert_held(&kbdev->pm.lock);
 303
 304         /* pm_current_policy will never be NULL while pm.lock is held */
 305         KBASE_DEBUG_ASSERT(kbdev->pm.pm_current_policy);
 306
 307         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 308
 309         active = kbdev->pm.pm_current_policy->get_core_active(kbdev);
 310
 311         if (active != MALI_FALSE) {
 312                 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 313
 314                 if (kbdev->pm.gpu_poweroff_pending) {
 315                         /* Cancel any pending power off request */
 316                         kbdev->pm.gpu_poweroff_pending = 0;
 317
 318                         /* If a request was pending then the GPU was still powered, so no need to continue */
 319                         return;
 320                 }
 321
 322                 if (!kbdev->pm.poweroff_timer_running && !kbdev->pm.gpu_powered) {
 323                         kbdev->pm.poweroff_timer_running = MALI_TRUE;
 324                         hrtimer_start(&kbdev->pm.gpu_poweroff_timer, kbdev->pm.gpu_poweroff_time, HRTIMER_MODE_REL);
 325                 }
 326
 327                 /* Power on the GPU and any cores requested by the policy */
 328                 kbase_pm_do_poweron(kbdev, MALI_FALSE);
 329         } else {
 330                 /* It is an error for the power policy to power off the GPU
 331                  * when there are contexts active */
 332                 KBASE_DEBUG_ASSERT(kbdev->pm.active_count == 0);
 333
 334                 if (kbdev->pm.shader_poweroff_pending) {
 335                         kbdev->pm.shader_poweroff_pending = 0;
 336                         kbdev->pm.shader_poweroff_pending_time = 0;
 337                 }
 338
 339                 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 340
 341
 342                 /* Request power off */
 343                 if (kbdev->pm.gpu_powered) {
 344                         kbdev->pm.gpu_poweroff_pending = kbdev->pm.poweroff_gpu_ticks;
 345                         if (!kbdev->pm.poweroff_timer_running) {
 346                                 /* Start timer if not running (eg if power policy has been changed from always_on
 347                                  * to something else). This will ensure the GPU is actually powered off */
 348                                 kbdev->pm.poweroff_timer_running = MALI_TRUE;
 349                                 hrtimer_start(&kbdev->pm.gpu_poweroff_timer, kbdev->pm.gpu_poweroff_time, HRTIMER_MODE_REL);
 350                         }
 351                 }
 352         }
 353 }
 354
 355 void kbase_pm_update_cores_state_nolock(kbase_device *kbdev)
 356 {
 357         u64 desired_bitmap;
 358         mali_bool cores_are_available;
 359
 360         lockdep_assert_held(&kbdev->pm.power_change_lock);
 361
 362         if (kbdev->pm.pm_current_policy == NULL)
 363                 return;
 364
 365         desired_bitmap = kbdev->pm.pm_current_policy->get_core_mask(kbdev);
 366         desired_bitmap &= kbase_pm_ca_get_core_mask(kbdev);
 367
 368         /* Enable core 0 if tiler required, regardless of core availability */
 369         if (kbdev->tiler_needed_cnt > 0 || kbdev->tiler_inuse_cnt > 0)
 370                 desired_bitmap |= 1;
 371
 372         if (kbdev->pm.desired_shader_state != desired_bitmap)
 373                 KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, NULL, 0u, (u32)desired_bitmap);
 374
 375         /* Are any cores being powered on? */
 376         if (~kbdev->pm.desired_shader_state & desired_bitmap ||
 377             kbdev->pm.ca_in_transition != MALI_FALSE) {
 378                 kbdev->pm.desired_shader_state = desired_bitmap;
 379
 380                 /* If any cores are being powered on, transition immediately */
 381                 cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
 382
 383                 /* Ensure timer does not power off wanted cores */
 384                 if (kbdev->pm.shader_poweroff_pending != 0) {
 385                         kbdev->pm.shader_poweroff_pending &= ~kbdev->pm.desired_shader_state;
 386                         if (kbdev->pm.shader_poweroff_pending == 0)
 387                                 kbdev->pm.shader_poweroff_pending_time = 0;
 388                 }
 389         } else if (kbdev->pm.desired_shader_state & ~desired_bitmap) {
 390                 /* Start timer to power off cores */
 391                 kbdev->pm.shader_poweroff_pending |= (kbdev->pm.desired_shader_state & ~desired_bitmap);
 392                 kbdev->pm.shader_poweroff_pending_time = kbdev->pm.poweroff_shader_ticks;
 393         } else if (kbdev->pm.active_count == 0 && desired_bitmap != 0 && kbdev->pm.poweroff_timer_running) {
 394                 /* If power policy is keeping cores on despite there being no active contexts
 395                  * then disable poweroff timer as it isn't required */
 396                 kbdev->pm.poweroff_timer_running = MALI_FALSE;
 397                 hrtimer_cancel(&kbdev->pm.gpu_poweroff_timer);
 398         }
 399
 400         /* Don't need 'cores_are_available', because we don't return anything */
 401         CSTD_UNUSED(cores_are_available);
 402 }
 403
 404 void kbase_pm_update_cores_state(kbase_device *kbdev)
 405 {
 406         unsigned long flags;
 407
 408         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 409
 410         kbase_pm_update_cores_state_nolock(kbdev);
 411
 412         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 413 }
 414
 415 int kbase_pm_list_policies(const kbase_pm_policy * const **list)
 416 {
 417         if (!list)
 418                 return POLICY_COUNT;
 419
 420         *list = policy_list;
 421
 422         return POLICY_COUNT;
 423 }
 424
 425 KBASE_EXPORT_TEST_API(kbase_pm_list_policies)
 426
 427 const kbase_pm_policy *kbase_pm_get_policy(kbase_device *kbdev)
 428 {
 429         KBASE_DEBUG_ASSERT(kbdev != NULL);
 430
 431         return kbdev->pm.pm_current_policy;
 432 }
 433
 434 KBASE_EXPORT_TEST_API(kbase_pm_get_policy)
 435
 436 void kbase_pm_set_policy(kbase_device *kbdev, const kbase_pm_policy *new_policy)
 437 {
 438         const kbase_pm_policy *old_policy;
 439         unsigned long flags;
 440
 441         KBASE_DEBUG_ASSERT(kbdev != NULL);
 442         KBASE_DEBUG_ASSERT(new_policy != NULL);
 443
 444         KBASE_TRACE_ADD(kbdev, PM_SET_POLICY, NULL, NULL, 0u, new_policy->id);
 445
 446         /* During a policy change we pretend the GPU is active */
 447         /* A suspend won't happen here, because we're in a syscall from a userspace thread */
 448         kbase_pm_context_active(kbdev);
 449
 450         mutex_lock(&kbdev->pm.lock);
 451
 452         /* Remove the policy to prevent IRQ handlers from working on it */
 453         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 454         old_policy = kbdev->pm.pm_current_policy;
 455         kbdev->pm.pm_current_policy = NULL;
 456         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 457
 458         KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u, old_policy->id);
 459         if (old_policy->term)
 460                 old_policy->term(kbdev);
 461
 462         KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u, new_policy->id);
 463         if (new_policy->init)
 464                 new_policy->init(kbdev);
 465
 466         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 467         kbdev->pm.pm_current_policy = new_policy;
 468         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 469
 470         /* If any core power state changes were previously attempted, but couldn't
 471          * be made because the policy was changing (current_policy was NULL), then
 472          * re-try them here. */
 473         kbase_pm_update_active(kbdev);
 474         kbase_pm_update_cores_state(kbdev);
 475
 476         mutex_unlock(&kbdev->pm.lock);
 477
 478         /* Now the policy change is finished, we release our fake context active reference */
 479         kbase_pm_context_idle(kbdev);
 480 }
 481
 482 KBASE_EXPORT_TEST_API(kbase_pm_set_policy)
 483
 484 /** Check whether a state change has finished, and trace it as completed */
 485 STATIC void kbase_pm_trace_check_and_finish_state_change(kbase_device *kbdev)
 486 {
 487         if ((kbdev->shader_available_bitmap & kbdev->pm.desired_shader_state) == kbdev->pm.desired_shader_state
 488                 && (kbdev->tiler_available_bitmap & kbdev->pm.desired_tiler_state) == kbdev->pm.desired_tiler_state)
 489                 kbase_timeline_pm_check_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
 490 }
 491
 492 void kbase_pm_request_cores(kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores)
 493 {
 494         unsigned long flags;
 495         u64 cores;
 496
 497         kbase_pm_change_state change_gpu_state = 0u;
 498
 499         KBASE_DEBUG_ASSERT(kbdev != NULL);
 500
 501         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 502
 503         cores = shader_cores;
 504         while (cores) {
 505                 int bitnum = fls64(cores) - 1;
 506                 u64 bit = 1ULL << bitnum;
 507
 508                 /* It should be almost impossible for this to overflow. It would require 2^32 atoms
 509                  * to request a particular core, which would require 2^24 contexts to submit. This
 510                  * would require an amount of memory that is impossible on a 32-bit system and
 511                  * extremely unlikely on a 64-bit system. */
 512                 int cnt = ++kbdev->shader_needed_cnt[bitnum];
 513
 514                 if (1 == cnt) {
 515                         kbdev->shader_needed_bitmap |= bit;
 516                         change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
 517                 }
 518
 519                 cores &= ~bit;
 520         }
 521
 522         if (tiler_required != MALI_FALSE) {
 523                 ++kbdev->tiler_needed_cnt;
 524
 525                 KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt != 0);
 526
 527                 /* For tiler jobs, we must make sure that core 0 is not turned off if it's already on.
 528                  * However, it's safe for core 0 to be left off and turned on later whilst a tiler job
 529                  * is running. Hence, we don't need to update the cores state immediately. Also,
 530                  * attempts to turn off cores will always check the tiler_needed/inuse state first anyway.
 531                  *
 532                  * Finally, kbase_js_choose_affinity() ensures core 0 is always requested for tiler jobs
 533                  * anyway. Hence when there's only a tiler job in the system, this will still cause
 534                  * kbase_pm_update_cores_state_nolock() to be called.
 535                  *
 536                  * Note that we still need to keep track of tiler_needed/inuse_cnt, to ensure that
 537                  * kbase_pm_update_cores_state_nolock() can override the core availability policy and
 538                  * force core 0 to be powered when a tiler job is in the system. */
 539         }
 540
 541         if (change_gpu_state) {
 542                 KBASE_TRACE_ADD(kbdev, PM_REQUEST_CHANGE_SHADER_NEEDED, NULL, NULL, 0u, (u32) kbdev->shader_needed_bitmap);
 543
 544                 kbase_timeline_pm_cores_func(kbdev, KBASE_PM_FUNC_ID_REQUEST_CORES_START, change_gpu_state);
 545                 kbase_pm_update_cores_state_nolock(kbdev);
 546                 kbase_timeline_pm_cores_func(kbdev, KBASE_PM_FUNC_ID_REQUEST_CORES_END, change_gpu_state);
 547         }
 548
 549         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 550 }
 551
 552 KBASE_EXPORT_TEST_API(kbase_pm_request_cores)
 553
 554 void kbase_pm_unrequest_cores(kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores)
 555 {
 556         unsigned long flags;
 557
 558         kbase_pm_change_state change_gpu_state = 0u;
 559
 560         KBASE_DEBUG_ASSERT(kbdev != NULL);
 561
 562         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 563
 564         while (shader_cores) {
 565                 int bitnum = fls64(shader_cores) - 1;
 566                 u64 bit = 1ULL << bitnum;
 567                 int cnt;
 568
 569                 KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);
 570
 571                 cnt = --kbdev->shader_needed_cnt[bitnum];
 572
 573                 if (0 == cnt) {
 574                         kbdev->shader_needed_bitmap &= ~bit;
 575
 576                         change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
 577                 }
 578
 579                 shader_cores &= ~bit;
 580         }
 581
 582         if (tiler_required != MALI_FALSE) {
 583                 KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);
 584
 585                 --kbdev->tiler_needed_cnt;
 586
 587                 /* Whilst tiler jobs must not allow core 0 to be turned off, we don't need to make an
 588                  * extra call to kbase_pm_update_cores_state_nolock() to ensure core 0 is turned off
 589                  * when the last tiler job unrequests cores: kbase_js_choose_affinity() ensures core 0
 590                  * was originally requested for tiler jobs. Hence when there's only a tiler job in the
 591                  * system, this will still cause kbase_pm_update_cores_state_nolock() to be called. */
 592         }
 593
 594         if (change_gpu_state) {
 595                 KBASE_TRACE_ADD(kbdev, PM_UNREQUEST_CHANGE_SHADER_NEEDED, NULL, NULL, 0u, (u32) kbdev->shader_needed_bitmap);
 596
 597                 kbase_pm_update_cores_state_nolock(kbdev);
 598
 599                 /* Trace that any state change effectively completes immediately -
 600                  * no-one will wait on the state change */
 601                 kbase_pm_trace_check_and_finish_state_change(kbdev);
 602         }
 603
 604         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 605 }
 606
 607 KBASE_EXPORT_TEST_API(kbase_pm_unrequest_cores)
 608
 609 kbase_pm_cores_ready kbase_pm_register_inuse_cores(kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores)
 610 {
 611         unsigned long flags;
 612         u64 prev_shader_needed; /* Just for tracing */
 613         u64 prev_shader_inuse;  /* Just for tracing */
 614
 615         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 616
 617         prev_shader_needed = kbdev->shader_needed_bitmap;
 618         prev_shader_inuse = kbdev->shader_inuse_bitmap;
 619
 620         /* If desired_shader_state does not contain the requested cores, then power
 621          * management is not attempting to powering those cores (most likely
 622          * due to core availability policy) and a new job affinity must be
 623          * chosen */
 624         if ((kbdev->pm.desired_shader_state & shader_cores) != shader_cores) {
 625                 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 626
 627                 return KBASE_NEW_AFFINITY;
 628         }
 629
 630         if ((kbdev->shader_available_bitmap & shader_cores) != shader_cores ||
 631             (tiler_required != MALI_FALSE && !kbdev->tiler_available_bitmap)) {
 632                 /* Trace ongoing core transition */
 633                 kbase_timeline_pm_l2_transition_start(kbdev);
 634                 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 635                 return KBASE_CORES_NOT_READY;
 636         }
 637
 638         /* If we started to trace a state change, then trace it has being finished
 639          * by now, at the very latest */
 640         kbase_pm_trace_check_and_finish_state_change(kbdev);
 641         /* Trace core transition done */
 642         kbase_timeline_pm_l2_transition_done(kbdev);
 643
 644         while (shader_cores) {
 645                 int bitnum = fls64(shader_cores) - 1;
 646                 u64 bit = 1ULL << bitnum;
 647                 int cnt;
 648
 649                 KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);
 650
 651                 cnt = --kbdev->shader_needed_cnt[bitnum];
 652
 653                 if (0 == cnt)
 654                         kbdev->shader_needed_bitmap &= ~bit;
 655
 656                 /* shader_inuse_cnt should not overflow because there can only be a
 657                  * very limited number of jobs on the h/w at one time */
 658
 659                 kbdev->shader_inuse_cnt[bitnum]++;
 660                 kbdev->shader_inuse_bitmap |= bit;
 661
 662                 shader_cores &= ~bit;
 663         }
 664
 665         if (tiler_required != MALI_FALSE) {
 666                 KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);
 667
 668                 --kbdev->tiler_needed_cnt;
 669
 670                 kbdev->tiler_inuse_cnt++;
 671
 672                 KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt != 0);
 673         }
 674
 675         if (prev_shader_needed != kbdev->shader_needed_bitmap)
 676                 KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_NEEDED, NULL, NULL, 0u, (u32) kbdev->shader_needed_bitmap);
 677
 678         if (prev_shader_inuse != kbdev->shader_inuse_bitmap)
 679                 KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_INUSE, NULL, NULL, 0u, (u32) kbdev->shader_inuse_bitmap);
 680
 681         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 682
 683         return KBASE_CORES_READY;
 684 }
 685
 686 KBASE_EXPORT_TEST_API(kbase_pm_register_inuse_cores)
 687
 688 void kbase_pm_release_cores(kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores)
 689 {
 690         unsigned long flags;
 691         kbase_pm_change_state change_gpu_state = 0u;
 692
 693         KBASE_DEBUG_ASSERT(kbdev != NULL);
 694
 695         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 696
 697         while (shader_cores) {
 698                 int bitnum = fls64(shader_cores) - 1;
 699                 u64 bit = 1ULL << bitnum;
 700                 int cnt;
 701
 702                 KBASE_DEBUG_ASSERT(kbdev->shader_inuse_cnt[bitnum] > 0);
 703
 704                 cnt = --kbdev->shader_inuse_cnt[bitnum];
 705
 706                 if (0 == cnt) {
 707                         kbdev->shader_inuse_bitmap &= ~bit;
 708                         change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
 709                 }
 710
 711                 shader_cores &= ~bit;
 712         }
 713
 714         if (tiler_required != MALI_FALSE) {
 715                 KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt > 0);
 716
 717                 --kbdev->tiler_inuse_cnt;
 718
 719                 /* Whilst tiler jobs must not allow core 0 to be turned off, we don't need to make an
 720                  * extra call to kbase_pm_update_cores_state_nolock() to ensure core 0 is turned off
 721                  * when the last tiler job finishes: kbase_js_choose_affinity() ensures core 0 was
 722                  * originally requested for tiler jobs. Hence when there's only a tiler job in the
 723                  * system, this will still cause kbase_pm_update_cores_state_nolock() to be called */
 724         }
 725
 726         if (change_gpu_state) {
 727                 KBASE_TRACE_ADD(kbdev, PM_RELEASE_CHANGE_SHADER_INUSE, NULL, NULL, 0u, (u32) kbdev->shader_inuse_bitmap);
 728
 729                 kbase_timeline_pm_cores_func(kbdev, KBASE_PM_FUNC_ID_RELEASE_CORES_START, change_gpu_state);
 730                 kbase_pm_update_cores_state_nolock(kbdev);
 731                 kbase_timeline_pm_cores_func(kbdev, KBASE_PM_FUNC_ID_RELEASE_CORES_END, change_gpu_state);
 732
 733                 /* Trace that any state change completed immediately */
 734                 kbase_pm_trace_check_and_finish_state_change(kbdev);
 735         }
 736
 737         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 738 }
 739
 740 KBASE_EXPORT_TEST_API(kbase_pm_release_cores)
 741
 742 void kbase_pm_request_cores_sync(struct kbase_device *kbdev, mali_bool tiler_required, u64 shader_cores)
 743 {
 744         kbase_pm_request_cores(kbdev, tiler_required, shader_cores);
 745
 746         kbase_pm_check_transitions_sync(kbdev);
 747 }
 748
 749 KBASE_EXPORT_TEST_API(kbase_pm_request_cores_sync)
 750
 751 void kbase_pm_request_l2_caches(kbase_device *kbdev)
 752 {
 753         unsigned long flags;
 754         u32 prior_l2_users_count;
 755         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 756
 757         prior_l2_users_count = kbdev->l2_users_count++;
 758
 759         KBASE_DEBUG_ASSERT(kbdev->l2_users_count != 0);
 760
 761         if (!prior_l2_users_count)
 762                 kbase_pm_update_cores_state_nolock(kbdev);
 763
 764         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 765         wait_event(kbdev->pm.l2_powered_wait, kbdev->pm.l2_powered == 1);
 766
 767         /* Trace that any state change completed immediately */
 768         kbase_pm_trace_check_and_finish_state_change(kbdev);
 769 }
 770
 771 KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches)
 772
 773 void kbase_pm_release_l2_caches(kbase_device *kbdev)
 774 {
 775         unsigned long flags;
 776         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 777
 778         KBASE_DEBUG_ASSERT(kbdev->l2_users_count > 0);
 779
 780         --kbdev->l2_users_count;
 781
 782         if (!kbdev->l2_users_count) {
 783                 kbase_pm_update_cores_state_nolock(kbdev);
 784                 /* Trace that any state change completed immediately */
 785                 kbase_pm_trace_check_and_finish_state_change(kbdev);
 786         }
 787
 788         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 789 }
 790
 791 KBASE_EXPORT_TEST_API(kbase_pm_release_l2_caches)
 792