drivers/gpu/arm/midgard_for_linux/backend/gpu/mali_kbase_pm_policy.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18 /*
  19  * Power policy API implementations
  20  */
  21
  22 #include <mali_kbase.h>
  23 #include <mali_midg_regmap.h>
  24 #include <mali_kbase_pm.h>
  25 #include <mali_kbase_config_defaults.h>
  26 #include <backend/gpu/mali_kbase_pm_internal.h>
  27
  28 static const struct kbase_pm_policy *const policy_list[] = {
  29 #ifdef CONFIG_MALI_NO_MALI
  30         &kbase_pm_always_on_policy_ops,
  31         &kbase_pm_demand_policy_ops,
  32         &kbase_pm_coarse_demand_policy_ops,
  33 #if !MALI_CUSTOMER_RELEASE
  34         &kbase_pm_demand_always_powered_policy_ops,
  35         &kbase_pm_fast_start_policy_ops,
  36 #endif
  37 #else                           /* CONFIG_MALI_NO_MALI */
  38         &kbase_pm_demand_policy_ops,
  39         &kbase_pm_always_on_policy_ops,
  40         &kbase_pm_coarse_demand_policy_ops,
  41 #if !MALI_CUSTOMER_RELEASE
  42         &kbase_pm_demand_always_powered_policy_ops,
  43         &kbase_pm_fast_start_policy_ops,
  44 #endif
  45 #endif /* CONFIG_MALI_NO_MALI */
  46 };
  47
  48 /* The number of policies available in the system.
  49  * This is derived from the number of functions listed in policy_get_functions.
  50  */
  51 #define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list))
  52
  53
  54 /* Function IDs for looking up Timeline Trace codes in
  55  * kbase_pm_change_state_trace_code */
  56 enum kbase_pm_func_id {
  57         KBASE_PM_FUNC_ID_REQUEST_CORES_START,
  58         KBASE_PM_FUNC_ID_REQUEST_CORES_END,
  59         KBASE_PM_FUNC_ID_RELEASE_CORES_START,
  60         KBASE_PM_FUNC_ID_RELEASE_CORES_END,
  61         /* Note: kbase_pm_unrequest_cores() is on the slow path, and we neither
  62          * expect to hit it nor tend to hit it very much anyway. We can detect
  63          * whether we need more instrumentation by a difference between
  64          * PM_CHECKTRANS events and PM_SEND/HANDLE_EVENT. */
  65
  66         /* Must be the last */
  67         KBASE_PM_FUNC_ID_COUNT
  68 };
  69
  70
  71 /* State changes during request/unrequest/release-ing cores */
  72 enum {
  73         KBASE_PM_CHANGE_STATE_SHADER = (1u << 0),
  74         KBASE_PM_CHANGE_STATE_TILER  = (1u << 1),
  75
  76         /* These two must be last */
  77         KBASE_PM_CHANGE_STATE_MASK = (KBASE_PM_CHANGE_STATE_TILER |
  78                                                 KBASE_PM_CHANGE_STATE_SHADER),
  79         KBASE_PM_CHANGE_STATE_COUNT = KBASE_PM_CHANGE_STATE_MASK + 1
  80 };
  81 typedef u32 kbase_pm_change_state;
  82
  83
  84 #ifdef CONFIG_MALI_TRACE_TIMELINE
  85 /* Timeline Trace code lookups for each function */
  86 static u32 kbase_pm_change_state_trace_code[KBASE_PM_FUNC_ID_COUNT]
  87                                         [KBASE_PM_CHANGE_STATE_COUNT] = {
  88         /* kbase_pm_request_cores */
  89         [KBASE_PM_FUNC_ID_REQUEST_CORES_START][0] = 0,
  90         [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] =
  91                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_START,
  92         [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_TILER] =
  93                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_START,
  94         [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER |
  95                                                 KBASE_PM_CHANGE_STATE_TILER] =
  96                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_START,
  97
  98         [KBASE_PM_FUNC_ID_REQUEST_CORES_END][0] = 0,
  99         [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] =
 100                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_END,
 101         [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_TILER] =
 102                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_END,
 103         [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER |
 104                                                 KBASE_PM_CHANGE_STATE_TILER] =
 105                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_END,
 106
 107         /* kbase_pm_release_cores */
 108         [KBASE_PM_FUNC_ID_RELEASE_CORES_START][0] = 0,
 109         [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] =
 110                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_START,
 111         [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_TILER] =
 112                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_START,
 113         [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER |
 114                                                 KBASE_PM_CHANGE_STATE_TILER] =
 115                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_START,
 116
 117         [KBASE_PM_FUNC_ID_RELEASE_CORES_END][0] = 0,
 118         [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] =
 119                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_END,
 120         [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_TILER] =
 121                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_END,
 122         [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER |
 123                                                 KBASE_PM_CHANGE_STATE_TILER] =
 124                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_END
 125 };
 126
 127 static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev,
 128                 enum kbase_pm_func_id func_id,
 129                 kbase_pm_change_state state)
 130 {
 131         int trace_code;
 132
 133         KBASE_DEBUG_ASSERT(func_id >= 0 && func_id < KBASE_PM_FUNC_ID_COUNT);
 134         KBASE_DEBUG_ASSERT(state != 0 && (state & KBASE_PM_CHANGE_STATE_MASK) ==
 135                                                                         state);
 136
 137         trace_code = kbase_pm_change_state_trace_code[func_id][state];
 138         KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code);
 139 }
 140
 141 #else /* CONFIG_MALI_TRACE_TIMELINE */
 142 static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev,
 143                 enum kbase_pm_func_id func_id, kbase_pm_change_state state)
 144 {
 145 }
 146
 147 #endif /* CONFIG_MALI_TRACE_TIMELINE */
 148
 149 /**
 150  * kbasep_pm_do_poweroff_cores - Process a poweroff request and power down any
 151  *                               requested shader cores
 152  * @kbdev: Device pointer
 153  */
 154 static void kbasep_pm_do_poweroff_cores(struct kbase_device *kbdev)
 155 {
 156         u64 prev_shader_state = kbdev->pm.backend.desired_shader_state;
 157         u64 prev_tiler_state = kbdev->pm.backend.desired_tiler_state;
 158
 159         lockdep_assert_held(&kbdev->pm.power_change_lock);
 160
 161         kbdev->pm.backend.desired_shader_state &=
 162                         ~kbdev->pm.backend.shader_poweroff_pending;
 163         kbdev->pm.backend.desired_tiler_state &=
 164                         ~kbdev->pm.backend.tiler_poweroff_pending;
 165
 166         kbdev->pm.backend.shader_poweroff_pending = 0;
 167         kbdev->pm.backend.tiler_poweroff_pending = 0;
 168
 169         if (prev_shader_state != kbdev->pm.backend.desired_shader_state ||
 170                         prev_tiler_state !=
 171                                 kbdev->pm.backend.desired_tiler_state ||
 172                         kbdev->pm.backend.ca_in_transition) {
 173                 bool cores_are_available;
 174
 175                 KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 176                         SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_START);
 177                 cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
 178                 KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 179                         SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_END);
 180
 181                 /* Don't need 'cores_are_available',
 182                  * because we don't return anything */
 183                 CSTD_UNUSED(cores_are_available);
 184         }
 185 }
 186
 187 static enum hrtimer_restart
 188 kbasep_pm_do_gpu_poweroff_callback(struct hrtimer *timer)
 189 {
 190         struct kbase_device *kbdev;
 191         unsigned long flags;
 192
 193         kbdev = container_of(timer, struct kbase_device,
 194                                                 pm.backend.gpu_poweroff_timer);
 195
 196         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 197
 198         /* It is safe for this call to do nothing if the work item is already
 199          * queued. The worker function will read the must up-to-date state of
 200          * kbdev->pm.backend.gpu_poweroff_pending under lock.
 201          *
 202          * If a state change occurs while the worker function is processing,
 203          * this call will succeed as a work item can be requeued once it has
 204          * started processing.
 205          */
 206         if (kbdev->pm.backend.gpu_poweroff_pending)
 207                 queue_work(kbdev->pm.backend.gpu_poweroff_wq,
 208                                         &kbdev->pm.backend.gpu_poweroff_work);
 209
 210         if (kbdev->pm.backend.shader_poweroff_pending ||
 211                         kbdev->pm.backend.tiler_poweroff_pending) {
 212                 kbdev->pm.backend.shader_poweroff_pending_time--;
 213
 214                 KBASE_DEBUG_ASSERT(
 215                                 kbdev->pm.backend.shader_poweroff_pending_time
 216                                                                         >= 0);
 217
 218                 if (!kbdev->pm.backend.shader_poweroff_pending_time)
 219                         kbasep_pm_do_poweroff_cores(kbdev);
 220         }
 221
 222         if (kbdev->pm.backend.poweroff_timer_needed) {
 223                 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 224
 225                 hrtimer_add_expires(timer, kbdev->pm.gpu_poweroff_time);
 226
 227                 return HRTIMER_RESTART;
 228         }
 229
 230         kbdev->pm.backend.poweroff_timer_running = false;
 231         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 232
 233         return HRTIMER_NORESTART;
 234 }
 235
 236 static void kbasep_pm_do_gpu_poweroff_wq(struct work_struct *data)
 237 {
 238         unsigned long flags;
 239         struct kbase_device *kbdev;
 240         bool do_poweroff = false;
 241
 242         kbdev = container_of(data, struct kbase_device,
 243                                                 pm.backend.gpu_poweroff_work);
 244
 245         mutex_lock(&kbdev->pm.lock);
 246
 247         if (kbdev->pm.backend.gpu_poweroff_pending == 0) {
 248                 mutex_unlock(&kbdev->pm.lock);
 249                 return;
 250         }
 251
 252         kbdev->pm.backend.gpu_poweroff_pending--;
 253
 254         if (kbdev->pm.backend.gpu_poweroff_pending > 0) {
 255                 mutex_unlock(&kbdev->pm.lock);
 256                 return;
 257         }
 258
 259         KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_poweroff_pending == 0);
 260
 261         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 262
 263         /* Only power off the GPU if a request is still pending */
 264         if (!kbdev->pm.backend.pm_current_policy->get_core_active(kbdev))
 265                 do_poweroff = true;
 266
 267         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 268
 269         if (do_poweroff) {
 270                 kbdev->pm.backend.poweroff_timer_needed = false;
 271                 hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer);
 272                 kbdev->pm.backend.poweroff_timer_running = false;
 273
 274                 /* Power off the GPU */
 275                 if (!kbase_pm_do_poweroff(kbdev, false)) {
 276                         /* GPU can not be powered off at present */
 277                         kbdev->pm.backend.poweroff_timer_needed = true;
 278                         kbdev->pm.backend.poweroff_timer_running = true;
 279                         hrtimer_start(&kbdev->pm.backend.gpu_poweroff_timer,
 280                                         kbdev->pm.gpu_poweroff_time,
 281                                         HRTIMER_MODE_REL);
 282                 }
 283         }
 284
 285         mutex_unlock(&kbdev->pm.lock);
 286 }
 287
 288 int kbase_pm_policy_init(struct kbase_device *kbdev)
 289 {
 290         struct workqueue_struct *wq;
 291
 292         wq = alloc_workqueue("kbase_pm_do_poweroff",
 293                         WQ_HIGHPRI | WQ_UNBOUND, 1);
 294         if (!wq)
 295                 return -ENOMEM;
 296
 297         kbdev->pm.backend.gpu_poweroff_wq = wq;
 298         INIT_WORK(&kbdev->pm.backend.gpu_poweroff_work,
 299                         kbasep_pm_do_gpu_poweroff_wq);
 300         hrtimer_init(&kbdev->pm.backend.gpu_poweroff_timer,
 301                         CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 302         kbdev->pm.backend.gpu_poweroff_timer.function =
 303                         kbasep_pm_do_gpu_poweroff_callback;
 304         kbdev->pm.backend.pm_current_policy = policy_list[0];
 305         kbdev->pm.backend.pm_current_policy->init(kbdev);
 306         kbdev->pm.gpu_poweroff_time =
 307                         HR_TIMER_DELAY_NSEC(DEFAULT_PM_GPU_POWEROFF_TICK_NS);
 308         kbdev->pm.poweroff_shader_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER;
 309         kbdev->pm.poweroff_gpu_ticks = DEFAULT_PM_POWEROFF_TICK_GPU;
 310
 311         return 0;
 312 }
 313
 314 void kbase_pm_policy_term(struct kbase_device *kbdev)
 315 {
 316         kbdev->pm.backend.pm_current_policy->term(kbdev);
 317         destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wq);
 318 }
 319
 320 void kbase_pm_cancel_deferred_poweroff(struct kbase_device *kbdev)
 321 {
 322         unsigned long flags;
 323
 324         lockdep_assert_held(&kbdev->pm.lock);
 325
 326         kbdev->pm.backend.poweroff_timer_needed = false;
 327         hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer);
 328         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 329         kbdev->pm.backend.poweroff_timer_running = false;
 330
 331         /* If wq is already running but is held off by pm.lock, make sure it has
 332          * no effect */
 333         kbdev->pm.backend.gpu_poweroff_pending = 0;
 334
 335         kbdev->pm.backend.shader_poweroff_pending = 0;
 336         kbdev->pm.backend.tiler_poweroff_pending = 0;
 337         kbdev->pm.backend.shader_poweroff_pending_time = 0;
 338
 339         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 340 }
 341
 342 void kbase_pm_update_active(struct kbase_device *kbdev)
 343 {
 344         struct kbase_pm_device_data *pm = &kbdev->pm;
 345         struct kbase_pm_backend_data *backend = &pm->backend;
 346         unsigned long flags;
 347         bool active;
 348
 349         lockdep_assert_held(&pm->lock);
 350
 351         /* pm_current_policy will never be NULL while pm.lock is held */
 352         KBASE_DEBUG_ASSERT(backend->pm_current_policy);
 353
 354         spin_lock_irqsave(&pm->power_change_lock, flags);
 355
 356         active = backend->pm_current_policy->get_core_active(kbdev);
 357
 358         if (active) {
 359                 if (backend->gpu_poweroff_pending) {
 360                         /* Cancel any pending power off request */
 361                         backend->gpu_poweroff_pending = 0;
 362
 363                         /* If a request was pending then the GPU was still
 364                          * powered, so no need to continue */
 365                         if (!kbdev->poweroff_pending) {
 366                                 spin_unlock_irqrestore(&pm->power_change_lock,
 367                                                 flags);
 368                                 return;
 369                         }
 370                 }
 371
 372                 if (!backend->poweroff_timer_running && !backend->gpu_powered &&
 373                                 (pm->poweroff_gpu_ticks ||
 374                                 pm->poweroff_shader_ticks)) {
 375                         backend->poweroff_timer_needed = true;
 376                         backend->poweroff_timer_running = true;
 377                         hrtimer_start(&backend->gpu_poweroff_timer,
 378                                         pm->gpu_poweroff_time,
 379                                         HRTIMER_MODE_REL);
 380                 }
 381
 382                 spin_unlock_irqrestore(&pm->power_change_lock, flags);
 383
 384                 /* Power on the GPU and any cores requested by the policy */
 385                 kbase_pm_do_poweron(kbdev, false);
 386         } else {
 387                 /* It is an error for the power policy to power off the GPU
 388                  * when there are contexts active */
 389                 KBASE_DEBUG_ASSERT(pm->active_count == 0);
 390
 391                 if (backend->shader_poweroff_pending ||
 392                                 backend->tiler_poweroff_pending) {
 393                         backend->shader_poweroff_pending = 0;
 394                         backend->tiler_poweroff_pending = 0;
 395                         backend->shader_poweroff_pending_time = 0;
 396                 }
 397
 398                 /* Request power off */
 399                 if (pm->backend.gpu_powered) {
 400                         if (pm->poweroff_gpu_ticks) {
 401                                 backend->gpu_poweroff_pending =
 402                                                 pm->poweroff_gpu_ticks;
 403                                 backend->poweroff_timer_needed = true;
 404                                 if (!backend->poweroff_timer_running) {
 405                                         /* Start timer if not running (eg if
 406                                          * power policy has been changed from
 407                                          * always_on to something else). This
 408                                          * will ensure the GPU is actually
 409                                          * powered off */
 410                                         backend->poweroff_timer_running
 411                                                         = true;
 412                                         hrtimer_start(
 413                                                 &backend->gpu_poweroff_timer,
 414                                                 pm->gpu_poweroff_time,
 415                                                 HRTIMER_MODE_REL);
 416                                 }
 417                                 spin_unlock_irqrestore(&pm->power_change_lock,
 418                                                 flags);
 419                         } else {
 420                                 spin_unlock_irqrestore(&pm->power_change_lock,
 421                                                 flags);
 422
 423                                 /* Power off the GPU immediately */
 424                                 if (!kbase_pm_do_poweroff(kbdev, false)) {
 425                                         /* GPU can not be powered off at present
 426                                          */
 427                                         spin_lock_irqsave(
 428                                                         &pm->power_change_lock,
 429                                                         flags);
 430                                         backend->poweroff_timer_needed = true;
 431                                         if (!backend->poweroff_timer_running) {
 432                                                 backend->poweroff_timer_running
 433                                                                 = true;
 434                                                 hrtimer_start(
 435                                                 &backend->gpu_poweroff_timer,
 436                                                         pm->gpu_poweroff_time,
 437                                                         HRTIMER_MODE_REL);
 438                                         }
 439                                         spin_unlock_irqrestore(
 440                                                         &pm->power_change_lock,
 441                                                         flags);
 442                                 }
 443                         }
 444                 } else {
 445                         spin_unlock_irqrestore(&pm->power_change_lock, flags);
 446                 }
 447         }
 448 }
 449
 450 void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev)
 451 {
 452         u64 desired_bitmap;
 453         u64 desired_tiler_bitmap;
 454         bool cores_are_available;
 455         bool do_poweroff = false;
 456
 457         lockdep_assert_held(&kbdev->pm.power_change_lock);
 458
 459         if (kbdev->pm.backend.pm_current_policy == NULL)
 460                 return;
 461
 462         desired_bitmap =
 463                 kbdev->pm.backend.pm_current_policy->get_core_mask(kbdev);
 464         desired_bitmap &= kbase_pm_ca_get_core_mask(kbdev);
 465
 466         if (kbdev->tiler_needed_cnt > 0 || kbdev->tiler_inuse_cnt > 0)
 467                 desired_tiler_bitmap = 1;
 468         else
 469                 desired_tiler_bitmap = 0;
 470
 471         if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) {
 472                 /* Unless XAFFINITY is supported, enable core 0 if tiler
 473                  * required, regardless of core availability */
 474                 if (kbdev->tiler_needed_cnt > 0 || kbdev->tiler_inuse_cnt > 0)
 475                         desired_bitmap |= 1;
 476         }
 477
 478         if (kbdev->pm.backend.desired_shader_state != desired_bitmap)
 479                 KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, NULL, 0u,
 480                                                         (u32)desired_bitmap);
 481         /* Are any cores being powered on? */
 482         if (~kbdev->pm.backend.desired_shader_state & desired_bitmap ||
 483             ~kbdev->pm.backend.desired_tiler_state & desired_tiler_bitmap ||
 484             kbdev->pm.backend.ca_in_transition) {
 485                 /* Check if we are powering off any cores before updating shader
 486                  * state */
 487                 if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap ||
 488                                 kbdev->pm.backend.desired_tiler_state &
 489                                 ~desired_tiler_bitmap) {
 490                         /* Start timer to power off cores */
 491                         kbdev->pm.backend.shader_poweroff_pending |=
 492                                 (kbdev->pm.backend.desired_shader_state &
 493                                                         ~desired_bitmap);
 494                         kbdev->pm.backend.tiler_poweroff_pending |=
 495                                 (kbdev->pm.backend.desired_tiler_state &
 496                                                         ~desired_tiler_bitmap);
 497
 498                         if (kbdev->pm.poweroff_shader_ticks)
 499                                 kbdev->pm.backend.shader_poweroff_pending_time =
 500                                                 kbdev->pm.poweroff_shader_ticks;
 501                         else
 502                                 do_poweroff = true;
 503                 }
 504
 505                 kbdev->pm.backend.desired_shader_state = desired_bitmap;
 506                 kbdev->pm.backend.desired_tiler_state = desired_tiler_bitmap;
 507
 508                 /* If any cores are being powered on, transition immediately */
 509                 cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
 510         } else if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap ||
 511                                 kbdev->pm.backend.desired_tiler_state &
 512                                 ~desired_tiler_bitmap) {
 513                 /* Start timer to power off cores */
 514                 kbdev->pm.backend.shader_poweroff_pending |=
 515                                 (kbdev->pm.backend.desired_shader_state &
 516                                                         ~desired_bitmap);
 517                 kbdev->pm.backend.tiler_poweroff_pending |=
 518                                 (kbdev->pm.backend.desired_tiler_state &
 519                                                         ~desired_tiler_bitmap);
 520                 if (kbdev->pm.poweroff_shader_ticks)
 521                         kbdev->pm.backend.shader_poweroff_pending_time =
 522                                         kbdev->pm.poweroff_shader_ticks;
 523                 else
 524                         kbasep_pm_do_poweroff_cores(kbdev);
 525         } else if (kbdev->pm.active_count == 0 && desired_bitmap != 0 &&
 526                         desired_tiler_bitmap != 0 &&
 527                         kbdev->pm.backend.poweroff_timer_needed) {
 528                 /* If power policy is keeping cores on despite there being no
 529                  * active contexts then disable poweroff timer as it isn't
 530                  * required.
 531                  * Only reset poweroff_timer_needed if we're not in the middle
 532                  * of the power off callback */
 533                 kbdev->pm.backend.poweroff_timer_needed = false;
 534         }
 535
 536         /* Ensure timer does not power off wanted cores and make sure to power
 537          * off unwanted cores */
 538         if (kbdev->pm.backend.shader_poweroff_pending ||
 539                         kbdev->pm.backend.tiler_poweroff_pending) {
 540                 kbdev->pm.backend.shader_poweroff_pending &=
 541                                 ~(kbdev->pm.backend.desired_shader_state &
 542                                                                 desired_bitmap);
 543                 kbdev->pm.backend.tiler_poweroff_pending &=
 544                                 ~(kbdev->pm.backend.desired_tiler_state &
 545                                 desired_tiler_bitmap);
 546
 547                 if (!kbdev->pm.backend.shader_poweroff_pending &&
 548                                 !kbdev->pm.backend.tiler_poweroff_pending)
 549                         kbdev->pm.backend.shader_poweroff_pending_time = 0;
 550         }
 551
 552         /* Shader poweroff is deferred to the end of the function, to eliminate
 553          * issues caused by the core availability policy recursing into this
 554          * function */
 555         if (do_poweroff)
 556                 kbasep_pm_do_poweroff_cores(kbdev);
 557
 558         /* Don't need 'cores_are_available', because we don't return anything */
 559         CSTD_UNUSED(cores_are_available);
 560 }
 561
 562 void kbase_pm_update_cores_state(struct kbase_device *kbdev)
 563 {
 564         unsigned long flags;
 565
 566         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 567
 568         kbase_pm_update_cores_state_nolock(kbdev);
 569
 570         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 571 }
 572
 573 int kbase_pm_list_policies(const struct kbase_pm_policy * const **list)
 574 {
 575         if (!list)
 576                 return POLICY_COUNT;
 577
 578         *list = policy_list;
 579
 580         return POLICY_COUNT;
 581 }
 582
 583 KBASE_EXPORT_TEST_API(kbase_pm_list_policies);
 584
 585 const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev)
 586 {
 587         KBASE_DEBUG_ASSERT(kbdev != NULL);
 588
 589         return kbdev->pm.backend.pm_current_policy;
 590 }
 591
 592 KBASE_EXPORT_TEST_API(kbase_pm_get_policy);
 593
 594 void kbase_pm_set_policy(struct kbase_device *kbdev,
 595                                 const struct kbase_pm_policy *new_policy)
 596 {
 597         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
 598         const struct kbase_pm_policy *old_policy;
 599         unsigned long flags;
 600
 601         KBASE_DEBUG_ASSERT(kbdev != NULL);
 602         KBASE_DEBUG_ASSERT(new_policy != NULL);
 603
 604         KBASE_TRACE_ADD(kbdev, PM_SET_POLICY, NULL, NULL, 0u, new_policy->id);
 605
 606         /* During a policy change we pretend the GPU is active */
 607         /* A suspend won't happen here, because we're in a syscall from a
 608          * userspace thread */
 609         kbase_pm_context_active(kbdev);
 610
 611         mutex_lock(&js_devdata->runpool_mutex);
 612         mutex_lock(&kbdev->pm.lock);
 613
 614         /* Remove the policy to prevent IRQ handlers from working on it */
 615         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 616         old_policy = kbdev->pm.backend.pm_current_policy;
 617         kbdev->pm.backend.pm_current_policy = NULL;
 618         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 619
 620         KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u,
 621                                                                 old_policy->id);
 622         if (old_policy->term)
 623                 old_policy->term(kbdev);
 624
 625         KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u,
 626                                                                 new_policy->id);
 627         if (new_policy->init)
 628                 new_policy->init(kbdev);
 629
 630         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 631         kbdev->pm.backend.pm_current_policy = new_policy;
 632         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 633
 634         /* If any core power state changes were previously attempted, but
 635          * couldn't be made because the policy was changing (current_policy was
 636          * NULL), then re-try them here. */
 637         kbase_pm_update_active(kbdev);
 638         kbase_pm_update_cores_state(kbdev);
 639
 640         mutex_unlock(&kbdev->pm.lock);
 641         mutex_unlock(&js_devdata->runpool_mutex);
 642
 643         /* Now the policy change is finished, we release our fake context active
 644          * reference */
 645         kbase_pm_context_idle(kbdev);
 646 }
 647
 648 KBASE_EXPORT_TEST_API(kbase_pm_set_policy);
 649
 650 /* Check whether a state change has finished, and trace it as completed */
 651 static void
 652 kbase_pm_trace_check_and_finish_state_change(struct kbase_device *kbdev)
 653 {
 654         if ((kbdev->shader_available_bitmap &
 655                                         kbdev->pm.backend.desired_shader_state)
 656                                 == kbdev->pm.backend.desired_shader_state &&
 657                 (kbdev->tiler_available_bitmap &
 658                                         kbdev->pm.backend.desired_tiler_state)
 659                                 == kbdev->pm.backend.desired_tiler_state)
 660                 kbase_timeline_pm_check_handle_event(kbdev,
 661                                 KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
 662 }
 663
 664 void kbase_pm_request_cores(struct kbase_device *kbdev,
 665                                 bool tiler_required, u64 shader_cores)
 666 {
 667         unsigned long flags;
 668         u64 cores;
 669
 670         kbase_pm_change_state change_gpu_state = 0u;
 671
 672         KBASE_DEBUG_ASSERT(kbdev != NULL);
 673
 674         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 675
 676         cores = shader_cores;
 677         while (cores) {
 678                 int bitnum = fls64(cores) - 1;
 679                 u64 bit = 1ULL << bitnum;
 680
 681                 /* It should be almost impossible for this to overflow. It would
 682                  * require 2^32 atoms to request a particular core, which would
 683                  * require 2^24 contexts to submit. This would require an amount
 684                  * of memory that is impossible on a 32-bit system and extremely
 685                  * unlikely on a 64-bit system. */
 686                 int cnt = ++kbdev->shader_needed_cnt[bitnum];
 687
 688                 if (1 == cnt) {
 689                         kbdev->shader_needed_bitmap |= bit;
 690                         change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
 691                 }
 692
 693                 cores &= ~bit;
 694         }
 695
 696         if (tiler_required) {
 697                 int cnt = ++kbdev->tiler_needed_cnt;
 698
 699                 if (1 == cnt)
 700                         change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER;
 701
 702                 KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt != 0);
 703         }
 704
 705         if (change_gpu_state) {
 706                 KBASE_TRACE_ADD(kbdev, PM_REQUEST_CHANGE_SHADER_NEEDED, NULL,
 707                                 NULL, 0u, (u32) kbdev->shader_needed_bitmap);
 708
 709                 kbase_timeline_pm_cores_func(kbdev,
 710                                         KBASE_PM_FUNC_ID_REQUEST_CORES_START,
 711                                                         change_gpu_state);
 712                 kbase_pm_update_cores_state_nolock(kbdev);
 713                 kbase_timeline_pm_cores_func(kbdev,
 714                                         KBASE_PM_FUNC_ID_REQUEST_CORES_END,
 715                                                         change_gpu_state);
 716         }
 717
 718         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 719 }
 720
 721 KBASE_EXPORT_TEST_API(kbase_pm_request_cores);
 722
 723 void kbase_pm_unrequest_cores(struct kbase_device *kbdev,
 724                                 bool tiler_required, u64 shader_cores)
 725 {
 726         unsigned long flags;
 727
 728         kbase_pm_change_state change_gpu_state = 0u;
 729
 730         KBASE_DEBUG_ASSERT(kbdev != NULL);
 731
 732         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 733
 734         while (shader_cores) {
 735                 int bitnum = fls64(shader_cores) - 1;
 736                 u64 bit = 1ULL << bitnum;
 737                 int cnt;
 738
 739                 KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);
 740
 741                 cnt = --kbdev->shader_needed_cnt[bitnum];
 742
 743                 if (0 == cnt) {
 744                         kbdev->shader_needed_bitmap &= ~bit;
 745
 746                         change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
 747                 }
 748
 749                 shader_cores &= ~bit;
 750         }
 751
 752         if (tiler_required) {
 753                 int cnt;
 754
 755                 KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);
 756
 757                 cnt = --kbdev->tiler_needed_cnt;
 758
 759                 if (0 == cnt)
 760                         change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER;
 761         }
 762
 763         if (change_gpu_state) {
 764                 KBASE_TRACE_ADD(kbdev, PM_UNREQUEST_CHANGE_SHADER_NEEDED, NULL,
 765                                 NULL, 0u, (u32) kbdev->shader_needed_bitmap);
 766
 767                 kbase_pm_update_cores_state_nolock(kbdev);
 768
 769                 /* Trace that any state change effectively completes immediately
 770                  * - no-one will wait on the state change */
 771                 kbase_pm_trace_check_and_finish_state_change(kbdev);
 772         }
 773
 774         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 775 }
 776
 777 KBASE_EXPORT_TEST_API(kbase_pm_unrequest_cores);
 778
 779 enum kbase_pm_cores_ready
 780 kbase_pm_register_inuse_cores(struct kbase_device *kbdev,
 781                                 bool tiler_required, u64 shader_cores)
 782 {
 783         unsigned long flags;
 784         u64 prev_shader_needed; /* Just for tracing */
 785         u64 prev_shader_inuse;  /* Just for tracing */
 786
 787         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 788
 789         prev_shader_needed = kbdev->shader_needed_bitmap;
 790         prev_shader_inuse = kbdev->shader_inuse_bitmap;
 791
 792         /* If desired_shader_state does not contain the requested cores, then
 793          * power management is not attempting to powering those cores (most
 794          * likely due to core availability policy) and a new job affinity must
 795          * be chosen */
 796         if ((kbdev->pm.backend.desired_shader_state & shader_cores) !=
 797                                                         shader_cores) {
 798                 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 799
 800                 return KBASE_NEW_AFFINITY;
 801         }
 802
 803         if ((kbdev->shader_available_bitmap & shader_cores) != shader_cores ||
 804             (tiler_required && !kbdev->tiler_available_bitmap)) {
 805                 /* Trace ongoing core transition */
 806                 kbase_timeline_pm_l2_transition_start(kbdev);
 807                 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 808                 return KBASE_CORES_NOT_READY;
 809         }
 810
 811         /* If we started to trace a state change, then trace it has being
 812          * finished by now, at the very latest */
 813         kbase_pm_trace_check_and_finish_state_change(kbdev);
 814         /* Trace core transition done */
 815         kbase_timeline_pm_l2_transition_done(kbdev);
 816
 817         while (shader_cores) {
 818                 int bitnum = fls64(shader_cores) - 1;
 819                 u64 bit = 1ULL << bitnum;
 820                 int cnt;
 821
 822                 KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);
 823
 824                 cnt = --kbdev->shader_needed_cnt[bitnum];
 825
 826                 if (0 == cnt)
 827                         kbdev->shader_needed_bitmap &= ~bit;
 828
 829                 /* shader_inuse_cnt should not overflow because there can only
 830                  * be a very limited number of jobs on the h/w at one time */
 831
 832                 kbdev->shader_inuse_cnt[bitnum]++;
 833                 kbdev->shader_inuse_bitmap |= bit;
 834
 835                 shader_cores &= ~bit;
 836         }
 837
 838         if (tiler_required) {
 839                 KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);
 840
 841                 --kbdev->tiler_needed_cnt;
 842
 843                 kbdev->tiler_inuse_cnt++;
 844
 845                 KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt != 0);
 846         }
 847
 848         if (prev_shader_needed != kbdev->shader_needed_bitmap)
 849                 KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_NEEDED, NULL,
 850                                 NULL, 0u, (u32) kbdev->shader_needed_bitmap);
 851
 852         if (prev_shader_inuse != kbdev->shader_inuse_bitmap)
 853                 KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_INUSE, NULL,
 854                                 NULL, 0u, (u32) kbdev->shader_inuse_bitmap);
 855
 856         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 857
 858         return KBASE_CORES_READY;
 859 }
 860
 861 KBASE_EXPORT_TEST_API(kbase_pm_register_inuse_cores);
 862
 863 void kbase_pm_release_cores(struct kbase_device *kbdev,
 864                                 bool tiler_required, u64 shader_cores)
 865 {
 866         unsigned long flags;
 867         kbase_pm_change_state change_gpu_state = 0u;
 868
 869         KBASE_DEBUG_ASSERT(kbdev != NULL);
 870
 871         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 872
 873         while (shader_cores) {
 874                 int bitnum = fls64(shader_cores) - 1;
 875                 u64 bit = 1ULL << bitnum;
 876                 int cnt;
 877
 878                 KBASE_DEBUG_ASSERT(kbdev->shader_inuse_cnt[bitnum] > 0);
 879
 880                 cnt = --kbdev->shader_inuse_cnt[bitnum];
 881
 882                 if (0 == cnt) {
 883                         kbdev->shader_inuse_bitmap &= ~bit;
 884                         change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
 885                 }
 886
 887                 shader_cores &= ~bit;
 888         }
 889
 890         if (tiler_required) {
 891                 int cnt;
 892
 893                 KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt > 0);
 894
 895                 cnt = --kbdev->tiler_inuse_cnt;
 896
 897                 if (0 == cnt)
 898                         change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER;
 899         }
 900
 901         if (change_gpu_state) {
 902                 KBASE_TRACE_ADD(kbdev, PM_RELEASE_CHANGE_SHADER_INUSE, NULL,
 903                                 NULL, 0u, (u32) kbdev->shader_inuse_bitmap);
 904
 905                 kbase_timeline_pm_cores_func(kbdev,
 906                                         KBASE_PM_FUNC_ID_RELEASE_CORES_START,
 907                                                         change_gpu_state);
 908                 kbase_pm_update_cores_state_nolock(kbdev);
 909                 kbase_timeline_pm_cores_func(kbdev,
 910                                         KBASE_PM_FUNC_ID_RELEASE_CORES_END,
 911                                                         change_gpu_state);
 912
 913                 /* Trace that any state change completed immediately */
 914                 kbase_pm_trace_check_and_finish_state_change(kbdev);
 915         }
 916
 917         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 918 }
 919
 920 KBASE_EXPORT_TEST_API(kbase_pm_release_cores);
 921
 922 void kbase_pm_request_cores_sync(struct kbase_device *kbdev,
 923                                         bool tiler_required,
 924                                         u64 shader_cores)
 925 {
 926         kbase_pm_request_cores(kbdev, tiler_required, shader_cores);
 927
 928         kbase_pm_check_transitions_sync(kbdev);
 929 }
 930
 931 KBASE_EXPORT_TEST_API(kbase_pm_request_cores_sync);
 932
 933 void kbase_pm_request_l2_caches(struct kbase_device *kbdev)
 934 {
 935         unsigned long flags;
 936         u32 prior_l2_users_count;
 937
 938         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 939
 940         prior_l2_users_count = kbdev->l2_users_count++;
 941
 942         KBASE_DEBUG_ASSERT(kbdev->l2_users_count != 0);
 943
 944         /* if the GPU is reset while the l2 is on, l2 will be off but
 945          * prior_l2_users_count will be > 0. l2_available_bitmap will have been
 946          * set to 0 though by kbase_pm_init_hw */
 947         if (!prior_l2_users_count || !kbdev->l2_available_bitmap)
 948                 kbase_pm_check_transitions_nolock(kbdev);
 949
 950         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 951         wait_event(kbdev->pm.backend.l2_powered_wait,
 952                                         kbdev->pm.backend.l2_powered == 1);
 953
 954         /* Trace that any state change completed immediately */
 955         kbase_pm_trace_check_and_finish_state_change(kbdev);
 956 }
 957
 958 KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches);
 959
 960 void kbase_pm_request_l2_caches_l2_is_on(struct kbase_device *kbdev)
 961 {
 962         unsigned long flags;
 963
 964         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 965
 966         kbdev->l2_users_count++;
 967
 968         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 969 }
 970
 971 KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches_l2_is_on);
 972
 973 void kbase_pm_release_l2_caches(struct kbase_device *kbdev)
 974 {
 975         unsigned long flags;
 976
 977         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 978
 979         KBASE_DEBUG_ASSERT(kbdev->l2_users_count > 0);
 980
 981         --kbdev->l2_users_count;
 982
 983         if (!kbdev->l2_users_count) {
 984                 kbase_pm_check_transitions_nolock(kbdev);
 985                 /* Trace that any state change completed immediately */
 986                 kbase_pm_trace_check_and_finish_state_change(kbdev);
 987         }
 988
 989         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 990 }
 991
 992 KBASE_EXPORT_TEST_API(kbase_pm_release_l2_caches);