drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18 /*
  19  * Power policy API implementations
  20  */
  21
  22 #include <mali_kbase.h>
  23 #include <mali_midg_regmap.h>
  24 #include <mali_kbase_pm.h>
  25 #include <mali_kbase_config_defaults.h>
  26 #include <backend/gpu/mali_kbase_pm_internal.h>
  27
  28 static const struct kbase_pm_policy *const policy_list[] = {
  29 #ifdef CONFIG_MALI_NO_MALI
  30         &kbase_pm_always_on_policy_ops,
  31         &kbase_pm_demand_policy_ops,
  32         &kbase_pm_coarse_demand_policy_ops,
  33 #if !MALI_CUSTOMER_RELEASE
  34         &kbase_pm_demand_always_powered_policy_ops,
  35         &kbase_pm_fast_start_policy_ops,
  36 #endif
  37 #else                           /* CONFIG_MALI_NO_MALI */
  38         &kbase_pm_demand_policy_ops,
  39         &kbase_pm_always_on_policy_ops,
  40         &kbase_pm_coarse_demand_policy_ops,
  41 #if !MALI_CUSTOMER_RELEASE
  42         &kbase_pm_demand_always_powered_policy_ops,
  43         &kbase_pm_fast_start_policy_ops,
  44 #endif
  45 #endif /* CONFIG_MALI_NO_MALI */
  46 };
  47
  48 /* The number of policies available in the system.
  49  * This is derived from the number of functions listed in policy_get_functions.
  50  */
  51 #define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list))
  52
  53
  54 /* Function IDs for looking up Timeline Trace codes in
  55  * kbase_pm_change_state_trace_code */
  56 enum kbase_pm_func_id {
  57         KBASE_PM_FUNC_ID_REQUEST_CORES_START,
  58         KBASE_PM_FUNC_ID_REQUEST_CORES_END,
  59         KBASE_PM_FUNC_ID_RELEASE_CORES_START,
  60         KBASE_PM_FUNC_ID_RELEASE_CORES_END,
  61         /* Note: kbase_pm_unrequest_cores() is on the slow path, and we neither
  62          * expect to hit it nor tend to hit it very much anyway. We can detect
  63          * whether we need more instrumentation by a difference between
  64          * PM_CHECKTRANS events and PM_SEND/HANDLE_EVENT. */
  65
  66         /* Must be the last */
  67         KBASE_PM_FUNC_ID_COUNT
  68 };
  69
  70
  71 /* State changes during request/unrequest/release-ing cores */
  72 enum {
  73         KBASE_PM_CHANGE_STATE_SHADER = (1u << 0),
  74         KBASE_PM_CHANGE_STATE_TILER  = (1u << 1),
  75
  76         /* These two must be last */
  77         KBASE_PM_CHANGE_STATE_MASK = (KBASE_PM_CHANGE_STATE_TILER |
  78                                                 KBASE_PM_CHANGE_STATE_SHADER),
  79         KBASE_PM_CHANGE_STATE_COUNT = KBASE_PM_CHANGE_STATE_MASK + 1
  80 };
  81 typedef u32 kbase_pm_change_state;
  82
  83
  84 #ifdef CONFIG_MALI_TRACE_TIMELINE
  85 /* Timeline Trace code lookups for each function */
  86 static u32 kbase_pm_change_state_trace_code[KBASE_PM_FUNC_ID_COUNT]
  87                                         [KBASE_PM_CHANGE_STATE_COUNT] = {
  88         /* kbase_pm_request_cores */
  89         [KBASE_PM_FUNC_ID_REQUEST_CORES_START][0] = 0,
  90         [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] =
  91                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_START,
  92         [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_TILER] =
  93                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_START,
  94         [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER |
  95                                                 KBASE_PM_CHANGE_STATE_TILER] =
  96                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_START,
  97
  98         [KBASE_PM_FUNC_ID_REQUEST_CORES_END][0] = 0,
  99         [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] =
 100                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_END,
 101         [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_TILER] =
 102                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_END,
 103         [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER |
 104                                                 KBASE_PM_CHANGE_STATE_TILER] =
 105                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_END,
 106
 107         /* kbase_pm_release_cores */
 108         [KBASE_PM_FUNC_ID_RELEASE_CORES_START][0] = 0,
 109         [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] =
 110                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_START,
 111         [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_TILER] =
 112                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_START,
 113         [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER |
 114                                                 KBASE_PM_CHANGE_STATE_TILER] =
 115                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_START,
 116
 117         [KBASE_PM_FUNC_ID_RELEASE_CORES_END][0] = 0,
 118         [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] =
 119                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_END,
 120         [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_TILER] =
 121                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_END,
 122         [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER |
 123                                                 KBASE_PM_CHANGE_STATE_TILER] =
 124                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_END
 125 };
 126
 127 static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev,
 128                 enum kbase_pm_func_id func_id,
 129                 kbase_pm_change_state state)
 130 {
 131         int trace_code;
 132
 133         KBASE_DEBUG_ASSERT(func_id >= 0 && func_id < KBASE_PM_FUNC_ID_COUNT);
 134         KBASE_DEBUG_ASSERT(state != 0 && (state & KBASE_PM_CHANGE_STATE_MASK) ==
 135                                                                         state);
 136
 137         trace_code = kbase_pm_change_state_trace_code[func_id][state];
 138         KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code);
 139 }
 140
 141 #else /* CONFIG_MALI_TRACE_TIMELINE */
 142 static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev,
 143                 enum kbase_pm_func_id func_id, kbase_pm_change_state state)
 144 {
 145 }
 146
 147 #endif /* CONFIG_MALI_TRACE_TIMELINE */
 148
 149 /**
 150  * kbasep_pm_do_poweroff_cores - Process a poweroff request and power down any
 151  *                               requested shader cores
 152  * @kbdev: Device pointer
 153  */
 154 static void kbasep_pm_do_poweroff_cores(struct kbase_device *kbdev)
 155 {
 156         u64 prev_shader_state = kbdev->pm.backend.desired_shader_state;
 157         u64 prev_tiler_state = kbdev->pm.backend.desired_tiler_state;
 158
 159         lockdep_assert_held(&kbdev->hwaccess_lock);
 160
 161         kbdev->pm.backend.desired_shader_state &=
 162                         ~kbdev->pm.backend.shader_poweroff_pending;
 163         kbdev->pm.backend.desired_tiler_state &=
 164                         ~kbdev->pm.backend.tiler_poweroff_pending;
 165
 166         kbdev->pm.backend.shader_poweroff_pending = 0;
 167         kbdev->pm.backend.tiler_poweroff_pending = 0;
 168
 169         if (prev_shader_state != kbdev->pm.backend.desired_shader_state ||
 170                         prev_tiler_state !=
 171                                 kbdev->pm.backend.desired_tiler_state ||
 172                         kbdev->pm.backend.ca_in_transition) {
 173                 bool cores_are_available;
 174
 175                 KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 176                         SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_START);
 177                 cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
 178                 KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 179                         SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_END);
 180
 181                 /* Don't need 'cores_are_available',
 182                  * because we don't return anything */
 183                 CSTD_UNUSED(cores_are_available);
 184         }
 185 }
 186
 187 static enum hrtimer_restart
 188 kbasep_pm_do_gpu_poweroff_callback(struct hrtimer *timer)
 189 {
 190         struct kbase_device *kbdev;
 191         unsigned long flags;
 192
 193         kbdev = container_of(timer, struct kbase_device,
 194                                                 pm.backend.gpu_poweroff_timer);
 195
 196         spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 197
 198         /* It is safe for this call to do nothing if the work item is already
 199          * queued. The worker function will read the must up-to-date state of
 200          * kbdev->pm.backend.gpu_poweroff_pending under lock.
 201          *
 202          * If a state change occurs while the worker function is processing,
 203          * this call will succeed as a work item can be requeued once it has
 204          * started processing.
 205          */
 206         if (kbdev->pm.backend.gpu_poweroff_pending)
 207                 queue_work(kbdev->pm.backend.gpu_poweroff_wq,
 208                                         &kbdev->pm.backend.gpu_poweroff_work);
 209
 210         if (kbdev->pm.backend.shader_poweroff_pending ||
 211                         kbdev->pm.backend.tiler_poweroff_pending) {
 212                 kbdev->pm.backend.shader_poweroff_pending_time--;
 213
 214                 KBASE_DEBUG_ASSERT(
 215                                 kbdev->pm.backend.shader_poweroff_pending_time
 216                                                                         >= 0);
 217
 218                 if (!kbdev->pm.backend.shader_poweroff_pending_time)
 219                         kbasep_pm_do_poweroff_cores(kbdev);
 220         }
 221
 222         if (kbdev->pm.backend.poweroff_timer_needed) {
 223                 spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 224
 225                 hrtimer_add_expires(timer, kbdev->pm.gpu_poweroff_time);
 226
 227                 return HRTIMER_RESTART;
 228         }
 229
 230         kbdev->pm.backend.poweroff_timer_running = false;
 231         spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 232
 233         return HRTIMER_NORESTART;
 234 }
 235
 236 static void kbasep_pm_do_gpu_poweroff_wq(struct work_struct *data)
 237 {
 238         unsigned long flags;
 239         struct kbase_device *kbdev;
 240         bool do_poweroff = false;
 241
 242         kbdev = container_of(data, struct kbase_device,
 243                                                 pm.backend.gpu_poweroff_work);
 244
 245         mutex_lock(&kbdev->pm.lock);
 246
 247         if (kbdev->pm.backend.gpu_poweroff_pending == 0) {
 248                 mutex_unlock(&kbdev->pm.lock);
 249                 return;
 250         }
 251
 252         kbdev->pm.backend.gpu_poweroff_pending--;
 253
 254         if (kbdev->pm.backend.gpu_poweroff_pending > 0) {
 255                 mutex_unlock(&kbdev->pm.lock);
 256                 return;
 257         }
 258
 259         KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_poweroff_pending == 0);
 260
 261         spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 262
 263         /* Only power off the GPU if a request is still pending */
 264         if (!kbdev->pm.backend.pm_current_policy->get_core_active(kbdev))
 265                 do_poweroff = true;
 266
 267         spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 268
 269         if (do_poweroff) {
 270                 kbdev->pm.backend.poweroff_timer_needed = false;
 271                 hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer);
 272                 kbdev->pm.backend.poweroff_timer_running = false;
 273
 274                 /* Power off the GPU */
 275                 kbase_pm_do_poweroff(kbdev, false);
 276         }
 277
 278         mutex_unlock(&kbdev->pm.lock);
 279 }
 280
 281 int kbase_pm_policy_init(struct kbase_device *kbdev)
 282 {
 283         struct workqueue_struct *wq;
 284
 285         wq = alloc_workqueue("kbase_pm_do_poweroff",
 286                         WQ_HIGHPRI | WQ_UNBOUND, 1);
 287         if (!wq)
 288                 return -ENOMEM;
 289
 290         kbdev->pm.backend.gpu_poweroff_wq = wq;
 291         INIT_WORK(&kbdev->pm.backend.gpu_poweroff_work,
 292                         kbasep_pm_do_gpu_poweroff_wq);
 293         hrtimer_init(&kbdev->pm.backend.gpu_poweroff_timer,
 294                         CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 295         kbdev->pm.backend.gpu_poweroff_timer.function =
 296                         kbasep_pm_do_gpu_poweroff_callback;
 297         kbdev->pm.backend.pm_current_policy = policy_list[0];
 298         kbdev->pm.backend.pm_current_policy->init(kbdev);
 299         kbdev->pm.gpu_poweroff_time =
 300                         HR_TIMER_DELAY_NSEC(DEFAULT_PM_GPU_POWEROFF_TICK_NS);
 301         kbdev->pm.poweroff_shader_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER;
 302         kbdev->pm.poweroff_gpu_ticks = DEFAULT_PM_POWEROFF_TICK_GPU;
 303
 304         return 0;
 305 }
 306
 307 void kbase_pm_policy_term(struct kbase_device *kbdev)
 308 {
 309         kbdev->pm.backend.pm_current_policy->term(kbdev);
 310         destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wq);
 311 }
 312
 313 void kbase_pm_cancel_deferred_poweroff(struct kbase_device *kbdev)
 314 {
 315         unsigned long flags;
 316
 317         lockdep_assert_held(&kbdev->pm.lock);
 318
 319         kbdev->pm.backend.poweroff_timer_needed = false;
 320         hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer);
 321         spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 322         kbdev->pm.backend.poweroff_timer_running = false;
 323
 324         /* If wq is already running but is held off by pm.lock, make sure it has
 325          * no effect */
 326         kbdev->pm.backend.gpu_poweroff_pending = 0;
 327
 328         kbdev->pm.backend.shader_poweroff_pending = 0;
 329         kbdev->pm.backend.tiler_poweroff_pending = 0;
 330         kbdev->pm.backend.shader_poweroff_pending_time = 0;
 331
 332         spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 333 }
 334
 335 void kbase_pm_update_active(struct kbase_device *kbdev)
 336 {
 337         struct kbase_pm_device_data *pm = &kbdev->pm;
 338         struct kbase_pm_backend_data *backend = &pm->backend;
 339         unsigned long flags;
 340         bool active;
 341
 342         lockdep_assert_held(&pm->lock);
 343
 344         /* pm_current_policy will never be NULL while pm.lock is held */
 345         KBASE_DEBUG_ASSERT(backend->pm_current_policy);
 346
 347         spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 348
 349         active = backend->pm_current_policy->get_core_active(kbdev);
 350
 351         if (active) {
 352                 if (backend->gpu_poweroff_pending) {
 353                         /* Cancel any pending power off request */
 354                         backend->gpu_poweroff_pending = 0;
 355
 356                         /* If a request was pending then the GPU was still
 357                          * powered, so no need to continue */
 358                         if (!kbdev->poweroff_pending) {
 359                                 spin_unlock_irqrestore(&kbdev->hwaccess_lock,
 360                                                 flags);
 361                                 return;
 362                         }
 363                 }
 364
 365                 if (!backend->poweroff_timer_running && !backend->gpu_powered &&
 366                                 (pm->poweroff_gpu_ticks ||
 367                                 pm->poweroff_shader_ticks)) {
 368                         backend->poweroff_timer_needed = true;
 369                         backend->poweroff_timer_running = true;
 370                         hrtimer_start(&backend->gpu_poweroff_timer,
 371                                         pm->gpu_poweroff_time,
 372                                         HRTIMER_MODE_REL);
 373                 }
 374
 375                 /* Power on the GPU and any cores requested by the policy */
 376                 if (pm->backend.poweroff_wait_in_progress) {
 377                         pm->backend.poweron_required = true;
 378                         spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 379                 } else {
 380                         spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 381                         kbase_pm_do_poweron(kbdev, false);
 382                 }
 383         } else {
 384                 /* It is an error for the power policy to power off the GPU
 385                  * when there are contexts active */
 386                 KBASE_DEBUG_ASSERT(pm->active_count == 0);
 387
 388                 if (backend->shader_poweroff_pending ||
 389                                 backend->tiler_poweroff_pending) {
 390                         backend->shader_poweroff_pending = 0;
 391                         backend->tiler_poweroff_pending = 0;
 392                         backend->shader_poweroff_pending_time = 0;
 393                 }
 394
 395                 /* Request power off */
 396                 if (pm->backend.gpu_powered) {
 397                         if (pm->poweroff_gpu_ticks) {
 398                                 backend->gpu_poweroff_pending =
 399                                                 pm->poweroff_gpu_ticks;
 400                                 backend->poweroff_timer_needed = true;
 401                                 if (!backend->poweroff_timer_running) {
 402                                         /* Start timer if not running (eg if
 403                                          * power policy has been changed from
 404                                          * always_on to something else). This
 405                                          * will ensure the GPU is actually
 406                                          * powered off */
 407                                         backend->poweroff_timer_running
 408                                                         = true;
 409                                         hrtimer_start(
 410                                                 &backend->gpu_poweroff_timer,
 411                                                 pm->gpu_poweroff_time,
 412                                                 HRTIMER_MODE_REL);
 413                                 }
 414                                 spin_unlock_irqrestore(&kbdev->hwaccess_lock,
 415                                                 flags);
 416                         } else {
 417                                 spin_unlock_irqrestore(&kbdev->hwaccess_lock,
 418                                                 flags);
 419
 420                                 /* Power off the GPU immediately */
 421                                 kbase_pm_do_poweroff(kbdev, false);
 422                         }
 423                 } else {
 424                         spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 425                 }
 426         }
 427 }
 428
 429 void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev)
 430 {
 431         u64 desired_bitmap;
 432         u64 desired_tiler_bitmap;
 433         bool cores_are_available;
 434         bool do_poweroff = false;
 435
 436         lockdep_assert_held(&kbdev->hwaccess_lock);
 437
 438         if (kbdev->pm.backend.pm_current_policy == NULL)
 439                 return;
 440         if (kbdev->pm.backend.poweroff_wait_in_progress)
 441                 return;
 442
 443         if (kbdev->protected_mode_transition && !kbdev->shader_needed_bitmap &&
 444                         !kbdev->shader_inuse_bitmap && !kbdev->tiler_needed_cnt
 445                         && !kbdev->tiler_inuse_cnt) {
 446                 /* We are trying to change in/out of protected mode - force all
 447                  * cores off so that the L2 powers down */
 448                 desired_bitmap = 0;
 449                 desired_tiler_bitmap = 0;
 450         } else {
 451                 desired_bitmap =
 452                 kbdev->pm.backend.pm_current_policy->get_core_mask(kbdev);
 453                 desired_bitmap &= kbase_pm_ca_get_core_mask(kbdev);
 454
 455                 if (kbdev->tiler_needed_cnt > 0 || kbdev->tiler_inuse_cnt > 0)
 456                         desired_tiler_bitmap = 1;
 457                 else
 458                         desired_tiler_bitmap = 0;
 459
 460                 if (!kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_XAFFINITY)) {
 461                         /* Unless XAFFINITY is supported, enable core 0 if tiler
 462                          * required, regardless of core availability */
 463                         if (kbdev->tiler_needed_cnt > 0 ||
 464                                         kbdev->tiler_inuse_cnt > 0)
 465                                 desired_bitmap |= 1;
 466                 }
 467         }
 468
 469         if (kbdev->pm.backend.desired_shader_state != desired_bitmap)
 470                 KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, NULL, 0u,
 471                                                         (u32)desired_bitmap);
 472         /* Are any cores being powered on? */
 473         if (~kbdev->pm.backend.desired_shader_state & desired_bitmap ||
 474             ~kbdev->pm.backend.desired_tiler_state & desired_tiler_bitmap ||
 475             kbdev->pm.backend.ca_in_transition) {
 476                 /* Check if we are powering off any cores before updating shader
 477                  * state */
 478                 if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap ||
 479                                 kbdev->pm.backend.desired_tiler_state &
 480                                 ~desired_tiler_bitmap) {
 481                         /* Start timer to power off cores */
 482                         kbdev->pm.backend.shader_poweroff_pending |=
 483                                 (kbdev->pm.backend.desired_shader_state &
 484                                                         ~desired_bitmap);
 485                         kbdev->pm.backend.tiler_poweroff_pending |=
 486                                 (kbdev->pm.backend.desired_tiler_state &
 487                                                         ~desired_tiler_bitmap);
 488
 489                         if (kbdev->pm.poweroff_shader_ticks &&
 490                                         !kbdev->protected_mode_transition)
 491                                 kbdev->pm.backend.shader_poweroff_pending_time =
 492                                                 kbdev->pm.poweroff_shader_ticks;
 493                         else
 494                                 do_poweroff = true;
 495                 }
 496
 497                 kbdev->pm.backend.desired_shader_state = desired_bitmap;
 498                 kbdev->pm.backend.desired_tiler_state = desired_tiler_bitmap;
 499
 500                 /* If any cores are being powered on, transition immediately */
 501                 cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
 502         } else if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap ||
 503                                 kbdev->pm.backend.desired_tiler_state &
 504                                 ~desired_tiler_bitmap) {
 505                 /* Start timer to power off cores */
 506                 kbdev->pm.backend.shader_poweroff_pending |=
 507                                 (kbdev->pm.backend.desired_shader_state &
 508                                                         ~desired_bitmap);
 509                 kbdev->pm.backend.tiler_poweroff_pending |=
 510                                 (kbdev->pm.backend.desired_tiler_state &
 511                                                         ~desired_tiler_bitmap);
 512                 if (kbdev->pm.poweroff_shader_ticks &&
 513                                 !kbdev->protected_mode_transition)
 514                         kbdev->pm.backend.shader_poweroff_pending_time =
 515                                         kbdev->pm.poweroff_shader_ticks;
 516                 else
 517                         kbasep_pm_do_poweroff_cores(kbdev);
 518         } else if (kbdev->pm.active_count == 0 && desired_bitmap != 0 &&
 519                         desired_tiler_bitmap != 0 &&
 520                         kbdev->pm.backend.poweroff_timer_needed) {
 521                 /* If power policy is keeping cores on despite there being no
 522                  * active contexts then disable poweroff timer as it isn't
 523                  * required.
 524                  * Only reset poweroff_timer_needed if we're not in the middle
 525                  * of the power off callback */
 526                 kbdev->pm.backend.poweroff_timer_needed = false;
 527         }
 528
 529         /* Ensure timer does not power off wanted cores and make sure to power
 530          * off unwanted cores */
 531         if (kbdev->pm.backend.shader_poweroff_pending ||
 532                         kbdev->pm.backend.tiler_poweroff_pending) {
 533                 kbdev->pm.backend.shader_poweroff_pending &=
 534                                 ~(kbdev->pm.backend.desired_shader_state &
 535                                                                 desired_bitmap);
 536                 kbdev->pm.backend.tiler_poweroff_pending &=
 537                                 ~(kbdev->pm.backend.desired_tiler_state &
 538                                 desired_tiler_bitmap);
 539
 540                 if (!kbdev->pm.backend.shader_poweroff_pending &&
 541                                 !kbdev->pm.backend.tiler_poweroff_pending)
 542                         kbdev->pm.backend.shader_poweroff_pending_time = 0;
 543         }
 544
 545         /* Shader poweroff is deferred to the end of the function, to eliminate
 546          * issues caused by the core availability policy recursing into this
 547          * function */
 548         if (do_poweroff)
 549                 kbasep_pm_do_poweroff_cores(kbdev);
 550
 551         /* Don't need 'cores_are_available', because we don't return anything */
 552         CSTD_UNUSED(cores_are_available);
 553 }
 554
 555 void kbase_pm_update_cores_state(struct kbase_device *kbdev)
 556 {
 557         unsigned long flags;
 558
 559         spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 560
 561         kbase_pm_update_cores_state_nolock(kbdev);
 562
 563         spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 564 }
 565
 566 int kbase_pm_list_policies(const struct kbase_pm_policy * const **list)
 567 {
 568         if (!list)
 569                 return POLICY_COUNT;
 570
 571         *list = policy_list;
 572
 573         return POLICY_COUNT;
 574 }
 575
 576 KBASE_EXPORT_TEST_API(kbase_pm_list_policies);
 577
 578 const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev)
 579 {
 580         KBASE_DEBUG_ASSERT(kbdev != NULL);
 581
 582         return kbdev->pm.backend.pm_current_policy;
 583 }
 584
 585 KBASE_EXPORT_TEST_API(kbase_pm_get_policy);
 586
 587 void kbase_pm_set_policy(struct kbase_device *kbdev,
 588                                 const struct kbase_pm_policy *new_policy)
 589 {
 590         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
 591         const struct kbase_pm_policy *old_policy;
 592         unsigned long flags;
 593
 594         KBASE_DEBUG_ASSERT(kbdev != NULL);
 595         KBASE_DEBUG_ASSERT(new_policy != NULL);
 596
 597         KBASE_TRACE_ADD(kbdev, PM_SET_POLICY, NULL, NULL, 0u, new_policy->id);
 598
 599         /* During a policy change we pretend the GPU is active */
 600         /* A suspend won't happen here, because we're in a syscall from a
 601          * userspace thread */
 602         kbase_pm_context_active(kbdev);
 603
 604         mutex_lock(&js_devdata->runpool_mutex);
 605         mutex_lock(&kbdev->pm.lock);
 606
 607         /* Remove the policy to prevent IRQ handlers from working on it */
 608         spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 609         old_policy = kbdev->pm.backend.pm_current_policy;
 610         kbdev->pm.backend.pm_current_policy = NULL;
 611         spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 612
 613         KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u,
 614                                                                 old_policy->id);
 615         if (old_policy->term)
 616                 old_policy->term(kbdev);
 617
 618         KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u,
 619                                                                 new_policy->id);
 620         if (new_policy->init)
 621                 new_policy->init(kbdev);
 622
 623         spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 624         kbdev->pm.backend.pm_current_policy = new_policy;
 625         spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 626
 627         /* If any core power state changes were previously attempted, but
 628          * couldn't be made because the policy was changing (current_policy was
 629          * NULL), then re-try them here. */
 630         kbase_pm_update_active(kbdev);
 631         kbase_pm_update_cores_state(kbdev);
 632
 633         mutex_unlock(&kbdev->pm.lock);
 634         mutex_unlock(&js_devdata->runpool_mutex);
 635
 636         /* Now the policy change is finished, we release our fake context active
 637          * reference */
 638         kbase_pm_context_idle(kbdev);
 639 }
 640
 641 KBASE_EXPORT_TEST_API(kbase_pm_set_policy);
 642
 643 /* Check whether a state change has finished, and trace it as completed */
 644 static void
 645 kbase_pm_trace_check_and_finish_state_change(struct kbase_device *kbdev)
 646 {
 647         if ((kbdev->shader_available_bitmap &
 648                                         kbdev->pm.backend.desired_shader_state)
 649                                 == kbdev->pm.backend.desired_shader_state &&
 650                 (kbdev->tiler_available_bitmap &
 651                                         kbdev->pm.backend.desired_tiler_state)
 652                                 == kbdev->pm.backend.desired_tiler_state)
 653                 kbase_timeline_pm_check_handle_event(kbdev,
 654                                 KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
 655 }
 656
 657 void kbase_pm_request_cores(struct kbase_device *kbdev,
 658                                 bool tiler_required, u64 shader_cores)
 659 {
 660         u64 cores;
 661
 662         kbase_pm_change_state change_gpu_state = 0u;
 663
 664         KBASE_DEBUG_ASSERT(kbdev != NULL);
 665
 666         lockdep_assert_held(&kbdev->hwaccess_lock);
 667
 668         cores = shader_cores;
 669         while (cores) {
 670                 int bitnum = fls64(cores) - 1;
 671                 u64 bit = 1ULL << bitnum;
 672
 673                 /* It should be almost impossible for this to overflow. It would
 674                  * require 2^32 atoms to request a particular core, which would
 675                  * require 2^24 contexts to submit. This would require an amount
 676                  * of memory that is impossible on a 32-bit system and extremely
 677                  * unlikely on a 64-bit system. */
 678                 int cnt = ++kbdev->shader_needed_cnt[bitnum];
 679
 680                 if (1 == cnt) {
 681                         kbdev->shader_needed_bitmap |= bit;
 682                         change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
 683                 }
 684
 685                 cores &= ~bit;
 686         }
 687
 688         if (tiler_required) {
 689                 int cnt = ++kbdev->tiler_needed_cnt;
 690
 691                 if (1 == cnt)
 692                         change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER;
 693
 694                 KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt != 0);
 695         }
 696
 697         if (change_gpu_state) {
 698                 KBASE_TRACE_ADD(kbdev, PM_REQUEST_CHANGE_SHADER_NEEDED, NULL,
 699                                 NULL, 0u, (u32) kbdev->shader_needed_bitmap);
 700
 701                 kbase_timeline_pm_cores_func(kbdev,
 702                                         KBASE_PM_FUNC_ID_REQUEST_CORES_START,
 703                                                         change_gpu_state);
 704                 kbase_pm_update_cores_state_nolock(kbdev);
 705                 kbase_timeline_pm_cores_func(kbdev,
 706                                         KBASE_PM_FUNC_ID_REQUEST_CORES_END,
 707                                                         change_gpu_state);
 708         }
 709 }
 710
 711 KBASE_EXPORT_TEST_API(kbase_pm_request_cores);
 712
 713 void kbase_pm_unrequest_cores(struct kbase_device *kbdev,
 714                                 bool tiler_required, u64 shader_cores)
 715 {
 716         kbase_pm_change_state change_gpu_state = 0u;
 717
 718         KBASE_DEBUG_ASSERT(kbdev != NULL);
 719
 720         lockdep_assert_held(&kbdev->hwaccess_lock);
 721
 722         while (shader_cores) {
 723                 int bitnum = fls64(shader_cores) - 1;
 724                 u64 bit = 1ULL << bitnum;
 725                 int cnt;
 726
 727                 KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);
 728
 729                 cnt = --kbdev->shader_needed_cnt[bitnum];
 730
 731                 if (0 == cnt) {
 732                         kbdev->shader_needed_bitmap &= ~bit;
 733
 734                         change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
 735                 }
 736
 737                 shader_cores &= ~bit;
 738         }
 739
 740         if (tiler_required) {
 741                 int cnt;
 742
 743                 KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);
 744
 745                 cnt = --kbdev->tiler_needed_cnt;
 746
 747                 if (0 == cnt)
 748                         change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER;
 749         }
 750
 751         if (change_gpu_state) {
 752                 KBASE_TRACE_ADD(kbdev, PM_UNREQUEST_CHANGE_SHADER_NEEDED, NULL,
 753                                 NULL, 0u, (u32) kbdev->shader_needed_bitmap);
 754
 755                 kbase_pm_update_cores_state_nolock(kbdev);
 756
 757                 /* Trace that any state change effectively completes immediately
 758                  * - no-one will wait on the state change */
 759                 kbase_pm_trace_check_and_finish_state_change(kbdev);
 760         }
 761 }
 762
 763 KBASE_EXPORT_TEST_API(kbase_pm_unrequest_cores);
 764
 765 enum kbase_pm_cores_ready
 766 kbase_pm_register_inuse_cores(struct kbase_device *kbdev,
 767                                 bool tiler_required, u64 shader_cores)
 768 {
 769         u64 prev_shader_needed; /* Just for tracing */
 770         u64 prev_shader_inuse;  /* Just for tracing */
 771
 772         lockdep_assert_held(&kbdev->hwaccess_lock);
 773
 774         prev_shader_needed = kbdev->shader_needed_bitmap;
 775         prev_shader_inuse = kbdev->shader_inuse_bitmap;
 776
 777         /* If desired_shader_state does not contain the requested cores, then
 778          * power management is not attempting to powering those cores (most
 779          * likely due to core availability policy) and a new job affinity must
 780          * be chosen */
 781         if ((kbdev->pm.backend.desired_shader_state & shader_cores) !=
 782                                                         shader_cores) {
 783                 return (kbdev->pm.backend.poweroff_wait_in_progress ||
 784                                 kbdev->pm.backend.pm_current_policy == NULL) ?
 785                                 KBASE_CORES_NOT_READY : KBASE_NEW_AFFINITY;
 786         }
 787
 788         if ((kbdev->shader_available_bitmap & shader_cores) != shader_cores ||
 789             (tiler_required && !kbdev->tiler_available_bitmap)) {
 790                 /* Trace ongoing core transition */
 791                 kbase_timeline_pm_l2_transition_start(kbdev);
 792                 return KBASE_CORES_NOT_READY;
 793         }
 794
 795         /* If we started to trace a state change, then trace it has being
 796          * finished by now, at the very latest */
 797         kbase_pm_trace_check_and_finish_state_change(kbdev);
 798         /* Trace core transition done */
 799         kbase_timeline_pm_l2_transition_done(kbdev);
 800
 801         while (shader_cores) {
 802                 int bitnum = fls64(shader_cores) - 1;
 803                 u64 bit = 1ULL << bitnum;
 804                 int cnt;
 805
 806                 KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);
 807
 808                 cnt = --kbdev->shader_needed_cnt[bitnum];
 809
 810                 if (0 == cnt)
 811                         kbdev->shader_needed_bitmap &= ~bit;
 812
 813                 /* shader_inuse_cnt should not overflow because there can only
 814                  * be a very limited number of jobs on the h/w at one time */
 815
 816                 kbdev->shader_inuse_cnt[bitnum]++;
 817                 kbdev->shader_inuse_bitmap |= bit;
 818
 819                 shader_cores &= ~bit;
 820         }
 821
 822         if (tiler_required) {
 823                 KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);
 824
 825                 --kbdev->tiler_needed_cnt;
 826
 827                 kbdev->tiler_inuse_cnt++;
 828
 829                 KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt != 0);
 830         }
 831
 832         if (prev_shader_needed != kbdev->shader_needed_bitmap)
 833                 KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_NEEDED, NULL,
 834                                 NULL, 0u, (u32) kbdev->shader_needed_bitmap);
 835
 836         if (prev_shader_inuse != kbdev->shader_inuse_bitmap)
 837                 KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_INUSE, NULL,
 838                                 NULL, 0u, (u32) kbdev->shader_inuse_bitmap);
 839
 840         return KBASE_CORES_READY;
 841 }
 842
 843 KBASE_EXPORT_TEST_API(kbase_pm_register_inuse_cores);
 844
 845 void kbase_pm_release_cores(struct kbase_device *kbdev,
 846                                 bool tiler_required, u64 shader_cores)
 847 {
 848         kbase_pm_change_state change_gpu_state = 0u;
 849
 850         KBASE_DEBUG_ASSERT(kbdev != NULL);
 851
 852         lockdep_assert_held(&kbdev->hwaccess_lock);
 853
 854         while (shader_cores) {
 855                 int bitnum = fls64(shader_cores) - 1;
 856                 u64 bit = 1ULL << bitnum;
 857                 int cnt;
 858
 859                 KBASE_DEBUG_ASSERT(kbdev->shader_inuse_cnt[bitnum] > 0);
 860
 861                 cnt = --kbdev->shader_inuse_cnt[bitnum];
 862
 863                 if (0 == cnt) {
 864                         kbdev->shader_inuse_bitmap &= ~bit;
 865                         change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
 866                 }
 867
 868                 shader_cores &= ~bit;
 869         }
 870
 871         if (tiler_required) {
 872                 int cnt;
 873
 874                 KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt > 0);
 875
 876                 cnt = --kbdev->tiler_inuse_cnt;
 877
 878                 if (0 == cnt)
 879                         change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER;
 880         }
 881
 882         if (change_gpu_state) {
 883                 KBASE_TRACE_ADD(kbdev, PM_RELEASE_CHANGE_SHADER_INUSE, NULL,
 884                                 NULL, 0u, (u32) kbdev->shader_inuse_bitmap);
 885
 886                 kbase_timeline_pm_cores_func(kbdev,
 887                                         KBASE_PM_FUNC_ID_RELEASE_CORES_START,
 888                                                         change_gpu_state);
 889                 kbase_pm_update_cores_state_nolock(kbdev);
 890                 kbase_timeline_pm_cores_func(kbdev,
 891                                         KBASE_PM_FUNC_ID_RELEASE_CORES_END,
 892                                                         change_gpu_state);
 893
 894                 /* Trace that any state change completed immediately */
 895                 kbase_pm_trace_check_and_finish_state_change(kbdev);
 896         }
 897 }
 898
 899 KBASE_EXPORT_TEST_API(kbase_pm_release_cores);
 900
 901 void kbase_pm_request_cores_sync(struct kbase_device *kbdev,
 902                                         bool tiler_required,
 903                                         u64 shader_cores)
 904 {
 905         unsigned long flags;
 906
 907         kbase_pm_wait_for_poweroff_complete(kbdev);
 908
 909         spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 910         kbase_pm_request_cores(kbdev, tiler_required, shader_cores);
 911         spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 912
 913         kbase_pm_check_transitions_sync(kbdev);
 914 }
 915
 916 KBASE_EXPORT_TEST_API(kbase_pm_request_cores_sync);
 917
 918 void kbase_pm_request_l2_caches(struct kbase_device *kbdev)
 919 {
 920         unsigned long flags;
 921         u32 prior_l2_users_count;
 922
 923         spin_lock_irqsave(&kbdev->hwaccess_lock, flags);
 924
 925         prior_l2_users_count = kbdev->l2_users_count++;
 926
 927         KBASE_DEBUG_ASSERT(kbdev->l2_users_count != 0);
 928
 929         /* if the GPU is reset while the l2 is on, l2 will be off but
 930          * prior_l2_users_count will be > 0. l2_available_bitmap will have been
 931          * set to 0 though by kbase_pm_init_hw */
 932         if (!prior_l2_users_count || !kbdev->l2_available_bitmap)
 933                 kbase_pm_check_transitions_nolock(kbdev);
 934
 935         spin_unlock_irqrestore(&kbdev->hwaccess_lock, flags);
 936         wait_event(kbdev->pm.backend.l2_powered_wait,
 937                                         kbdev->pm.backend.l2_powered == 1);
 938
 939         /* Trace that any state change completed immediately */
 940         kbase_pm_trace_check_and_finish_state_change(kbdev);
 941 }
 942
 943 KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches);
 944
 945 void kbase_pm_request_l2_caches_l2_is_on(struct kbase_device *kbdev)
 946 {
 947         lockdep_assert_held(&kbdev->hwaccess_lock);
 948
 949         kbdev->l2_users_count++;
 950 }
 951
 952 KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches_l2_is_on);
 953
 954 void kbase_pm_release_l2_caches(struct kbase_device *kbdev)
 955 {
 956         lockdep_assert_held(&kbdev->hwaccess_lock);
 957
 958         KBASE_DEBUG_ASSERT(kbdev->l2_users_count > 0);
 959
 960         --kbdev->l2_users_count;
 961
 962         if (!kbdev->l2_users_count) {
 963                 kbase_pm_check_transitions_nolock(kbdev);
 964                 /* Trace that any state change completed immediately */
 965                 kbase_pm_trace_check_and_finish_state_change(kbdev);
 966         }
 967 }
 968
 969 KBASE_EXPORT_TEST_API(kbase_pm_release_l2_caches);