drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_policy.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18 /*
  19  * Power policy API implementations
  20  */
  21
  22 #include <mali_kbase.h>
  23 #include <mali_midg_regmap.h>
  24 #include <mali_kbase_gator.h>
  25 #include <mali_kbase_pm.h>
  26 #include <mali_kbase_config_defaults.h>
  27 #include <backend/gpu/mali_kbase_pm_internal.h>
  28
  29 static const struct kbase_pm_policy *const policy_list[] = {
  30 #ifdef CONFIG_MALI_NO_MALI
  31         &kbase_pm_always_on_policy_ops,
  32         &kbase_pm_demand_policy_ops,
  33         &kbase_pm_coarse_demand_policy_ops,
  34 #if !MALI_CUSTOMER_RELEASE
  35         &kbase_pm_demand_always_powered_policy_ops,
  36         &kbase_pm_fast_start_policy_ops,
  37 #endif
  38 #else                           /* CONFIG_MALI_NO_MALI */
  39         &kbase_pm_demand_policy_ops,
  40         &kbase_pm_always_on_policy_ops,
  41         &kbase_pm_coarse_demand_policy_ops,
  42 #if !MALI_CUSTOMER_RELEASE
  43         &kbase_pm_demand_always_powered_policy_ops,
  44         &kbase_pm_fast_start_policy_ops,
  45 #endif
  46 #endif /* CONFIG_MALI_NO_MALI */
  47 };
  48
  49 /* The number of policies available in the system.
  50  * This is derived from the number of functions listed in policy_get_functions.
  51  */
  52 #define POLICY_COUNT (sizeof(policy_list)/sizeof(*policy_list))
  53
  54
  55 /* Function IDs for looking up Timeline Trace codes in
  56  * kbase_pm_change_state_trace_code */
  57 enum kbase_pm_func_id {
  58         KBASE_PM_FUNC_ID_REQUEST_CORES_START,
  59         KBASE_PM_FUNC_ID_REQUEST_CORES_END,
  60         KBASE_PM_FUNC_ID_RELEASE_CORES_START,
  61         KBASE_PM_FUNC_ID_RELEASE_CORES_END,
  62         /* Note: kbase_pm_unrequest_cores() is on the slow path, and we neither
  63          * expect to hit it nor tend to hit it very much anyway. We can detect
  64          * whether we need more instrumentation by a difference between
  65          * PM_CHECKTRANS events and PM_SEND/HANDLE_EVENT. */
  66
  67         /* Must be the last */
  68         KBASE_PM_FUNC_ID_COUNT
  69 };
  70
  71
  72 /* State changes during request/unrequest/release-ing cores */
  73 enum {
  74         KBASE_PM_CHANGE_STATE_SHADER = (1u << 0),
  75         KBASE_PM_CHANGE_STATE_TILER  = (1u << 1),
  76
  77         /* These two must be last */
  78         KBASE_PM_CHANGE_STATE_MASK = (KBASE_PM_CHANGE_STATE_TILER |
  79                                                 KBASE_PM_CHANGE_STATE_SHADER),
  80         KBASE_PM_CHANGE_STATE_COUNT = KBASE_PM_CHANGE_STATE_MASK + 1
  81 };
  82 typedef u32 kbase_pm_change_state;
  83
  84
  85 #ifdef CONFIG_MALI_TRACE_TIMELINE
  86 /* Timeline Trace code lookups for each function */
  87 static u32 kbase_pm_change_state_trace_code[KBASE_PM_FUNC_ID_COUNT]
  88                                         [KBASE_PM_CHANGE_STATE_COUNT] = {
  89         /* kbase_pm_request_cores */
  90         [KBASE_PM_FUNC_ID_REQUEST_CORES_START][0] = 0,
  91         [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] =
  92                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_START,
  93         [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_TILER] =
  94                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_START,
  95         [KBASE_PM_FUNC_ID_REQUEST_CORES_START][KBASE_PM_CHANGE_STATE_SHADER |
  96                                                 KBASE_PM_CHANGE_STATE_TILER] =
  97                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_START,
  98
  99         [KBASE_PM_FUNC_ID_REQUEST_CORES_END][0] = 0,
 100         [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] =
 101                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_END,
 102         [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_TILER] =
 103                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_TILER_END,
 104         [KBASE_PM_FUNC_ID_REQUEST_CORES_END][KBASE_PM_CHANGE_STATE_SHADER |
 105                                                 KBASE_PM_CHANGE_STATE_TILER] =
 106                 SW_FLOW_PM_CHECKTRANS_PM_REQUEST_CORES_SHADER_TILER_END,
 107
 108         /* kbase_pm_release_cores */
 109         [KBASE_PM_FUNC_ID_RELEASE_CORES_START][0] = 0,
 110         [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER] =
 111                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_START,
 112         [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_TILER] =
 113                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_START,
 114         [KBASE_PM_FUNC_ID_RELEASE_CORES_START][KBASE_PM_CHANGE_STATE_SHADER |
 115                                                 KBASE_PM_CHANGE_STATE_TILER] =
 116                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_START,
 117
 118         [KBASE_PM_FUNC_ID_RELEASE_CORES_END][0] = 0,
 119         [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER] =
 120                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_END,
 121         [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_TILER] =
 122                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_TILER_END,
 123         [KBASE_PM_FUNC_ID_RELEASE_CORES_END][KBASE_PM_CHANGE_STATE_SHADER |
 124                                                 KBASE_PM_CHANGE_STATE_TILER] =
 125                 SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_SHADER_TILER_END
 126 };
 127
 128 static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev,
 129                 enum kbase_pm_func_id func_id,
 130                 kbase_pm_change_state state)
 131 {
 132         int trace_code;
 133
 134         KBASE_DEBUG_ASSERT(func_id >= 0 && func_id < KBASE_PM_FUNC_ID_COUNT);
 135         KBASE_DEBUG_ASSERT(state != 0 && (state & KBASE_PM_CHANGE_STATE_MASK) ==
 136                                                                         state);
 137
 138         trace_code = kbase_pm_change_state_trace_code[func_id][state];
 139         KBASE_TIMELINE_PM_CHECKTRANS(kbdev, trace_code);
 140 }
 141
 142 #else /* CONFIG_MALI_TRACE_TIMELINE */
 143 static inline void kbase_timeline_pm_cores_func(struct kbase_device *kbdev,
 144                 enum kbase_pm_func_id func_id, kbase_pm_change_state state)
 145 {
 146 }
 147
 148 #endif /* CONFIG_MALI_TRACE_TIMELINE */
 149
 150 /**
 151  * kbasep_pm_do_poweroff_cores - Process a poweroff request and power down any
 152  *                               requested shader cores
 153  * @kbdev: Device pointer
 154  */
 155 static void kbasep_pm_do_poweroff_cores(struct kbase_device *kbdev)
 156 {
 157         u64 prev_shader_state = kbdev->pm.backend.desired_shader_state;
 158
 159         lockdep_assert_held(&kbdev->pm.power_change_lock);
 160
 161         kbdev->pm.backend.desired_shader_state &=
 162                         ~kbdev->pm.backend.shader_poweroff_pending;
 163
 164         kbdev->pm.backend.shader_poweroff_pending = 0;
 165
 166         if (prev_shader_state != kbdev->pm.backend.desired_shader_state
 167                         || kbdev->pm.backend.ca_in_transition) {
 168                 bool cores_are_available;
 169
 170                 KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 171                         SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_START);
 172                 cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
 173                 KBASE_TIMELINE_PM_CHECKTRANS(kbdev,
 174                         SW_FLOW_PM_CHECKTRANS_PM_RELEASE_CORES_DEFERRED_END);
 175
 176                 /* Don't need 'cores_are_available',
 177                  * because we don't return anything */
 178                 CSTD_UNUSED(cores_are_available);
 179         }
 180 }
 181
 182 static enum hrtimer_restart
 183 kbasep_pm_do_gpu_poweroff_callback(struct hrtimer *timer)
 184 {
 185         struct kbase_device *kbdev;
 186         unsigned long flags;
 187
 188         kbdev = container_of(timer, struct kbase_device,
 189                                                 pm.backend.gpu_poweroff_timer);
 190
 191         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 192
 193         /* It is safe for this call to do nothing if the work item is already
 194          * queued. The worker function will read the must up-to-date state of
 195          * kbdev->pm.backend.gpu_poweroff_pending under lock.
 196          *
 197          * If a state change occurs while the worker function is processing,
 198          * this call will succeed as a work item can be requeued once it has
 199          * started processing.
 200          */
 201         if (kbdev->pm.backend.gpu_poweroff_pending)
 202                 queue_work(kbdev->pm.backend.gpu_poweroff_wq,
 203                                         &kbdev->pm.backend.gpu_poweroff_work);
 204
 205         if (kbdev->pm.backend.shader_poweroff_pending) {
 206                 kbdev->pm.backend.shader_poweroff_pending_time--;
 207
 208                 KBASE_DEBUG_ASSERT(
 209                                 kbdev->pm.backend.shader_poweroff_pending_time
 210                                                                         >= 0);
 211
 212                 if (!kbdev->pm.backend.shader_poweroff_pending_time)
 213                         kbasep_pm_do_poweroff_cores(kbdev);
 214         }
 215
 216         if (kbdev->pm.backend.poweroff_timer_needed) {
 217                 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 218
 219                 hrtimer_add_expires(timer, kbdev->pm.gpu_poweroff_time);
 220
 221                 return HRTIMER_RESTART;
 222         }
 223
 224         kbdev->pm.backend.poweroff_timer_running = false;
 225         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 226
 227         return HRTIMER_NORESTART;
 228 }
 229
 230 static void kbasep_pm_do_gpu_poweroff_wq(struct work_struct *data)
 231 {
 232         unsigned long flags;
 233         struct kbase_device *kbdev;
 234         bool do_poweroff = false;
 235
 236         kbdev = container_of(data, struct kbase_device,
 237                                                 pm.backend.gpu_poweroff_work);
 238
 239         mutex_lock(&kbdev->pm.lock);
 240
 241         if (kbdev->pm.backend.gpu_poweroff_pending == 0) {
 242                 mutex_unlock(&kbdev->pm.lock);
 243                 return;
 244         }
 245
 246         kbdev->pm.backend.gpu_poweroff_pending--;
 247
 248         if (kbdev->pm.backend.gpu_poweroff_pending > 0) {
 249                 mutex_unlock(&kbdev->pm.lock);
 250                 return;
 251         }
 252
 253         KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_poweroff_pending == 0);
 254
 255         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 256
 257         /* Only power off the GPU if a request is still pending */
 258         if (!kbdev->pm.backend.pm_current_policy->get_core_active(kbdev))
 259                 do_poweroff = true;
 260
 261         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 262
 263         if (do_poweroff) {
 264                 kbdev->pm.backend.poweroff_timer_needed = false;
 265                 hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer);
 266                 kbdev->pm.backend.poweroff_timer_running = false;
 267
 268                 /* Power off the GPU */
 269                 if (!kbase_pm_do_poweroff(kbdev, false)) {
 270                         /* GPU can not be powered off at present */
 271                         kbdev->pm.backend.poweroff_timer_needed = true;
 272                         kbdev->pm.backend.poweroff_timer_running = true;
 273                         hrtimer_start(&kbdev->pm.backend.gpu_poweroff_timer,
 274                                         kbdev->pm.gpu_poweroff_time,
 275                                         HRTIMER_MODE_REL);
 276                 }
 277         }
 278
 279         mutex_unlock(&kbdev->pm.lock);
 280 }
 281
 282 int kbase_pm_policy_init(struct kbase_device *kbdev)
 283 {
 284         struct workqueue_struct *wq;
 285
 286         wq = alloc_workqueue("kbase_pm_do_poweroff",
 287                         WQ_HIGHPRI | WQ_UNBOUND, 1);
 288         if (!wq)
 289                 return -ENOMEM;
 290
 291         kbdev->pm.backend.gpu_poweroff_wq = wq;
 292         INIT_WORK(&kbdev->pm.backend.gpu_poweroff_work,
 293                         kbasep_pm_do_gpu_poweroff_wq);
 294         hrtimer_init(&kbdev->pm.backend.gpu_poweroff_timer,
 295                         CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 296         kbdev->pm.backend.gpu_poweroff_timer.function =
 297                         kbasep_pm_do_gpu_poweroff_callback;
 298         kbdev->pm.backend.pm_current_policy = policy_list[0];
 299         kbdev->pm.backend.pm_current_policy->init(kbdev);
 300         kbdev->pm.gpu_poweroff_time =
 301                         HR_TIMER_DELAY_NSEC(DEFAULT_PM_GPU_POWEROFF_TICK_NS);
 302         kbdev->pm.poweroff_shader_ticks = DEFAULT_PM_POWEROFF_TICK_SHADER;
 303         kbdev->pm.poweroff_gpu_ticks = DEFAULT_PM_POWEROFF_TICK_GPU;
 304
 305         return 0;
 306 }
 307
 308 void kbase_pm_policy_term(struct kbase_device *kbdev)
 309 {
 310         kbdev->pm.backend.pm_current_policy->term(kbdev);
 311         destroy_workqueue(kbdev->pm.backend.gpu_poweroff_wq);
 312 }
 313
 314 void kbase_pm_cancel_deferred_poweroff(struct kbase_device *kbdev)
 315 {
 316         unsigned long flags;
 317
 318         lockdep_assert_held(&kbdev->pm.lock);
 319
 320         kbdev->pm.backend.poweroff_timer_needed = false;
 321         hrtimer_cancel(&kbdev->pm.backend.gpu_poweroff_timer);
 322         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 323         kbdev->pm.backend.poweroff_timer_running = false;
 324
 325         /* If wq is already running but is held off by pm.lock, make sure it has
 326          * no effect */
 327         kbdev->pm.backend.gpu_poweroff_pending = 0;
 328
 329         kbdev->pm.backend.shader_poweroff_pending = 0;
 330         kbdev->pm.backend.shader_poweroff_pending_time = 0;
 331
 332         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 333 }
 334
 335 void kbase_pm_update_active(struct kbase_device *kbdev)
 336 {
 337         struct kbase_pm_device_data *pm = &kbdev->pm;
 338         struct kbase_pm_backend_data *backend = &pm->backend;
 339         unsigned long flags;
 340         bool active;
 341
 342         lockdep_assert_held(&pm->lock);
 343
 344         /* pm_current_policy will never be NULL while pm.lock is held */
 345         KBASE_DEBUG_ASSERT(backend->pm_current_policy);
 346
 347         spin_lock_irqsave(&pm->power_change_lock, flags);
 348
 349         active = backend->pm_current_policy->get_core_active(kbdev);
 350
 351         if (active) {
 352                 if (backend->gpu_poweroff_pending) {
 353                         /* Cancel any pending power off request */
 354                         backend->gpu_poweroff_pending = 0;
 355
 356                         /* If a request was pending then the GPU was still
 357                          * powered, so no need to continue */
 358                         if (!kbdev->poweroff_pending) {
 359                                 spin_unlock_irqrestore(&pm->power_change_lock,
 360                                                 flags);
 361                                 return;
 362                         }
 363                 }
 364
 365                 if (!backend->poweroff_timer_running && !backend->gpu_powered &&
 366                                 (pm->poweroff_gpu_ticks ||
 367                                 pm->poweroff_shader_ticks)) {
 368                         backend->poweroff_timer_needed = true;
 369                         backend->poweroff_timer_running = true;
 370                         hrtimer_start(&backend->gpu_poweroff_timer,
 371                                         pm->gpu_poweroff_time,
 372                                         HRTIMER_MODE_REL);
 373                 }
 374
 375                 spin_unlock_irqrestore(&pm->power_change_lock, flags);
 376
 377                 /* Power on the GPU and any cores requested by the policy */
 378                 kbase_pm_do_poweron(kbdev, false);
 379         } else {
 380                 /* It is an error for the power policy to power off the GPU
 381                  * when there are contexts active */
 382                 KBASE_DEBUG_ASSERT(pm->active_count == 0);
 383
 384                 if (backend->shader_poweroff_pending) {
 385                         backend->shader_poweroff_pending = 0;
 386                         backend->shader_poweroff_pending_time = 0;
 387                 }
 388
 389                 /* Request power off */
 390                 if (pm->backend.gpu_powered) {
 391                         if (pm->poweroff_gpu_ticks) {
 392                                 backend->gpu_poweroff_pending =
 393                                                 pm->poweroff_gpu_ticks;
 394                                 backend->poweroff_timer_needed = true;
 395                                 if (!backend->poweroff_timer_running) {
 396                                         /* Start timer if not running (eg if
 397                                          * power policy has been changed from
 398                                          * always_on to something else). This
 399                                          * will ensure the GPU is actually
 400                                          * powered off */
 401                                         backend->poweroff_timer_running
 402                                                         = true;
 403                                         hrtimer_start(
 404                                                 &backend->gpu_poweroff_timer,
 405                                                 pm->gpu_poweroff_time,
 406                                                 HRTIMER_MODE_REL);
 407                                 }
 408                                 spin_unlock_irqrestore(&pm->power_change_lock,
 409                                                 flags);
 410                         } else {
 411                                 spin_unlock_irqrestore(&pm->power_change_lock,
 412                                                 flags);
 413
 414                                 /* Power off the GPU immediately */
 415                                 if (!kbase_pm_do_poweroff(kbdev, false)) {
 416                                         /* GPU can not be powered off at present
 417                                          */
 418                                         spin_lock_irqsave(
 419                                                         &pm->power_change_lock,
 420                                                         flags);
 421                                         backend->poweroff_timer_needed = true;
 422                                         if (!backend->poweroff_timer_running) {
 423                                                 backend->poweroff_timer_running
 424                                                                 = true;
 425                                                 hrtimer_start(
 426                                                 &backend->gpu_poweroff_timer,
 427                                                         pm->gpu_poweroff_time,
 428                                                         HRTIMER_MODE_REL);
 429                                         }
 430                                         spin_unlock_irqrestore(
 431                                                         &pm->power_change_lock,
 432                                                         flags);
 433                                 }
 434                         }
 435                 } else {
 436                         spin_unlock_irqrestore(&pm->power_change_lock, flags);
 437                 }
 438         }
 439 }
 440
 441 void kbase_pm_update_cores_state_nolock(struct kbase_device *kbdev)
 442 {
 443         u64 desired_bitmap;
 444         bool cores_are_available;
 445         bool do_poweroff = false;
 446
 447         lockdep_assert_held(&kbdev->pm.power_change_lock);
 448
 449         if (kbdev->pm.backend.pm_current_policy == NULL)
 450                 return;
 451
 452         desired_bitmap =
 453                 kbdev->pm.backend.pm_current_policy->get_core_mask(kbdev);
 454         desired_bitmap &= kbase_pm_ca_get_core_mask(kbdev);
 455
 456         /* Enable core 0 if tiler required, regardless of core availability */
 457         if (kbdev->tiler_needed_cnt > 0 || kbdev->tiler_inuse_cnt > 0)
 458                 desired_bitmap |= 1;
 459
 460         if (kbdev->pm.backend.desired_shader_state != desired_bitmap)
 461                 KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_DESIRED, NULL, NULL, 0u,
 462                                                         (u32)desired_bitmap);
 463         /* Are any cores being powered on? */
 464         if (~kbdev->pm.backend.desired_shader_state & desired_bitmap ||
 465             kbdev->pm.backend.ca_in_transition) {
 466                 /* Check if we are powering off any cores before updating shader
 467                  * state */
 468                 if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap) {
 469                         /* Start timer to power off cores */
 470                         kbdev->pm.backend.shader_poweroff_pending |=
 471                                 (kbdev->pm.backend.desired_shader_state &
 472                                                         ~desired_bitmap);
 473
 474                         if (kbdev->pm.poweroff_shader_ticks)
 475                                 kbdev->pm.backend.shader_poweroff_pending_time =
 476                                                 kbdev->pm.poweroff_shader_ticks;
 477                         else
 478                                 do_poweroff = true;
 479                 }
 480
 481                 kbdev->pm.backend.desired_shader_state = desired_bitmap;
 482
 483                 /* If any cores are being powered on, transition immediately */
 484                 cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
 485         } else if (kbdev->pm.backend.desired_shader_state & ~desired_bitmap) {
 486                 /* Start timer to power off cores */
 487                 kbdev->pm.backend.shader_poweroff_pending |=
 488                                 (kbdev->pm.backend.desired_shader_state &
 489                                                         ~desired_bitmap);
 490                 if (kbdev->pm.poweroff_shader_ticks)
 491                         kbdev->pm.backend.shader_poweroff_pending_time =
 492                                         kbdev->pm.poweroff_shader_ticks;
 493                 else
 494                         kbasep_pm_do_poweroff_cores(kbdev);
 495         } else if (kbdev->pm.active_count == 0 && desired_bitmap != 0 &&
 496                                 kbdev->pm.backend.poweroff_timer_needed) {
 497                 /* If power policy is keeping cores on despite there being no
 498                  * active contexts then disable poweroff timer as it isn't
 499                  * required.
 500                  * Only reset poweroff_timer_needed if we're not in the middle
 501                  * of the power off callback */
 502                 kbdev->pm.backend.poweroff_timer_needed = false;
 503         }
 504
 505         /* Ensure timer does not power off wanted cores and make sure to power
 506          * off unwanted cores */
 507         if (kbdev->pm.backend.shader_poweroff_pending != 0) {
 508                 kbdev->pm.backend.shader_poweroff_pending &=
 509                                 ~(kbdev->pm.backend.desired_shader_state &
 510                                                                 desired_bitmap);
 511                 if (kbdev->pm.backend.shader_poweroff_pending == 0)
 512                         kbdev->pm.backend.shader_poweroff_pending_time = 0;
 513         }
 514
 515         /* Shader poweroff is deferred to the end of the function, to eliminate
 516          * issues caused by the core availability policy recursing into this
 517          * function */
 518         if (do_poweroff)
 519                 kbasep_pm_do_poweroff_cores(kbdev);
 520
 521         /* Don't need 'cores_are_available', because we don't return anything */
 522         CSTD_UNUSED(cores_are_available);
 523 }
 524
 525 void kbase_pm_update_cores_state(struct kbase_device *kbdev)
 526 {
 527         unsigned long flags;
 528
 529         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 530
 531         kbase_pm_update_cores_state_nolock(kbdev);
 532
 533         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 534 }
 535
 536 int kbase_pm_list_policies(const struct kbase_pm_policy * const **list)
 537 {
 538         if (!list)
 539                 return POLICY_COUNT;
 540
 541         *list = policy_list;
 542
 543         return POLICY_COUNT;
 544 }
 545
 546 KBASE_EXPORT_TEST_API(kbase_pm_list_policies);
 547
 548 const struct kbase_pm_policy *kbase_pm_get_policy(struct kbase_device *kbdev)
 549 {
 550         KBASE_DEBUG_ASSERT(kbdev != NULL);
 551
 552         return kbdev->pm.backend.pm_current_policy;
 553 }
 554
 555 KBASE_EXPORT_TEST_API(kbase_pm_get_policy);
 556
 557 void kbase_pm_set_policy(struct kbase_device *kbdev,
 558                                 const struct kbase_pm_policy *new_policy)
 559 {
 560         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
 561         const struct kbase_pm_policy *old_policy;
 562         unsigned long flags;
 563
 564         KBASE_DEBUG_ASSERT(kbdev != NULL);
 565         KBASE_DEBUG_ASSERT(new_policy != NULL);
 566
 567         KBASE_TRACE_ADD(kbdev, PM_SET_POLICY, NULL, NULL, 0u, new_policy->id);
 568
 569         /* During a policy change we pretend the GPU is active */
 570         /* A suspend won't happen here, because we're in a syscall from a
 571          * userspace thread */
 572         kbase_pm_context_active(kbdev);
 573
 574         mutex_lock(&js_devdata->runpool_mutex);
 575         mutex_lock(&kbdev->pm.lock);
 576
 577         /* Remove the policy to prevent IRQ handlers from working on it */
 578         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 579         old_policy = kbdev->pm.backend.pm_current_policy;
 580         kbdev->pm.backend.pm_current_policy = NULL;
 581         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 582
 583         KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_TERM, NULL, NULL, 0u,
 584                                                                 old_policy->id);
 585         if (old_policy->term)
 586                 old_policy->term(kbdev);
 587
 588         KBASE_TRACE_ADD(kbdev, PM_CURRENT_POLICY_INIT, NULL, NULL, 0u,
 589                                                                 new_policy->id);
 590         if (new_policy->init)
 591                 new_policy->init(kbdev);
 592
 593         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 594         kbdev->pm.backend.pm_current_policy = new_policy;
 595         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 596
 597         /* If any core power state changes were previously attempted, but
 598          * couldn't be made because the policy was changing (current_policy was
 599          * NULL), then re-try them here. */
 600         kbase_pm_update_active(kbdev);
 601         kbase_pm_update_cores_state(kbdev);
 602
 603         mutex_unlock(&kbdev->pm.lock);
 604         mutex_unlock(&js_devdata->runpool_mutex);
 605
 606         /* Now the policy change is finished, we release our fake context active
 607          * reference */
 608         kbase_pm_context_idle(kbdev);
 609 }
 610
 611 KBASE_EXPORT_TEST_API(kbase_pm_set_policy);
 612
 613 /* Check whether a state change has finished, and trace it as completed */
 614 static void
 615 kbase_pm_trace_check_and_finish_state_change(struct kbase_device *kbdev)
 616 {
 617         if ((kbdev->shader_available_bitmap &
 618                                         kbdev->pm.backend.desired_shader_state)
 619                                 == kbdev->pm.backend.desired_shader_state &&
 620                 (kbdev->tiler_available_bitmap &
 621                                         kbdev->pm.backend.desired_tiler_state)
 622                                 == kbdev->pm.backend.desired_tiler_state)
 623                 kbase_timeline_pm_check_handle_event(kbdev,
 624                                 KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
 625 }
 626
 627 void kbase_pm_request_cores(struct kbase_device *kbdev,
 628                                 bool tiler_required, u64 shader_cores)
 629 {
 630         unsigned long flags;
 631         u64 cores;
 632
 633         kbase_pm_change_state change_gpu_state = 0u;
 634
 635         KBASE_DEBUG_ASSERT(kbdev != NULL);
 636
 637         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 638
 639         cores = shader_cores;
 640         while (cores) {
 641                 int bitnum = fls64(cores) - 1;
 642                 u64 bit = 1ULL << bitnum;
 643
 644                 /* It should be almost impossible for this to overflow. It would
 645                  * require 2^32 atoms to request a particular core, which would
 646                  * require 2^24 contexts to submit. This would require an amount
 647                  * of memory that is impossible on a 32-bit system and extremely
 648                  * unlikely on a 64-bit system. */
 649                 int cnt = ++kbdev->shader_needed_cnt[bitnum];
 650
 651                 if (1 == cnt) {
 652                         kbdev->shader_needed_bitmap |= bit;
 653                         change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
 654                 }
 655
 656                 cores &= ~bit;
 657         }
 658
 659         if (tiler_required) {
 660                 int cnt = ++kbdev->tiler_needed_cnt;
 661
 662                 if (1 == cnt)
 663                         change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER;
 664
 665                 KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt != 0);
 666         }
 667
 668         if (change_gpu_state) {
 669                 KBASE_TRACE_ADD(kbdev, PM_REQUEST_CHANGE_SHADER_NEEDED, NULL,
 670                                 NULL, 0u, (u32) kbdev->shader_needed_bitmap);
 671
 672                 kbase_timeline_pm_cores_func(kbdev,
 673                                         KBASE_PM_FUNC_ID_REQUEST_CORES_START,
 674                                                         change_gpu_state);
 675                 kbase_pm_update_cores_state_nolock(kbdev);
 676                 kbase_timeline_pm_cores_func(kbdev,
 677                                         KBASE_PM_FUNC_ID_REQUEST_CORES_END,
 678                                                         change_gpu_state);
 679         }
 680
 681         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 682 }
 683
 684 KBASE_EXPORT_TEST_API(kbase_pm_request_cores);
 685
 686 void kbase_pm_unrequest_cores(struct kbase_device *kbdev,
 687                                 bool tiler_required, u64 shader_cores)
 688 {
 689         unsigned long flags;
 690
 691         kbase_pm_change_state change_gpu_state = 0u;
 692
 693         KBASE_DEBUG_ASSERT(kbdev != NULL);
 694
 695         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 696
 697         while (shader_cores) {
 698                 int bitnum = fls64(shader_cores) - 1;
 699                 u64 bit = 1ULL << bitnum;
 700                 int cnt;
 701
 702                 KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);
 703
 704                 cnt = --kbdev->shader_needed_cnt[bitnum];
 705
 706                 if (0 == cnt) {
 707                         kbdev->shader_needed_bitmap &= ~bit;
 708
 709                         change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
 710                 }
 711
 712                 shader_cores &= ~bit;
 713         }
 714
 715         if (tiler_required) {
 716                 int cnt;
 717
 718                 KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);
 719
 720                 cnt = --kbdev->tiler_needed_cnt;
 721
 722                 if (0 == cnt)
 723                         change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER;
 724         }
 725
 726         if (change_gpu_state) {
 727                 KBASE_TRACE_ADD(kbdev, PM_UNREQUEST_CHANGE_SHADER_NEEDED, NULL,
 728                                 NULL, 0u, (u32) kbdev->shader_needed_bitmap);
 729
 730                 kbase_pm_update_cores_state_nolock(kbdev);
 731
 732                 /* Trace that any state change effectively completes immediately
 733                  * - no-one will wait on the state change */
 734                 kbase_pm_trace_check_and_finish_state_change(kbdev);
 735         }
 736
 737         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 738 }
 739
 740 KBASE_EXPORT_TEST_API(kbase_pm_unrequest_cores);
 741
 742 enum kbase_pm_cores_ready
 743 kbase_pm_register_inuse_cores(struct kbase_device *kbdev,
 744                                 bool tiler_required, u64 shader_cores)
 745 {
 746         unsigned long flags;
 747         u64 prev_shader_needed; /* Just for tracing */
 748         u64 prev_shader_inuse;  /* Just for tracing */
 749
 750         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 751
 752         prev_shader_needed = kbdev->shader_needed_bitmap;
 753         prev_shader_inuse = kbdev->shader_inuse_bitmap;
 754
 755         /* If desired_shader_state does not contain the requested cores, then
 756          * power management is not attempting to powering those cores (most
 757          * likely due to core availability policy) and a new job affinity must
 758          * be chosen */
 759         if ((kbdev->pm.backend.desired_shader_state & shader_cores) !=
 760                                                         shader_cores) {
 761                 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 762
 763                 return KBASE_NEW_AFFINITY;
 764         }
 765
 766         if ((kbdev->shader_available_bitmap & shader_cores) != shader_cores ||
 767             (tiler_required && !kbdev->tiler_available_bitmap)) {
 768                 /* Trace ongoing core transition */
 769                 kbase_timeline_pm_l2_transition_start(kbdev);
 770                 spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 771                 return KBASE_CORES_NOT_READY;
 772         }
 773
 774         /* If we started to trace a state change, then trace it has being
 775          * finished by now, at the very latest */
 776         kbase_pm_trace_check_and_finish_state_change(kbdev);
 777         /* Trace core transition done */
 778         kbase_timeline_pm_l2_transition_done(kbdev);
 779
 780         while (shader_cores) {
 781                 int bitnum = fls64(shader_cores) - 1;
 782                 u64 bit = 1ULL << bitnum;
 783                 int cnt;
 784
 785                 KBASE_DEBUG_ASSERT(kbdev->shader_needed_cnt[bitnum] > 0);
 786
 787                 cnt = --kbdev->shader_needed_cnt[bitnum];
 788
 789                 if (0 == cnt)
 790                         kbdev->shader_needed_bitmap &= ~bit;
 791
 792                 /* shader_inuse_cnt should not overflow because there can only
 793                  * be a very limited number of jobs on the h/w at one time */
 794
 795                 kbdev->shader_inuse_cnt[bitnum]++;
 796                 kbdev->shader_inuse_bitmap |= bit;
 797
 798                 shader_cores &= ~bit;
 799         }
 800
 801         if (tiler_required) {
 802                 KBASE_DEBUG_ASSERT(kbdev->tiler_needed_cnt > 0);
 803
 804                 --kbdev->tiler_needed_cnt;
 805
 806                 kbdev->tiler_inuse_cnt++;
 807
 808                 KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt != 0);
 809         }
 810
 811         if (prev_shader_needed != kbdev->shader_needed_bitmap)
 812                 KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_NEEDED, NULL,
 813                                 NULL, 0u, (u32) kbdev->shader_needed_bitmap);
 814
 815         if (prev_shader_inuse != kbdev->shader_inuse_bitmap)
 816                 KBASE_TRACE_ADD(kbdev, PM_REGISTER_CHANGE_SHADER_INUSE, NULL,
 817                                 NULL, 0u, (u32) kbdev->shader_inuse_bitmap);
 818
 819         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 820
 821         return KBASE_CORES_READY;
 822 }
 823
 824 KBASE_EXPORT_TEST_API(kbase_pm_register_inuse_cores);
 825
 826 void kbase_pm_release_cores(struct kbase_device *kbdev,
 827                                 bool tiler_required, u64 shader_cores)
 828 {
 829         unsigned long flags;
 830         kbase_pm_change_state change_gpu_state = 0u;
 831
 832         KBASE_DEBUG_ASSERT(kbdev != NULL);
 833
 834         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 835
 836         while (shader_cores) {
 837                 int bitnum = fls64(shader_cores) - 1;
 838                 u64 bit = 1ULL << bitnum;
 839                 int cnt;
 840
 841                 KBASE_DEBUG_ASSERT(kbdev->shader_inuse_cnt[bitnum] > 0);
 842
 843                 cnt = --kbdev->shader_inuse_cnt[bitnum];
 844
 845                 if (0 == cnt) {
 846                         kbdev->shader_inuse_bitmap &= ~bit;
 847                         change_gpu_state |= KBASE_PM_CHANGE_STATE_SHADER;
 848                 }
 849
 850                 shader_cores &= ~bit;
 851         }
 852
 853         if (tiler_required) {
 854                 int cnt;
 855
 856                 KBASE_DEBUG_ASSERT(kbdev->tiler_inuse_cnt > 0);
 857
 858                 cnt = --kbdev->tiler_inuse_cnt;
 859
 860                 if (0 == cnt)
 861                         change_gpu_state |= KBASE_PM_CHANGE_STATE_TILER;
 862         }
 863
 864         if (change_gpu_state) {
 865                 KBASE_TRACE_ADD(kbdev, PM_RELEASE_CHANGE_SHADER_INUSE, NULL,
 866                                 NULL, 0u, (u32) kbdev->shader_inuse_bitmap);
 867
 868                 kbase_timeline_pm_cores_func(kbdev,
 869                                         KBASE_PM_FUNC_ID_RELEASE_CORES_START,
 870                                                         change_gpu_state);
 871                 kbase_pm_update_cores_state_nolock(kbdev);
 872                 kbase_timeline_pm_cores_func(kbdev,
 873                                         KBASE_PM_FUNC_ID_RELEASE_CORES_END,
 874                                                         change_gpu_state);
 875
 876                 /* Trace that any state change completed immediately */
 877                 kbase_pm_trace_check_and_finish_state_change(kbdev);
 878         }
 879
 880         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 881 }
 882
 883 KBASE_EXPORT_TEST_API(kbase_pm_release_cores);
 884
 885 void kbase_pm_request_cores_sync(struct kbase_device *kbdev,
 886                                         bool tiler_required,
 887                                         u64 shader_cores)
 888 {
 889         kbase_pm_request_cores(kbdev, tiler_required, shader_cores);
 890
 891         kbase_pm_check_transitions_sync(kbdev);
 892 }
 893
 894 KBASE_EXPORT_TEST_API(kbase_pm_request_cores_sync);
 895
 896 void kbase_pm_request_l2_caches(struct kbase_device *kbdev)
 897 {
 898         unsigned long flags;
 899         u32 prior_l2_users_count;
 900
 901         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 902
 903         prior_l2_users_count = kbdev->l2_users_count++;
 904
 905         KBASE_DEBUG_ASSERT(kbdev->l2_users_count != 0);
 906
 907         /* if the GPU is reset while the l2 is on, l2 will be off but
 908          * prior_l2_users_count will be > 0. l2_available_bitmap will have been
 909          * set to 0 though by kbase_pm_init_hw */
 910         if (!prior_l2_users_count || !kbdev->l2_available_bitmap)
 911                 kbase_pm_check_transitions_nolock(kbdev);
 912
 913         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 914         wait_event(kbdev->pm.backend.l2_powered_wait,
 915                                         kbdev->pm.backend.l2_powered == 1);
 916
 917         /* Trace that any state change completed immediately */
 918         kbase_pm_trace_check_and_finish_state_change(kbdev);
 919 }
 920
 921 KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches);
 922
 923 void kbase_pm_request_l2_caches_l2_is_on(struct kbase_device *kbdev)
 924 {
 925         unsigned long flags;
 926
 927         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 928
 929         kbdev->l2_users_count++;
 930
 931         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 932 }
 933
 934 KBASE_EXPORT_TEST_API(kbase_pm_request_l2_caches_l2_is_on);
 935
 936 void kbase_pm_release_l2_caches(struct kbase_device *kbdev)
 937 {
 938         unsigned long flags;
 939
 940         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 941
 942         KBASE_DEBUG_ASSERT(kbdev->l2_users_count > 0);
 943
 944         --kbdev->l2_users_count;
 945
 946         if (!kbdev->l2_users_count) {
 947                 kbase_pm_check_transitions_nolock(kbdev);
 948                 /* Trace that any state change completed immediately */
 949                 kbase_pm_trace_check_and_finish_state_change(kbdev);
 950         }
 951
 952         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 953 }
 954
 955 KBASE_EXPORT_TEST_API(kbase_pm_release_l2_caches);