drivers/gpu/arm/t6xx/kbase/src/common/mali_kbase_pm_driver.c

   1 /*
   2  *
   3  * (C) COPYRIGHT ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18
  19
  20 /**
  21  * @file mali_kbase_pm_driver.c
  22  * Base kernel Power Management hardware control
  23  */
  24
  25 #include <kbase/src/common/mali_kbase.h>
  26 #include <kbase/src/common/mali_midg_regmap.h>
  27 #include <kbase/src/common/mali_kbase_gator.h>
  28 #include <kbase/src/common/mali_kbase_pm.h>
  29
  30 #if MALI_MOCK_TEST
  31 #define MOCKABLE(function) function##_original
  32 #else
  33 #define MOCKABLE(function) function
  34 #endif                          /* MALI_MOCK_TEST */
  35
  36 /** Actions that can be performed on a core.
  37  *
  38  * This enumeration is private to the file. Its values are set to allow @ref core_type_to_reg function,
  39  * which decodes this enumeration, to be simpler and more efficient.
  40  */
  41 typedef enum kbasep_pm_action {
  42         ACTION_PRESENT = 0,
  43         ACTION_READY = (SHADER_READY_LO - SHADER_PRESENT_LO),
  44         ACTION_PWRON = (SHADER_PWRON_LO - SHADER_PRESENT_LO),
  45         ACTION_PWROFF = (SHADER_PWROFF_LO - SHADER_PRESENT_LO),
  46         ACTION_PWRTRANS = (SHADER_PWRTRANS_LO - SHADER_PRESENT_LO),
  47         ACTION_PWRACTIVE = (SHADER_PWRACTIVE_LO - SHADER_PRESENT_LO)
  48 } kbasep_pm_action;
  49
  50 /** Decode a core type and action to a register.
  51  *
  52  * Given a core type (defined by @ref kbase_pm_core_type) and an action (defined by @ref kbasep_pm_action) this
  53  * function will return the register offset that will perform the action on the core type. The register returned is
  54  * the \c _LO register and an offset must be applied to use the \c _HI register.
  55  *
  56  * @param core_type The type of core
  57  * @param action    The type of action
  58  *
  59  * @return The register offset of the \c _LO register that performs an action of type \c action on a core of type \c
  60  * core_type.
  61  */
  62 static u32 core_type_to_reg(kbase_pm_core_type core_type, kbasep_pm_action action)
  63 {
  64         return core_type + action;
  65 }
  66
  67 /** Invokes an action on a core set
  68  *
  69  * This function performs the action given by \c action on a set of cores of a type given by \c core_type. It is a
  70  * static function used by @ref kbase_pm_transition_core_type
  71  *
  72  * @param kbdev     The kbase device structure of the device
  73  * @param core_type The type of core that the action should be performed on
  74  * @param cores     A bit mask of cores to perform the action on (low 32 bits)
  75  * @param action    The action to perform on the cores
  76  */
  77 STATIC void kbase_pm_invoke(kbase_device *kbdev, kbase_pm_core_type core_type, u64 cores, kbasep_pm_action action)
  78 {
  79         u32 reg;
  80         u32 lo = cores & 0xFFFFFFFF;
  81         u32 hi = (cores >> 32) & 0xFFFFFFFF;
  82
  83         lockdep_assert_held(&kbdev->pm.power_change_lock);
  84
  85         reg = core_type_to_reg(core_type, action);
  86
  87         KBASE_DEBUG_ASSERT(reg);
  88 #ifdef CONFIG_MALI_GATOR_SUPPORT
  89         if (cores) {
  90                 if (action == ACTION_PWRON)
  91                         kbase_trace_mali_pm_power_on(core_type, cores);
  92                 else if (action == ACTION_PWROFF)
  93                         kbase_trace_mali_pm_power_off(core_type, cores);
  94         }
  95 #endif                          /* CONFIG_MALI_GATOR_SUPPORT */
  96         /* Tracing */
  97         if (cores) {
  98                 if (action == ACTION_PWRON)
  99                         switch (core_type) {
 100                                 case KBASE_PM_CORE_SHADER:
 101                                         KBASE_TRACE_ADD(kbdev, PM_PWRON, NULL, NULL, 0u, lo);
 102                                         break;
 103                                 case KBASE_PM_CORE_TILER:
 104                                         KBASE_TRACE_ADD(kbdev, PM_PWRON_TILER, NULL, NULL, 0u, lo);
 105                                         break;
 106                                 case KBASE_PM_CORE_L2:
 107                                         KBASE_TRACE_ADD(kbdev, PM_PWRON_L2, NULL, NULL, 0u, lo);
 108                                         break;
 109                                 default:
 110                                         /* L3 not handled */
 111                                         break;
 112                         }
 113                 else if (action == ACTION_PWROFF)
 114                         switch (core_type) {
 115                                 case KBASE_PM_CORE_SHADER:
 116                                         KBASE_TRACE_ADD(kbdev, PM_PWROFF, NULL, NULL, 0u, lo);
 117                                         break;
 118                                 case KBASE_PM_CORE_TILER:
 119                                         KBASE_TRACE_ADD(kbdev, PM_PWROFF_TILER, NULL, NULL, 0u, lo);
 120                                         break;
 121                                 case KBASE_PM_CORE_L2:
 122                                         KBASE_TRACE_ADD(kbdev, PM_PWROFF_L2, NULL, NULL, 0u, lo);
 123                                         break;
 124                                 default:
 125                                         /* L3 not handled */
 126                                         break;
 127                         }
 128         }
 129
 130         if (lo != 0)
 131                 kbase_reg_write(kbdev, GPU_CONTROL_REG(reg), lo, NULL);
 132
 133         if (hi != 0)
 134                 kbase_reg_write(kbdev, GPU_CONTROL_REG(reg + 4), hi, NULL);
 135 }
 136
 137 /** Get information about a core set
 138  *
 139  * This function gets information (chosen by \c action) about a set of cores of a type given by \c core_type. It is a
 140  * static function used by @ref kbase_pm_get_present_cores, @ref kbase_pm_get_active_cores, @ref
 141  * kbase_pm_get_trans_cores and @ref kbase_pm_get_ready_cores.
 142  *
 143  * @param kbdev     The kbase device structure of the device
 144  * @param core_type The type of core that the should be queried
 145  * @param action    The property of the cores to query
 146  *
 147  * @return A bit mask specifying the state of the cores
 148  */
 149 static u64 kbase_pm_get_state(kbase_device *kbdev, kbase_pm_core_type core_type, kbasep_pm_action action)
 150 {
 151         u32 reg;
 152         u32 lo, hi;
 153
 154         reg = core_type_to_reg(core_type, action);
 155
 156         KBASE_DEBUG_ASSERT(reg);
 157
 158         lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg), NULL);
 159         hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg + 4), NULL);
 160
 161         return (((u64) hi) << 32) | ((u64) lo);
 162 }
 163
 164 void kbasep_pm_read_present_cores(kbase_device *kbdev)
 165 {
 166         kbdev->shader_present_bitmap = kbase_pm_get_state(kbdev, KBASE_PM_CORE_SHADER, ACTION_PRESENT);
 167         kbdev->tiler_present_bitmap = kbase_pm_get_state(kbdev, KBASE_PM_CORE_TILER, ACTION_PRESENT);
 168         kbdev->l2_present_bitmap = kbase_pm_get_state(kbdev, KBASE_PM_CORE_L2, ACTION_PRESENT);
 169         kbdev->l3_present_bitmap = kbase_pm_get_state(kbdev, KBASE_PM_CORE_L3, ACTION_PRESENT);
 170
 171         kbdev->shader_inuse_bitmap = 0;
 172         kbdev->shader_needed_bitmap = 0;
 173         kbdev->shader_available_bitmap = 0;
 174         kbdev->tiler_available_bitmap = 0;
 175         kbdev->l2_users_count = 0;
 176         kbdev->l2_available_bitmap = 0;
 177         kbdev->tiler_needed_cnt = 0;
 178         kbdev->tiler_inuse_cnt = 0;
 179
 180         memset(kbdev->shader_needed_cnt, 0, sizeof(kbdev->shader_needed_cnt));
 181 }
 182
 183 KBASE_EXPORT_TEST_API(kbasep_pm_read_present_cores)
 184
 185 /** Get the cores that are present
 186  */
 187 u64 kbase_pm_get_present_cores(kbase_device *kbdev, kbase_pm_core_type type)
 188 {
 189         KBASE_DEBUG_ASSERT(kbdev != NULL);
 190
 191         switch (type) {
 192         case KBASE_PM_CORE_L3:
 193                 return kbdev->l3_present_bitmap;
 194                 break;
 195         case KBASE_PM_CORE_L2:
 196                 return kbdev->l2_present_bitmap;
 197                 break;
 198         case KBASE_PM_CORE_SHADER:
 199                 return kbdev->shader_present_bitmap;
 200                 break;
 201         case KBASE_PM_CORE_TILER:
 202                 return kbdev->tiler_present_bitmap;
 203                 break;
 204         }
 205         KBASE_DEBUG_ASSERT(0);
 206         return 0;
 207 }
 208
 209 KBASE_EXPORT_TEST_API(kbase_pm_get_present_cores)
 210
 211 /** Get the cores that are "active" (busy processing work)
 212  */
 213 u64 kbase_pm_get_active_cores(kbase_device *kbdev, kbase_pm_core_type type)
 214 {
 215         return kbase_pm_get_state(kbdev, type, ACTION_PWRACTIVE);
 216 }
 217
 218 KBASE_EXPORT_TEST_API(kbase_pm_get_active_cores)
 219
 220 /** Get the cores that are transitioning between power states
 221  */
 222 u64 kbase_pm_get_trans_cores(kbase_device *kbdev, kbase_pm_core_type type)
 223 {
 224         return kbase_pm_get_state(kbdev, type, ACTION_PWRTRANS);
 225 }
 226
 227 KBASE_EXPORT_TEST_API(kbase_pm_get_trans_cores)
 228 /** Get the cores that are powered on
 229  */
 230 u64 kbase_pm_get_ready_cores(kbase_device *kbdev, kbase_pm_core_type type)
 231 {
 232         u64 result;
 233         result = kbase_pm_get_state(kbdev, type, ACTION_READY);
 234
 235         switch (type) {
 236                 case KBASE_PM_CORE_SHADER:
 237                         KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED, NULL, NULL, 0u, (u32) result);
 238                         break;
 239                 case KBASE_PM_CORE_TILER:
 240                         KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, NULL, 0u, (u32) result);
 241                         break;
 242                 case KBASE_PM_CORE_L2:
 243                         KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, NULL, 0u, (u32) result);
 244                         break;
 245                 default:
 246                         /* NB: L3 not currently traced */
 247                         break;
 248         }
 249
 250         return result;
 251 }
 252
 253 KBASE_EXPORT_TEST_API(kbase_pm_get_ready_cores)
 254
 255 /** Perform power transitions for a particular core type.
 256  *
 257  * This function will perform any available power transitions to make the actual hardware state closer to the desired
 258  * state. If a core is currently transitioning then changes to the power state of that call cannot be made until the
 259  * transition has finished. Cores which are not present in the hardware are ignored if they are specified in the
 260  * desired_state bitmask, however the return value will always be 0 in this case.
 261  *
 262  * @param kbdev             The kbase device
 263  * @param type              The core type to perform transitions for
 264  * @param desired_state     A bit mask of the desired state of the cores
 265  * @param in_use            A bit mask of the cores that are currently running jobs.
 266  *                          These cores have to be kept powered up because there are jobs
 267  *                          running (or about to run) on them.
 268  * @param[out] available    Receives a bit mask of the cores that the job scheduler can use to submit jobs to.
 269  *                          May be NULL if this is not needed.
 270  * @param[in,out] powering_on Bit mask to update with cores that are transitioning to a power-on state.
 271  *
 272  * @return MALI_TRUE if the desired state has been reached, MALI_FALSE otherwise
 273  */
 274 STATIC mali_bool kbase_pm_transition_core_type(kbase_device *kbdev, kbase_pm_core_type type, u64 desired_state,
 275                                                u64 in_use, u64 * const available, u64 *powering_on)
 276 {
 277         u64 present;
 278         u64 ready;
 279         u64 trans;
 280         u64 powerup;
 281         u64 powerdown;
 282         u64 powering_on_trans;
 283         u64 desired_state_in_use;
 284
 285         lockdep_assert_held(&kbdev->pm.power_change_lock);
 286
 287         /* Get current state */
 288         present = kbase_pm_get_present_cores(kbdev, type);
 289         trans = kbase_pm_get_trans_cores(kbdev, type);
 290         ready = kbase_pm_get_ready_cores(kbdev, type);
 291
 292         powering_on_trans = trans & *powering_on;
 293         *powering_on = powering_on_trans;
 294
 295         if (available != NULL)
 296                 *available = (ready | powering_on_trans) & desired_state;
 297
 298         /* Update desired state to include the in-use cores. These have to be kept powered up because there are jobs
 299          * running or about to run on these cores
 300          */
 301         desired_state_in_use = desired_state | in_use;
 302
 303         /* Update state of whether l2 caches are powered */
 304         if (type == KBASE_PM_CORE_L2) {
 305                 if ((ready == present) && (desired_state_in_use == ready) && (trans == 0)) {
 306                         /* All are ready, none will be turned off, and none are transitioning */
 307                         kbdev->pm.l2_powered = 1;
 308                         if (kbdev->l2_users_count > 0) {
 309                                 /* Notify any registered l2 cache users (optimized out when no users waiting) */
 310                                 wake_up(&kbdev->pm.l2_powered_wait);
 311                         }
 312                 } else {
 313                         kbdev->pm.l2_powered = 0;
 314                 }
 315         }
 316
 317         if (desired_state_in_use == ready && (trans == 0))
 318                 return MALI_TRUE;
 319
 320         /* Restrict the cores to those that are actually present */
 321         powerup = desired_state_in_use & present;
 322         powerdown = (~desired_state_in_use) & present;
 323
 324         /* Restrict to cores that are not already in the desired state */
 325         powerup &= ~ready;
 326         powerdown &= ready;
 327
 328         /* Don't transition any cores that are already transitioning, except for
 329          * Mali cores that support the following case:
 330          *
 331          * If the SHADER_PWRON or TILER_PWRON registers are written to turn on
 332          * a core that is currently transitioning to power off, then this is
 333          * remembered and the shader core is automatically powered up again once
 334          * the original transition completes. Once the automatic power on is
 335          * complete any job scheduled on the shader core should start.
 336          */
 337         powerdown &= ~trans;
 338
 339         if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS))
 340                 if (KBASE_PM_CORE_SHADER == type || KBASE_PM_CORE_TILER == type)
 341                         trans = powering_on_trans; /* for exception cases, only mask off cores in power on transitions */
 342
 343         powerup &= ~trans;
 344
 345         /* Perform transitions if any */
 346         kbase_pm_invoke(kbdev, type, powerup, ACTION_PWRON);
 347         kbase_pm_invoke(kbdev, type, powerdown, ACTION_PWROFF);
 348
 349         /* Recalculate cores transitioning on, and re-evaluate our state */
 350         powering_on_trans |= powerup;
 351         *powering_on = powering_on_trans;
 352         if (available != NULL)
 353                 *available = (ready | powering_on_trans) & desired_state;
 354
 355         return MALI_FALSE;
 356 }
 357
 358 KBASE_EXPORT_TEST_API(kbase_pm_transition_core_type)
 359
 360 /** Determine which caches should be on for a particular core state.
 361  *
 362  * This function takes a bit mask of the present caches and the cores (or caches) that are attached to the caches that
 363  * will be powered. It then computes which caches should be turned on to allow the cores requested to be powered up.
 364  *
 365  * @param present       The bit mask of present caches
 366  * @param cores_powered A bit mask of cores (or L2 caches) that are desired to be powered
 367  *
 368  * @return A bit mask of the caches that should be turned on
 369  */
 370 STATIC u64 get_desired_cache_status(u64 present, u64 cores_powered)
 371 {
 372         u64 desired = 0;
 373
 374         while (present) {
 375                 /* Find out which is the highest set bit */
 376                 u64 bit = fls64(present) - 1;
 377                 u64 bit_mask = 1ull << bit;
 378                 /* Create a mask which has all bits from 'bit' upwards set */
 379
 380                 u64 mask = ~(bit_mask - 1);
 381
 382                 /* If there are any cores powered at this bit or above (that haven't previously been processed) then we need
 383                  * this core on */
 384                 if (cores_powered & mask)
 385                         desired |= bit_mask;
 386
 387                 /* Remove bits from cores_powered and present */
 388                 cores_powered &= ~mask;
 389                 present &= ~bit_mask;
 390         }
 391
 392         return desired;
 393 }
 394
 395 KBASE_EXPORT_TEST_API(get_desired_cache_status)
 396
 397 mali_bool MOCKABLE(kbase_pm_check_transitions_nolock) (struct kbase_device *kbdev)
 398 {
 399         mali_bool cores_are_available = MALI_FALSE;
 400         mali_bool in_desired_state = MALI_TRUE;
 401         u64 desired_l2_state;
 402         u64 desired_l3_state;
 403         u64 cores_powered;
 404         u64 tiler_available_bitmap;
 405         u64 shader_available_bitmap;
 406         u64 shader_ready_bitmap;
 407         u64 shader_transitioning_bitmap;
 408         u64 l2_available_bitmap;
 409
 410         KBASE_DEBUG_ASSERT(NULL != kbdev);
 411         lockdep_assert_held(&kbdev->pm.power_change_lock);
 412
 413         spin_lock(&kbdev->pm.gpu_powered_lock);
 414         if (kbdev->pm.gpu_powered == MALI_FALSE) {
 415                 spin_unlock(&kbdev->pm.gpu_powered_lock);
 416                 if (kbdev->pm.desired_shader_state == 0 && kbdev->pm.desired_tiler_state == 0)
 417                         return MALI_TRUE;
 418                 return MALI_FALSE;
 419         }
 420
 421         /* Trace that a change-state is being requested, and that it took
 422          * (effectively) no time to start it. This is useful for counting how many
 423          * state changes occurred, in a way that's backwards-compatible with
 424          * processing the trace data */
 425         kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE);
 426         kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE);
 427
 428         /* If any cores are already powered then, we must keep the caches on */
 429         cores_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER);
 430
 431         cores_powered |= kbdev->pm.desired_shader_state;
 432
 433         /* If there are l2 cache users registered, keep all l2s powered even if all other cores are off. */
 434         if (kbdev->l2_users_count > 0)
 435                 cores_powered |= kbdev->l2_present_bitmap;
 436
 437         desired_l2_state = get_desired_cache_status(kbdev->l2_present_bitmap, cores_powered);
 438
 439         /* If any l2 cache is on, then enable l2 #0, for use by job manager */
 440         if (0 != desired_l2_state) {
 441                 desired_l2_state |= 1;
 442                 /* Also enable tiler if l2 cache is powered */
 443                 kbdev->pm.desired_tiler_state = kbdev->tiler_present_bitmap;
 444         } else {
 445                 kbdev->pm.desired_tiler_state = 0;
 446         }
 447
 448         desired_l3_state = get_desired_cache_status(kbdev->l3_present_bitmap, desired_l2_state);
 449
 450         in_desired_state &= kbase_pm_transition_core_type(kbdev, KBASE_PM_CORE_L3, desired_l3_state, 0, NULL, &kbdev->pm.powering_on_l3_state);
 451         in_desired_state &= kbase_pm_transition_core_type(kbdev, KBASE_PM_CORE_L2, desired_l2_state, 0, &l2_available_bitmap, &kbdev->pm.powering_on_l2_state);
 452
 453         if( kbdev->l2_available_bitmap != l2_available_bitmap)
 454         {
 455                 KBASE_TIMELINE_POWER_L2(kbdev,l2_available_bitmap);
 456         }
 457
 458         kbdev->l2_available_bitmap = l2_available_bitmap;
 459
 460         if (in_desired_state) {
 461
 462                 in_desired_state &= kbase_pm_transition_core_type(kbdev, KBASE_PM_CORE_TILER, kbdev->pm.desired_tiler_state, 0, &tiler_available_bitmap, &kbdev->pm.powering_on_tiler_state);
 463                 in_desired_state &= kbase_pm_transition_core_type(kbdev, KBASE_PM_CORE_SHADER, kbdev->pm.desired_shader_state, kbdev->shader_inuse_bitmap, &shader_available_bitmap, &kbdev->pm.powering_on_shader_state);
 464
 465                 if (kbdev->shader_available_bitmap != shader_available_bitmap) {
 466                         KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, NULL, 0u, (u32) shader_available_bitmap);
 467                         KBASE_TIMELINE_POWER_SHADER(kbdev, shader_available_bitmap);
 468                 }
 469
 470                 kbdev->shader_available_bitmap = shader_available_bitmap;
 471
 472                 if (kbdev->tiler_available_bitmap != tiler_available_bitmap) {
 473                         KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, NULL, 0u, (u32) tiler_available_bitmap);
 474                         KBASE_TIMELINE_POWER_TILER(kbdev, tiler_available_bitmap);
 475                 }
 476
 477                 kbdev->tiler_available_bitmap = tiler_available_bitmap;
 478
 479         } else if ((l2_available_bitmap & kbdev->tiler_present_bitmap) != kbdev->tiler_present_bitmap) {
 480                 tiler_available_bitmap = 0;
 481
 482                 if (kbdev->tiler_available_bitmap != tiler_available_bitmap) {
 483                         KBASE_TIMELINE_POWER_TILER(kbdev, tiler_available_bitmap);
 484                 }
 485
 486                 kbdev->tiler_available_bitmap = tiler_available_bitmap;
 487         }
 488
 489         /* State updated for slow-path waiters */
 490         kbdev->pm.gpu_in_desired_state = in_desired_state;
 491
 492         shader_ready_bitmap = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER);
 493         shader_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev, KBASE_PM_CORE_SHADER);
 494
 495         /* Determine whether the cores are now available (even if the set of
 496          * available cores is empty). Note that they can be available even if we've
 497          * not finished transitioning to the desired state */
 498         if ((kbdev->shader_available_bitmap & kbdev->pm.desired_shader_state) == kbdev->pm.desired_shader_state
 499                 && (kbdev->tiler_available_bitmap & kbdev->pm.desired_tiler_state) == kbdev->pm.desired_tiler_state) {
 500                 cores_are_available = MALI_TRUE;
 501
 502                 KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE, NULL, NULL, 0u, (u32)(kbdev->shader_available_bitmap & kbdev->pm.desired_shader_state));
 503                 KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE_TILER, NULL, NULL, 0u, (u32)(kbdev->tiler_available_bitmap & kbdev->pm.desired_tiler_state));
 504
 505                 /* Log timelining information about handling events that power up
 506                  * cores, to match up either with immediate submission either because
 507                  * cores already available, or from PM IRQ */
 508                 if (!in_desired_state)
 509                         kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
 510         }
 511
 512         if (in_desired_state) {
 513                 KBASE_DEBUG_ASSERT(cores_are_available);
 514
 515 #ifdef CONFIG_MALI_GATOR_SUPPORT
 516                 kbase_trace_mali_pm_status(KBASE_PM_CORE_L3, kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L3));
 517                 kbase_trace_mali_pm_status(KBASE_PM_CORE_L2, kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_L2));
 518                 kbase_trace_mali_pm_status(KBASE_PM_CORE_SHADER, kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER));
 519                 kbase_trace_mali_pm_status(KBASE_PM_CORE_TILER, kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_TILER));
 520 #endif                          /* CONFIG_MALI_GATOR_SUPPORT */
 521
 522                 KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, NULL, kbdev->pm.gpu_in_desired_state, (u32)kbdev->pm.desired_shader_state);
 523                 KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED_TILER, NULL, NULL, 0u, (u32)kbdev->pm.desired_tiler_state);
 524
 525                 /* Log timelining information for synchronous waiters */
 526                 kbase_timeline_pm_send_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
 527                 /* Wake slow-path waiters. Job scheduler does not use this. */
 528                 KBASE_TRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, NULL, 0u, 0);
 529                 wake_up(&kbdev->pm.gpu_in_desired_state_wait);
 530         }
 531
 532         spin_unlock(&kbdev->pm.gpu_powered_lock);
 533
 534         /* kbase_pm_ca_update_core_status can cause one-level recursion into
 535          * this function, so it must only be called once all changes to kbdev
 536          * have been committed, and after the gpu_powered_lock has been
 537          * dropped. */
 538         if (kbdev->shader_ready_bitmap != shader_ready_bitmap ||
 539             kbdev->shader_transitioning_bitmap != shader_transitioning_bitmap) {
 540                 kbdev->shader_ready_bitmap = shader_ready_bitmap;
 541                 kbdev->shader_transitioning_bitmap = shader_transitioning_bitmap;
 542
 543                 kbase_pm_ca_update_core_status(kbdev, shader_ready_bitmap, shader_transitioning_bitmap);
 544         }
 545
 546         /* The core availability policy is not allowed to keep core group 0 off */
 547         if (!((shader_ready_bitmap | shader_transitioning_bitmap) & kbdev->gpu_props.props.coherency_info.group[0].core_mask) &&
 548             !(kbase_pm_ca_get_core_mask(kbdev) & kbdev->gpu_props.props.coherency_info.group[0].core_mask))
 549                 BUG();
 550
 551         /* The core availability policy is allowed to keep core group 1 off,
 552          * but all jobs specifically targeting CG1 must fail */
 553         if (!((shader_ready_bitmap | shader_transitioning_bitmap) & kbdev->gpu_props.props.coherency_info.group[1].core_mask) &&
 554             !(kbase_pm_ca_get_core_mask(kbdev) & kbdev->gpu_props.props.coherency_info.group[1].core_mask))
 555                 kbdev->pm.cg1_disabled = MALI_TRUE;
 556         else
 557                 kbdev->pm.cg1_disabled = MALI_FALSE;
 558
 559         return cores_are_available;
 560 }
 561 KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_nolock)
 562
 563 void kbase_pm_check_transitions_sync(struct kbase_device *kbdev)
 564 {
 565         unsigned long flags;
 566         mali_bool cores_are_available;
 567         /* Force the transition to be checked and reported - the cores may be
 568          * 'available' (for job submission) but not fully powered up. */
 569         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 570         cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
 571         /* Don't need 'cores_are_available', because we don't return anything */
 572         CSTD_UNUSED(cores_are_available);
 573         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 574
 575         /* Wait for cores */
 576         wait_event(kbdev->pm.gpu_in_desired_state_wait, kbdev->pm.gpu_in_desired_state);
 577
 578         /* Log timelining information that a change in state has completed */
 579         kbase_timeline_pm_handle_event(kbdev, KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
 580 }
 581 KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_sync)
 582
 583 void kbase_pm_enable_interrupts(kbase_device *kbdev)
 584 {
 585         unsigned long flags;
 586
 587         KBASE_DEBUG_ASSERT(NULL != kbdev);
 588         /*
 589          * Clear all interrupts,
 590          * and unmask them all.
 591          */
 592         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 593         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL, NULL);
 594         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), GPU_IRQ_REG_ALL, NULL);
 595         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 596
 597         kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF, NULL);
 598         kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0xFFFFFFFF, NULL);
 599
 600         kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL);
 601         kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFFFFFF, NULL);
 602 }
 603
 604 KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts)
 605
 606 void kbase_pm_disable_interrupts(kbase_device *kbdev)
 607 {
 608         unsigned long flags;
 609
 610         KBASE_DEBUG_ASSERT(NULL != kbdev);
 611         /*
 612          * Mask all interrupts,
 613          * and clear them all.
 614          */
 615         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 616         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), 0, NULL);
 617         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL, NULL);
 618         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 619
 620         kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0, NULL);
 621         kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF, NULL);
 622
 623         kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL);
 624         kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL);
 625 }
 626
 627 KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts)
 628
 629 /*
 630  * pmu layout:
 631  * 0x0000: PMU TAG (RO) (0xCAFECAFE)
 632  * 0x0004: PMU VERSION ID (RO) (0x00000000)
 633  * 0x0008: CLOCK ENABLE (RW) (31:1 SBZ, 0 CLOCK STATE)
 634  */
 635 void kbase_pm_clock_on(kbase_device *kbdev, mali_bool is_resume)
 636 {
 637         mali_bool reset_required = is_resume;
 638         unsigned long flags;
 639         KBASE_DEBUG_ASSERT(NULL != kbdev);
 640         lockdep_assert_held(&kbdev->pm.lock);
 641
 642         if (kbdev->pm.gpu_powered) {
 643                 /* Already turned on */
 644                 KBASE_DEBUG_ASSERT(!is_resume);
 645                 return;
 646         }
 647
 648         KBASE_TRACE_ADD(kbdev, PM_GPU_ON, NULL, NULL, 0u, 0u);
 649
 650         if (is_resume && kbdev->pm.callback_power_resume) {
 651                 kbdev->pm.callback_power_resume(kbdev);
 652         } else if (kbdev->pm.callback_power_on) {
 653                 if (kbdev->pm.callback_power_on(kbdev))
 654                         reset_required = MALI_TRUE;
 655         }
 656
 657         if (reset_required) {
 658                 /* GPU state was lost, reset GPU to ensure it is in a
 659                  * consistent state */
 660                 kbase_pm_init_hw(kbdev, MALI_TRUE);
 661         }
 662
 663         spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags);
 664         kbdev->pm.gpu_powered = MALI_TRUE;
 665         spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
 666
 667         /* Lastly, enable the interrupts */
 668         kbase_pm_enable_interrupts(kbdev);
 669 }
 670
 671 KBASE_EXPORT_TEST_API(kbase_pm_clock_on)
 672
 673 void kbase_pm_clock_off(kbase_device *kbdev, mali_bool is_suspend)
 674 {
 675         unsigned long flags;
 676         KBASE_DEBUG_ASSERT(NULL != kbdev);
 677         lockdep_assert_held(&kbdev->pm.lock);
 678
 679         /* ASSERT that the cores should now be unavailable. No lock needed. */
 680         KBASE_DEBUG_ASSERT(kbdev->shader_available_bitmap == 0u);
 681
 682         if (!kbdev->pm.gpu_powered) {
 683                 /* Already turned off */
 684                 if (is_suspend && kbdev->pm.callback_power_suspend)
 685                         kbdev->pm.callback_power_suspend(kbdev);
 686                 return;
 687         }
 688
 689         KBASE_TRACE_ADD(kbdev, PM_GPU_OFF, NULL, NULL, 0u, 0u);
 690
 691         /* Disable interrupts. This also clears any outstanding interrupts */
 692         kbase_pm_disable_interrupts(kbdev);
 693         /* Ensure that any IRQ handlers have finished */
 694         kbase_synchronize_irqs(kbdev);
 695
 696         /* The GPU power may be turned off from this point */
 697         spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags);
 698         kbdev->pm.gpu_powered = MALI_FALSE;
 699         spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
 700
 701         if (is_suspend && kbdev->pm.callback_power_suspend)
 702                 kbdev->pm.callback_power_suspend(kbdev);
 703         else if (kbdev->pm.callback_power_off)
 704                 kbdev->pm.callback_power_off(kbdev);
 705 }
 706
 707 KBASE_EXPORT_TEST_API(kbase_pm_clock_off)
 708
 709 struct kbasep_reset_timeout_data {
 710         struct hrtimer timer;
 711         mali_bool timed_out;
 712         kbase_device *kbdev;
 713 };
 714
 715 void kbase_pm_reset_done(kbase_device *kbdev)
 716 {
 717         KBASE_DEBUG_ASSERT(kbdev != NULL);
 718         kbdev->pm.reset_done = MALI_TRUE;
 719         wake_up(&kbdev->pm.reset_done_wait);
 720 }
 721
 722 /**
 723  * Wait for the RESET_COMPLETED IRQ to occur, then reset the waiting state.
 724  */
 725 STATIC void kbase_pm_wait_for_reset(kbase_device *kbdev)
 726 {
 727         lockdep_assert_held(&kbdev->pm.lock);
 728
 729         wait_event(kbdev->pm.reset_done_wait, (kbdev->pm.reset_done));
 730         kbdev->pm.reset_done = MALI_FALSE;
 731 }
 732
 733 KBASE_EXPORT_TEST_API(kbase_pm_reset_done)
 734
 735 static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer)
 736 {
 737         struct kbasep_reset_timeout_data *rtdata = container_of(timer, struct kbasep_reset_timeout_data, timer);
 738
 739         rtdata->timed_out = 1;
 740
 741         /* Set the wait queue to wake up kbase_pm_init_hw even though the reset hasn't completed */
 742         kbase_pm_reset_done(rtdata->kbdev);
 743
 744         return HRTIMER_NORESTART;
 745 }
 746
 747 static void kbase_pm_hw_issues(kbase_device *kbdev)
 748 {
 749         u32 value = 0;
 750         u32 config_value;
 751
 752         /* Needed due to MIDBASE-1494: LS_PAUSEBUFFER_DISABLE. See PRLAM-8443. */
 753         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8443))
 754                 value |= SC_LS_PAUSEBUFFER_DISABLE;
 755
 756         /* Needed due to MIDBASE-2054: SDC_DISABLE_OQ_DISCARD. See PRLAM-10327. */
 757         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10327))
 758                 value |= SC_SDC_DISABLE_OQ_DISCARD;
 759
 760         /* Enable alternative hardware counter selection if configured. */
 761         if (kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_ALTERNATIVE_HWC))
 762                 value |= SC_ALT_COUNTERS;
 763
 764         /* Use software control of forward pixel kill when needed. See MIDEUR-174. */
 765         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_2121))
 766                 value |= SC_OVERRIDE_FWD_PIXEL_KILL;
 767
 768         /* Needed due to MIDBASE-2795. ENABLE_TEXGRD_FLAGS. See PRLAM-10797. */
 769         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10797))
 770                 value |= SC_ENABLE_TEXGRD_FLAGS;
 771
 772         if (value != 0)
 773                 kbase_reg_write(kbdev, GPU_CONTROL_REG(SHADER_CONFIG), value, NULL);
 774
 775         /* Limit the GPU bus bandwidth if the platform needs this. */
 776         value = kbase_reg_read(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), NULL);
 777
 778         /* Limit read ID width for AXI */
 779         config_value = (u32) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_ARID_LIMIT);
 780         value &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_READS);
 781         value |= (config_value & 0x3) << L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT;
 782
 783         /* Limit write ID width for AXI */
 784         config_value = (u32) kbasep_get_config_value(kbdev, kbdev->config_attributes, KBASE_CONFIG_ATTR_AWID_LIMIT);
 785         value &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES);
 786         value |= (config_value & 0x3) << L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT;
 787
 788         kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG), value, NULL);
 789 }
 790
 791 mali_error kbase_pm_init_hw(kbase_device *kbdev, mali_bool enable_irqs )
 792 {
 793         unsigned long flags;
 794         struct kbasep_reset_timeout_data rtdata;
 795
 796         KBASE_DEBUG_ASSERT(NULL != kbdev);
 797         lockdep_assert_held(&kbdev->pm.lock);
 798
 799         /* Ensure the clock is on before attempting to access the hardware */
 800         if (!kbdev->pm.gpu_powered) {
 801                 if (kbdev->pm.callback_power_on)
 802                         kbdev->pm.callback_power_on(kbdev);
 803
 804                 spin_lock_irqsave(&kbdev->pm.gpu_powered_lock, flags);
 805                 kbdev->pm.gpu_powered = MALI_TRUE;
 806                 spin_unlock_irqrestore(&kbdev->pm.gpu_powered_lock, flags);
 807         }
 808
 809         /* Ensure interrupts are off to begin with, this also clears any outstanding interrupts */
 810         kbase_pm_disable_interrupts(kbdev);
 811
 812         /* Prepare for the soft-reset */
 813         kbdev->pm.reset_done = MALI_FALSE;
 814
 815         /* The cores should be made unavailable due to the reset */
 816         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 817         if (kbdev->shader_available_bitmap != 0u)
 818                         KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL, NULL, 0u, (u32)0u);
 819         if (kbdev->tiler_available_bitmap != 0u)
 820                         KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER, NULL, NULL, 0u, (u32)0u);
 821         kbdev->shader_available_bitmap = 0u;
 822         kbdev->tiler_available_bitmap = 0u;
 823         kbdev->l2_available_bitmap = 0u;
 824         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 825
 826         /* Soft reset the GPU */
 827         KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0);
 828         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_SOFT_RESET, NULL);
 829
 830         /* Unmask the reset complete interrupt only */
 831         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), RESET_COMPLETED, NULL);
 832
 833         /* Initialize a structure for tracking the status of the reset */
 834         rtdata.kbdev = kbdev;
 835         rtdata.timed_out = 0;
 836
 837         /* Create a timer to use as a timeout on the reset */
 838         hrtimer_init_on_stack(&rtdata.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 839         rtdata.timer.function = kbasep_reset_timeout;
 840
 841         hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), HRTIMER_MODE_REL);
 842
 843         /* Wait for the RESET_COMPLETED interrupt to be raised */
 844         kbase_pm_wait_for_reset(kbdev);
 845
 846         if (rtdata.timed_out == 0) {
 847                 /* GPU has been reset */
 848                 hrtimer_cancel(&rtdata.timer);
 849                 destroy_hrtimer_on_stack(&rtdata.timer);
 850                 goto out;
 851         }
 852
 853         /* No interrupt has been received - check if the RAWSTAT register says the reset has completed */
 854         if (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) & RESET_COMPLETED) {
 855                 /* The interrupt is set in the RAWSTAT; this suggests that the interrupts are not getting to the CPU */
 856                 KBASE_DEBUG_PRINT_WARN(KBASE_PM, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n");
 857                 /* If interrupts aren't working we can't continue. */
 858                 destroy_hrtimer_on_stack(&rtdata.timer);
 859                 goto out;
 860         }
 861
 862         /* The GPU doesn't seem to be responding to the reset so try a hard reset */
 863         KBASE_DEBUG_PRINT_ERROR(KBASE_PM, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n", RESET_TIMEOUT);
 864         KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0);
 865         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_HARD_RESET, NULL);
 866
 867         /* Restart the timer to wait for the hard reset to complete */
 868         rtdata.timed_out = 0;
 869
 870         hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT), HRTIMER_MODE_REL);
 871
 872         /* Wait for the RESET_COMPLETED interrupt to be raised */
 873         kbase_pm_wait_for_reset(kbdev);
 874
 875         if (rtdata.timed_out == 0) {
 876                 /* GPU has been reset */
 877                 hrtimer_cancel(&rtdata.timer);
 878                 destroy_hrtimer_on_stack(&rtdata.timer);
 879                 goto out;
 880         }
 881
 882         destroy_hrtimer_on_stack(&rtdata.timer);
 883
 884         KBASE_DEBUG_PRINT_ERROR(KBASE_PM, "Failed to hard-reset the GPU (timed out after %d ms)\n", RESET_TIMEOUT);
 885
 886         /* The GPU still hasn't reset, give up */
 887         return MALI_ERROR_FUNCTION_FAILED;
 888
 889  out:
 890         /* Re-enable interrupts if requested*/
 891         if ( enable_irqs )
 892         {
 893                 kbase_pm_enable_interrupts(kbdev);
 894         }
 895         /* If cycle counter was in use-re enable it */
 896         spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
 897
 898         if (kbdev->pm.gpu_cycle_counter_requests)
 899                 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_START, NULL);
 900
 901         spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
 902
 903         kbase_pm_hw_issues(kbdev);
 904
 905         return MALI_ERROR_NONE;
 906 }
 907
 908 KBASE_EXPORT_TEST_API(kbase_pm_init_hw)
 909
 910 void kbase_pm_request_gpu_cycle_counter(kbase_device *kbdev)
 911 {
 912         unsigned long flags;
 913         KBASE_DEBUG_ASSERT(kbdev != NULL);
 914
 915         KBASE_DEBUG_ASSERT(kbdev->pm.gpu_powered);
 916
 917         spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
 918
 919         KBASE_DEBUG_ASSERT(kbdev->pm.gpu_cycle_counter_requests < INT_MAX);
 920
 921         ++kbdev->pm.gpu_cycle_counter_requests;
 922
 923         if (1 == kbdev->pm.gpu_cycle_counter_requests)
 924                 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_START, NULL);
 925
 926         spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
 927 }
 928
 929 KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter)
 930
 931 void kbase_pm_release_gpu_cycle_counter(kbase_device *kbdev)
 932 {
 933         unsigned long flags;
 934         KBASE_DEBUG_ASSERT(kbdev != NULL);
 935
 936         spin_lock_irqsave(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
 937
 938         KBASE_DEBUG_ASSERT(kbdev->pm.gpu_cycle_counter_requests > 0);
 939
 940         --kbdev->pm.gpu_cycle_counter_requests;
 941
 942         if (0 == kbdev->pm.gpu_cycle_counter_requests)
 943                 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND), GPU_COMMAND_CYCLE_COUNT_STOP, NULL);
 944
 945         spin_unlock_irqrestore(&kbdev->pm.gpu_cycle_counter_requests_lock, flags);
 946 }
 947
 948 KBASE_EXPORT_TEST_API(kbase_pm_release_gpu_cycle_counter)