drivers/gpu/arm/midgard_for_linux/backend/gpu/mali_kbase_pm_driver.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2010-2015 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18
  19
  20 /*
  21  * Base kernel Power Management hardware control
  22  */
  23
  24 #include <mali_kbase.h>
  25 #include <mali_kbase_config_defaults.h>
  26 #include <mali_midg_regmap.h>
  27 #if defined(CONFIG_MALI_GATOR_SUPPORT)
  28 #include <mali_kbase_gator.h>
  29 #endif
  30 #if defined(CONFIG_MALI_MIPE_ENABLED)
  31 #include <mali_kbase_tlstream.h>
  32 #endif
  33 #include <mali_kbase_pm.h>
  34 #include <mali_kbase_cache_policy.h>
  35 #include <mali_kbase_config_defaults.h>
  36 #include <mali_kbase_smc.h>
  37 #include <mali_kbase_hwaccess_jm.h>
  38 #include <backend/gpu/mali_kbase_cache_policy_backend.h>
  39 #include <backend/gpu/mali_kbase_device_internal.h>
  40 #include <backend/gpu/mali_kbase_irq_internal.h>
  41 #include <backend/gpu/mali_kbase_pm_internal.h>
  42
  43 #include <linux/of.h>
  44
  45 #if MALI_MOCK_TEST
  46 #define MOCKABLE(function) function##_original
  47 #else
  48 #define MOCKABLE(function) function
  49 #endif                          /* MALI_MOCK_TEST */
  50
  51 /* Special value to indicate that the JM_CONFIG reg isn't currently used. */
  52 #define KBASE_JM_CONFIG_UNUSED (1<<31)
  53
  54 /**
  55  * enum kbasep_pm_action - Actions that can be performed on a core.
  56  *
  57  * This enumeration is private to the file. Its values are set to allow
  58  * core_type_to_reg() function, which decodes this enumeration, to be simpler
  59  * and more efficient.
  60  *
  61  * @ACTION_PRESENT: The cores that are present
  62  * @ACTION_READY: The cores that are ready
  63  * @ACTION_PWRON: Power on the cores specified
  64  * @ACTION_PWROFF: Power off the cores specified
  65  * @ACTION_PWRTRANS: The cores that are transitioning
  66  * @ACTION_PWRACTIVE: The cores that are active
  67  */
  68 enum kbasep_pm_action {
  69         ACTION_PRESENT = 0,
  70         ACTION_READY = (SHADER_READY_LO - SHADER_PRESENT_LO),
  71         ACTION_PWRON = (SHADER_PWRON_LO - SHADER_PRESENT_LO),
  72         ACTION_PWROFF = (SHADER_PWROFF_LO - SHADER_PRESENT_LO),
  73         ACTION_PWRTRANS = (SHADER_PWRTRANS_LO - SHADER_PRESENT_LO),
  74         ACTION_PWRACTIVE = (SHADER_PWRACTIVE_LO - SHADER_PRESENT_LO)
  75 };
  76
  77 static u64 kbase_pm_get_state(
  78                 struct kbase_device *kbdev,
  79                 enum kbase_pm_core_type core_type,
  80                 enum kbasep_pm_action action);
  81
  82 /**
  83  * core_type_to_reg - Decode a core type and action to a register.
  84  *
  85  * Given a core type (defined by kbase_pm_core_type) and an action (defined
  86  * by kbasep_pm_action) this function will return the register offset that
  87  * will perform the action on the core type. The register returned is the _LO
  88  * register and an offset must be applied to use the _HI register.
  89  *
  90  * @core_type: The type of core
  91  * @action:    The type of action
  92  *
  93  * Return: The register offset of the _LO register that performs an action of
  94  * type @action on a core of type @core_type.
  95  */
  96 static u32 core_type_to_reg(enum kbase_pm_core_type core_type,
  97                                                 enum kbasep_pm_action action)
  98 {
  99         return (u32)core_type + (u32)action;
 100 }
 101
 102
 103 /**
 104  * kbase_pm_invoke - Invokes an action on a core set
 105  *
 106  * This function performs the action given by @action on a set of cores of a
 107  * type given by @core_type. It is a static function used by
 108  * kbase_pm_transition_core_type()
 109  *
 110  * @kbdev:     The kbase device structure of the device
 111  * @core_type: The type of core that the action should be performed on
 112  * @cores:     A bit mask of cores to perform the action on (low 32 bits)
 113  * @action:    The action to perform on the cores
 114  */
 115 static void kbase_pm_invoke(struct kbase_device *kbdev,
 116                                         enum kbase_pm_core_type core_type,
 117                                         u64 cores,
 118                                         enum kbasep_pm_action action)
 119 {
 120         u32 reg;
 121         u32 lo = cores & 0xFFFFFFFF;
 122         u32 hi = (cores >> 32) & 0xFFFFFFFF;
 123
 124         lockdep_assert_held(&kbdev->pm.power_change_lock);
 125
 126         reg = core_type_to_reg(core_type, action);
 127
 128         KBASE_DEBUG_ASSERT(reg);
 129 #if defined(CONFIG_MALI_GATOR_SUPPORT)
 130         if (cores) {
 131                 if (action == ACTION_PWRON)
 132                         kbase_trace_mali_pm_power_on(core_type, cores);
 133                 else if (action == ACTION_PWROFF)
 134                         kbase_trace_mali_pm_power_off(core_type, cores);
 135         }
 136 #endif
 137 #if defined(CONFIG_MALI_MIPE_ENABLED)
 138         if (cores) {
 139                 u64 state = kbase_pm_get_state(kbdev, core_type, ACTION_READY);
 140
 141                 if (action == ACTION_PWRON)
 142                         state |= cores;
 143                 else if (action == ACTION_PWROFF)
 144                         state &= ~cores;
 145                 kbase_tlstream_aux_pm_state(core_type, state);
 146         }
 147 #endif
 148         /* Tracing */
 149         if (cores) {
 150                 if (action == ACTION_PWRON)
 151                         switch (core_type) {
 152                         case KBASE_PM_CORE_SHADER:
 153                                 KBASE_TRACE_ADD(kbdev, PM_PWRON, NULL, NULL, 0u,
 154                                                                         lo);
 155                                 break;
 156                         case KBASE_PM_CORE_TILER:
 157                                 KBASE_TRACE_ADD(kbdev, PM_PWRON_TILER, NULL,
 158                                                                 NULL, 0u, lo);
 159                                 break;
 160                         case KBASE_PM_CORE_L2:
 161                                 KBASE_TRACE_ADD(kbdev, PM_PWRON_L2, NULL, NULL,
 162                                                                         0u, lo);
 163                                 break;
 164                         default:
 165                                 break;
 166                         }
 167                 else if (action == ACTION_PWROFF)
 168                         switch (core_type) {
 169                         case KBASE_PM_CORE_SHADER:
 170                                 KBASE_TRACE_ADD(kbdev, PM_PWROFF, NULL, NULL,
 171                                                                         0u, lo);
 172                                 break;
 173                         case KBASE_PM_CORE_TILER:
 174                                 KBASE_TRACE_ADD(kbdev, PM_PWROFF_TILER, NULL,
 175                                                                 NULL, 0u, lo);
 176                                 break;
 177                         case KBASE_PM_CORE_L2:
 178                                 KBASE_TRACE_ADD(kbdev, PM_PWROFF_L2, NULL, NULL,
 179                                                                         0u, lo);
 180                                 break;
 181                         default:
 182                                 break;
 183                         }
 184         }
 185
 186         if (lo != 0)
 187                 kbase_reg_write(kbdev, GPU_CONTROL_REG(reg), lo, NULL);
 188
 189         if (hi != 0)
 190                 kbase_reg_write(kbdev, GPU_CONTROL_REG(reg + 4), hi, NULL);
 191 }
 192
 193 /**
 194  * kbase_pm_get_state - Get information about a core set
 195  *
 196  * This function gets information (chosen by @action) about a set of cores of
 197  * a type given by @core_type. It is a static function used by
 198  * kbase_pm_get_present_cores(), kbase_pm_get_active_cores(),
 199  * kbase_pm_get_trans_cores() and kbase_pm_get_ready_cores().
 200  *
 201  * @kbdev:     The kbase device structure of the device
 202  * @core_type: The type of core that the should be queried
 203  * @action:    The property of the cores to query
 204  *
 205  * Return: A bit mask specifying the state of the cores
 206  */
 207 static u64 kbase_pm_get_state(struct kbase_device *kbdev,
 208                                         enum kbase_pm_core_type core_type,
 209                                         enum kbasep_pm_action action)
 210 {
 211         u32 reg;
 212         u32 lo, hi;
 213
 214         reg = core_type_to_reg(core_type, action);
 215
 216         KBASE_DEBUG_ASSERT(reg);
 217
 218         lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg), NULL);
 219         hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg + 4), NULL);
 220
 221         return (((u64) hi) << 32) | ((u64) lo);
 222 }
 223
 224 void kbasep_pm_read_present_cores(struct kbase_device *kbdev)
 225 {
 226         kbdev->shader_inuse_bitmap = 0;
 227         kbdev->shader_needed_bitmap = 0;
 228         kbdev->shader_available_bitmap = 0;
 229         kbdev->tiler_available_bitmap = 0;
 230         kbdev->l2_users_count = 0;
 231         kbdev->l2_available_bitmap = 0;
 232         kbdev->tiler_needed_cnt = 0;
 233         kbdev->tiler_inuse_cnt = 0;
 234
 235         memset(kbdev->shader_needed_cnt, 0, sizeof(kbdev->shader_needed_cnt));
 236 }
 237
 238 KBASE_EXPORT_TEST_API(kbasep_pm_read_present_cores);
 239
 240 /**
 241  * kbase_pm_get_present_cores - Get the cores that are present
 242  *
 243  * @kbdev: Kbase device
 244  * @type: The type of cores to query
 245  *
 246  * Return: Bitmask of the cores that are present
 247  */
 248 u64 kbase_pm_get_present_cores(struct kbase_device *kbdev,
 249                                                 enum kbase_pm_core_type type)
 250 {
 251         KBASE_DEBUG_ASSERT(kbdev != NULL);
 252
 253         switch (type) {
 254         case KBASE_PM_CORE_L2:
 255                 return kbdev->gpu_props.props.raw_props.l2_present;
 256         case KBASE_PM_CORE_SHADER:
 257                 return kbdev->gpu_props.props.raw_props.shader_present;
 258         case KBASE_PM_CORE_TILER:
 259                 return kbdev->gpu_props.props.raw_props.tiler_present;
 260         }
 261         KBASE_DEBUG_ASSERT(0);
 262         return 0;
 263 }
 264
 265 KBASE_EXPORT_TEST_API(kbase_pm_get_present_cores);
 266
 267 /**
 268  * kbase_pm_get_active_cores - Get the cores that are "active"
 269  *                             (busy processing work)
 270  *
 271  * @kbdev: Kbase device
 272  * @type: The type of cores to query
 273  *
 274  * Return: Bitmask of cores that are active
 275  */
 276 u64 kbase_pm_get_active_cores(struct kbase_device *kbdev,
 277                                                 enum kbase_pm_core_type type)
 278 {
 279         return kbase_pm_get_state(kbdev, type, ACTION_PWRACTIVE);
 280 }
 281
 282 KBASE_EXPORT_TEST_API(kbase_pm_get_active_cores);
 283
 284 /**
 285  * kbase_pm_get_trans_cores - Get the cores that are transitioning between
 286  *                            power states
 287  *
 288  * @kbdev: Kbase device
 289  * @type: The type of cores to query
 290  *
 291  * Return: Bitmask of cores that are transitioning
 292  */
 293 u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev,
 294                                                 enum kbase_pm_core_type type)
 295 {
 296         return kbase_pm_get_state(kbdev, type, ACTION_PWRTRANS);
 297 }
 298
 299 KBASE_EXPORT_TEST_API(kbase_pm_get_trans_cores);
 300
 301 /**
 302  * kbase_pm_get_ready_cores - Get the cores that are powered on
 303  *
 304  * @kbdev: Kbase device
 305  * @type: The type of cores to query
 306  *
 307  * Return: Bitmask of cores that are ready (powered on)
 308  */
 309 u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev,
 310                                                 enum kbase_pm_core_type type)
 311 {
 312         u64 result;
 313
 314         result = kbase_pm_get_state(kbdev, type, ACTION_READY);
 315
 316         switch (type) {
 317         case KBASE_PM_CORE_SHADER:
 318                 KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED, NULL, NULL, 0u,
 319                                                                 (u32) result);
 320                 break;
 321         case KBASE_PM_CORE_TILER:
 322                 KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, NULL, 0u,
 323                                                                 (u32) result);
 324                 break;
 325         case KBASE_PM_CORE_L2:
 326                 KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, NULL, 0u,
 327                                                                 (u32) result);
 328                 break;
 329         default:
 330                 break;
 331         }
 332
 333         return result;
 334 }
 335
 336 KBASE_EXPORT_TEST_API(kbase_pm_get_ready_cores);
 337
 338 /**
 339  * kbase_pm_transition_core_type - Perform power transitions for a particular
 340  *                                 core type.
 341  *
 342  * This function will perform any available power transitions to make the actual
 343  * hardware state closer to the desired state. If a core is currently
 344  * transitioning then changes to the power state of that call cannot be made
 345  * until the transition has finished. Cores which are not present in the
 346  * hardware are ignored if they are specified in the desired_state bitmask,
 347  * however the return value will always be 0 in this case.
 348  *
 349  * @kbdev:             The kbase device
 350  * @type:              The core type to perform transitions for
 351  * @desired_state:     A bit mask of the desired state of the cores
 352  * @in_use:            A bit mask of the cores that are currently running
 353  *                     jobs. These cores have to be kept powered up because
 354  *                     there are jobs running (or about to run) on them.
 355  * @available:         Receives a bit mask of the cores that the job
 356  *                     scheduler can use to submit jobs to. May be NULL if
 357  *                     this is not needed.
 358  * @powering_on:       Bit mask to update with cores that are
 359  *                    transitioning to a power-on state.
 360  *
 361  * Return: true if the desired state has been reached, false otherwise
 362  */
 363 static bool kbase_pm_transition_core_type(struct kbase_device *kbdev,
 364                                                 enum kbase_pm_core_type type,
 365                                                 u64 desired_state,
 366                                                 u64 in_use,
 367                                                 u64 * const available,
 368                                                 u64 *powering_on)
 369 {
 370         u64 present;
 371         u64 ready;
 372         u64 trans;
 373         u64 powerup;
 374         u64 powerdown;
 375         u64 powering_on_trans;
 376         u64 desired_state_in_use;
 377
 378         lockdep_assert_held(&kbdev->pm.power_change_lock);
 379
 380         /* Get current state */
 381         present = kbase_pm_get_present_cores(kbdev, type);
 382         trans = kbase_pm_get_trans_cores(kbdev, type);
 383         ready = kbase_pm_get_ready_cores(kbdev, type);
 384         /* mask off ready from trans in case transitions finished between the
 385          * register reads */
 386         trans &= ~ready;
 387
 388         powering_on_trans = trans & *powering_on;
 389         *powering_on = powering_on_trans;
 390
 391         if (available != NULL)
 392                 *available = (ready | powering_on_trans) & desired_state;
 393
 394         /* Update desired state to include the in-use cores. These have to be
 395          * kept powered up because there are jobs running or about to run on
 396          * these cores
 397          */
 398         desired_state_in_use = desired_state | in_use;
 399
 400         /* Update state of whether l2 caches are powered */
 401         if (type == KBASE_PM_CORE_L2) {
 402                 if ((ready == present) && (desired_state_in_use == ready) &&
 403                                                                 (trans == 0)) {
 404                         /* All are ready, none will be turned off, and none are
 405                          * transitioning */
 406                         kbdev->pm.backend.l2_powered = 1;
 407                         if (kbdev->l2_users_count > 0) {
 408                                 /* Notify any registered l2 cache users
 409                                  * (optimized out when no users waiting) */
 410                                 wake_up(&kbdev->pm.backend.l2_powered_wait);
 411                         }
 412                 } else
 413                         kbdev->pm.backend.l2_powered = 0;
 414         }
 415
 416         if (desired_state_in_use == ready && (trans == 0))
 417                 return true;
 418
 419         /* Restrict the cores to those that are actually present */
 420         powerup = desired_state_in_use & present;
 421         powerdown = (~desired_state_in_use) & present;
 422
 423         /* Restrict to cores that are not already in the desired state */
 424         powerup &= ~ready;
 425         powerdown &= ready;
 426
 427         /* Don't transition any cores that are already transitioning, except for
 428          * Mali cores that support the following case:
 429          *
 430          * If the SHADER_PWRON or TILER_PWRON registers are written to turn on
 431          * a core that is currently transitioning to power off, then this is
 432          * remembered and the shader core is automatically powered up again once
 433          * the original transition completes. Once the automatic power on is
 434          * complete any job scheduled on the shader core should start.
 435          */
 436         powerdown &= ~trans;
 437
 438         if (kbase_hw_has_feature(kbdev,
 439                                 BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS))
 440                 if (KBASE_PM_CORE_SHADER == type || KBASE_PM_CORE_TILER == type)
 441                         trans = powering_on_trans; /* for exception cases, only
 442                                                     * mask off cores in power on
 443                                                     * transitions */
 444
 445         powerup &= ~trans;
 446
 447         /* Perform transitions if any */
 448         kbase_pm_invoke(kbdev, type, powerup, ACTION_PWRON);
 449         kbase_pm_invoke(kbdev, type, powerdown, ACTION_PWROFF);
 450
 451         /* Recalculate cores transitioning on, and re-evaluate our state */
 452         powering_on_trans |= powerup;
 453         *powering_on = powering_on_trans;
 454         if (available != NULL)
 455                 *available = (ready | powering_on_trans) & desired_state;
 456
 457         return false;
 458 }
 459
 460 KBASE_EXPORT_TEST_API(kbase_pm_transition_core_type);
 461
 462 /**
 463  * get_desired_cache_status - Determine which caches should be on for a
 464  *                            particular core state
 465  *
 466  * This function takes a bit mask of the present caches and the cores (or
 467  * caches) that are attached to the caches that will be powered. It then
 468  * computes which caches should be turned on to allow the cores requested to be
 469  * powered up.
 470  *
 471  * @present:       The bit mask of present caches
 472  * @cores_powered: A bit mask of cores (or L2 caches) that are desired to
 473  *                 be powered
 474  *
 475  * Return: A bit mask of the caches that should be turned on
 476  */
 477 static u64 get_desired_cache_status(u64 present, u64 cores_powered)
 478 {
 479         u64 desired = 0;
 480
 481         while (present) {
 482                 /* Find out which is the highest set bit */
 483                 u64 bit = fls64(present) - 1;
 484                 u64 bit_mask = 1ull << bit;
 485                 /* Create a mask which has all bits from 'bit' upwards set */
 486
 487                 u64 mask = ~(bit_mask - 1);
 488
 489                 /* If there are any cores powered at this bit or above (that
 490                  * haven't previously been processed) then we need this core on
 491                  */
 492                 if (cores_powered & mask)
 493                         desired |= bit_mask;
 494
 495                 /* Remove bits from cores_powered and present */
 496                 cores_powered &= ~mask;
 497                 present &= ~bit_mask;
 498         }
 499
 500         return desired;
 501 }
 502
 503 KBASE_EXPORT_TEST_API(get_desired_cache_status);
 504
 505 bool
 506 MOCKABLE(kbase_pm_check_transitions_nolock) (struct kbase_device *kbdev)
 507 {
 508         bool cores_are_available = false;
 509         bool in_desired_state = true;
 510         u64 desired_l2_state;
 511         u64 cores_powered;
 512         u64 tiler_available_bitmap;
 513         u64 shader_available_bitmap;
 514         u64 shader_ready_bitmap;
 515         u64 shader_transitioning_bitmap;
 516         u64 l2_available_bitmap;
 517         u64 prev_l2_available_bitmap;
 518
 519         KBASE_DEBUG_ASSERT(NULL != kbdev);
 520         lockdep_assert_held(&kbdev->pm.power_change_lock);
 521
 522         spin_lock(&kbdev->pm.backend.gpu_powered_lock);
 523         if (kbdev->pm.backend.gpu_powered == false) {
 524                 spin_unlock(&kbdev->pm.backend.gpu_powered_lock);
 525                 if (kbdev->pm.backend.desired_shader_state == 0 &&
 526                                 kbdev->pm.backend.desired_tiler_state == 0)
 527                         return true;
 528                 return false;
 529         }
 530
 531         /* Trace that a change-state is being requested, and that it took
 532          * (effectively) no time to start it. This is useful for counting how
 533          * many state changes occurred, in a way that's backwards-compatible
 534          * with processing the trace data */
 535         kbase_timeline_pm_send_event(kbdev,
 536                                 KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE);
 537         kbase_timeline_pm_handle_event(kbdev,
 538                                 KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE);
 539
 540         /* If any cores are already powered then, we must keep the caches on */
 541         cores_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER);
 542
 543         cores_powered |= kbdev->pm.backend.desired_shader_state;
 544
 545         /* If there are l2 cache users registered, keep all l2s powered even if
 546          * all other cores are off. */
 547         if (kbdev->l2_users_count > 0)
 548                 cores_powered |= kbdev->gpu_props.props.raw_props.l2_present;
 549
 550         desired_l2_state = get_desired_cache_status(
 551                         kbdev->gpu_props.props.raw_props.l2_present,
 552                         cores_powered);
 553
 554         /* If any l2 cache is on, then enable l2 #0, for use by job manager */
 555         if (0 != desired_l2_state) {
 556                 desired_l2_state |= 1;
 557                 /* Also enable tiler if l2 cache is powered */
 558                 kbdev->pm.backend.desired_tiler_state =
 559                         kbdev->gpu_props.props.raw_props.tiler_present;
 560         } else {
 561                 kbdev->pm.backend.desired_tiler_state = 0;
 562         }
 563
 564         prev_l2_available_bitmap = kbdev->l2_available_bitmap;
 565         in_desired_state &= kbase_pm_transition_core_type(kbdev,
 566                                 KBASE_PM_CORE_L2, desired_l2_state, 0,
 567                                 &l2_available_bitmap,
 568                                 &kbdev->pm.backend.powering_on_l2_state);
 569
 570         if (kbdev->l2_available_bitmap != l2_available_bitmap)
 571                 KBASE_TIMELINE_POWER_L2(kbdev, l2_available_bitmap);
 572
 573         kbdev->l2_available_bitmap = l2_available_bitmap;
 574
 575         if (in_desired_state) {
 576                 in_desired_state &= kbase_pm_transition_core_type(kbdev,
 577                                 KBASE_PM_CORE_TILER,
 578                                 kbdev->pm.backend.desired_tiler_state,
 579                                 0, &tiler_available_bitmap,
 580                                 &kbdev->pm.backend.powering_on_tiler_state);
 581                 in_desired_state &= kbase_pm_transition_core_type(kbdev,
 582                                 KBASE_PM_CORE_SHADER,
 583                                 kbdev->pm.backend.desired_shader_state,
 584                                 kbdev->shader_inuse_bitmap,
 585                                 &shader_available_bitmap,
 586                                 &kbdev->pm.backend.powering_on_shader_state);
 587
 588                 if (kbdev->shader_available_bitmap != shader_available_bitmap) {
 589                         KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL,
 590                                                 NULL, 0u,
 591                                                 (u32) shader_available_bitmap);
 592                         KBASE_TIMELINE_POWER_SHADER(kbdev,
 593                                                 shader_available_bitmap);
 594                 }
 595
 596                 kbdev->shader_available_bitmap = shader_available_bitmap;
 597
 598                 if (kbdev->tiler_available_bitmap != tiler_available_bitmap) {
 599                         KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER,
 600                                                 NULL, NULL, 0u,
 601                                                 (u32) tiler_available_bitmap);
 602                         KBASE_TIMELINE_POWER_TILER(kbdev,
 603                                                         tiler_available_bitmap);
 604                 }
 605
 606                 kbdev->tiler_available_bitmap = tiler_available_bitmap;
 607
 608         } else if ((l2_available_bitmap &
 609                         kbdev->gpu_props.props.raw_props.tiler_present) !=
 610                         kbdev->gpu_props.props.raw_props.tiler_present) {
 611                 tiler_available_bitmap = 0;
 612
 613                 if (kbdev->tiler_available_bitmap != tiler_available_bitmap)
 614                         KBASE_TIMELINE_POWER_TILER(kbdev,
 615                                                         tiler_available_bitmap);
 616
 617                 kbdev->tiler_available_bitmap = tiler_available_bitmap;
 618         }
 619
 620         /* State updated for slow-path waiters */
 621         kbdev->pm.backend.gpu_in_desired_state = in_desired_state;
 622
 623         shader_ready_bitmap = kbase_pm_get_ready_cores(kbdev,
 624                                                         KBASE_PM_CORE_SHADER);
 625         shader_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev,
 626                                                         KBASE_PM_CORE_SHADER);
 627
 628         /* Determine whether the cores are now available (even if the set of
 629          * available cores is empty). Note that they can be available even if
 630          * we've not finished transitioning to the desired state */
 631         if ((kbdev->shader_available_bitmap &
 632                                         kbdev->pm.backend.desired_shader_state)
 633                                 == kbdev->pm.backend.desired_shader_state &&
 634                 (kbdev->tiler_available_bitmap &
 635                                         kbdev->pm.backend.desired_tiler_state)
 636                                 == kbdev->pm.backend.desired_tiler_state) {
 637                 cores_are_available = true;
 638
 639                 KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE, NULL, NULL, 0u,
 640                                 (u32)(kbdev->shader_available_bitmap &
 641                                 kbdev->pm.backend.desired_shader_state));
 642                 KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE_TILER, NULL, NULL, 0u,
 643                                 (u32)(kbdev->tiler_available_bitmap &
 644                                 kbdev->pm.backend.desired_tiler_state));
 645
 646                 /* Log timelining information about handling events that power
 647                  * up cores, to match up either with immediate submission either
 648                  * because cores already available, or from PM IRQ */
 649                 if (!in_desired_state)
 650                         kbase_timeline_pm_send_event(kbdev,
 651                                 KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
 652         }
 653
 654         if (in_desired_state) {
 655                 KBASE_DEBUG_ASSERT(cores_are_available);
 656
 657 #if defined(CONFIG_MALI_GATOR_SUPPORT)
 658                 kbase_trace_mali_pm_status(KBASE_PM_CORE_L2,
 659                                                 kbase_pm_get_ready_cores(kbdev,
 660                                                         KBASE_PM_CORE_L2));
 661                 kbase_trace_mali_pm_status(KBASE_PM_CORE_SHADER,
 662                                                 kbase_pm_get_ready_cores(kbdev,
 663                                                         KBASE_PM_CORE_SHADER));
 664                 kbase_trace_mali_pm_status(KBASE_PM_CORE_TILER,
 665                                                 kbase_pm_get_ready_cores(kbdev,
 666                                                         KBASE_PM_CORE_TILER));
 667 #endif
 668 #if defined(CONFIG_MALI_MIPE_ENABLED)
 669                 kbase_tlstream_aux_pm_state(
 670                                 KBASE_PM_CORE_L2,
 671                                 kbase_pm_get_ready_cores(
 672                                         kbdev, KBASE_PM_CORE_L2));
 673                 kbase_tlstream_aux_pm_state(
 674                                 KBASE_PM_CORE_SHADER,
 675                                 kbase_pm_get_ready_cores(
 676                                         kbdev, KBASE_PM_CORE_SHADER));
 677                 kbase_tlstream_aux_pm_state(
 678                                 KBASE_PM_CORE_TILER,
 679                                 kbase_pm_get_ready_cores(
 680                                         kbdev,
 681                                         KBASE_PM_CORE_TILER));
 682 #endif
 683
 684                 KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, NULL,
 685                                 kbdev->pm.backend.gpu_in_desired_state,
 686                                 (u32)kbdev->pm.backend.desired_shader_state);
 687                 KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED_TILER, NULL, NULL, 0u,
 688                                 (u32)kbdev->pm.backend.desired_tiler_state);
 689
 690                 /* Log timelining information for synchronous waiters */
 691                 kbase_timeline_pm_send_event(kbdev,
 692                                 KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
 693                 /* Wake slow-path waiters. Job scheduler does not use this. */
 694                 KBASE_TRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, NULL, 0u, 0);
 695                 wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait);
 696         }
 697
 698         spin_unlock(&kbdev->pm.backend.gpu_powered_lock);
 699
 700         /* kbase_pm_ca_update_core_status can cause one-level recursion into
 701          * this function, so it must only be called once all changes to kbdev
 702          * have been committed, and after the gpu_powered_lock has been
 703          * dropped. */
 704         if (kbdev->shader_ready_bitmap != shader_ready_bitmap ||
 705             kbdev->shader_transitioning_bitmap != shader_transitioning_bitmap) {
 706                 kbdev->shader_ready_bitmap = shader_ready_bitmap;
 707                 kbdev->shader_transitioning_bitmap =
 708                                                 shader_transitioning_bitmap;
 709
 710                 kbase_pm_ca_update_core_status(kbdev, shader_ready_bitmap,
 711                                                 shader_transitioning_bitmap);
 712         }
 713
 714         /* The core availability policy is not allowed to keep core group 0
 715          * turned off (unless it was changing the l2 power state) */
 716         if (!((shader_ready_bitmap | shader_transitioning_bitmap) &
 717                 kbdev->gpu_props.props.coherency_info.group[0].core_mask) &&
 718                 (prev_l2_available_bitmap == desired_l2_state) &&
 719                 !(kbase_pm_ca_get_core_mask(kbdev) &
 720                 kbdev->gpu_props.props.coherency_info.group[0].core_mask))
 721                 BUG();
 722
 723         /* The core availability policy is allowed to keep core group 1 off,
 724          * but all jobs specifically targeting CG1 must fail */
 725         if (!((shader_ready_bitmap | shader_transitioning_bitmap) &
 726                 kbdev->gpu_props.props.coherency_info.group[1].core_mask) &&
 727                 !(kbase_pm_ca_get_core_mask(kbdev) &
 728                 kbdev->gpu_props.props.coherency_info.group[1].core_mask))
 729                 kbdev->pm.backend.cg1_disabled = true;
 730         else
 731                 kbdev->pm.backend.cg1_disabled = false;
 732
 733         return cores_are_available;
 734 }
 735 KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_nolock);
 736
 737 /* Timeout for kbase_pm_check_transitions_sync when wait_event_killable has
 738  * aborted due to a fatal signal. If the time spent waiting has exceeded this
 739  * threshold then there is most likely a hardware issue. */
 740 #define PM_TIMEOUT (5*HZ) /* 5s */
 741
 742 void kbase_pm_check_transitions_sync(struct kbase_device *kbdev)
 743 {
 744         unsigned long flags;
 745         unsigned long timeout;
 746         bool cores_are_available;
 747         int ret;
 748
 749         /* Force the transition to be checked and reported - the cores may be
 750          * 'available' (for job submission) but not fully powered up. */
 751         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 752         cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
 753         /* Don't need 'cores_are_available', because we don't return anything */
 754         CSTD_UNUSED(cores_are_available);
 755         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 756
 757         timeout = jiffies + PM_TIMEOUT;
 758
 759         /* Wait for cores */
 760         ret = wait_event_killable(kbdev->pm.backend.gpu_in_desired_state_wait,
 761                         kbdev->pm.backend.gpu_in_desired_state);
 762
 763         if (ret < 0 && time_after(jiffies, timeout)) {
 764                 dev_err(kbdev->dev, "Power transition timed out unexpectedly\n");
 765                 dev_err(kbdev->dev, "Desired state :\n");
 766                 dev_err(kbdev->dev, "\tShader=%016llx\n",
 767                                 kbdev->pm.backend.desired_shader_state);
 768                 dev_err(kbdev->dev, "\tTiler =%016llx\n",
 769                                 kbdev->pm.backend.desired_tiler_state);
 770                 dev_err(kbdev->dev, "Current state :\n");
 771                 dev_err(kbdev->dev, "\tShader=%08x%08x\n",
 772                                 kbase_reg_read(kbdev,
 773                                         GPU_CONTROL_REG(SHADER_READY_HI), NULL),
 774                                 kbase_reg_read(kbdev,
 775                                         GPU_CONTROL_REG(SHADER_READY_LO),
 776                                         NULL));
 777                 dev_err(kbdev->dev, "\tTiler =%08x%08x\n",
 778                                 kbase_reg_read(kbdev,
 779                                         GPU_CONTROL_REG(TILER_READY_HI), NULL),
 780                                 kbase_reg_read(kbdev,
 781                                         GPU_CONTROL_REG(TILER_READY_LO), NULL));
 782                 dev_err(kbdev->dev, "\tL2    =%08x%08x\n",
 783                                 kbase_reg_read(kbdev,
 784                                         GPU_CONTROL_REG(L2_READY_HI), NULL),
 785                                 kbase_reg_read(kbdev,
 786                                         GPU_CONTROL_REG(L2_READY_LO), NULL));
 787                 dev_err(kbdev->dev, "Cores transitioning :\n");
 788                 dev_err(kbdev->dev, "\tShader=%08x%08x\n",
 789                                 kbase_reg_read(kbdev, GPU_CONTROL_REG(
 790                                                 SHADER_PWRTRANS_HI), NULL),
 791                                 kbase_reg_read(kbdev, GPU_CONTROL_REG(
 792                                                 SHADER_PWRTRANS_LO), NULL));
 793                 dev_err(kbdev->dev, "\tTiler =%08x%08x\n",
 794                                 kbase_reg_read(kbdev, GPU_CONTROL_REG(
 795                                                 TILER_PWRTRANS_HI), NULL),
 796                                 kbase_reg_read(kbdev, GPU_CONTROL_REG(
 797                                                 TILER_PWRTRANS_LO), NULL));
 798                 dev_err(kbdev->dev, "\tL2    =%08x%08x\n",
 799                                 kbase_reg_read(kbdev, GPU_CONTROL_REG(
 800                                                 L2_PWRTRANS_HI), NULL),
 801                                 kbase_reg_read(kbdev, GPU_CONTROL_REG(
 802                                                 L2_PWRTRANS_LO), NULL));
 803 #if KBASE_GPU_RESET_EN
 804                 dev_err(kbdev->dev, "Sending reset to GPU - all running jobs will be lost\n");
 805                 if (kbase_prepare_to_reset_gpu(kbdev))
 806                         kbase_reset_gpu(kbdev);
 807 #endif /* KBASE_GPU_RESET_EN */
 808         } else {
 809                 /* Log timelining information that a change in state has
 810                  * completed */
 811                 kbase_timeline_pm_handle_event(kbdev,
 812                                 KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
 813         }
 814 }
 815 KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_sync);
 816
 817 void kbase_pm_enable_interrupts(struct kbase_device *kbdev)
 818 {
 819         unsigned long flags;
 820
 821         KBASE_DEBUG_ASSERT(NULL != kbdev);
 822         /*
 823          * Clear all interrupts,
 824          * and unmask them all.
 825          */
 826         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 827         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL,
 828                                                                         NULL);
 829         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), GPU_IRQ_REG_ALL,
 830                                                                         NULL);
 831         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 832
 833         kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF,
 834                                                                         NULL);
 835         kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0xFFFFFFFF, NULL);
 836
 837         kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL);
 838         kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFFFFFF, NULL);
 839 }
 840
 841 KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts);
 842
 843 void kbase_pm_disable_interrupts(struct kbase_device *kbdev)
 844 {
 845         unsigned long flags;
 846
 847         KBASE_DEBUG_ASSERT(NULL != kbdev);
 848         /*
 849          * Mask all interrupts,
 850          * and clear them all.
 851          */
 852         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 853         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), 0, NULL);
 854         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL,
 855                                                                         NULL);
 856         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 857
 858         kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0, NULL);
 859         kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF,
 860                                                                         NULL);
 861
 862         kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL);
 863         kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL);
 864 }
 865
 866 KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts);
 867
 868 /*
 869  * pmu layout:
 870  * 0x0000: PMU TAG (RO) (0xCAFECAFE)
 871  * 0x0004: PMU VERSION ID (RO) (0x00000000)
 872  * 0x0008: CLOCK ENABLE (RW) (31:1 SBZ, 0 CLOCK STATE)
 873  */
 874 void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume)
 875 {
 876         bool reset_required = is_resume;
 877         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
 878         unsigned long flags;
 879         int i;
 880
 881         KBASE_DEBUG_ASSERT(NULL != kbdev);
 882         lockdep_assert_held(&js_devdata->runpool_mutex);
 883         lockdep_assert_held(&kbdev->pm.lock);
 884
 885         if (kbdev->pm.backend.gpu_powered) {
 886                 /* Already turned on */
 887                 if (kbdev->poweroff_pending)
 888                         kbase_pm_enable_interrupts(kbdev);
 889                 kbdev->poweroff_pending = false;
 890                 KBASE_DEBUG_ASSERT(!is_resume);
 891                 return;
 892         }
 893
 894         kbdev->poweroff_pending = false;
 895
 896         KBASE_TRACE_ADD(kbdev, PM_GPU_ON, NULL, NULL, 0u, 0u);
 897
 898         if (is_resume && kbdev->pm.backend.callback_power_resume) {
 899                 kbdev->pm.backend.callback_power_resume(kbdev);
 900                 return;
 901         } else if (kbdev->pm.backend.callback_power_on) {
 902                 kbdev->pm.backend.callback_power_on(kbdev);
 903                 /* If your platform properly keeps the GPU state you may use the
 904                  * return value of the callback_power_on function to
 905                  * conditionally reset the GPU on power up. Currently we are
 906                  * conservative and always reset the GPU. */
 907                 reset_required = true;
 908         }
 909
 910         spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags);
 911         kbdev->pm.backend.gpu_powered = true;
 912         spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags);
 913
 914         if (reset_required) {
 915                 /* GPU state was lost, reset GPU to ensure it is in a
 916                  * consistent state */
 917                 kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS);
 918         }
 919
 920         /* Reprogram the GPU's MMU */
 921         for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
 922                 struct kbase_as *as = &kbdev->as[i];
 923
 924                 mutex_lock(&as->transaction_mutex);
 925                 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
 926
 927                 if (js_devdata->runpool_irq.per_as_data[i].kctx)
 928                         kbase_mmu_update(
 929                                 js_devdata->runpool_irq.per_as_data[i].kctx);
 930                 else
 931                         kbase_mmu_disable_as(kbdev, i);
 932
 933                 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
 934                 mutex_unlock(&as->transaction_mutex);
 935         }
 936
 937         /* Lastly, enable the interrupts */
 938         kbase_pm_enable_interrupts(kbdev);
 939 }
 940
 941 KBASE_EXPORT_TEST_API(kbase_pm_clock_on);
 942
 943 bool kbase_pm_clock_off(struct kbase_device *kbdev, bool is_suspend)
 944 {
 945         unsigned long flags;
 946
 947         KBASE_DEBUG_ASSERT(NULL != kbdev);
 948         lockdep_assert_held(&kbdev->pm.lock);
 949
 950         /* ASSERT that the cores should now be unavailable. No lock needed. */
 951         KBASE_DEBUG_ASSERT(kbdev->shader_available_bitmap == 0u);
 952
 953         kbdev->poweroff_pending = true;
 954
 955         if (!kbdev->pm.backend.gpu_powered) {
 956                 /* Already turned off */
 957                 if (is_suspend && kbdev->pm.backend.callback_power_suspend)
 958                         kbdev->pm.backend.callback_power_suspend(kbdev);
 959                 return true;
 960         }
 961
 962         KBASE_TRACE_ADD(kbdev, PM_GPU_OFF, NULL, NULL, 0u, 0u);
 963
 964         /* Disable interrupts. This also clears any outstanding interrupts */
 965         kbase_pm_disable_interrupts(kbdev);
 966         /* Ensure that any IRQ handlers have finished */
 967         kbase_synchronize_irqs(kbdev);
 968
 969         spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags);
 970
 971         if (atomic_read(&kbdev->faults_pending)) {
 972                 /* Page/bus faults are still being processed. The GPU can not
 973                  * be powered off until they have completed */
 974                 spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock,
 975                                                                         flags);
 976                 return false;
 977         }
 978
 979
 980         /* The GPU power may be turned off from this point */
 981         kbdev->pm.backend.gpu_powered = false;
 982         spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags);
 983
 984         if (is_suspend && kbdev->pm.backend.callback_power_suspend)
 985                 kbdev->pm.backend.callback_power_suspend(kbdev);
 986         else if (kbdev->pm.backend.callback_power_off)
 987                 kbdev->pm.backend.callback_power_off(kbdev);
 988         return true;
 989 }
 990
 991 KBASE_EXPORT_TEST_API(kbase_pm_clock_off);
 992
 993 struct kbasep_reset_timeout_data {
 994         struct hrtimer timer;
 995         bool timed_out;
 996         struct kbase_device *kbdev;
 997 };
 998
 999 void kbase_pm_reset_done(struct kbase_device *kbdev)
1000 {
1001         KBASE_DEBUG_ASSERT(kbdev != NULL);
1002         kbdev->pm.backend.reset_done = true;
1003         wake_up(&kbdev->pm.backend.reset_done_wait);
1004 }
1005
1006 /**
1007  * kbase_pm_wait_for_reset - Wait for a reset to happen
1008  *
1009  * Wait for the %RESET_COMPLETED IRQ to occur, then reset the waiting state.
1010  *
1011  * @kbdev: Kbase device
1012  */
1013 static void kbase_pm_wait_for_reset(struct kbase_device *kbdev)
1014 {
1015         lockdep_assert_held(&kbdev->pm.lock);
1016
1017         wait_event(kbdev->pm.backend.reset_done_wait,
1018                                                 (kbdev->pm.backend.reset_done));
1019         kbdev->pm.backend.reset_done = false;
1020 }
1021
1022 KBASE_EXPORT_TEST_API(kbase_pm_reset_done);
1023
1024 static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer)
1025 {
1026         struct kbasep_reset_timeout_data *rtdata =
1027                 container_of(timer, struct kbasep_reset_timeout_data, timer);
1028
1029         rtdata->timed_out = 1;
1030
1031         /* Set the wait queue to wake up kbase_pm_init_hw even though the reset
1032          * hasn't completed */
1033         kbase_pm_reset_done(rtdata->kbdev);
1034
1035         return HRTIMER_NORESTART;
1036 }
1037
1038 static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev)
1039 {
1040         struct device_node *np = kbdev->dev->of_node;
1041         u32 jm_values[4];
1042         const u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id;
1043         const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >>
1044                 GPU_ID_VERSION_PRODUCT_ID_SHIFT;
1045         const u32 major = (gpu_id & GPU_ID_VERSION_MAJOR) >>
1046                 GPU_ID_VERSION_MAJOR_SHIFT;
1047
1048         kbdev->hw_quirks_sc = 0;
1049
1050         /* Needed due to MIDBASE-1494: LS_PAUSEBUFFER_DISABLE. See PRLAM-8443.
1051          * and needed due to MIDGLES-3539. See PRLAM-11035 */
1052         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8443) ||
1053                         kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11035))
1054                 kbdev->hw_quirks_sc |= SC_LS_PAUSEBUFFER_DISABLE;
1055
1056         /* Needed due to MIDBASE-2054: SDC_DISABLE_OQ_DISCARD. See PRLAM-10327.
1057          */
1058         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10327))
1059                 kbdev->hw_quirks_sc |= SC_SDC_DISABLE_OQ_DISCARD;
1060
1061         /* Enable alternative hardware counter selection if configured. */
1062         if (DEFAULT_ALTERNATIVE_HWC)
1063                 kbdev->hw_quirks_sc |= SC_ALT_COUNTERS;
1064
1065         /* Needed due to MIDBASE-2795. ENABLE_TEXGRD_FLAGS. See PRLAM-10797. */
1066         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10797))
1067                 kbdev->hw_quirks_sc |= SC_ENABLE_TEXGRD_FLAGS;
1068
1069         if (!kbase_hw_has_issue(kbdev, GPUCORE_1619)) {
1070                 if (prod_id < 0x760 || prod_id == 0x6956) /* T60x, T62x, T72x */
1071                         kbdev->hw_quirks_sc |= SC_LS_ATTR_CHECK_DISABLE;
1072                 else if (prod_id >= 0x760 && prod_id <= 0x880) /* T76x, T8xx */
1073                         kbdev->hw_quirks_sc |= SC_LS_ALLOW_ATTR_TYPES;
1074         }
1075
1076         kbdev->hw_quirks_tiler = kbase_reg_read(kbdev,
1077                         GPU_CONTROL_REG(TILER_CONFIG), NULL);
1078
1079         /* Set tiler clock gate override if required */
1080         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3953))
1081                 kbdev->hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE;
1082
1083         /* Limit the GPU bus bandwidth if the platform needs this. */
1084         kbdev->hw_quirks_mmu = kbase_reg_read(kbdev,
1085                         GPU_CONTROL_REG(L2_MMU_CONFIG), NULL);
1086
1087         /* Limit read ID width for AXI */
1088         kbdev->hw_quirks_mmu &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_READS);
1089         kbdev->hw_quirks_mmu |= (DEFAULT_ARID_LIMIT & 0x3) <<
1090                                 L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT;
1091
1092         /* Limit write ID width for AXI */
1093         kbdev->hw_quirks_mmu &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES);
1094         kbdev->hw_quirks_mmu |= (DEFAULT_AWID_LIMIT & 0x3) <<
1095                                 L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT;
1096
1097
1098         /* Only for T86x/T88x-based products after r2p0 */
1099         if (prod_id >= 0x860 && prod_id <= 0x880 && major >= 2) {
1100                 /* The JM_CONFIG register is specified as follows in the
1101                  T86x/T88x Engineering Specification Supplement:
1102                  The values are read from device tree in order.
1103                 */
1104 #define TIMESTAMP_OVERRIDE  1
1105 #define CLOCK_GATE_OVERRIDE (1<<1)
1106 #define JOB_THROTTLE_ENABLE (1<<2)
1107 #define JOB_THROTTLE_LIMIT_SHIFT 3
1108
1109                 /* 6 bits in the register */
1110                 const u32 jm_max_limit = 0x3F;
1111
1112                 if (of_property_read_u32_array(np,
1113                                         "jm_config",
1114                                         &jm_values[0],
1115                                         ARRAY_SIZE(jm_values))) {
1116                         /* Entry not in device tree, use defaults  */
1117                         jm_values[0] = 0;
1118                         jm_values[1] = 0;
1119                         jm_values[2] = 0;
1120                         jm_values[3] = jm_max_limit; /* Max value */
1121                 }
1122
1123                 /* Limit throttle limit to 6 bits*/
1124                 if (jm_values[3] > jm_max_limit) {
1125                         dev_dbg(kbdev->dev, "JOB_THROTTLE_LIMIT supplied in device tree is too large. Limiting to MAX (63).");
1126                         jm_values[3] = jm_max_limit;
1127                 }
1128
1129                 /* Aggregate to one integer. */
1130                 kbdev->hw_quirks_jm = (jm_values[0] ? TIMESTAMP_OVERRIDE : 0);
1131                 kbdev->hw_quirks_jm |= (jm_values[1] ? CLOCK_GATE_OVERRIDE : 0);
1132                 kbdev->hw_quirks_jm |= (jm_values[2] ? JOB_THROTTLE_ENABLE : 0);
1133                 kbdev->hw_quirks_jm |= (jm_values[3] <<
1134                                 JOB_THROTTLE_LIMIT_SHIFT);
1135         } else {
1136                 kbdev->hw_quirks_jm = KBASE_JM_CONFIG_UNUSED;
1137         }
1138
1139
1140 }
1141
1142 static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev)
1143 {
1144         if (kbdev->hw_quirks_sc)
1145                 kbase_reg_write(kbdev, GPU_CONTROL_REG(SHADER_CONFIG),
1146                                 kbdev->hw_quirks_sc, NULL);
1147
1148         kbase_reg_write(kbdev, GPU_CONTROL_REG(TILER_CONFIG),
1149                         kbdev->hw_quirks_tiler, NULL);
1150
1151         kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG),
1152                         kbdev->hw_quirks_mmu, NULL);
1153
1154
1155         if (kbdev->hw_quirks_jm != KBASE_JM_CONFIG_UNUSED)
1156                 kbase_reg_write(kbdev, GPU_CONTROL_REG(JM_CONFIG),
1157                                 kbdev->hw_quirks_jm, NULL);
1158
1159 }
1160
1161
1162 int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags)
1163 {
1164         unsigned long irq_flags;
1165         struct kbasep_reset_timeout_data rtdata;
1166
1167         KBASE_DEBUG_ASSERT(NULL != kbdev);
1168         lockdep_assert_held(&kbdev->pm.lock);
1169
1170         /* Ensure the clock is on before attempting to access the hardware */
1171         if (!kbdev->pm.backend.gpu_powered) {
1172                 if (kbdev->pm.backend.callback_power_on)
1173                         kbdev->pm.backend.callback_power_on(kbdev);
1174
1175                 spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock,
1176                                                                 irq_flags);
1177                 kbdev->pm.backend.gpu_powered = true;
1178                 spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock,
1179                                                                 irq_flags);
1180         }
1181
1182         /* Ensure interrupts are off to begin with, this also clears any
1183          * outstanding interrupts */
1184         kbase_pm_disable_interrupts(kbdev);
1185         /* Prepare for the soft-reset */
1186         kbdev->pm.backend.reset_done = false;
1187
1188         /* The cores should be made unavailable due to the reset */
1189         spin_lock_irqsave(&kbdev->pm.power_change_lock, irq_flags);
1190         if (kbdev->shader_available_bitmap != 0u)
1191                         KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL,
1192                                                 NULL, 0u, (u32)0u);
1193         if (kbdev->tiler_available_bitmap != 0u)
1194                         KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER,
1195                                                 NULL, NULL, 0u, (u32)0u);
1196         kbdev->shader_available_bitmap = 0u;
1197         kbdev->tiler_available_bitmap = 0u;
1198         kbdev->l2_available_bitmap = 0u;
1199         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, irq_flags);
1200
1201         /* Soft reset the GPU */
1202         KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0);
1203 #if defined(CONFIG_MALI_MIPE_ENABLED)
1204         kbase_tlstream_jd_gpu_soft_reset(kbdev);
1205 #endif
1206         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1207                                                 GPU_COMMAND_SOFT_RESET, NULL);
1208
1209         /* Unmask the reset complete interrupt only */
1210         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), RESET_COMPLETED,
1211                                                                         NULL);
1212
1213         /* Initialize a structure for tracking the status of the reset */
1214         rtdata.kbdev = kbdev;
1215         rtdata.timed_out = 0;
1216
1217         /* Create a timer to use as a timeout on the reset */
1218         hrtimer_init_on_stack(&rtdata.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1219         rtdata.timer.function = kbasep_reset_timeout;
1220
1221         hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT),
1222                                                         HRTIMER_MODE_REL);
1223
1224         /* Wait for the RESET_COMPLETED interrupt to be raised */
1225         kbase_pm_wait_for_reset(kbdev);
1226
1227         if (rtdata.timed_out == 0) {
1228                 /* GPU has been reset */
1229                 hrtimer_cancel(&rtdata.timer);
1230                 destroy_hrtimer_on_stack(&rtdata.timer);
1231                 goto out;
1232         }
1233
1234         /* No interrupt has been received - check if the RAWSTAT register says
1235          * the reset has completed */
1236         if (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) &
1237                                                         RESET_COMPLETED) {
1238                 /* The interrupt is set in the RAWSTAT; this suggests that the
1239                  * interrupts are not getting to the CPU */
1240                 dev_err(kbdev->dev, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n");
1241                 /* If interrupts aren't working we can't continue. */
1242                 destroy_hrtimer_on_stack(&rtdata.timer);
1243                 return -EINVAL;
1244         }
1245
1246         /* The GPU doesn't seem to be responding to the reset so try a hard
1247          * reset */
1248         dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n",
1249                                                                 RESET_TIMEOUT);
1250         KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0);
1251         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1252                                                 GPU_COMMAND_HARD_RESET, NULL);
1253
1254         /* Restart the timer to wait for the hard reset to complete */
1255         rtdata.timed_out = 0;
1256
1257         hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT),
1258                                                         HRTIMER_MODE_REL);
1259
1260         /* Wait for the RESET_COMPLETED interrupt to be raised */
1261         kbase_pm_wait_for_reset(kbdev);
1262
1263         if (rtdata.timed_out == 0) {
1264                 /* GPU has been reset */
1265                 hrtimer_cancel(&rtdata.timer);
1266                 destroy_hrtimer_on_stack(&rtdata.timer);
1267                 goto out;
1268         }
1269
1270         destroy_hrtimer_on_stack(&rtdata.timer);
1271
1272         dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n",
1273                                                                 RESET_TIMEOUT);
1274
1275         /* The GPU still hasn't reset, give up */
1276         return -EINVAL;
1277
1278 out:
1279
1280         if (flags & PM_HW_ISSUES_DETECT)
1281                 kbase_pm_hw_issues_detect(kbdev);
1282
1283         kbase_pm_hw_issues_apply(kbdev);
1284
1285
1286         /* If cycle counter was in use re-enable it, enable_irqs will only be
1287          * false when called from kbase_pm_powerup */
1288         if (kbdev->pm.backend.gpu_cycle_counter_requests &&
1289                                                 (flags & PM_ENABLE_IRQS)) {
1290                 /* enable interrupts as the L2 may have to be powered on */
1291                 kbase_pm_enable_interrupts(kbdev);
1292                 kbase_pm_request_l2_caches(kbdev);
1293
1294                 /* Re-enable the counters if we need to */
1295                 spin_lock_irqsave(
1296                         &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
1297                                                                 irq_flags);
1298                 if (kbdev->pm.backend.gpu_cycle_counter_requests)
1299                         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1300                                         GPU_COMMAND_CYCLE_COUNT_START, NULL);
1301                 spin_unlock_irqrestore(
1302                         &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
1303                                                                 irq_flags);
1304
1305                 kbase_pm_release_l2_caches(kbdev);
1306                 kbase_pm_disable_interrupts(kbdev);
1307         }
1308
1309         if (flags & PM_ENABLE_IRQS)
1310                 kbase_pm_enable_interrupts(kbdev);
1311
1312         return 0;
1313 }
1314
1315 /**
1316  * kbase_pm_request_gpu_cycle_counter_do_request - Request cycle counters
1317  *
1318  * Increase the count of cycle counter users and turn the cycle counters on if
1319  * they were previously off
1320  *
1321  * This function is designed to be called by
1322  * kbase_pm_request_gpu_cycle_counter() or
1323  * kbase_pm_request_gpu_cycle_counter_l2_is_on() only
1324  *
1325  * When this function is called the l2 cache must be on and the l2 cache users
1326  * count must have been incremented by a call to (
1327  * kbase_pm_request_l2_caches() or kbase_pm_request_l2_caches_l2_on() )
1328  *
1329  * @kbdev:     The kbase device structure of the device
1330  */
1331 static void
1332 kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *kbdev)
1333 {
1334         unsigned long flags;
1335
1336         spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock,
1337                                                                         flags);
1338
1339         ++kbdev->pm.backend.gpu_cycle_counter_requests;
1340
1341         if (1 == kbdev->pm.backend.gpu_cycle_counter_requests)
1342                 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1343                                         GPU_COMMAND_CYCLE_COUNT_START, NULL);
1344
1345         spin_unlock_irqrestore(
1346                         &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
1347                                                                         flags);
1348 }
1349
1350 void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev)
1351 {
1352         KBASE_DEBUG_ASSERT(kbdev != NULL);
1353
1354         KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered);
1355
1356         KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests <
1357                                                                 INT_MAX);
1358
1359         kbase_pm_request_l2_caches(kbdev);
1360
1361         kbase_pm_request_gpu_cycle_counter_do_request(kbdev);
1362 }
1363
1364 KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter);
1365
1366 void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev)
1367 {
1368         KBASE_DEBUG_ASSERT(kbdev != NULL);
1369
1370         KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered);
1371
1372         KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests <
1373                                                                 INT_MAX);
1374
1375         kbase_pm_request_l2_caches_l2_is_on(kbdev);
1376
1377         kbase_pm_request_gpu_cycle_counter_do_request(kbdev);
1378 }
1379
1380 KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter_l2_is_on);
1381
1382 void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev)
1383 {
1384         unsigned long flags;
1385
1386         KBASE_DEBUG_ASSERT(kbdev != NULL);
1387
1388         spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock,
1389                                                                         flags);
1390
1391         KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests > 0);
1392
1393         --kbdev->pm.backend.gpu_cycle_counter_requests;
1394
1395         if (0 == kbdev->pm.backend.gpu_cycle_counter_requests)
1396                 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1397                                         GPU_COMMAND_CYCLE_COUNT_STOP, NULL);
1398
1399         spin_unlock_irqrestore(
1400                         &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
1401                                                                         flags);
1402
1403         kbase_pm_release_l2_caches(kbdev);
1404 }
1405
1406 KBASE_EXPORT_TEST_API(kbase_pm_release_gpu_cycle_counter);