drivers/gpu/arm/midgard/backend/gpu/mali_kbase_pm_driver.c

   1 /*
   2  *
   3  * (C) COPYRIGHT 2010-2016 ARM Limited. All rights reserved.
   4  *
   5  * This program is free software and is provided to you under the terms of the
   6  * GNU General Public License version 2 as published by the Free Software
   7  * Foundation, and any use by you of this program is subject to the terms
   8  * of such GNU licence.
   9  *
  10  * A copy of the licence is included with the program, and can also be obtained
  11  * from Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  12  * Boston, MA  02110-1301, USA.
  13  *
  14  */
  15
  16
  17
  18
  19
  20 /*
  21  * Base kernel Power Management hardware control
  22  */
  23
  24 #include <mali_kbase.h>
  25 #include <mali_kbase_config_defaults.h>
  26 #include <mali_midg_regmap.h>
  27 #if defined(CONFIG_MALI_GATOR_SUPPORT)
  28 #include <mali_kbase_gator.h>
  29 #endif
  30 #include <mali_kbase_tlstream.h>
  31 #include <mali_kbase_pm.h>
  32 #include <mali_kbase_config_defaults.h>
  33 #include <mali_kbase_smc.h>
  34 #include <mali_kbase_hwaccess_jm.h>
  35 #include <backend/gpu/mali_kbase_cache_policy_backend.h>
  36 #include <backend/gpu/mali_kbase_device_internal.h>
  37 #include <backend/gpu/mali_kbase_irq_internal.h>
  38 #include <backend/gpu/mali_kbase_pm_internal.h>
  39
  40 #include <linux/of.h>
  41
  42 #if MALI_MOCK_TEST
  43 #define MOCKABLE(function) function##_original
  44 #else
  45 #define MOCKABLE(function) function
  46 #endif                          /* MALI_MOCK_TEST */
  47
  48 /* Special value to indicate that the JM_CONFIG reg isn't currently used. */
  49 #define KBASE_JM_CONFIG_UNUSED (1<<31)
  50
  51 /**
  52  * enum kbasep_pm_action - Actions that can be performed on a core.
  53  *
  54  * This enumeration is private to the file. Its values are set to allow
  55  * core_type_to_reg() function, which decodes this enumeration, to be simpler
  56  * and more efficient.
  57  *
  58  * @ACTION_PRESENT: The cores that are present
  59  * @ACTION_READY: The cores that are ready
  60  * @ACTION_PWRON: Power on the cores specified
  61  * @ACTION_PWROFF: Power off the cores specified
  62  * @ACTION_PWRTRANS: The cores that are transitioning
  63  * @ACTION_PWRACTIVE: The cores that are active
  64  */
  65 enum kbasep_pm_action {
  66         ACTION_PRESENT = 0,
  67         ACTION_READY = (SHADER_READY_LO - SHADER_PRESENT_LO),
  68         ACTION_PWRON = (SHADER_PWRON_LO - SHADER_PRESENT_LO),
  69         ACTION_PWROFF = (SHADER_PWROFF_LO - SHADER_PRESENT_LO),
  70         ACTION_PWRTRANS = (SHADER_PWRTRANS_LO - SHADER_PRESENT_LO),
  71         ACTION_PWRACTIVE = (SHADER_PWRACTIVE_LO - SHADER_PRESENT_LO)
  72 };
  73
  74 static u64 kbase_pm_get_state(
  75                 struct kbase_device *kbdev,
  76                 enum kbase_pm_core_type core_type,
  77                 enum kbasep_pm_action action);
  78
  79 /**
  80  * core_type_to_reg - Decode a core type and action to a register.
  81  *
  82  * Given a core type (defined by kbase_pm_core_type) and an action (defined
  83  * by kbasep_pm_action) this function will return the register offset that
  84  * will perform the action on the core type. The register returned is the _LO
  85  * register and an offset must be applied to use the _HI register.
  86  *
  87  * @core_type: The type of core
  88  * @action:    The type of action
  89  *
  90  * Return: The register offset of the _LO register that performs an action of
  91  * type @action on a core of type @core_type.
  92  */
  93 static u32 core_type_to_reg(enum kbase_pm_core_type core_type,
  94                                                 enum kbasep_pm_action action)
  95 {
  96         return (u32)core_type + (u32)action;
  97 }
  98
  99 #ifdef CONFIG_ARM64
 100 static void mali_cci_flush_l2(struct kbase_device *kbdev)
 101 {
 102         const u32 mask = CLEAN_CACHES_COMPLETED | RESET_COMPLETED;
 103         u32 loops = KBASE_CLEAN_CACHE_MAX_LOOPS;
 104         u32 raw;
 105
 106         /*
 107          * Note that we don't take the cache flush mutex here since
 108          * we expect to be the last user of the L2, all other L2 users
 109          * would have dropped their references, to initiate L2 power
 110          * down, L2 power down being the only valid place for this
 111          * to be called from.
 112          */
 113
 114         kbase_reg_write(kbdev,
 115                         GPU_CONTROL_REG(GPU_COMMAND),
 116                         GPU_COMMAND_CLEAN_INV_CACHES,
 117                         NULL);
 118
 119         raw = kbase_reg_read(kbdev,
 120                 GPU_CONTROL_REG(GPU_IRQ_RAWSTAT),
 121                 NULL);
 122
 123         /* Wait for cache flush to complete before continuing, exit on
 124          * gpu resets or loop expiry. */
 125         while (((raw & mask) == 0) && --loops) {
 126                 raw = kbase_reg_read(kbdev,
 127                                         GPU_CONTROL_REG(GPU_IRQ_RAWSTAT),
 128                                         NULL);
 129         }
 130 }
 131 #endif
 132
 133 /**
 134  * kbase_pm_invoke - Invokes an action on a core set
 135  *
 136  * This function performs the action given by @action on a set of cores of a
 137  * type given by @core_type. It is a static function used by
 138  * kbase_pm_transition_core_type()
 139  *
 140  * @kbdev:     The kbase device structure of the device
 141  * @core_type: The type of core that the action should be performed on
 142  * @cores:     A bit mask of cores to perform the action on (low 32 bits)
 143  * @action:    The action to perform on the cores
 144  */
 145 static void kbase_pm_invoke(struct kbase_device *kbdev,
 146                                         enum kbase_pm_core_type core_type,
 147                                         u64 cores,
 148                                         enum kbasep_pm_action action)
 149 {
 150         u32 reg;
 151         u32 lo = cores & 0xFFFFFFFF;
 152         u32 hi = (cores >> 32) & 0xFFFFFFFF;
 153
 154         lockdep_assert_held(&kbdev->pm.power_change_lock);
 155
 156         reg = core_type_to_reg(core_type, action);
 157
 158         KBASE_DEBUG_ASSERT(reg);
 159 #if defined(CONFIG_MALI_GATOR_SUPPORT)
 160         if (cores) {
 161                 if (action == ACTION_PWRON)
 162                         kbase_trace_mali_pm_power_on(core_type, cores);
 163                 else if (action == ACTION_PWROFF)
 164                         kbase_trace_mali_pm_power_off(core_type, cores);
 165         }
 166 #endif
 167
 168         if (cores) {
 169                 u64 state = kbase_pm_get_state(kbdev, core_type, ACTION_READY);
 170
 171                 if (action == ACTION_PWRON)
 172                         state |= cores;
 173                 else if (action == ACTION_PWROFF)
 174                         state &= ~cores;
 175                 kbase_tlstream_aux_pm_state(core_type, state);
 176         }
 177
 178         /* Tracing */
 179         if (cores) {
 180                 if (action == ACTION_PWRON)
 181                         switch (core_type) {
 182                         case KBASE_PM_CORE_SHADER:
 183                                 KBASE_TRACE_ADD(kbdev, PM_PWRON, NULL, NULL, 0u,
 184                                                                         lo);
 185                                 break;
 186                         case KBASE_PM_CORE_TILER:
 187                                 KBASE_TRACE_ADD(kbdev, PM_PWRON_TILER, NULL,
 188                                                                 NULL, 0u, lo);
 189                                 break;
 190                         case KBASE_PM_CORE_L2:
 191                                 KBASE_TRACE_ADD(kbdev, PM_PWRON_L2, NULL, NULL,
 192                                                                         0u, lo);
 193                                 break;
 194                         default:
 195                                 break;
 196                         }
 197                 else if (action == ACTION_PWROFF)
 198                         switch (core_type) {
 199                         case KBASE_PM_CORE_SHADER:
 200                                 KBASE_TRACE_ADD(kbdev, PM_PWROFF, NULL, NULL,
 201                                                                         0u, lo);
 202                                 break;
 203                         case KBASE_PM_CORE_TILER:
 204                                 KBASE_TRACE_ADD(kbdev, PM_PWROFF_TILER, NULL,
 205                                                                 NULL, 0u, lo);
 206                                 break;
 207                         case KBASE_PM_CORE_L2:
 208                                 KBASE_TRACE_ADD(kbdev, PM_PWROFF_L2, NULL, NULL,
 209                                                                         0u, lo);
 210                                 /* disable snoops before L2 is turned off */
 211                                 kbase_pm_cache_snoop_disable(kbdev);
 212                                 break;
 213                         default:
 214                                 break;
 215                         }
 216         }
 217
 218         if (lo != 0)
 219                 kbase_reg_write(kbdev, GPU_CONTROL_REG(reg), lo, NULL);
 220
 221         if (hi != 0)
 222                 kbase_reg_write(kbdev, GPU_CONTROL_REG(reg + 4), hi, NULL);
 223 }
 224
 225 /**
 226  * kbase_pm_get_state - Get information about a core set
 227  *
 228  * This function gets information (chosen by @action) about a set of cores of
 229  * a type given by @core_type. It is a static function used by
 230  * kbase_pm_get_present_cores(), kbase_pm_get_active_cores(),
 231  * kbase_pm_get_trans_cores() and kbase_pm_get_ready_cores().
 232  *
 233  * @kbdev:     The kbase device structure of the device
 234  * @core_type: The type of core that the should be queried
 235  * @action:    The property of the cores to query
 236  *
 237  * Return: A bit mask specifying the state of the cores
 238  */
 239 static u64 kbase_pm_get_state(struct kbase_device *kbdev,
 240                                         enum kbase_pm_core_type core_type,
 241                                         enum kbasep_pm_action action)
 242 {
 243         u32 reg;
 244         u32 lo, hi;
 245
 246         reg = core_type_to_reg(core_type, action);
 247
 248         KBASE_DEBUG_ASSERT(reg);
 249
 250         lo = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg), NULL);
 251         hi = kbase_reg_read(kbdev, GPU_CONTROL_REG(reg + 4), NULL);
 252
 253         return (((u64) hi) << 32) | ((u64) lo);
 254 }
 255
 256 void kbasep_pm_read_present_cores(struct kbase_device *kbdev)
 257 {
 258         kbdev->shader_inuse_bitmap = 0;
 259         kbdev->shader_needed_bitmap = 0;
 260         kbdev->shader_available_bitmap = 0;
 261         kbdev->tiler_available_bitmap = 0;
 262         kbdev->l2_users_count = 0;
 263         kbdev->l2_available_bitmap = 0;
 264         kbdev->tiler_needed_cnt = 0;
 265         kbdev->tiler_inuse_cnt = 0;
 266
 267         memset(kbdev->shader_needed_cnt, 0, sizeof(kbdev->shader_needed_cnt));
 268 }
 269
 270 KBASE_EXPORT_TEST_API(kbasep_pm_read_present_cores);
 271
 272 /**
 273  * kbase_pm_get_present_cores - Get the cores that are present
 274  *
 275  * @kbdev: Kbase device
 276  * @type: The type of cores to query
 277  *
 278  * Return: Bitmask of the cores that are present
 279  */
 280 u64 kbase_pm_get_present_cores(struct kbase_device *kbdev,
 281                                                 enum kbase_pm_core_type type)
 282 {
 283         KBASE_DEBUG_ASSERT(kbdev != NULL);
 284
 285         switch (type) {
 286         case KBASE_PM_CORE_L2:
 287                 return kbdev->gpu_props.props.raw_props.l2_present;
 288         case KBASE_PM_CORE_SHADER:
 289                 return kbdev->gpu_props.props.raw_props.shader_present;
 290         case KBASE_PM_CORE_TILER:
 291                 return kbdev->gpu_props.props.raw_props.tiler_present;
 292         }
 293         KBASE_DEBUG_ASSERT(0);
 294         return 0;
 295 }
 296
 297 KBASE_EXPORT_TEST_API(kbase_pm_get_present_cores);
 298
 299 /**
 300  * kbase_pm_get_active_cores - Get the cores that are "active"
 301  *                             (busy processing work)
 302  *
 303  * @kbdev: Kbase device
 304  * @type: The type of cores to query
 305  *
 306  * Return: Bitmask of cores that are active
 307  */
 308 u64 kbase_pm_get_active_cores(struct kbase_device *kbdev,
 309                                                 enum kbase_pm_core_type type)
 310 {
 311         return kbase_pm_get_state(kbdev, type, ACTION_PWRACTIVE);
 312 }
 313
 314 KBASE_EXPORT_TEST_API(kbase_pm_get_active_cores);
 315
 316 /**
 317  * kbase_pm_get_trans_cores - Get the cores that are transitioning between
 318  *                            power states
 319  *
 320  * @kbdev: Kbase device
 321  * @type: The type of cores to query
 322  *
 323  * Return: Bitmask of cores that are transitioning
 324  */
 325 u64 kbase_pm_get_trans_cores(struct kbase_device *kbdev,
 326                                                 enum kbase_pm_core_type type)
 327 {
 328         return kbase_pm_get_state(kbdev, type, ACTION_PWRTRANS);
 329 }
 330
 331 KBASE_EXPORT_TEST_API(kbase_pm_get_trans_cores);
 332
 333 /**
 334  * kbase_pm_get_ready_cores - Get the cores that are powered on
 335  *
 336  * @kbdev: Kbase device
 337  * @type: The type of cores to query
 338  *
 339  * Return: Bitmask of cores that are ready (powered on)
 340  */
 341 u64 kbase_pm_get_ready_cores(struct kbase_device *kbdev,
 342                                                 enum kbase_pm_core_type type)
 343 {
 344         u64 result;
 345
 346         result = kbase_pm_get_state(kbdev, type, ACTION_READY);
 347
 348         switch (type) {
 349         case KBASE_PM_CORE_SHADER:
 350                 KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED, NULL, NULL, 0u,
 351                                                                 (u32) result);
 352                 break;
 353         case KBASE_PM_CORE_TILER:
 354                 KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_TILER, NULL, NULL, 0u,
 355                                                                 (u32) result);
 356                 break;
 357         case KBASE_PM_CORE_L2:
 358                 KBASE_TRACE_ADD(kbdev, PM_CORES_POWERED_L2, NULL, NULL, 0u,
 359                                                                 (u32) result);
 360                 break;
 361         default:
 362                 break;
 363         }
 364
 365         return result;
 366 }
 367
 368 KBASE_EXPORT_TEST_API(kbase_pm_get_ready_cores);
 369
 370 /**
 371  * kbase_pm_transition_core_type - Perform power transitions for a particular
 372  *                                 core type.
 373  *
 374  * This function will perform any available power transitions to make the actual
 375  * hardware state closer to the desired state. If a core is currently
 376  * transitioning then changes to the power state of that call cannot be made
 377  * until the transition has finished. Cores which are not present in the
 378  * hardware are ignored if they are specified in the desired_state bitmask,
 379  * however the return value will always be 0 in this case.
 380  *
 381  * @kbdev:             The kbase device
 382  * @type:              The core type to perform transitions for
 383  * @desired_state:     A bit mask of the desired state of the cores
 384  * @in_use:            A bit mask of the cores that are currently running
 385  *                     jobs. These cores have to be kept powered up because
 386  *                     there are jobs running (or about to run) on them.
 387  * @available:         Receives a bit mask of the cores that the job
 388  *                     scheduler can use to submit jobs to. May be NULL if
 389  *                     this is not needed.
 390  * @powering_on:       Bit mask to update with cores that are
 391  *                    transitioning to a power-on state.
 392  *
 393  * Return: true if the desired state has been reached, false otherwise
 394  */
 395 static bool kbase_pm_transition_core_type(struct kbase_device *kbdev,
 396                                                 enum kbase_pm_core_type type,
 397                                                 u64 desired_state,
 398                                                 u64 in_use,
 399                                                 u64 * const available,
 400                                                 u64 *powering_on)
 401 {
 402         u64 present;
 403         u64 ready;
 404         u64 trans;
 405         u64 powerup;
 406         u64 powerdown;
 407         u64 powering_on_trans;
 408         u64 desired_state_in_use;
 409
 410         lockdep_assert_held(&kbdev->pm.power_change_lock);
 411
 412         /* Get current state */
 413         present = kbase_pm_get_present_cores(kbdev, type);
 414         trans = kbase_pm_get_trans_cores(kbdev, type);
 415         ready = kbase_pm_get_ready_cores(kbdev, type);
 416         /* mask off ready from trans in case transitions finished between the
 417          * register reads */
 418         trans &= ~ready;
 419
 420         powering_on_trans = trans & *powering_on;
 421         *powering_on = powering_on_trans;
 422
 423         if (available != NULL)
 424                 *available = (ready | powering_on_trans) & desired_state;
 425
 426         /* Update desired state to include the in-use cores. These have to be
 427          * kept powered up because there are jobs running or about to run on
 428          * these cores
 429          */
 430         desired_state_in_use = desired_state | in_use;
 431
 432         /* Update state of whether l2 caches are powered */
 433         if (type == KBASE_PM_CORE_L2) {
 434                 if ((ready == present) && (desired_state_in_use == ready) &&
 435                                                                 (trans == 0)) {
 436                         /* All are ready, none will be turned off, and none are
 437                          * transitioning */
 438                         kbdev->pm.backend.l2_powered = 1;
 439                         /*
 440                          * Ensure snoops are enabled after L2 is powered up,
 441                          * note that kbase keeps track of the snoop state, so
 442                          * safe to repeatedly call.
 443                          */
 444                         kbase_pm_cache_snoop_enable(kbdev);
 445                         if (kbdev->l2_users_count > 0) {
 446                                 /* Notify any registered l2 cache users
 447                                  * (optimized out when no users waiting) */
 448                                 wake_up(&kbdev->pm.backend.l2_powered_wait);
 449                         }
 450                 } else
 451                         kbdev->pm.backend.l2_powered = 0;
 452         }
 453
 454         if (desired_state_in_use == ready && (trans == 0))
 455                 return true;
 456
 457         /* Restrict the cores to those that are actually present */
 458         powerup = desired_state_in_use & present;
 459         powerdown = (~desired_state_in_use) & present;
 460
 461         /* Restrict to cores that are not already in the desired state */
 462         powerup &= ~ready;
 463         powerdown &= ready;
 464
 465         /* Don't transition any cores that are already transitioning, except for
 466          * Mali cores that support the following case:
 467          *
 468          * If the SHADER_PWRON or TILER_PWRON registers are written to turn on
 469          * a core that is currently transitioning to power off, then this is
 470          * remembered and the shader core is automatically powered up again once
 471          * the original transition completes. Once the automatic power on is
 472          * complete any job scheduled on the shader core should start.
 473          */
 474         powerdown &= ~trans;
 475
 476         if (kbase_hw_has_feature(kbdev,
 477                                 BASE_HW_FEATURE_PWRON_DURING_PWROFF_TRANS))
 478                 if (KBASE_PM_CORE_SHADER == type || KBASE_PM_CORE_TILER == type)
 479                         trans = powering_on_trans; /* for exception cases, only
 480                                                     * mask off cores in power on
 481                                                     * transitions */
 482
 483         powerup &= ~trans;
 484
 485         /* Perform transitions if any */
 486         kbase_pm_invoke(kbdev, type, powerup, ACTION_PWRON);
 487         kbase_pm_invoke(kbdev, type, powerdown, ACTION_PWROFF);
 488
 489         /* Recalculate cores transitioning on, and re-evaluate our state */
 490         powering_on_trans |= powerup;
 491         *powering_on = powering_on_trans;
 492         if (available != NULL)
 493                 *available = (ready | powering_on_trans) & desired_state;
 494
 495         return false;
 496 }
 497
 498 KBASE_EXPORT_TEST_API(kbase_pm_transition_core_type);
 499
 500 /**
 501  * get_desired_cache_status - Determine which caches should be on for a
 502  *                            particular core state
 503  *
 504  * This function takes a bit mask of the present caches and the cores (or
 505  * caches) that are attached to the caches that will be powered. It then
 506  * computes which caches should be turned on to allow the cores requested to be
 507  * powered up.
 508  *
 509  * @present:       The bit mask of present caches
 510  * @cores_powered: A bit mask of cores (or L2 caches) that are desired to
 511  *                 be powered
 512  * @tilers_powered: The bit mask of tilers that are desired to be powered
 513  *
 514  * Return: A bit mask of the caches that should be turned on
 515  */
 516 static u64 get_desired_cache_status(u64 present, u64 cores_powered,
 517                 u64 tilers_powered)
 518 {
 519         u64 desired = 0;
 520
 521         while (present) {
 522                 /* Find out which is the highest set bit */
 523                 u64 bit = fls64(present) - 1;
 524                 u64 bit_mask = 1ull << bit;
 525                 /* Create a mask which has all bits from 'bit' upwards set */
 526
 527                 u64 mask = ~(bit_mask - 1);
 528
 529                 /* If there are any cores powered at this bit or above (that
 530                  * haven't previously been processed) then we need this core on
 531                  */
 532                 if (cores_powered & mask)
 533                         desired |= bit_mask;
 534
 535                 /* Remove bits from cores_powered and present */
 536                 cores_powered &= ~mask;
 537                 present &= ~bit_mask;
 538         }
 539
 540         /* Power up the required L2(s) for the tiler */
 541         if (tilers_powered)
 542                 desired |= 1;
 543
 544         return desired;
 545 }
 546
 547 KBASE_EXPORT_TEST_API(get_desired_cache_status);
 548
 549 bool
 550 MOCKABLE(kbase_pm_check_transitions_nolock) (struct kbase_device *kbdev)
 551 {
 552         bool cores_are_available = false;
 553         bool in_desired_state = true;
 554         u64 desired_l2_state;
 555         u64 cores_powered;
 556         u64 tilers_powered;
 557         u64 tiler_available_bitmap;
 558         u64 shader_available_bitmap;
 559         u64 shader_ready_bitmap;
 560         u64 shader_transitioning_bitmap;
 561         u64 l2_available_bitmap;
 562         u64 prev_l2_available_bitmap;
 563
 564         KBASE_DEBUG_ASSERT(NULL != kbdev);
 565         lockdep_assert_held(&kbdev->pm.power_change_lock);
 566
 567         spin_lock(&kbdev->pm.backend.gpu_powered_lock);
 568         if (kbdev->pm.backend.gpu_powered == false) {
 569                 spin_unlock(&kbdev->pm.backend.gpu_powered_lock);
 570                 if (kbdev->pm.backend.desired_shader_state == 0 &&
 571                                 kbdev->pm.backend.desired_tiler_state == 0)
 572                         return true;
 573                 return false;
 574         }
 575
 576         /* Trace that a change-state is being requested, and that it took
 577          * (effectively) no time to start it. This is useful for counting how
 578          * many state changes occurred, in a way that's backwards-compatible
 579          * with processing the trace data */
 580         kbase_timeline_pm_send_event(kbdev,
 581                                 KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE);
 582         kbase_timeline_pm_handle_event(kbdev,
 583                                 KBASE_TIMELINE_PM_EVENT_CHANGE_GPU_STATE);
 584
 585         /* If any cores are already powered then, we must keep the caches on */
 586         cores_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_SHADER);
 587
 588         cores_powered |= kbdev->pm.backend.desired_shader_state;
 589
 590         /* Work out which tilers want to be powered */
 591         tilers_powered = kbase_pm_get_ready_cores(kbdev, KBASE_PM_CORE_TILER);
 592         tilers_powered |= kbdev->pm.backend.desired_tiler_state;
 593
 594         /* If there are l2 cache users registered, keep all l2s powered even if
 595          * all other cores are off. */
 596         if (kbdev->l2_users_count > 0)
 597                 cores_powered |= kbdev->gpu_props.props.raw_props.l2_present;
 598
 599         desired_l2_state = get_desired_cache_status(
 600                         kbdev->gpu_props.props.raw_props.l2_present,
 601                         cores_powered, tilers_powered);
 602
 603         /* If any l2 cache is on, then enable l2 #0, for use by job manager */
 604         if (0 != desired_l2_state)
 605                 desired_l2_state |= 1;
 606
 607         prev_l2_available_bitmap = kbdev->l2_available_bitmap;
 608         in_desired_state &= kbase_pm_transition_core_type(kbdev,
 609                                 KBASE_PM_CORE_L2, desired_l2_state, 0,
 610                                 &l2_available_bitmap,
 611                                 &kbdev->pm.backend.powering_on_l2_state);
 612
 613         if (kbdev->l2_available_bitmap != l2_available_bitmap)
 614                 KBASE_TIMELINE_POWER_L2(kbdev, l2_available_bitmap);
 615
 616         kbdev->l2_available_bitmap = l2_available_bitmap;
 617
 618         if (in_desired_state) {
 619                 in_desired_state &= kbase_pm_transition_core_type(kbdev,
 620                                 KBASE_PM_CORE_TILER,
 621                                 kbdev->pm.backend.desired_tiler_state,
 622                                 0, &tiler_available_bitmap,
 623                                 &kbdev->pm.backend.powering_on_tiler_state);
 624                 in_desired_state &= kbase_pm_transition_core_type(kbdev,
 625                                 KBASE_PM_CORE_SHADER,
 626                                 kbdev->pm.backend.desired_shader_state,
 627                                 kbdev->shader_inuse_bitmap,
 628                                 &shader_available_bitmap,
 629                                 &kbdev->pm.backend.powering_on_shader_state);
 630
 631                 if (kbdev->shader_available_bitmap != shader_available_bitmap) {
 632                         KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL,
 633                                                 NULL, 0u,
 634                                                 (u32) shader_available_bitmap);
 635                         KBASE_TIMELINE_POWER_SHADER(kbdev,
 636                                                 shader_available_bitmap);
 637                 }
 638
 639                 kbdev->shader_available_bitmap = shader_available_bitmap;
 640
 641                 if (kbdev->tiler_available_bitmap != tiler_available_bitmap) {
 642                         KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER,
 643                                                 NULL, NULL, 0u,
 644                                                 (u32) tiler_available_bitmap);
 645                         KBASE_TIMELINE_POWER_TILER(kbdev,
 646                                                         tiler_available_bitmap);
 647                 }
 648
 649                 kbdev->tiler_available_bitmap = tiler_available_bitmap;
 650
 651         } else if ((l2_available_bitmap &
 652                         kbdev->gpu_props.props.raw_props.tiler_present) !=
 653                         kbdev->gpu_props.props.raw_props.tiler_present) {
 654                 tiler_available_bitmap = 0;
 655
 656                 if (kbdev->tiler_available_bitmap != tiler_available_bitmap)
 657                         KBASE_TIMELINE_POWER_TILER(kbdev,
 658                                                         tiler_available_bitmap);
 659
 660                 kbdev->tiler_available_bitmap = tiler_available_bitmap;
 661         }
 662
 663         /* State updated for slow-path waiters */
 664         kbdev->pm.backend.gpu_in_desired_state = in_desired_state;
 665
 666         shader_ready_bitmap = kbase_pm_get_ready_cores(kbdev,
 667                                                         KBASE_PM_CORE_SHADER);
 668         shader_transitioning_bitmap = kbase_pm_get_trans_cores(kbdev,
 669                                                         KBASE_PM_CORE_SHADER);
 670
 671         /* Determine whether the cores are now available (even if the set of
 672          * available cores is empty). Note that they can be available even if
 673          * we've not finished transitioning to the desired state */
 674         if ((kbdev->shader_available_bitmap &
 675                                         kbdev->pm.backend.desired_shader_state)
 676                                 == kbdev->pm.backend.desired_shader_state &&
 677                 (kbdev->tiler_available_bitmap &
 678                                         kbdev->pm.backend.desired_tiler_state)
 679                                 == kbdev->pm.backend.desired_tiler_state) {
 680                 cores_are_available = true;
 681
 682                 KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE, NULL, NULL, 0u,
 683                                 (u32)(kbdev->shader_available_bitmap &
 684                                 kbdev->pm.backend.desired_shader_state));
 685                 KBASE_TRACE_ADD(kbdev, PM_CORES_AVAILABLE_TILER, NULL, NULL, 0u,
 686                                 (u32)(kbdev->tiler_available_bitmap &
 687                                 kbdev->pm.backend.desired_tiler_state));
 688
 689                 /* Log timelining information about handling events that power
 690                  * up cores, to match up either with immediate submission either
 691                  * because cores already available, or from PM IRQ */
 692                 if (!in_desired_state)
 693                         kbase_timeline_pm_send_event(kbdev,
 694                                 KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
 695         }
 696
 697         if (in_desired_state) {
 698                 KBASE_DEBUG_ASSERT(cores_are_available);
 699
 700 #if defined(CONFIG_MALI_GATOR_SUPPORT)
 701                 kbase_trace_mali_pm_status(KBASE_PM_CORE_L2,
 702                                                 kbase_pm_get_ready_cores(kbdev,
 703                                                         KBASE_PM_CORE_L2));
 704                 kbase_trace_mali_pm_status(KBASE_PM_CORE_SHADER,
 705                                                 kbase_pm_get_ready_cores(kbdev,
 706                                                         KBASE_PM_CORE_SHADER));
 707                 kbase_trace_mali_pm_status(KBASE_PM_CORE_TILER,
 708                                                 kbase_pm_get_ready_cores(kbdev,
 709                                                         KBASE_PM_CORE_TILER));
 710 #endif
 711
 712                 kbase_tlstream_aux_pm_state(
 713                                 KBASE_PM_CORE_L2,
 714                                 kbase_pm_get_ready_cores(
 715                                         kbdev, KBASE_PM_CORE_L2));
 716                 kbase_tlstream_aux_pm_state(
 717                                 KBASE_PM_CORE_SHADER,
 718                                 kbase_pm_get_ready_cores(
 719                                         kbdev, KBASE_PM_CORE_SHADER));
 720                 kbase_tlstream_aux_pm_state(
 721                                 KBASE_PM_CORE_TILER,
 722                                 kbase_pm_get_ready_cores(
 723                                         kbdev,
 724                                         KBASE_PM_CORE_TILER));
 725
 726                 KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED, NULL, NULL,
 727                                 kbdev->pm.backend.gpu_in_desired_state,
 728                                 (u32)kbdev->pm.backend.desired_shader_state);
 729                 KBASE_TRACE_ADD(kbdev, PM_DESIRED_REACHED_TILER, NULL, NULL, 0u,
 730                                 (u32)kbdev->pm.backend.desired_tiler_state);
 731
 732                 /* Log timelining information for synchronous waiters */
 733                 kbase_timeline_pm_send_event(kbdev,
 734                                 KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
 735                 /* Wake slow-path waiters. Job scheduler does not use this. */
 736                 KBASE_TRACE_ADD(kbdev, PM_WAKE_WAITERS, NULL, NULL, 0u, 0);
 737                 wake_up(&kbdev->pm.backend.gpu_in_desired_state_wait);
 738         }
 739
 740         spin_unlock(&kbdev->pm.backend.gpu_powered_lock);
 741
 742         /* kbase_pm_ca_update_core_status can cause one-level recursion into
 743          * this function, so it must only be called once all changes to kbdev
 744          * have been committed, and after the gpu_powered_lock has been
 745          * dropped. */
 746         if (kbdev->shader_ready_bitmap != shader_ready_bitmap ||
 747             kbdev->shader_transitioning_bitmap != shader_transitioning_bitmap) {
 748                 kbdev->shader_ready_bitmap = shader_ready_bitmap;
 749                 kbdev->shader_transitioning_bitmap =
 750                                                 shader_transitioning_bitmap;
 751
 752                 kbase_pm_ca_update_core_status(kbdev, shader_ready_bitmap,
 753                                                 shader_transitioning_bitmap);
 754         }
 755
 756         /* The core availability policy is not allowed to keep core group 0
 757          * turned off (unless it was changing the l2 power state) */
 758         if (!((shader_ready_bitmap | shader_transitioning_bitmap) &
 759                 kbdev->gpu_props.props.coherency_info.group[0].core_mask) &&
 760                 (prev_l2_available_bitmap == desired_l2_state) &&
 761                 !(kbase_pm_ca_get_core_mask(kbdev) &
 762                 kbdev->gpu_props.props.coherency_info.group[0].core_mask))
 763                 BUG();
 764
 765         /* The core availability policy is allowed to keep core group 1 off,
 766          * but all jobs specifically targeting CG1 must fail */
 767         if (!((shader_ready_bitmap | shader_transitioning_bitmap) &
 768                 kbdev->gpu_props.props.coherency_info.group[1].core_mask) &&
 769                 !(kbase_pm_ca_get_core_mask(kbdev) &
 770                 kbdev->gpu_props.props.coherency_info.group[1].core_mask))
 771                 kbdev->pm.backend.cg1_disabled = true;
 772         else
 773                 kbdev->pm.backend.cg1_disabled = false;
 774
 775         return cores_are_available;
 776 }
 777 KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_nolock);
 778
 779 /* Timeout for kbase_pm_check_transitions_sync when wait_event_killable has
 780  * aborted due to a fatal signal. If the time spent waiting has exceeded this
 781  * threshold then there is most likely a hardware issue. */
 782 #define PM_TIMEOUT (5*HZ) /* 5s */
 783
 784 void kbase_pm_check_transitions_sync(struct kbase_device *kbdev)
 785 {
 786         unsigned long flags;
 787         unsigned long timeout;
 788         bool cores_are_available;
 789         int ret;
 790
 791         /* Force the transition to be checked and reported - the cores may be
 792          * 'available' (for job submission) but not fully powered up. */
 793         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 794         cores_are_available = kbase_pm_check_transitions_nolock(kbdev);
 795         /* Don't need 'cores_are_available', because we don't return anything */
 796         CSTD_UNUSED(cores_are_available);
 797         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 798
 799         timeout = jiffies + PM_TIMEOUT;
 800
 801         /* Wait for cores */
 802         ret = wait_event_killable(kbdev->pm.backend.gpu_in_desired_state_wait,
 803                         kbdev->pm.backend.gpu_in_desired_state);
 804
 805         if (ret < 0 && time_after(jiffies, timeout)) {
 806                 dev_err(kbdev->dev, "Power transition timed out unexpectedly\n");
 807                 dev_err(kbdev->dev, "Desired state :\n");
 808                 dev_err(kbdev->dev, "\tShader=%016llx\n",
 809                                 kbdev->pm.backend.desired_shader_state);
 810                 dev_err(kbdev->dev, "\tTiler =%016llx\n",
 811                                 kbdev->pm.backend.desired_tiler_state);
 812                 dev_err(kbdev->dev, "Current state :\n");
 813                 dev_err(kbdev->dev, "\tShader=%08x%08x\n",
 814                                 kbase_reg_read(kbdev,
 815                                         GPU_CONTROL_REG(SHADER_READY_HI), NULL),
 816                                 kbase_reg_read(kbdev,
 817                                         GPU_CONTROL_REG(SHADER_READY_LO),
 818                                         NULL));
 819                 dev_err(kbdev->dev, "\tTiler =%08x%08x\n",
 820                                 kbase_reg_read(kbdev,
 821                                         GPU_CONTROL_REG(TILER_READY_HI), NULL),
 822                                 kbase_reg_read(kbdev,
 823                                         GPU_CONTROL_REG(TILER_READY_LO), NULL));
 824                 dev_err(kbdev->dev, "\tL2    =%08x%08x\n",
 825                                 kbase_reg_read(kbdev,
 826                                         GPU_CONTROL_REG(L2_READY_HI), NULL),
 827                                 kbase_reg_read(kbdev,
 828                                         GPU_CONTROL_REG(L2_READY_LO), NULL));
 829                 dev_err(kbdev->dev, "Cores transitioning :\n");
 830                 dev_err(kbdev->dev, "\tShader=%08x%08x\n",
 831                                 kbase_reg_read(kbdev, GPU_CONTROL_REG(
 832                                                 SHADER_PWRTRANS_HI), NULL),
 833                                 kbase_reg_read(kbdev, GPU_CONTROL_REG(
 834                                                 SHADER_PWRTRANS_LO), NULL));
 835                 dev_err(kbdev->dev, "\tTiler =%08x%08x\n",
 836                                 kbase_reg_read(kbdev, GPU_CONTROL_REG(
 837                                                 TILER_PWRTRANS_HI), NULL),
 838                                 kbase_reg_read(kbdev, GPU_CONTROL_REG(
 839                                                 TILER_PWRTRANS_LO), NULL));
 840                 dev_err(kbdev->dev, "\tL2    =%08x%08x\n",
 841                                 kbase_reg_read(kbdev, GPU_CONTROL_REG(
 842                                                 L2_PWRTRANS_HI), NULL),
 843                                 kbase_reg_read(kbdev, GPU_CONTROL_REG(
 844                                                 L2_PWRTRANS_LO), NULL));
 845 #if KBASE_GPU_RESET_EN
 846                 dev_err(kbdev->dev, "Sending reset to GPU - all running jobs will be lost\n");
 847                 if (kbase_prepare_to_reset_gpu(kbdev))
 848                         kbase_reset_gpu(kbdev);
 849 #endif /* KBASE_GPU_RESET_EN */
 850         } else {
 851                 /* Log timelining information that a change in state has
 852                  * completed */
 853                 kbase_timeline_pm_handle_event(kbdev,
 854                                 KBASE_TIMELINE_PM_EVENT_GPU_STATE_CHANGED);
 855         }
 856 }
 857 KBASE_EXPORT_TEST_API(kbase_pm_check_transitions_sync);
 858
 859 void kbase_pm_enable_interrupts(struct kbase_device *kbdev)
 860 {
 861         unsigned long flags;
 862
 863         KBASE_DEBUG_ASSERT(NULL != kbdev);
 864         /*
 865          * Clear all interrupts,
 866          * and unmask them all.
 867          */
 868         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 869         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL,
 870                                                                         NULL);
 871         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), GPU_IRQ_REG_ALL,
 872                                                                         NULL);
 873         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 874
 875         kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF,
 876                                                                         NULL);
 877         kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0xFFFFFFFF, NULL);
 878
 879         kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL);
 880         kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0xFFFFFFFF, NULL);
 881 }
 882
 883 KBASE_EXPORT_TEST_API(kbase_pm_enable_interrupts);
 884
 885 void kbase_pm_disable_interrupts(struct kbase_device *kbdev)
 886 {
 887         unsigned long flags;
 888
 889         KBASE_DEBUG_ASSERT(NULL != kbdev);
 890         /*
 891          * Mask all interrupts,
 892          * and clear them all.
 893          */
 894         spin_lock_irqsave(&kbdev->pm.power_change_lock, flags);
 895         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), 0, NULL);
 896         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_CLEAR), GPU_IRQ_REG_ALL,
 897                                                                         NULL);
 898         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, flags);
 899
 900         kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_MASK), 0, NULL);
 901         kbase_reg_write(kbdev, JOB_CONTROL_REG(JOB_IRQ_CLEAR), 0xFFFFFFFF,
 902                                                                         NULL);
 903
 904         kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_MASK), 0, NULL);
 905         kbase_reg_write(kbdev, MMU_REG(MMU_IRQ_CLEAR), 0xFFFFFFFF, NULL);
 906 }
 907
 908 KBASE_EXPORT_TEST_API(kbase_pm_disable_interrupts);
 909
 910 /*
 911  * pmu layout:
 912  * 0x0000: PMU TAG (RO) (0xCAFECAFE)
 913  * 0x0004: PMU VERSION ID (RO) (0x00000000)
 914  * 0x0008: CLOCK ENABLE (RW) (31:1 SBZ, 0 CLOCK STATE)
 915  */
 916 void kbase_pm_clock_on(struct kbase_device *kbdev, bool is_resume)
 917 {
 918         bool reset_required = is_resume;
 919         struct kbasep_js_device_data *js_devdata = &kbdev->js_data;
 920         unsigned long flags;
 921         int i;
 922
 923         KBASE_DEBUG_ASSERT(NULL != kbdev);
 924         lockdep_assert_held(&js_devdata->runpool_mutex);
 925         lockdep_assert_held(&kbdev->pm.lock);
 926
 927         if (kbdev->pm.backend.gpu_powered) {
 928                 /* Already turned on */
 929                 if (kbdev->poweroff_pending)
 930                         kbase_pm_enable_interrupts(kbdev);
 931                 kbdev->poweroff_pending = false;
 932                 KBASE_DEBUG_ASSERT(!is_resume);
 933                 return;
 934         }
 935
 936         kbdev->poweroff_pending = false;
 937
 938         KBASE_TRACE_ADD(kbdev, PM_GPU_ON, NULL, NULL, 0u, 0u);
 939
 940         if (is_resume && kbdev->pm.backend.callback_power_resume) {
 941                 kbdev->pm.backend.callback_power_resume(kbdev);
 942                 return;
 943         } else if (kbdev->pm.backend.callback_power_on) {
 944                 kbdev->pm.backend.callback_power_on(kbdev);
 945                 /* If your platform properly keeps the GPU state you may use the
 946                  * return value of the callback_power_on function to
 947                  * conditionally reset the GPU on power up. Currently we are
 948                  * conservative and always reset the GPU. */
 949                 reset_required = true;
 950         }
 951
 952         spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags);
 953         kbdev->pm.backend.gpu_powered = true;
 954         spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags);
 955
 956         if (reset_required) {
 957                 /* GPU state was lost, reset GPU to ensure it is in a
 958                  * consistent state */
 959                 kbase_pm_init_hw(kbdev, PM_ENABLE_IRQS);
 960         }
 961
 962         /* Reprogram the GPU's MMU */
 963         for (i = 0; i < kbdev->nr_hw_address_spaces; i++) {
 964                 struct kbase_as *as = &kbdev->as[i];
 965
 966                 mutex_lock(&as->transaction_mutex);
 967                 spin_lock_irqsave(&js_devdata->runpool_irq.lock, flags);
 968
 969                 if (js_devdata->runpool_irq.per_as_data[i].kctx)
 970                         kbase_mmu_update(
 971                                 js_devdata->runpool_irq.per_as_data[i].kctx);
 972                 else
 973                         kbase_mmu_disable_as(kbdev, i);
 974
 975                 spin_unlock_irqrestore(&js_devdata->runpool_irq.lock, flags);
 976                 mutex_unlock(&as->transaction_mutex);
 977         }
 978
 979         /* Lastly, enable the interrupts */
 980         kbase_pm_enable_interrupts(kbdev);
 981 }
 982
 983 KBASE_EXPORT_TEST_API(kbase_pm_clock_on);
 984
 985 bool kbase_pm_clock_off(struct kbase_device *kbdev, bool is_suspend)
 986 {
 987         unsigned long flags;
 988
 989         KBASE_DEBUG_ASSERT(NULL != kbdev);
 990         lockdep_assert_held(&kbdev->pm.lock);
 991
 992         /* ASSERT that the cores should now be unavailable. No lock needed. */
 993         KBASE_DEBUG_ASSERT(kbdev->shader_available_bitmap == 0u);
 994
 995         kbdev->poweroff_pending = true;
 996
 997         if (!kbdev->pm.backend.gpu_powered) {
 998                 /* Already turned off */
 999                 if (is_suspend && kbdev->pm.backend.callback_power_suspend)
1000                         kbdev->pm.backend.callback_power_suspend(kbdev);
1001                 return true;
1002         }
1003
1004         KBASE_TRACE_ADD(kbdev, PM_GPU_OFF, NULL, NULL, 0u, 0u);
1005
1006         /* Disable interrupts. This also clears any outstanding interrupts */
1007         kbase_pm_disable_interrupts(kbdev);
1008         /* Ensure that any IRQ handlers have finished */
1009         kbase_synchronize_irqs(kbdev);
1010
1011         spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock, flags);
1012
1013         if (atomic_read(&kbdev->faults_pending)) {
1014                 /* Page/bus faults are still being processed. The GPU can not
1015                  * be powered off until they have completed */
1016                 spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock,
1017                                                                         flags);
1018                 return false;
1019         }
1020
1021         kbase_pm_cache_snoop_disable(kbdev);
1022
1023         /* The GPU power may be turned off from this point */
1024         kbdev->pm.backend.gpu_powered = false;
1025         spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock, flags);
1026
1027         if (is_suspend && kbdev->pm.backend.callback_power_suspend)
1028                 kbdev->pm.backend.callback_power_suspend(kbdev);
1029         else if (kbdev->pm.backend.callback_power_off)
1030                 kbdev->pm.backend.callback_power_off(kbdev);
1031         return true;
1032 }
1033
1034 KBASE_EXPORT_TEST_API(kbase_pm_clock_off);
1035
1036 struct kbasep_reset_timeout_data {
1037         struct hrtimer timer;
1038         bool timed_out;
1039         struct kbase_device *kbdev;
1040 };
1041
1042 void kbase_pm_reset_done(struct kbase_device *kbdev)
1043 {
1044         KBASE_DEBUG_ASSERT(kbdev != NULL);
1045         kbdev->pm.backend.reset_done = true;
1046         wake_up(&kbdev->pm.backend.reset_done_wait);
1047 }
1048
1049 /**
1050  * kbase_pm_wait_for_reset - Wait for a reset to happen
1051  *
1052  * Wait for the %RESET_COMPLETED IRQ to occur, then reset the waiting state.
1053  *
1054  * @kbdev: Kbase device
1055  */
1056 static void kbase_pm_wait_for_reset(struct kbase_device *kbdev)
1057 {
1058         lockdep_assert_held(&kbdev->pm.lock);
1059
1060         wait_event(kbdev->pm.backend.reset_done_wait,
1061                                                 (kbdev->pm.backend.reset_done));
1062         kbdev->pm.backend.reset_done = false;
1063 }
1064
1065 KBASE_EXPORT_TEST_API(kbase_pm_reset_done);
1066
1067 static enum hrtimer_restart kbasep_reset_timeout(struct hrtimer *timer)
1068 {
1069         struct kbasep_reset_timeout_data *rtdata =
1070                 container_of(timer, struct kbasep_reset_timeout_data, timer);
1071
1072         rtdata->timed_out = 1;
1073
1074         /* Set the wait queue to wake up kbase_pm_init_hw even though the reset
1075          * hasn't completed */
1076         kbase_pm_reset_done(rtdata->kbdev);
1077
1078         return HRTIMER_NORESTART;
1079 }
1080
1081 static void kbase_pm_hw_issues_detect(struct kbase_device *kbdev)
1082 {
1083         struct device_node *np = kbdev->dev->of_node;
1084         u32 jm_values[4];
1085         const u32 gpu_id = kbdev->gpu_props.props.raw_props.gpu_id;
1086         const u32 prod_id = (gpu_id & GPU_ID_VERSION_PRODUCT_ID) >>
1087                 GPU_ID_VERSION_PRODUCT_ID_SHIFT;
1088         const u32 major = (gpu_id & GPU_ID_VERSION_MAJOR) >>
1089                 GPU_ID_VERSION_MAJOR_SHIFT;
1090
1091         kbdev->hw_quirks_sc = 0;
1092
1093         /* Needed due to MIDBASE-1494: LS_PAUSEBUFFER_DISABLE. See PRLAM-8443.
1094          * and needed due to MIDGLES-3539. See PRLAM-11035 */
1095         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_8443) ||
1096                         kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_11035))
1097                 kbdev->hw_quirks_sc |= SC_LS_PAUSEBUFFER_DISABLE;
1098
1099         /* Needed due to MIDBASE-2054: SDC_DISABLE_OQ_DISCARD. See PRLAM-10327.
1100          */
1101         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10327))
1102                 kbdev->hw_quirks_sc |= SC_SDC_DISABLE_OQ_DISCARD;
1103
1104 #ifdef CONFIG_MALI_PRFCNT_SET_SECONDARY
1105         /* Enable alternative hardware counter selection if configured. */
1106         if (!GPU_ID_IS_NEW_FORMAT(prod_id))
1107                 kbdev->hw_quirks_sc |= SC_ALT_COUNTERS;
1108 #endif
1109
1110         /* Needed due to MIDBASE-2795. ENABLE_TEXGRD_FLAGS. See PRLAM-10797. */
1111         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_10797))
1112                 kbdev->hw_quirks_sc |= SC_ENABLE_TEXGRD_FLAGS;
1113
1114         if (!kbase_hw_has_issue(kbdev, GPUCORE_1619)) {
1115                 if (prod_id < 0x750 || prod_id == 0x6956) /* T60x, T62x, T72x */
1116                         kbdev->hw_quirks_sc |= SC_LS_ATTR_CHECK_DISABLE;
1117                 else if (prod_id >= 0x750 && prod_id <= 0x880) /* T76x, T8xx */
1118                         kbdev->hw_quirks_sc |= SC_LS_ALLOW_ATTR_TYPES;
1119         }
1120
1121         kbdev->hw_quirks_tiler = kbase_reg_read(kbdev,
1122                         GPU_CONTROL_REG(TILER_CONFIG), NULL);
1123
1124         /* Set tiler clock gate override if required */
1125         if (kbase_hw_has_issue(kbdev, BASE_HW_ISSUE_T76X_3953))
1126                 kbdev->hw_quirks_tiler |= TC_CLOCK_GATE_OVERRIDE;
1127
1128         /* Limit the GPU bus bandwidth if the platform needs this. */
1129         kbdev->hw_quirks_mmu = kbase_reg_read(kbdev,
1130                         GPU_CONTROL_REG(L2_MMU_CONFIG), NULL);
1131
1132         /* Limit read ID width for AXI */
1133         kbdev->hw_quirks_mmu &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_READS);
1134         kbdev->hw_quirks_mmu |= (DEFAULT_ARID_LIMIT & 0x3) <<
1135                                 L2_MMU_CONFIG_LIMIT_EXTERNAL_READS_SHIFT;
1136
1137         /* Limit write ID width for AXI */
1138         kbdev->hw_quirks_mmu &= ~(L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES);
1139         kbdev->hw_quirks_mmu |= (DEFAULT_AWID_LIMIT & 0x3) <<
1140                                 L2_MMU_CONFIG_LIMIT_EXTERNAL_WRITES_SHIFT;
1141
1142         if (kbdev->system_coherency == COHERENCY_ACE) {
1143                 /* Allow memory configuration disparity to be ignored, we
1144                  * optimize the use of shared memory and thus we expect
1145                  * some disparity in the memory configuration */
1146                 kbdev->hw_quirks_mmu |= L2_MMU_CONFIG_ALLOW_SNOOP_DISPARITY;
1147         }
1148
1149         /* Only for T86x/T88x-based products after r2p0 */
1150         if (prod_id >= 0x860 && prod_id <= 0x880 && major >= 2) {
1151                 /* The JM_CONFIG register is specified as follows in the
1152                  T86x/T88x Engineering Specification Supplement:
1153                  The values are read from device tree in order.
1154                 */
1155 #define TIMESTAMP_OVERRIDE  1
1156 #define CLOCK_GATE_OVERRIDE (1<<1)
1157 #define JOB_THROTTLE_ENABLE (1<<2)
1158 #define JOB_THROTTLE_LIMIT_SHIFT 3
1159
1160                 /* 6 bits in the register */
1161                 const u32 jm_max_limit = 0x3F;
1162
1163                 if (of_property_read_u32_array(np,
1164                                         "jm_config",
1165                                         &jm_values[0],
1166                                         ARRAY_SIZE(jm_values))) {
1167                         /* Entry not in device tree, use defaults  */
1168                         jm_values[0] = 0;
1169                         jm_values[1] = 0;
1170                         jm_values[2] = 0;
1171                         jm_values[3] = jm_max_limit; /* Max value */
1172                 }
1173
1174                 /* Limit throttle limit to 6 bits*/
1175                 if (jm_values[3] > jm_max_limit) {
1176                         dev_dbg(kbdev->dev, "JOB_THROTTLE_LIMIT supplied in device tree is too large. Limiting to MAX (63).");
1177                         jm_values[3] = jm_max_limit;
1178                 }
1179
1180                 /* Aggregate to one integer. */
1181                 kbdev->hw_quirks_jm = (jm_values[0] ? TIMESTAMP_OVERRIDE : 0);
1182                 kbdev->hw_quirks_jm |= (jm_values[1] ? CLOCK_GATE_OVERRIDE : 0);
1183                 kbdev->hw_quirks_jm |= (jm_values[2] ? JOB_THROTTLE_ENABLE : 0);
1184                 kbdev->hw_quirks_jm |= (jm_values[3] <<
1185                                 JOB_THROTTLE_LIMIT_SHIFT);
1186         } else {
1187                 kbdev->hw_quirks_jm = KBASE_JM_CONFIG_UNUSED;
1188         }
1189
1190
1191 }
1192
1193 static void kbase_pm_hw_issues_apply(struct kbase_device *kbdev)
1194 {
1195         if (kbdev->hw_quirks_sc)
1196                 kbase_reg_write(kbdev, GPU_CONTROL_REG(SHADER_CONFIG),
1197                                 kbdev->hw_quirks_sc, NULL);
1198
1199         kbase_reg_write(kbdev, GPU_CONTROL_REG(TILER_CONFIG),
1200                         kbdev->hw_quirks_tiler, NULL);
1201
1202         kbase_reg_write(kbdev, GPU_CONTROL_REG(L2_MMU_CONFIG),
1203                         kbdev->hw_quirks_mmu, NULL);
1204
1205
1206         if (kbdev->hw_quirks_jm != KBASE_JM_CONFIG_UNUSED)
1207                 kbase_reg_write(kbdev, GPU_CONTROL_REG(JM_CONFIG),
1208                                 kbdev->hw_quirks_jm, NULL);
1209
1210 }
1211
1212 void kbase_pm_cache_snoop_enable(struct kbase_device *kbdev)
1213 {
1214         if ((kbdev->system_coherency == COHERENCY_ACE) &&
1215                 !kbdev->cci_snoop_enabled) {
1216 #ifdef CONFIG_ARM64
1217                 if (kbdev->snoop_enable_smc != 0)
1218                         kbase_invoke_smc_fid(kbdev->snoop_enable_smc, 0, 0, 0);
1219 #endif /* CONFIG_ARM64 */
1220                 dev_dbg(kbdev->dev, "MALI - CCI Snoops - Enabled\n");
1221                 kbdev->cci_snoop_enabled = true;
1222         }
1223 }
1224
1225 void kbase_pm_cache_snoop_disable(struct kbase_device *kbdev)
1226 {
1227         if ((kbdev->system_coherency == COHERENCY_ACE) &&
1228                 kbdev->cci_snoop_enabled) {
1229 #ifdef CONFIG_ARM64
1230                 if (kbdev->snoop_disable_smc != 0) {
1231                         mali_cci_flush_l2(kbdev);
1232                         kbase_invoke_smc_fid(kbdev->snoop_disable_smc, 0, 0, 0);
1233                 }
1234 #endif /* CONFIG_ARM64 */
1235                 dev_dbg(kbdev->dev, "MALI - CCI Snoops Disabled\n");
1236                 kbdev->cci_snoop_enabled = false;
1237         }
1238 }
1239
1240 static int kbase_pm_reset_do_normal(struct kbase_device *kbdev)
1241 {
1242         struct kbasep_reset_timeout_data rtdata;
1243
1244         KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0);
1245
1246         kbase_tlstream_jd_gpu_soft_reset(kbdev);
1247
1248         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1249                                                 GPU_COMMAND_SOFT_RESET, NULL);
1250
1251         /* Unmask the reset complete interrupt only */
1252         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_IRQ_MASK), RESET_COMPLETED,
1253                                                                         NULL);
1254
1255         /* Initialize a structure for tracking the status of the reset */
1256         rtdata.kbdev = kbdev;
1257         rtdata.timed_out = 0;
1258
1259         /* Create a timer to use as a timeout on the reset */
1260         hrtimer_init_on_stack(&rtdata.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1261         rtdata.timer.function = kbasep_reset_timeout;
1262
1263         hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT),
1264                                                         HRTIMER_MODE_REL);
1265
1266         /* Wait for the RESET_COMPLETED interrupt to be raised */
1267         kbase_pm_wait_for_reset(kbdev);
1268
1269         if (rtdata.timed_out == 0) {
1270                 /* GPU has been reset */
1271                 hrtimer_cancel(&rtdata.timer);
1272                 destroy_hrtimer_on_stack(&rtdata.timer);
1273                 return 0;
1274         }
1275
1276         /* No interrupt has been received - check if the RAWSTAT register says
1277          * the reset has completed */
1278         if (kbase_reg_read(kbdev, GPU_CONTROL_REG(GPU_IRQ_RAWSTAT), NULL) &
1279                                                         RESET_COMPLETED) {
1280                 /* The interrupt is set in the RAWSTAT; this suggests that the
1281                  * interrupts are not getting to the CPU */
1282                 dev_err(kbdev->dev, "Reset interrupt didn't reach CPU. Check interrupt assignments.\n");
1283                 /* If interrupts aren't working we can't continue. */
1284                 destroy_hrtimer_on_stack(&rtdata.timer);
1285                 return -EINVAL;
1286         }
1287
1288         /* The GPU doesn't seem to be responding to the reset so try a hard
1289          * reset */
1290         dev_err(kbdev->dev, "Failed to soft-reset GPU (timed out after %d ms), now attempting a hard reset\n",
1291                                                                 RESET_TIMEOUT);
1292         KBASE_TRACE_ADD(kbdev, CORE_GPU_HARD_RESET, NULL, NULL, 0u, 0);
1293         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1294                                                 GPU_COMMAND_HARD_RESET, NULL);
1295
1296         /* Restart the timer to wait for the hard reset to complete */
1297         rtdata.timed_out = 0;
1298
1299         hrtimer_start(&rtdata.timer, HR_TIMER_DELAY_MSEC(RESET_TIMEOUT),
1300                                                         HRTIMER_MODE_REL);
1301
1302         /* Wait for the RESET_COMPLETED interrupt to be raised */
1303         kbase_pm_wait_for_reset(kbdev);
1304
1305         if (rtdata.timed_out == 0) {
1306                 /* GPU has been reset */
1307                 hrtimer_cancel(&rtdata.timer);
1308                 destroy_hrtimer_on_stack(&rtdata.timer);
1309                 return 0;
1310         }
1311
1312         destroy_hrtimer_on_stack(&rtdata.timer);
1313
1314         dev_err(kbdev->dev, "Failed to hard-reset the GPU (timed out after %d ms)\n",
1315                                                                 RESET_TIMEOUT);
1316
1317         return -EINVAL;
1318 }
1319
1320 static int kbase_pm_reset_do_protected(struct kbase_device *kbdev)
1321 {
1322         KBASE_TRACE_ADD(kbdev, CORE_GPU_SOFT_RESET, NULL, NULL, 0u, 0);
1323         kbase_tlstream_jd_gpu_soft_reset(kbdev);
1324
1325         return kbdev->protected_ops->protected_mode_reset(kbdev);
1326 }
1327
1328 int kbase_pm_init_hw(struct kbase_device *kbdev, unsigned int flags)
1329 {
1330         unsigned long irq_flags;
1331         int err;
1332         bool resume_vinstr = false;
1333
1334         KBASE_DEBUG_ASSERT(NULL != kbdev);
1335         lockdep_assert_held(&kbdev->pm.lock);
1336
1337         /* Ensure the clock is on before attempting to access the hardware */
1338         if (!kbdev->pm.backend.gpu_powered) {
1339                 if (kbdev->pm.backend.callback_power_on)
1340                         kbdev->pm.backend.callback_power_on(kbdev);
1341
1342                 spin_lock_irqsave(&kbdev->pm.backend.gpu_powered_lock,
1343                                                                 irq_flags);
1344                 kbdev->pm.backend.gpu_powered = true;
1345                 spin_unlock_irqrestore(&kbdev->pm.backend.gpu_powered_lock,
1346                                                                 irq_flags);
1347         }
1348
1349         /* Ensure interrupts are off to begin with, this also clears any
1350          * outstanding interrupts */
1351         kbase_pm_disable_interrupts(kbdev);
1352         /* Ensure cache snoops are disabled before reset. */
1353         kbase_pm_cache_snoop_disable(kbdev);
1354         /* Prepare for the soft-reset */
1355         kbdev->pm.backend.reset_done = false;
1356
1357         /* The cores should be made unavailable due to the reset */
1358         spin_lock_irqsave(&kbdev->pm.power_change_lock, irq_flags);
1359         if (kbdev->shader_available_bitmap != 0u)
1360                         KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE, NULL,
1361                                                 NULL, 0u, (u32)0u);
1362         if (kbdev->tiler_available_bitmap != 0u)
1363                         KBASE_TRACE_ADD(kbdev, PM_CORES_CHANGE_AVAILABLE_TILER,
1364                                                 NULL, NULL, 0u, (u32)0u);
1365         kbdev->shader_available_bitmap = 0u;
1366         kbdev->tiler_available_bitmap = 0u;
1367         kbdev->l2_available_bitmap = 0u;
1368         spin_unlock_irqrestore(&kbdev->pm.power_change_lock, irq_flags);
1369
1370         /* Soft reset the GPU */
1371         if (kbdev->protected_mode_support &&
1372                         kbdev->protected_ops->protected_mode_reset)
1373                 err = kbase_pm_reset_do_protected(kbdev);
1374         else
1375                 err = kbase_pm_reset_do_normal(kbdev);
1376
1377         spin_lock_irqsave(&kbdev->js_data.runpool_irq.lock, irq_flags);
1378         if (kbdev->protected_mode)
1379                 resume_vinstr = true;
1380         kbdev->protected_mode_transition = false;
1381         kbdev->protected_mode = false;
1382         spin_unlock_irqrestore(&kbdev->js_data.runpool_irq.lock, irq_flags);
1383
1384         if (err)
1385                 goto exit;
1386
1387         if (flags & PM_HW_ISSUES_DETECT)
1388                 kbase_pm_hw_issues_detect(kbdev);
1389
1390         kbase_pm_hw_issues_apply(kbdev);
1391
1392         kbase_cache_set_coherency_mode(kbdev, kbdev->system_coherency);
1393
1394         /* Sanity check protected mode was left after reset */
1395         if (kbase_hw_has_feature(kbdev, BASE_HW_FEATURE_PROTECTED_MODE)) {
1396                 u32 gpu_status = kbase_reg_read(kbdev,
1397                                 GPU_CONTROL_REG(GPU_STATUS), NULL);
1398
1399                 WARN_ON(gpu_status & GPU_STATUS_PROTECTED_MODE_ACTIVE);
1400         }
1401
1402         /* If cycle counter was in use re-enable it, enable_irqs will only be
1403          * false when called from kbase_pm_powerup */
1404         if (kbdev->pm.backend.gpu_cycle_counter_requests &&
1405                                                 (flags & PM_ENABLE_IRQS)) {
1406                 /* enable interrupts as the L2 may have to be powered on */
1407                 kbase_pm_enable_interrupts(kbdev);
1408                 kbase_pm_request_l2_caches(kbdev);
1409
1410                 /* Re-enable the counters if we need to */
1411                 spin_lock_irqsave(
1412                         &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
1413                                                                 irq_flags);
1414                 if (kbdev->pm.backend.gpu_cycle_counter_requests)
1415                         kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1416                                         GPU_COMMAND_CYCLE_COUNT_START, NULL);
1417                 spin_unlock_irqrestore(
1418                         &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
1419                                                                 irq_flags);
1420
1421                 kbase_pm_release_l2_caches(kbdev);
1422                 kbase_pm_disable_interrupts(kbdev);
1423         }
1424
1425         if (flags & PM_ENABLE_IRQS)
1426                 kbase_pm_enable_interrupts(kbdev);
1427
1428 exit:
1429         /* If GPU is leaving protected mode resume vinstr operation. */
1430         if (kbdev->vinstr_ctx && resume_vinstr)
1431                 kbase_vinstr_resume(kbdev->vinstr_ctx);
1432
1433         return err;
1434 }
1435
1436 /**
1437  * kbase_pm_request_gpu_cycle_counter_do_request - Request cycle counters
1438  *
1439  * Increase the count of cycle counter users and turn the cycle counters on if
1440  * they were previously off
1441  *
1442  * This function is designed to be called by
1443  * kbase_pm_request_gpu_cycle_counter() or
1444  * kbase_pm_request_gpu_cycle_counter_l2_is_on() only
1445  *
1446  * When this function is called the l2 cache must be on and the l2 cache users
1447  * count must have been incremented by a call to (
1448  * kbase_pm_request_l2_caches() or kbase_pm_request_l2_caches_l2_on() )
1449  *
1450  * @kbdev:     The kbase device structure of the device
1451  */
1452 static void
1453 kbase_pm_request_gpu_cycle_counter_do_request(struct kbase_device *kbdev)
1454 {
1455         unsigned long flags;
1456
1457         spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock,
1458                                                                         flags);
1459
1460         ++kbdev->pm.backend.gpu_cycle_counter_requests;
1461
1462         if (1 == kbdev->pm.backend.gpu_cycle_counter_requests)
1463                 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1464                                         GPU_COMMAND_CYCLE_COUNT_START, NULL);
1465
1466         spin_unlock_irqrestore(
1467                         &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
1468                                                                         flags);
1469 }
1470
1471 void kbase_pm_request_gpu_cycle_counter(struct kbase_device *kbdev)
1472 {
1473         KBASE_DEBUG_ASSERT(kbdev != NULL);
1474
1475         KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered);
1476
1477         KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests <
1478                                                                 INT_MAX);
1479
1480         kbase_pm_request_l2_caches(kbdev);
1481
1482         kbase_pm_request_gpu_cycle_counter_do_request(kbdev);
1483 }
1484
1485 KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter);
1486
1487 void kbase_pm_request_gpu_cycle_counter_l2_is_on(struct kbase_device *kbdev)
1488 {
1489         KBASE_DEBUG_ASSERT(kbdev != NULL);
1490
1491         KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_powered);
1492
1493         KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests <
1494                                                                 INT_MAX);
1495
1496         kbase_pm_request_l2_caches_l2_is_on(kbdev);
1497
1498         kbase_pm_request_gpu_cycle_counter_do_request(kbdev);
1499 }
1500
1501 KBASE_EXPORT_TEST_API(kbase_pm_request_gpu_cycle_counter_l2_is_on);
1502
1503 void kbase_pm_release_gpu_cycle_counter(struct kbase_device *kbdev)
1504 {
1505         unsigned long flags;
1506
1507         KBASE_DEBUG_ASSERT(kbdev != NULL);
1508
1509         spin_lock_irqsave(&kbdev->pm.backend.gpu_cycle_counter_requests_lock,
1510                                                                         flags);
1511
1512         KBASE_DEBUG_ASSERT(kbdev->pm.backend.gpu_cycle_counter_requests > 0);
1513
1514         --kbdev->pm.backend.gpu_cycle_counter_requests;
1515
1516         if (0 == kbdev->pm.backend.gpu_cycle_counter_requests)
1517                 kbase_reg_write(kbdev, GPU_CONTROL_REG(GPU_COMMAND),
1518                                         GPU_COMMAND_CYCLE_COUNT_STOP, NULL);
1519
1520         spin_unlock_irqrestore(
1521                         &kbdev->pm.backend.gpu_cycle_counter_requests_lock,
1522                                                                         flags);
1523
1524         kbase_pm_release_l2_caches(kbdev);
1525 }
1526
1527 KBASE_EXPORT_TEST_API(kbase_pm_release_gpu_cycle_counter);