drivers/gpu/rogue_m/services/server/devices/rgx/rgxpower.c

   1 /*************************************************************************/ /*!
   2 @File
   3 @Title          Device specific power routines
   4 @Copyright      Copyright (c) Imagination Technologies Ltd. All Rights Reserved
   5 @Description    Device specific functions
   6 @License        Dual MIT/GPLv2
   7
   8 The contents of this file are subject to the MIT license as set out below.
   9
  10 Permission is hereby granted, free of charge, to any person obtaining a copy
  11 of this software and associated documentation files (the "Software"), to deal
  12 in the Software without restriction, including without limitation the rights
  13 to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  14 copies of the Software, and to permit persons to whom the Software is
  15 furnished to do so, subject to the following conditions:
  16
  17 The above copyright notice and this permission notice shall be included in
  18 all copies or substantial portions of the Software.
  19
  20 Alternatively, the contents of this file may be used under the terms of
  21 the GNU General Public License Version 2 ("GPL") in which case the provisions
  22 of GPL are applicable instead of those above.
  23
  24 If you wish to allow use of your version of this file only under the terms of
  25 GPL, and not to allow others to use your version of this file under the terms
  26 of the MIT license, indicate your decision by deleting the provisions above
  27 and replace them with the notice and other provisions required by GPL as set
  28 out in the file called "GPL-COPYING" included in this distribution. If you do
  29 not delete the provisions above, a recipient may use your version of this file
  30 under the terms of either the MIT license or GPL.
  31
  32 This License is also included in this distribution in the file called
  33 "MIT-COPYING".
  34
  35 EXCEPT AS OTHERWISE STATED IN A NEGOTIATED AGREEMENT: (A) THE SOFTWARE IS
  36 PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
  37 BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  38 PURPOSE AND NONINFRINGEMENT; AND (B) IN NO EVENT SHALL THE AUTHORS OR
  39 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
  40 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  41 CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  42 */ /**************************************************************************/
  43
  44 #include <stddef.h>
  45
  46 #include "rgxpower.h"
  47 #include "rgxinit.h"
  48 #include "rgx_fwif_km.h"
  49 #include "rgxutils.h"
  50 #include "rgxfwutils.h"
  51 #include "pdump_km.h"
  52 #include "rgxdefs_km.h"
  53 #include "pvrsrv.h"
  54 #include "pvr_debug.h"
  55 #include "osfunc.h"
  56 #include "rgxdebug.h"
  57 #include "rgx_meta.h"
  58 #include "devicemem_pdump.h"
  59 #include "rgxapi_km.h"
  60 #include "rgxtimecorr.h"
  61
  62 #include "process_stats.h"
  63
  64 extern IMG_UINT32 g_ui32HostSampleIRQCount;
  65
  66 #if ! defined(FIX_HW_BRN_37453)
  67 /*!
  68 *******************************************************************************
  69
  70  @Function      RGXEnableClocks
  71
  72  @Description Enable RGX Clocks
  73
  74  @Input psDevInfo - device info structure
  75
  76  @Return   IMG_VOID
  77
  78 ******************************************************************************/
  79 static IMG_VOID RGXEnableClocks(PVRSRV_RGXDEV_INFO      *psDevInfo)
  80 {
  81         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGX clock: use default (automatic clock gating)");
  82 }
  83 #endif
  84 #if defined(PVR_DVFS)
  85 #include "pvr_dvfs_device.h"
  86 #endif
  87
  88
  89 /*!
  90 *******************************************************************************
  91
  92  @Function      _RGXInitSLC
  93
  94  @Description Initialise RGX SLC
  95
  96  @Input psDevInfo - device info structure
  97
  98  @Return   IMG_VOID
  99
 100 ******************************************************************************/
 101 #if !defined(RGX_FEATURE_S7_CACHE_HIERARCHY)
 102
 103 #define RGX_INIT_SLC _RGXInitSLC
 104
 105 static IMG_VOID _RGXInitSLC(PVRSRV_RGXDEV_INFO  *psDevInfo)
 106 {
 107         IMG_UINT32      ui32Reg;
 108         IMG_UINT32      ui32RegVal;
 109
 110 #if defined(FIX_HW_BRN_36492)
 111         /* Because the WA for this BRN forbids using SLC reset, need to inval it instead */
 112         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "Invalidate the SLC");
 113         OSWriteHWReg32(psDevInfo->pvRegsBaseKM,
 114                         RGX_CR_SLC_CTRL_FLUSH_INVAL, RGX_CR_SLC_CTRL_FLUSH_INVAL_ALL_EN);
 115         PDUMPREG32(RGX_PDUMPREG_NAME,
 116                         RGX_CR_SLC_CTRL_FLUSH_INVAL, RGX_CR_SLC_CTRL_FLUSH_INVAL_ALL_EN,
 117                         PDUMP_FLAGS_CONTINUOUS);
 118
 119         /* poll for completion */
 120         PVRSRVPollForValueKM((IMG_UINT32 *)((IMG_UINT8*)psDevInfo->pvRegsBaseKM + RGX_CR_SLC_STATUS0),
 121                                                          0x0,
 122                                                          RGX_CR_SLC_STATUS0_INVAL_PENDING_EN);
 123
 124         PDUMPREGPOL(RGX_PDUMPREG_NAME,
 125                                 RGX_CR_SLC_STATUS0,
 126                                 0x0,
 127                                 RGX_CR_SLC_STATUS0_INVAL_PENDING_EN,
 128                                 PDUMP_FLAGS_CONTINUOUS,
 129                                 PDUMP_POLL_OPERATOR_EQUAL);
 130 #endif
 131
 132         if (!PVRSRVSystemSnoopingOfCPUCache() && !PVRSRVSystemSnoopingOfDeviceCache())
 133         {
 134                 PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "System has NO cache snooping");
 135         }
 136         else
 137         {
 138                 if (PVRSRVSystemSnoopingOfCPUCache())
 139                 {
 140                         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "System has CPU cache snooping");
 141                 }
 142                 if (PVRSRVSystemSnoopingOfDeviceCache())
 143                 {
 144                         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "System has DEVICE cache snooping");
 145                 }
 146         }
 147
 148 #if (RGX_FEATURE_SLC_SIZE_IN_BYTES < (128*1024))
 149         /*
 150          * SLC Bypass control
 151          */
 152         ui32Reg = RGX_CR_SLC_CTRL_BYPASS;
 153
 154         /* Bypass SLC for textures if the SLC size is less than 128kB */
 155         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "Bypass SLC for TPU");
 156         ui32RegVal = RGX_CR_SLC_CTRL_BYPASS_REQ_TPU_EN;
 157
 158         OSWriteHWReg32(psDevInfo->pvRegsBaseKM, ui32Reg, ui32RegVal);
 159         PDUMPREG32(RGX_PDUMPREG_NAME, ui32Reg, ui32RegVal, PDUMP_FLAGS_CONTINUOUS);
 160 #endif
 161
 162         /*
 163          * SLC Bypass control
 164          */
 165         ui32Reg = RGX_CR_SLC_CTRL_MISC;
 166         ui32RegVal = RGX_CR_SLC_CTRL_MISC_ADDR_DECODE_MODE_PVR_HASH1;
 167
 168         /* Bypass burst combiner if SLC line size is smaller than 1024 bits */
 169 #if (RGX_FEATURE_SLC_CACHE_LINE_SIZE_BITS < 1024)
 170         ui32RegVal |= RGX_CR_SLC_CTRL_MISC_BYPASS_BURST_COMBINER_EN;
 171 #endif
 172         OSWriteHWReg32(psDevInfo->pvRegsBaseKM, ui32Reg, ui32RegVal);
 173         PDUMPREG32(RGX_PDUMPREG_NAME, ui32Reg, ui32RegVal, PDUMP_FLAGS_CONTINUOUS);
 174
 175 }
 176 #endif /* RGX_FEATURE_S7_CACHE_HIERARCHY */
 177
 178
 179 /*!
 180 *******************************************************************************
 181
 182  @Function      RGXInitSLC3
 183
 184  @Description Initialise RGX SLC3
 185
 186  @Input psDevInfo - device info structure
 187
 188  @Return   IMG_VOID
 189
 190 ******************************************************************************/
 191 #if defined(RGX_FEATURE_S7_CACHE_HIERARCHY)
 192
 193 #define RGX_INIT_SLC _RGXInitSLC3
 194
 195 static IMG_VOID _RGXInitSLC3(PVRSRV_RGXDEV_INFO *psDevInfo)
 196 {
 197         IMG_UINT32      ui32Reg;
 198         IMG_UINT32      ui32RegVal;
 199
 200
 201 #if defined(HW_ERN_51468)
 202     /*
 203      * SLC control
 204      */
 205         ui32Reg = RGX_CR_SLC3_CTRL_MISC;
 206         ui32RegVal = RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_WEAVED_HASH;
 207         OSWriteHWReg32(psDevInfo->pvRegsBaseKM, ui32Reg, ui32RegVal);
 208         PDUMPREG32(RGX_PDUMPREG_NAME, ui32Reg, ui32RegVal, PDUMP_FLAGS_CONTINUOUS);
 209
 210 #else
 211
 212     /*
 213      * SLC control
 214      */
 215         ui32Reg = RGX_CR_SLC3_CTRL_MISC;
 216         ui32RegVal = RGX_CR_SLC3_CTRL_MISC_ADDR_DECODE_MODE_SCRAMBLE_PVR_HASH;
 217         OSWriteHWReg32(psDevInfo->pvRegsBaseKM, ui32Reg, ui32RegVal);
 218         PDUMPREG32(RGX_PDUMPREG_NAME, ui32Reg, ui32RegVal, PDUMP_FLAGS_CONTINUOUS);
 219
 220         /*
 221          * SLC scramble bits
 222          */
 223         {
 224                 IMG_UINT32 i;
 225                 IMG_UINT32 aui32ScrambleRegs[] = {
 226                     RGX_CR_SLC3_SCRAMBLE,
 227                     RGX_CR_SLC3_SCRAMBLE2,
 228                     RGX_CR_SLC3_SCRAMBLE3,
 229                     RGX_CR_SLC3_SCRAMBLE4};
 230
 231                 IMG_UINT64 aui64ScrambleValues[] = {
 232 #if (RGX_FEATURE_SLC_BANKS == 2)
 233                    IMG_UINT64_C(0x6965a99a55696a6a),
 234                    IMG_UINT64_C(0x6aa9aa66959aaa9a),
 235                    IMG_UINT64_C(0x9a5665965a99a566),
 236                    IMG_UINT64_C(0x5aa69596aa66669a)
 237 #elif (RGX_FEATURE_SLC_BANKS == 4)
 238                    IMG_UINT64_C(0xc6788d722dd29ce4),
 239                    IMG_UINT64_C(0x7272e4e11b279372),
 240                    IMG_UINT64_C(0x87d872d26c6c4be1),
 241                    IMG_UINT64_C(0xe1b4878d4b36e478)
 242 #elif (RGX_FEATURE_SLC_BANKS == 8)
 243                    IMG_UINT64_C(0x859d6569e8fac688),
 244                    IMG_UINT64_C(0xf285e1eae4299d33),
 245                    IMG_UINT64_C(0x1e1af2be3c0aa447)
 246 #endif
 247                 };
 248
 249                 for (i = 0;
 250                      i < sizeof(aui64ScrambleValues)/sizeof(IMG_UINT64);
 251                          i++)
 252                 {
 253                         IMG_UINT32 ui32Reg = aui32ScrambleRegs[i];
 254                         IMG_UINT64 ui64Value = aui64ScrambleValues[i];
 255
 256                         OSWriteHWReg64(psDevInfo->pvRegsBaseKM, ui32Reg, ui64Value);
 257                         PDUMPREG64(RGX_PDUMPREG_NAME, ui32Reg, ui64Value, PDUMP_FLAGS_CONTINUOUS);
 258                 }
 259         }
 260 #endif
 261
 262 #if defined(HW_ERN_45914)
 263         /* Disable the forced SLC coherency which the hardware enables for compatibility with older pdumps */
 264         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXStart: disable forced SLC coherency");
 265         OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_GARTEN_SLC, 0);
 266         PDUMPREG64(RGX_PDUMPREG_NAME, RGX_CR_GARTEN_SLC, 0, PDUMP_FLAGS_CONTINUOUS);
 267 #endif
 268
 269 }
 270 #endif
 271
 272
 273 /*!
 274 *******************************************************************************
 275
 276  @Function      RGXInitBIF
 277
 278  @Description Initialise RGX BIF
 279
 280  @Input psDevInfo - device info structure
 281
 282  @Return   IMG_VOID
 283
 284 ******************************************************************************/
 285 static IMG_VOID RGXInitBIF(PVRSRV_RGXDEV_INFO   *psDevInfo)
 286 {
 287         PVRSRV_ERROR    eError;
 288         IMG_DEV_PHYADDR sPCAddr;
 289
 290         /*
 291                 Acquire the address of the Kernel Page Catalogue.
 292         */
 293         eError = MMU_AcquireBaseAddr(psDevInfo->psKernelMMUCtx, &sPCAddr);
 294         PVR_ASSERT(eError == PVRSRV_OK);
 295
 296         /* Sanity check Cat-Base address */
 297         PVR_ASSERT((((sPCAddr.uiAddr
 298                         >> psDevInfo->ui32KernelCatBaseAlignShift)
 299                         << psDevInfo->ui32KernelCatBaseShift)
 300                         & ~psDevInfo->ui64KernelCatBaseMask) == 0x0UL);
 301
 302         /*
 303                 Write the kernel catalogue base.
 304         */
 305         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGX firmware MMU Page Catalogue");
 306
 307         if (psDevInfo->ui32KernelCatBaseIdReg != -1)
 308         {
 309                 /* Set the mapping index */
 310                 OSWriteHWReg32(psDevInfo->pvRegsBaseKM,
 311                                                 psDevInfo->ui32KernelCatBaseIdReg,
 312                                                 psDevInfo->ui32KernelCatBaseId);
 313
 314                 /* pdump mapping context */
 315                 PDUMPREG32(RGX_PDUMPREG_NAME,
 316                                                         psDevInfo->ui32KernelCatBaseIdReg,
 317                                                         psDevInfo->ui32KernelCatBaseId,
 318                                                         PDUMP_FLAGS_CONTINUOUS);
 319         }
 320
 321         if (psDevInfo->ui32KernelCatBaseWordSize == 8)
 322         {
 323                 /* Write the cat-base address */
 324                 OSWriteHWReg64(psDevInfo->pvRegsBaseKM,
 325                                                 psDevInfo->ui32KernelCatBaseReg,
 326                                                 ((sPCAddr.uiAddr
 327                                                         >> psDevInfo->ui32KernelCatBaseAlignShift)
 328                                                         << psDevInfo->ui32KernelCatBaseShift)
 329                                                         & psDevInfo->ui64KernelCatBaseMask);
 330         }
 331         else
 332         {
 333                 /* Write the cat-base address */
 334                 OSWriteHWReg32(psDevInfo->pvRegsBaseKM,
 335                                                 psDevInfo->ui32KernelCatBaseReg,
 336                                                 (IMG_UINT32)(((sPCAddr.uiAddr
 337                                                         >> psDevInfo->ui32KernelCatBaseAlignShift)
 338                                                         << psDevInfo->ui32KernelCatBaseShift)
 339                                                         & psDevInfo->ui64KernelCatBaseMask));
 340         }
 341
 342         /* pdump catbase address */
 343         MMU_PDumpWritePageCatBase(psDevInfo->psKernelMMUCtx,
 344                                                           RGX_PDUMPREG_NAME,
 345                                                           psDevInfo->ui32KernelCatBaseReg,
 346                                                           psDevInfo->ui32KernelCatBaseWordSize,
 347                                                           psDevInfo->ui32KernelCatBaseAlignShift,
 348                                                           psDevInfo->ui32KernelCatBaseShift,
 349                                                           PDUMP_FLAGS_CONTINUOUS);
 350
 351         /*
 352          * Trusted META boot
 353          */
 354 #if defined(SUPPORT_TRUSTED_DEVICE)
 355         #if defined(TRUSTED_DEVICE_DEFAULT_ENABLED)
 356         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXInitBIF: Trusted Device enabled");
 357         OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_TRUST, RGX_CR_BIF_TRUST_ENABLE_EN);
 358         PDUMPREG32(RGX_PDUMPREG_NAME, RGX_CR_BIF_TRUST, RGX_CR_BIF_TRUST_ENABLE_EN, PDUMP_FLAGS_CONTINUOUS);
 359         OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_SYS_BUS_SECURE, RGX_CR_SYS_BUS_SECURE_ENABLE_EN);
 360         PDUMPREG32(RGX_PDUMPREG_NAME, RGX_CR_SYS_BUS_SECURE, RGX_CR_SYS_BUS_SECURE_ENABLE_EN, PDUMP_FLAGS_CONTINUOUS);
 361         #else /* ! defined(TRUSTED_DEVICE_DEFAULT_ENABLED) */
 362         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXInitBIF: Trusted Device disabled");
 363         OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_BIF_TRUST, 0);
 364         PDUMPREG32(RGX_PDUMPREG_NAME, RGX_CR_BIF_TRUST, 0, PDUMP_FLAGS_CONTINUOUS);
 365         OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_SYS_BUS_SECURE, 0);
 366         PDUMPREG32(RGX_PDUMPREG_NAME, RGX_CR_SYS_BUS_SECURE, 0, PDUMP_FLAGS_CONTINUOUS);
 367         #endif /* TRUSTED_DEVICE_DEFAULT_ENABLED */
 368 #endif
 369
 370 }
 371
 372 #if defined(RGX_FEATURE_AXI_ACELITE)
 373 /*!
 374 *******************************************************************************
 375
 376  @Function      RGXAXIACELiteInit
 377
 378  @Description Initialise AXI-ACE Lite interface
 379
 380  @Input psDevInfo - device info structure
 381
 382  @Return   IMG_VOID
 383
 384 ******************************************************************************/
 385 static IMG_VOID RGXAXIACELiteInit(PVRSRV_RGXDEV_INFO *psDevInfo)
 386 {
 387         IMG_UINT32 ui32RegAddr;
 388         IMG_UINT64 ui64RegVal;
 389
 390         ui32RegAddr = RGX_CR_AXI_ACE_LITE_CONFIGURATION;
 391
 392         /* Setup AXI-ACE config. Set everything to outer cache */
 393         ui64RegVal =   (3U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_NON_SNOOPING_SHIFT) |
 394                                    (3U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_NON_SNOOPING_SHIFT) |
 395                                    (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_CACHE_MAINTENANCE_SHIFT)  |
 396                                    (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWDOMAIN_COHERENT_SHIFT) |
 397                                    (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARDOMAIN_COHERENT_SHIFT) |
 398                                    (((IMG_UINT64) 1) << RGX_CR_AXI_ACE_LITE_CONFIGURATION_DISABLE_COHERENT_WRITELINEUNIQUE_SHIFT) |
 399                                    (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_AWCACHE_COHERENT_SHIFT) |
 400                                    (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_COHERENT_SHIFT) |
 401                                    (2U << RGX_CR_AXI_ACE_LITE_CONFIGURATION_ARCACHE_CACHE_MAINTENANCE_SHIFT);
 402
 403         OSWriteHWReg64(psDevInfo->pvRegsBaseKM,
 404                                    ui32RegAddr,
 405                                    ui64RegVal);
 406         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "Init AXI-ACE interface");
 407         PDUMPREG64(RGX_PDUMPREG_NAME, ui32RegAddr, ui64RegVal, PDUMP_FLAGS_CONTINUOUS);
 408 }
 409 #endif
 410
 411
 412 /*!
 413 *******************************************************************************
 414
 415  @Function      RGXStart
 416
 417  @Description
 418
 419  (client invoked) chip-reset and initialisation
 420
 421  @Input psDevInfo - device info structure
 422
 423  @Return   PVRSRV_ERROR
 424
 425 ******************************************************************************/
 426 static PVRSRV_ERROR RGXStart(PVRSRV_RGXDEV_INFO *psDevInfo, PVRSRV_DEVICE_CONFIG *psDevConfig)
 427 {
 428         PVRSRV_ERROR    eError = PVRSRV_OK;
 429         RGXFWIF_INIT    *psRGXFWInit;
 430
 431 #if defined(FIX_HW_BRN_37453)
 432         /* Force all clocks on*/
 433         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXStart: force all clocks on");
 434         OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_CLK_CTRL, RGX_CR_CLK_CTRL_ALL_ON);
 435         PDUMPREG64(RGX_PDUMPREG_NAME, RGX_CR_CLK_CTRL, RGX_CR_CLK_CTRL_ALL_ON, PDUMP_FLAGS_CONTINUOUS);
 436 #endif
 437
 438 #if defined(SUPPORT_SHARED_SLC) && !defined(FIX_HW_BRN_36492)
 439         /* When the SLC is shared, the SLC reset is performed by the System layer when calling
 440          * RGXInitSLC (before any device uses it), therefore mask out the SLC bit to avoid
 441          * soft_resetting it here. If HW_BRN_36492, the bit is already masked out.
 442          */
 443 #define RGX_CR_SOFT_RESET_ALL   (RGX_CR_SOFT_RESET_MASKFULL ^ RGX_CR_SOFT_RESET_SLC_EN)
 444         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXStart: Shared SLC (don't reset SLC as part of RGX reset)");
 445 #else
 446 #define RGX_CR_SOFT_RESET_ALL   (RGX_CR_SOFT_RESET_MASKFULL)
 447 #endif
 448
 449 #if defined(RGX_FEATURE_S7_TOP_INFRASTRUCTURE)
 450         /* Set RGX in soft-reset */
 451         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXStart: soft reset assert step 1");
 452         OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET, RGX_S7_SOFT_RESET_DUSTS);
 453         PDUMPREG64(RGX_PDUMPREG_NAME, RGX_CR_SOFT_RESET, RGX_S7_SOFT_RESET_DUSTS, PDUMP_FLAGS_CONTINUOUS);
 454
 455         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXStart: soft reset assert step 2");
 456         OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET, RGX_S7_SOFT_RESET_JONES_ALL | RGX_S7_SOFT_RESET_DUSTS);
 457         PDUMPREG64(RGX_PDUMPREG_NAME, RGX_CR_SOFT_RESET, RGX_S7_SOFT_RESET_JONES_ALL | RGX_S7_SOFT_RESET_DUSTS, PDUMP_FLAGS_CONTINUOUS);
 458
 459         OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET2, RGX_S7_SOFT_RESET2);
 460         PDUMPREG64(RGX_PDUMPREG_NAME, RGX_CR_SOFT_RESET2, RGX_S7_SOFT_RESET2, PDUMP_FLAGS_CONTINUOUS);
 461
 462         /* Read soft-reset to fence previous write in order to clear the SOCIF pipeline */
 463         (IMG_VOID) OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET);
 464         PDUMPREGREAD64(RGX_PDUMPREG_NAME, RGX_CR_SOFT_RESET, PDUMP_FLAGS_CONTINUOUS);
 465
 466         /* Take everything out of reset but META */
 467         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXStart: soft reset de-assert step 1 excluding META");
 468         OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET, RGX_S7_SOFT_RESET_DUSTS | RGX_CR_SOFT_RESET_GARTEN_EN);
 469         PDUMPREG64(RGX_PDUMPREG_NAME, RGX_CR_SOFT_RESET, RGX_S7_SOFT_RESET_DUSTS | RGX_CR_SOFT_RESET_GARTEN_EN, PDUMP_FLAGS_CONTINUOUS);
 470
 471         OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET2, 0x0);
 472         PDUMPREG64(RGX_PDUMPREG_NAME, RGX_CR_SOFT_RESET2, 0x0, PDUMP_FLAGS_CONTINUOUS);
 473         (IMG_VOID) OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET);
 474         PDUMPREGREAD64(RGX_PDUMPREG_NAME, RGX_CR_SOFT_RESET, PDUMP_FLAGS_CONTINUOUS);
 475
 476         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXStart: soft reset de-assert step 2 excluding META");
 477         OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET, RGX_CR_SOFT_RESET_GARTEN_EN);
 478         PDUMPREG64(RGX_PDUMPREG_NAME, RGX_CR_SOFT_RESET, RGX_CR_SOFT_RESET_GARTEN_EN, PDUMP_FLAGS_CONTINUOUS);
 479         (IMG_VOID) OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET);
 480         PDUMPREGREAD64(RGX_PDUMPREG_NAME, RGX_CR_SOFT_RESET, PDUMP_FLAGS_CONTINUOUS);
 481 #else
 482         /* Set RGX in soft-reset */
 483         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXStart: soft reset everything");
 484         OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET, RGX_CR_SOFT_RESET_ALL);
 485         PDUMPREG64(RGX_PDUMPREG_NAME, RGX_CR_SOFT_RESET, RGX_CR_SOFT_RESET_ALL, PDUMP_FLAGS_CONTINUOUS);
 486
 487         /* Take Rascal and Dust out of reset */
 488         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXStart: Rascal and Dust out of reset");
 489         OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET, RGX_CR_SOFT_RESET_ALL ^ RGX_CR_SOFT_RESET_RASCALDUSTS_EN);
 490         PDUMPREG64(RGX_PDUMPREG_NAME, RGX_CR_SOFT_RESET, RGX_CR_SOFT_RESET_ALL ^ RGX_CR_SOFT_RESET_RASCALDUSTS_EN, PDUMP_FLAGS_CONTINUOUS);
 491
 492         /* Read soft-reset to fence previous write in order to clear the SOCIF pipeline */
 493         (IMG_VOID) OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET);
 494         PDUMPREGREAD64(RGX_PDUMPREG_NAME, RGX_CR_SOFT_RESET, PDUMP_FLAGS_CONTINUOUS);
 495
 496         /* Take everything out of reset but META */
 497         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXStart: Take everything out of reset but META");
 498         OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET, RGX_CR_SOFT_RESET_GARTEN_EN);
 499         PDUMPREG64(RGX_PDUMPREG_NAME, RGX_CR_SOFT_RESET, RGX_CR_SOFT_RESET_GARTEN_EN, PDUMP_FLAGS_CONTINUOUS);
 500 #endif
 501
 502 #if ! defined(FIX_HW_BRN_37453)
 503         /*
 504          * Enable clocks.
 505          */
 506         RGXEnableClocks(psDevInfo);
 507 #endif
 508
 509         /*
 510          * Initialise SLC.
 511          */
 512 #if !defined(SUPPORT_SHARED_SLC)
 513         RGX_INIT_SLC(psDevInfo);
 514 #endif
 515
 516 #if !defined(SUPPORT_META_SLAVE_BOOT)
 517         /* Configure META to Master boot */
 518         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXStart: META Master boot");
 519         OSWriteHWReg32(psDevInfo->pvRegsBaseKM, RGX_CR_META_BOOT, RGX_CR_META_BOOT_MODE_EN);
 520         PDUMPREG32(RGX_PDUMPREG_NAME, RGX_CR_META_BOOT, RGX_CR_META_BOOT_MODE_EN, PDUMP_FLAGS_CONTINUOUS);
 521 #endif
 522
 523         /* Set Garten IDLE to META idle and Set the Garten Wrapper BIF Fence address */
 524         {
 525                 IMG_UINT64 ui64GartenConfig;
 526
 527                 /* Garten IDLE bit controlled by META */
 528                 ui64GartenConfig = RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_IDLE_CTRL_META;
 529
 530                 /* Set fence addr to the bootloader */
 531                 ui64GartenConfig |= (RGXFW_BOOTLDR_DEVV_ADDR & ~RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_ADDR_CLRMSK);
 532
 533                 /* Set PC = 0 for fences */
 534                 ui64GartenConfig &= RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_PC_BASE_CLRMSK;
 535
 536 #if defined(RGX_FEATURE_SLC_VIVT)
 537 #if !defined(FIX_HW_BRN_51281)
 538                 /* Ensure the META fences go all the way to external memory */
 539                 ui64GartenConfig |= RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_SLC_COHERENT_EN;    /* SLC Coherent 1 */
 540                 ui64GartenConfig &= RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_PERSISTENCE_CLRMSK; /* SLC Persistence 0 */
 541 #endif
 542
 543 #else
 544                 /* Set SLC DM=META */
 545                 ui64GartenConfig |= ((IMG_UINT64) RGXFW_SEGMMU_META_DM_ID) << RGX_CR_MTS_GARTEN_WRAPPER_CONFIG_FENCE_DM_SHIFT;
 546
 547 #endif
 548
 549                 PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXStart: Configure META wrapper");
 550                 OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG, ui64GartenConfig);
 551                 PDUMPREG64(RGX_PDUMPREG_NAME, RGX_CR_MTS_GARTEN_WRAPPER_CONFIG, ui64GartenConfig, PDUMP_FLAGS_CONTINUOUS);
 552         }
 553
 554 #if defined(RGX_FEATURE_AXI_ACELITE)
 555         /*
 556                 We must init the AXI-ACE interface before 1st BIF transaction
 557         */
 558         RGXAXIACELiteInit(psDevInfo);
 559 #endif
 560
 561         /*
 562          * Initialise BIF.
 563          */
 564         RGXInitBIF(psDevInfo);
 565
 566         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXStart: Take META out of reset");
 567         /* need to wait for at least 16 cycles before taking meta out of reset ... */
 568         PVRSRVSystemWaitCycles(psDevConfig, 32);
 569         PDUMPIDLWITHFLAGS(32, PDUMP_FLAGS_CONTINUOUS);
 570
 571         OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET, 0x0);
 572         PDUMPREG64(RGX_PDUMPREG_NAME, RGX_CR_SOFT_RESET, 0x0, PDUMP_FLAGS_CONTINUOUS);
 573
 574         (IMG_VOID) OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET);
 575         PDUMPREGREAD64(RGX_PDUMPREG_NAME, RGX_CR_SOFT_RESET, PDUMP_FLAGS_CONTINUOUS);
 576
 577         /* ... and afterwards */
 578         PVRSRVSystemWaitCycles(psDevConfig, 32);
 579         PDUMPIDLWITHFLAGS(32, PDUMP_FLAGS_CONTINUOUS);
 580 #if defined(FIX_HW_BRN_37453)
 581         /* we rely on the 32 clk sleep from above */
 582
 583         /* switch clocks back to auto */
 584         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXStart: set clocks back to auto");
 585         OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_CLK_CTRL, RGX_CR_CLK_CTRL_ALL_AUTO);
 586         PDUMPREG64(RGX_PDUMPREG_NAME, RGX_CR_CLK_CTRL, RGX_CR_CLK_CTRL_ALL_AUTO, PDUMP_FLAGS_CONTINUOUS);
 587 #endif
 588
 589         /*
 590          * Start the firmware.
 591          */
 592 #if defined(SUPPORT_META_SLAVE_BOOT)
 593         RGXStartFirmware(psDevInfo);
 594 #else
 595         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXStart: RGX Firmware Master boot Start");
 596 #endif
 597
 598         OSMemoryBarrier();
 599
 600         /* Check whether the FW has started by polling on bFirmwareStarted flag */
 601         eError = DevmemAcquireCpuVirtAddr(psDevInfo->psRGXFWIfInitMemDesc,
 602                                                                           (IMG_VOID **)&psRGXFWInit);
 603         if (eError != PVRSRV_OK)
 604         {
 605                 PVR_DPF((PVR_DBG_ERROR,"RGXStart: Failed to acquire kernel fw if ctl (%u)", eError));
 606                 return eError;
 607         }
 608
 609         if (PVRSRVPollForValueKM((IMG_UINT32 *)&psRGXFWInit->bFirmwareStarted,
 610                                                          IMG_TRUE,
 611                                                          0xFFFFFFFF) != PVRSRV_OK)
 612         {
 613                 PVR_DPF((PVR_DBG_ERROR, "RGXStart: Polling for 'FW started' flag failed."));
 614                 eError = PVRSRV_ERROR_TIMEOUT;
 615                 DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfInitMemDesc);
 616                 return eError;
 617         }
 618
 619 #if defined(PDUMP)
 620         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "Wait for the Firmware to start.");
 621         eError = DevmemPDumpDevmemPol32(psDevInfo->psRGXFWIfInitMemDesc,
 622                                                                                         offsetof(RGXFWIF_INIT, bFirmwareStarted),
 623                                                                                         IMG_TRUE,
 624                                                                                         0xFFFFFFFFU,
 625                                                                                         PDUMP_POLL_OPERATOR_EQUAL,
 626                                                                                         PDUMP_FLAGS_CONTINUOUS);
 627
 628         if (eError != PVRSRV_OK)
 629         {
 630                 PVR_DPF((PVR_DBG_ERROR, "RGXStart: problem pdumping POL for psRGXFWIfInitMemDesc (%d)", eError));
 631                 DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfInitMemDesc);
 632                 return eError;
 633         }
 634 #endif
 635
 636         SetFirmwareStartTime(psRGXFWInit->ui32FirmwareStartedTimeStamp);
 637
 638         DevmemReleaseCpuVirtAddr(psDevInfo->psRGXFWIfInitMemDesc);
 639
 640         return eError;
 641 }
 642
 643
 644 /*!
 645 *******************************************************************************
 646
 647  @Function      RGXStop
 648
 649  @Description Stop RGX in preparation for power down
 650
 651  @Input psDevInfo - RGX device info
 652
 653  @Return   PVRSRV_ERROR
 654
 655 ******************************************************************************/
 656 static PVRSRV_ERROR RGXStop(PVRSRV_RGXDEV_INFO  *psDevInfo)
 657
 658 {
 659         PVRSRV_ERROR            eError;
 660
 661
 662         eError = RGXRunScript(psDevInfo, psDevInfo->psScripts->asDeinitCommands, RGX_MAX_DEINIT_COMMANDS, PDUMP_FLAGS_CONTINUOUS, IMG_NULL);
 663         if (eError != PVRSRV_OK)
 664         {
 665                 PVR_DPF((PVR_DBG_ERROR,"RGXStop: RGXRunScript failed (%d)", eError));
 666                 return eError;
 667         }
 668
 669
 670         return PVRSRV_OK;
 671 }
 672
 673 /*
 674         RGXInitSLC
 675 */
 676 #if defined(SUPPORT_SHARED_SLC)
 677 PVRSRV_ERROR RGXInitSLC(IMG_HANDLE hDevHandle)
 678 {
 679
 680         PVRSRV_DEVICE_NODE      *psDeviceNode = hDevHandle;
 681         PVRSRV_RGXDEV_INFO      *psDevInfo;
 682
 683         if (psDeviceNode == IMG_NULL)
 684         {
 685                 return PVRSRV_ERROR_INVALID_PARAMS;
 686         }
 687
 688         psDevInfo = psDeviceNode->pvDevice;
 689
 690 #if !defined(FIX_HW_BRN_36492)
 691
 692         /* reset the SLC */
 693         PDUMPCOMMENTWITHFLAGS(PDUMP_FLAGS_CONTINUOUS, "RGXInitSLC: soft reset SLC");
 694         OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET, RGX_CR_SOFT_RESET_SLC_EN);
 695         PDUMPREG64(RGX_PDUMPREG_NAME, RGX_CR_SOFT_RESET, RGX_CR_SOFT_RESET_SLC_EN, PDUMP_FLAGS_CONTINUOUS);
 696
 697         /* Read soft-reset to fence previous write in order to clear the SOCIF pipeline */
 698         OSReadHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET);
 699         PDUMPREGREAD64(RGX_PDUMPREG_NAME, RGX_CR_SOFT_RESET, PDUMP_FLAGS_CONTINUOUS);
 700
 701         /* Take everything out of reset */
 702         OSWriteHWReg64(psDevInfo->pvRegsBaseKM, RGX_CR_SOFT_RESET, 0);
 703         PDUMPREG64(RGX_PDUMPREG_NAME, RGX_CR_SOFT_RESET, 0, PDUMP_FLAGS_CONTINUOUS);
 704 #endif
 705
 706         RGX_INIT_SLC(psDevInfo);
 707
 708         return PVRSRV_OK;
 709 }
 710 #endif
 711
 712
 713 static IMG_VOID _RGXUpdateGPUUtilStats(PVRSRV_RGXDEV_INFO *psDevInfo)
 714 {
 715         RGXFWIF_GPU_UTIL_FWCB *psUtilFWCb;
 716         IMG_UINT64 *paui64StatsCounters;
 717         IMG_UINT64 ui64LastPeriod;
 718         IMG_UINT64 ui64LastState;
 719         IMG_UINT64 ui64LastTime;
 720         IMG_UINT64 ui64TimeNow;
 721
 722         psUtilFWCb = psDevInfo->psRGXFWIfGpuUtilFWCb;
 723         paui64StatsCounters = &psUtilFWCb->aui64StatsCounters[0];
 724
 725         OSLockAcquire(psDevInfo->hGPUUtilLock);
 726
 727         ui64TimeNow = RGXFWIF_GPU_UTIL_GET_TIME(OSClockns64());
 728
 729         /* Update counters to account for the time since the last update */
 730         ui64LastState  = RGXFWIF_GPU_UTIL_GET_STATE(psUtilFWCb->ui64LastWord);
 731         ui64LastTime   = RGXFWIF_GPU_UTIL_GET_TIME(psUtilFWCb->ui64LastWord);
 732         ui64LastPeriod = RGXFWIF_GPU_UTIL_GET_PERIOD(ui64TimeNow, ui64LastTime);
 733         paui64StatsCounters[ui64LastState] += ui64LastPeriod;
 734
 735         /* Update state and time of the latest update */
 736         psUtilFWCb->ui64LastWord = RGXFWIF_GPU_UTIL_MAKE_WORD(ui64TimeNow, ui64LastState);
 737
 738         OSLockRelease(psDevInfo->hGPUUtilLock);
 739 }
 740
 741
 742 /*
 743         RGXPrePowerState
 744 */
 745 PVRSRV_ERROR RGXPrePowerState (IMG_HANDLE                               hDevHandle,
 746                                                            PVRSRV_DEV_POWER_STATE       eNewPowerState,
 747                                                            PVRSRV_DEV_POWER_STATE       eCurrentPowerState,
 748                                                            IMG_BOOL                                     bForced)
 749 {
 750         PVRSRV_ERROR eError = PVRSRV_OK;
 751
 752         if ((eNewPowerState != eCurrentPowerState) &&
 753                 (eNewPowerState != PVRSRV_DEV_POWER_STATE_ON))
 754         {
 755                 PVRSRV_DEVICE_NODE      *psDeviceNode = hDevHandle;
 756                 PVRSRV_RGXDEV_INFO      *psDevInfo = psDeviceNode->pvDevice;
 757                 RGXFWIF_KCCB_CMD        sPowCmd;
 758                 RGXFWIF_TRACEBUF        *psFWTraceBuf = psDevInfo->psRGXFWIfTraceBuf;
 759                 IMG_UINT32                      ui32DM;
 760
 761                 /* Send the Power off request to the FW */
 762                 sPowCmd.eCmdType = RGXFWIF_KCCB_CMD_POW;
 763                 sPowCmd.uCmdData.sPowData.ePowType = RGXFWIF_POW_OFF_REQ;
 764                 sPowCmd.uCmdData.sPowData.uPoweReqData.bForced = bForced;
 765
 766                 SyncPrimSet(psDevInfo->psPowSyncPrim, 0);
 767
 768                 /* Send one pow command to each DM to make sure we flush all the DMs pipelines */
 769                 for (ui32DM = 0; ui32DM < RGXFWIF_DM_MAX; ui32DM++)
 770                 {
 771                         eError = RGXSendCommandRaw(psDevInfo,
 772                                         ui32DM,
 773                                         &sPowCmd,
 774                                         sizeof(sPowCmd),
 775                                         0);
 776                         if (eError != PVRSRV_OK)
 777                         {
 778                                 PVR_DPF((PVR_DBG_ERROR,"RGXPrePowerState: Failed to send Power off request for DM%d", ui32DM));
 779                                 return eError;
 780                         }
 781                 }
 782
 783                 /* Wait for the firmware to complete processing. It cannot use PVRSRVWaitForValueKM as it relies
 784                    on the EventObject which is signalled in this MISR */
 785                 eError = PVRSRVPollForValueKM(psDevInfo->psPowSyncPrim->pui32LinAddr, 0x1, 0xFFFFFFFF);
 786
 787                 /* Check the Power state after the answer */
 788                 if (eError == PVRSRV_OK)
 789                 {
 790                         /* Finally, de-initialise some registers. */
 791                         if (psFWTraceBuf->ePowState == RGXFWIF_POW_OFF)
 792                         {
 793 #if !defined(NO_HARDWARE)
 794                                 /* Wait for the pending META to host interrupts to come back. */
 795                                 eError = PVRSRVPollForValueKM(&g_ui32HostSampleIRQCount,
 796                                                                                   psDevInfo->psRGXFWIfTraceBuf->ui32InterruptCount,
 797                                                                                   0xffffffff);
 798
 799                                 if (eError != PVRSRV_OK)
 800                                 {
 801                                         PVR_DPF((PVR_DBG_ERROR,"RGXPrePowerState: Wait for pending interrupts failed. Host:%d, FW: %d",
 802                                         g_ui32HostSampleIRQCount,
 803                                         psDevInfo->psRGXFWIfTraceBuf->ui32InterruptCount));
 804
 805                                         RGX_WaitForInterruptsTimeout(psDevInfo);
 806                                 }
 807 #endif /* NO_HARDWARE */
 808
 809                                 /* Update GPU frequency and timer correlation related data */
 810                                 RGXGPUFreqCalibratePrePowerState(psDeviceNode);
 811
 812                                 /* Update GPU state counters */
 813                                 _RGXUpdateGPUUtilStats(psDevInfo);
 814
 815 #if defined(PVR_DVFS)
 816                                 eError = SuspendDVFS();
 817                                 if (eError != PVRSRV_OK)
 818                                 {
 819                                         PVR_DPF((PVR_DBG_ERROR,"RGXPostPowerState: Failed to suspend DVFS"));
 820                                         return eError;
 821                                 }
 822 #endif
 823                                 eError = RGXStop(psDevInfo);
 824                                 if (eError != PVRSRV_OK)
 825                                 {
 826                                         PVR_DPF((PVR_DBG_ERROR,"RGXPrePowerState: RGXStop failed (%s)", PVRSRVGetErrorStringKM(eError)));
 827                                         eError = PVRSRV_ERROR_DEVICE_POWER_CHANGE_FAILURE;
 828                                 }
 829                                 psDevInfo->bIgnoreFurtherIRQs = IMG_TRUE;
 830                         }
 831                         else
 832                         {
 833                                 /* the sync was updated bu the pow state isn't off -> the FW denied the transition */
 834                                 eError = PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED;
 835
 836                                 if (bForced)
 837                                 {       /* It is an error for a forced request to be denied */
 838                                         PVR_DPF((PVR_DBG_ERROR,"RGXPrePowerState: Failure to power off during a forced power off. FW: %d", psFWTraceBuf->ePowState));
 839                                 }
 840                         }
 841                 }
 842                 else if (eError == PVRSRV_ERROR_TIMEOUT)
 843                 {
 844                         /* timeout waiting for the FW to ack the request: return timeout */
 845                         PVR_DPF((PVR_DBG_WARNING,"RGXPrePowerState: Timeout waiting for powoff ack from the FW"));
 846                 }
 847                 else
 848                 {
 849                         PVR_DPF((PVR_DBG_ERROR,"RGXPrePowerState: Error waiting for powoff ack from the FW (%s)", PVRSRVGetErrorStringKM(eError)));
 850                         eError = PVRSRV_ERROR_DEVICE_POWER_CHANGE_FAILURE;
 851                 }
 852
 853         }
 854
 855         return eError;
 856 }
 857
 858
 859 /*
 860         RGXPostPowerState
 861 */
 862 PVRSRV_ERROR RGXPostPowerState (IMG_HANDLE                              hDevHandle,
 863                                                                 PVRSRV_DEV_POWER_STATE  eNewPowerState,
 864                                                                 PVRSRV_DEV_POWER_STATE  eCurrentPowerState,
 865                                                                 IMG_BOOL                                bForced)
 866 {
 867         if ((eNewPowerState != eCurrentPowerState) &&
 868                 (eCurrentPowerState != PVRSRV_DEV_POWER_STATE_ON))
 869         {
 870                 PVRSRV_ERROR             eError;
 871                 PVRSRV_DEVICE_NODE       *psDeviceNode = hDevHandle;
 872                 PVRSRV_RGXDEV_INFO       *psDevInfo = psDeviceNode->pvDevice;
 873                 PVRSRV_DEVICE_CONFIG *psDevConfig = psDeviceNode->psDevConfig;
 874
 875                 if (eCurrentPowerState == PVRSRV_DEV_POWER_STATE_OFF)
 876                 {
 877                         /* Update GPU frequency and timer correlation related data */
 878                         RGXGPUFreqCalibratePostPowerState(psDeviceNode);
 879
 880                         /* Update GPU state counters */
 881                         _RGXUpdateGPUUtilStats(psDevInfo);
 882
 883                         /*
 884                                 Run the RGX init script.
 885                         */
 886                         eError = RGXStart(psDevInfo, psDevConfig);
 887                         if (eError != PVRSRV_OK)
 888                         {
 889                                 PVR_DPF((PVR_DBG_ERROR,"RGXPostPowerState: RGXStart failed"));
 890                                 return eError;
 891                         }
 892
 893                         /* Coming up from off, re-allow RGX interrupts.  */
 894                         psDevInfo->bIgnoreFurtherIRQs = IMG_FALSE;
 895
 896 #if defined(PVR_DVFS)
 897                         eError = ResumeDVFS();
 898                         if (eError != PVRSRV_OK)
 899                         {
 900                                 PVR_DPF((PVR_DBG_ERROR,"RGXPostPowerState: Failed to resume DVFS"));
 901                                 return eError;
 902                         }
 903 #endif
 904                 }
 905         }
 906
 907         PDUMPCOMMENT("RGXPostPowerState: Current state: %d, New state: %d", eCurrentPowerState, eNewPowerState);
 908
 909         return PVRSRV_OK;
 910 }
 911
 912
 913 /*
 914         RGXPreClockSpeedChange
 915 */
 916 PVRSRV_ERROR RGXPreClockSpeedChange (IMG_HANDLE                         hDevHandle,
 917                                                                          PVRSRV_DEV_POWER_STATE eCurrentPowerState)
 918 {
 919         PVRSRV_ERROR            eError = PVRSRV_OK;
 920         PVRSRV_DEVICE_NODE      *psDeviceNode = hDevHandle;
 921         PVRSRV_RGXDEV_INFO      *psDevInfo = psDeviceNode->pvDevice;
 922         RGX_DATA                        *psRGXData = (RGX_DATA*)psDeviceNode->psDevConfig->hDevData;
 923         RGXFWIF_TRACEBUF        *psFWTraceBuf = psDevInfo->psRGXFWIfTraceBuf;
 924
 925         PVR_UNREFERENCED_PARAMETER(psRGXData);
 926
 927         PVR_DPF((PVR_DBG_MESSAGE,"RGXPreClockSpeedChange: RGX clock speed was %uHz",
 928                         psRGXData->psRGXTimingInfo->ui32CoreClockSpeed));
 929
 930     if ((eCurrentPowerState != PVRSRV_DEV_POWER_STATE_OFF)
 931                 && (psFWTraceBuf->ePowState != RGXFWIF_POW_OFF))
 932         {
 933                 /* Update GPU frequency and timer correlation related data */
 934                 RGXGPUFreqCalibratePreClockSpeedChange(psDeviceNode);
 935         }
 936
 937         return eError;
 938 }
 939
 940
 941 /*
 942         RGXPostClockSpeedChange
 943 */
 944 PVRSRV_ERROR RGXPostClockSpeedChange (IMG_HANDLE                                hDevHandle,
 945                                                                           PVRSRV_DEV_POWER_STATE        eCurrentPowerState)
 946 {
 947         PVRSRV_DEVICE_NODE      *psDeviceNode = hDevHandle;
 948         PVRSRV_RGXDEV_INFO      *psDevInfo = psDeviceNode->pvDevice;
 949         RGX_DATA                        *psRGXData = (RGX_DATA*)psDeviceNode->psDevConfig->hDevData;
 950         PVRSRV_ERROR            eError = PVRSRV_OK;
 951         RGXFWIF_TRACEBUF        *psFWTraceBuf = psDevInfo->psRGXFWIfTraceBuf;
 952         IMG_UINT32              ui32NewClockSpeed = psRGXData->psRGXTimingInfo->ui32CoreClockSpeed;
 953
 954         /* Update runtime configuration with the new value */
 955         psDevInfo->psRGXFWIfRuntimeCfg->ui32CoreClockSpeed = ui32NewClockSpeed;
 956
 957     if ((eCurrentPowerState != PVRSRV_DEV_POWER_STATE_OFF)
 958                 && (psFWTraceBuf->ePowState != RGXFWIF_POW_OFF))
 959         {
 960                 RGXFWIF_KCCB_CMD        sCOREClkSpeedChangeCmd;
 961
 962                 RGXGPUFreqCalibratePostClockSpeedChange(psDeviceNode, ui32NewClockSpeed);
 963
 964                 sCOREClkSpeedChangeCmd.eCmdType = RGXFWIF_KCCB_CMD_CORECLKSPEEDCHANGE;
 965                 sCOREClkSpeedChangeCmd.uCmdData.sCORECLKSPEEDCHANGEData.ui32NewClockSpeed = ui32NewClockSpeed;
 966
 967                 /* Ensure the new clock speed is written to memory before requesting the FW to read it */
 968                 OSMemoryBarrier();
 969
 970                 PDUMPCOMMENT("Scheduling CORE clock speed change command");
 971
 972                 PDUMPPOWCMDSTART();
 973                 eError = RGXSendCommandRaw(psDeviceNode->pvDevice,
 974                                            RGXFWIF_DM_GP,
 975                                            &sCOREClkSpeedChangeCmd,
 976                                            sizeof(sCOREClkSpeedChangeCmd),
 977                                            0);
 978                 PDUMPPOWCMDEND();
 979
 980                 if (eError != PVRSRV_OK)
 981                 {
 982                         PDUMPCOMMENT("Scheduling CORE clock speed change command failed");
 983                         PVR_DPF((PVR_DBG_ERROR, "RGXPostClockSpeedChange: Scheduling KCCB command failed. Error:%u", eError));
 984                         return eError;
 985                 }
 986
 987                 PVR_DPF((PVR_DBG_MESSAGE,"RGXPostClockSpeedChange: RGX clock speed changed to %uHz",
 988                                 psRGXData->psRGXTimingInfo->ui32CoreClockSpeed));
 989         }
 990
 991         return eError;
 992 }
 993
 994
 995 /*!
 996 ******************************************************************************
 997
 998  @Function      RGXDustCountChange
 999
1000  @Description
1001
1002         Does change of number of DUSTs
1003
1004  @Input    hDevHandle : RGX Device Node
1005  @Input    ui32NumberOfDusts : Number of DUSTs to make transition to
1006
1007  @Return   PVRSRV_ERROR :
1008
1009 ******************************************************************************/
1010 PVRSRV_ERROR RGXDustCountChange(IMG_HANDLE                              hDevHandle,
1011                                                                 IMG_UINT32                              ui32NumberOfDusts)
1012 {
1013
1014         PVRSRV_DEVICE_NODE      *psDeviceNode = hDevHandle;
1015         PVRSRV_RGXDEV_INFO      *psDevInfo = psDeviceNode->pvDevice;
1016         PVRSRV_ERROR            eError;
1017         RGXFWIF_KCCB_CMD        sDustCountChange;
1018         IMG_UINT32                      ui32MaxAvailableDusts = RGX_FEATURE_NUM_CLUSTERS / 2;
1019
1020 #if !defined(NO_HARDWARE)
1021         RGXFWIF_TRACEBUF        *psFWTraceBuf = psDevInfo->psRGXFWIfTraceBuf;
1022
1023         if (psFWTraceBuf->ePowState != RGXFWIF_POW_FORCED_IDLE)
1024         {
1025                 eError = PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED;
1026                 PVR_DPF((PVR_DBG_ERROR,"RGXDustCountChange: Attempt to change dust count when not IDLE"));
1027                 return eError;
1028         }
1029 #endif
1030
1031         PVR_ASSERT(ui32MaxAvailableDusts > 1);
1032
1033         if (ui32NumberOfDusts > ui32MaxAvailableDusts)
1034         {
1035                 eError = PVRSRV_ERROR_INVALID_PARAMS;
1036                 PVR_DPF((PVR_DBG_ERROR,
1037                                 "RGXDustCountChange: Invalid number of DUSTs (%u) while expecting value within <0,%u>. Error:%u",
1038                                 ui32NumberOfDusts,
1039                                 ui32MaxAvailableDusts,
1040                                 eError));
1041                 return eError;
1042         }
1043
1044         SyncPrimSet(psDevInfo->psPowSyncPrim, 0);
1045
1046         sDustCountChange.eCmdType = RGXFWIF_KCCB_CMD_POW;
1047         sDustCountChange.uCmdData.sPowData.ePowType = RGXFWIF_POW_NUMDUST_CHANGE;
1048         sDustCountChange.uCmdData.sPowData.uPoweReqData.ui32NumOfDusts = ui32NumberOfDusts;
1049
1050         PDUMPCOMMENT("Scheduling command to change Dust Count to %u", ui32NumberOfDusts);
1051         eError = RGXSendCommandRaw(psDeviceNode->pvDevice,
1052                                 RGXFWIF_DM_GP,
1053                                 &sDustCountChange,
1054                                 sizeof(sDustCountChange),
1055                                 0);
1056
1057         if (eError != PVRSRV_OK)
1058         {
1059                 PDUMPCOMMENT("Scheduling command to change Dust Count failed. Error:%u", eError);
1060                 PVR_DPF((PVR_DBG_ERROR, "RGXDustCountChange: Scheduling KCCB to change Dust Count failed. Error:%u", eError));
1061                 return eError;
1062         }
1063
1064         /* Wait for the firmware to answer. */
1065         eError = PVRSRVPollForValueKM(psDevInfo->psPowSyncPrim->pui32LinAddr, 0x1, 0xFFFFFFFF);
1066
1067         if (eError != PVRSRV_OK)
1068         {
1069                 PVR_DPF((PVR_DBG_ERROR,"RGXDustCountChange: Timeout waiting for idle request"));
1070                 return eError;
1071         }
1072
1073 #if defined(PDUMP)
1074         PDUMPCOMMENT("RGXDustCountChange: Poll for Kernel SyncPrim [0x%p] on DM %d ", psDevInfo->psPowSyncPrim->pui32LinAddr, RGXFWIF_DM_GP);
1075
1076         SyncPrimPDumpPol(psDevInfo->psPowSyncPrim,
1077                                         1,
1078                                         0xffffffff,
1079                                         PDUMP_POLL_OPERATOR_EQUAL,
1080                                         0);
1081 #endif
1082
1083         return PVRSRV_OK;
1084 }
1085 /*
1086  @Function      RGXAPMLatencyChange
1087 */
1088 PVRSRV_ERROR RGXAPMLatencyChange(IMG_HANDLE                             hDevHandle,
1089                                 IMG_UINT32                              ui32ActivePMLatencyms,
1090                                 IMG_BOOL                                bActivePMLatencyPersistant)
1091 {
1092
1093         PVRSRV_DEVICE_NODE      *psDeviceNode = hDevHandle;
1094         PVRSRV_RGXDEV_INFO      *psDevInfo = psDeviceNode->pvDevice;
1095         PVRSRV_ERROR            eError;
1096         RGXFWIF_RUNTIME_CFG     *psRuntimeCfg = psDevInfo->psRGXFWIfRuntimeCfg;
1097         PVRSRV_DEV_POWER_STATE  ePowerState;
1098
1099         eError = PVRSRVPowerLock();
1100         if (eError != PVRSRV_OK)
1101         {
1102                 PVR_DPF((PVR_DBG_ERROR,"RGXAPMLatencyChange: Failed to acquire power lock"));
1103                 return eError;
1104         }
1105
1106         /* Update runtime configuration with the new values */
1107         psRuntimeCfg->ui32ActivePMLatencyms = ui32ActivePMLatencyms;
1108         psRuntimeCfg->bActivePMLatencyPersistant = bActivePMLatencyPersistant;
1109
1110         eError = PVRSRVGetDevicePowerState(psDeviceNode->sDevId.ui32DeviceIndex, &ePowerState);
1111
1112         if ((eError == PVRSRV_OK) && (ePowerState != PVRSRV_DEV_POWER_STATE_OFF))
1113         {
1114                 RGXFWIF_KCCB_CMD        sActivePMLatencyChange;
1115                 sActivePMLatencyChange.eCmdType = RGXFWIF_KCCB_CMD_POW;
1116                 sActivePMLatencyChange.uCmdData.sPowData.ePowType = RGXFWIF_POW_APM_LATENCY_CHANGE;
1117                 sActivePMLatencyChange.uCmdData.sPowData.uPoweReqData.ui32ActivePMLatencyms = ui32ActivePMLatencyms;
1118
1119                 /* Ensure the new APM latency is written to memory before requesting the FW to read it */
1120                 OSMemoryBarrier();
1121
1122                 PDUMPCOMMENT("Scheduling command to change APM latency to %u", ui32ActivePMLatencyms);
1123                 eError = RGXSendCommandRaw(psDeviceNode->pvDevice,
1124                                         RGXFWIF_DM_GP,
1125                                         &sActivePMLatencyChange,
1126                                         sizeof(sActivePMLatencyChange),
1127                                         0);
1128
1129                 if (eError != PVRSRV_OK)
1130                 {
1131                         PDUMPCOMMENT("Scheduling command to change APM latency failed. Error:%u", eError);
1132                         PVR_DPF((PVR_DBG_ERROR, "RGXAPMLatencyChange: Scheduling KCCB to change APM latency failed. Error:%u", eError));
1133                         return eError;
1134                 }
1135         }
1136
1137         PVRSRVPowerUnlock();
1138
1139         return PVRSRV_OK;
1140 }
1141
1142 /*
1143         RGXActivePowerRequest
1144 */
1145 PVRSRV_ERROR RGXActivePowerRequest(IMG_HANDLE hDevHandle)
1146 {
1147         PVRSRV_ERROR eError = PVRSRV_OK;
1148         PVRSRV_DEVICE_NODE      *psDeviceNode = hDevHandle;
1149
1150         PVRSRV_RGXDEV_INFO *psDevInfo = psDeviceNode->pvDevice;
1151         RGXFWIF_TRACEBUF *psFWTraceBuf = psDevInfo->psRGXFWIfTraceBuf;
1152
1153         OSAcquireBridgeLock();
1154         /* NOTE: If this function were to wait for event object attempt should be
1155            made to prevent releasing bridge lock during sleep. Bridge lock should
1156            be held during sleep. */
1157
1158         /* Powerlock to avoid further requests from racing with the FW hand-shake from now on
1159            (previous kicks to this point are detected by the FW) */
1160         eError = PVRSRVPowerLock();
1161         if(eError != PVRSRV_OK)
1162         {
1163                 PVR_DPF((PVR_DBG_ERROR,"RGXActivePowerRequest: Failed to acquire PowerLock (device index: %d, error: %s)",
1164                                         psDeviceNode->sDevId.ui32DeviceIndex,
1165                                         PVRSRVGetErrorStringKM(eError)));
1166                 goto _RGXActivePowerRequest_PowerLock_failed;
1167         }
1168
1169         /* Check again for IDLE once we have the power lock */
1170         if (psFWTraceBuf->ePowState == RGXFWIF_POW_IDLE)
1171         {
1172
1173                 psDevInfo->ui32ActivePMReqTotal++;
1174
1175         SetFirmwareHandshakeIdleTime(RGXReadHWTimerReg(psDevInfo)-psFWTraceBuf->ui64StartIdleTime);
1176
1177                 PDUMPPOWCMDSTART();
1178                 eError =
1179                         PVRSRVSetDevicePowerStateKM(psDeviceNode->sDevId.ui32DeviceIndex,
1180                                         PVRSRV_DEV_POWER_STATE_OFF,
1181                                         IMG_FALSE); /* forced */
1182                 PDUMPPOWCMDEND();
1183
1184                 if (eError == PVRSRV_OK)
1185                 {
1186                         psDevInfo->ui32ActivePMReqOk++;
1187                 }
1188                 else if (eError == PVRSRV_ERROR_DEVICE_POWER_CHANGE_DENIED)
1189                 {
1190                         psDevInfo->ui32ActivePMReqDenied++;
1191                 }
1192
1193         }
1194
1195         PVRSRVPowerUnlock();
1196
1197 _RGXActivePowerRequest_PowerLock_failed:
1198         OSReleaseBridgeLock();
1199
1200         return eError;
1201
1202 }
1203 /*
1204         RGXForcedIdleRequest
1205 */
1206
1207 #define RGX_FORCED_IDLE_RETRY_COUNT 10
1208
1209 PVRSRV_ERROR RGXForcedIdleRequest(IMG_HANDLE hDevHandle, IMG_BOOL bDeviceOffPermitted)
1210 {
1211         PVRSRV_DEVICE_NODE      *psDeviceNode = hDevHandle;
1212         PVRSRV_RGXDEV_INFO      *psDevInfo = psDeviceNode->pvDevice;
1213         RGXFWIF_KCCB_CMD        sPowCmd;
1214         PVRSRV_ERROR            eError;
1215         IMG_UINT32              ui32RetryCount = 0;
1216
1217 #if !defined(NO_HARDWARE)
1218         RGXFWIF_TRACEBUF        *psFWTraceBuf = psDevInfo->psRGXFWIfTraceBuf;
1219
1220         /* Firmware already forced idle */
1221         if (psFWTraceBuf->ePowState == RGXFWIF_POW_FORCED_IDLE)
1222         {
1223                 return PVRSRV_OK;
1224         }
1225
1226         /* Firmware is not powered. Sometimes this is permitted, for instance we were forcing idle to power down. */
1227         if (psFWTraceBuf->ePowState == RGXFWIF_POW_OFF)
1228         {
1229                 return (bDeviceOffPermitted) ? PVRSRV_OK : PVRSRV_ERROR_DEVICE_IDLE_REQUEST_DENIED;
1230         }
1231 #endif
1232
1233         SyncPrimSet(psDevInfo->psPowSyncPrim, 0);
1234         sPowCmd.eCmdType = RGXFWIF_KCCB_CMD_POW;
1235         sPowCmd.uCmdData.sPowData.ePowType = RGXFWIF_POW_FORCED_IDLE_REQ;
1236         sPowCmd.uCmdData.sPowData.uPoweReqData.bCancelForcedIdle = IMG_FALSE;
1237
1238         PDUMPCOMMENT("RGXForcedIdleRequest: Sending forced idle command");
1239
1240         /* Send one forced IDLE command to GP */
1241         eError = RGXSendCommandRaw(psDevInfo,
1242                         RGXFWIF_DM_GP,
1243                         &sPowCmd,
1244                         sizeof(sPowCmd),
1245                         0);
1246
1247         if (eError != PVRSRV_OK)
1248         {
1249                 PVR_DPF((PVR_DBG_ERROR,"RGXForcedIdleRequest: Failed to send idle request"));
1250                 return eError;
1251         }
1252
1253         /* Wait for GPU to finish current workload */
1254         do {
1255                 eError = PVRSRVPollForValueKM(psDevInfo->psPowSyncPrim->pui32LinAddr, 0x1, 0xFFFFFFFF);
1256                 if ((eError == PVRSRV_OK) || (ui32RetryCount == RGX_FORCED_IDLE_RETRY_COUNT))
1257                 {
1258                         break;
1259                 }
1260                 ui32RetryCount++;
1261                 PVR_DPF((PVR_DBG_WARNING,"RGXForcedIdleRequest: Request timeout. Retry %d of %d", ui32RetryCount, RGX_FORCED_IDLE_RETRY_COUNT));
1262         } while (IMG_TRUE);
1263
1264         if (eError != PVRSRV_OK)
1265         {
1266                 PVR_DPF((PVR_DBG_ERROR,"RGXForcedIdleRequest: Idle request failed. Firmware potentially left in forced idle state"));
1267                 return eError;
1268         }
1269
1270 #if defined(PDUMP)
1271         PDUMPCOMMENT("RGXForcedIdleRequest: Poll for Kernel SyncPrim [0x%p] on DM %d ", psDevInfo->psPowSyncPrim->pui32LinAddr, RGXFWIF_DM_GP);
1272
1273         SyncPrimPDumpPol(psDevInfo->psPowSyncPrim,
1274                                         1,
1275                                         0xffffffff,
1276                                         PDUMP_POLL_OPERATOR_EQUAL,
1277                                         0);
1278 #endif
1279
1280 #if !defined(NO_HARDWARE)
1281         /* Check the firmware state for idleness */
1282         if (psFWTraceBuf->ePowState != RGXFWIF_POW_FORCED_IDLE)
1283         {
1284                 PVR_DPF((PVR_DBG_ERROR,"RGXForcedIdleRequest: Failed to force IDLE"));
1285
1286                 return PVRSRV_ERROR_DEVICE_IDLE_REQUEST_DENIED;
1287         }
1288 #endif
1289
1290         return PVRSRV_OK;
1291 }
1292
1293 /*
1294         RGXCancelForcedIdleRequest
1295 */
1296 PVRSRV_ERROR RGXCancelForcedIdleRequest(IMG_HANDLE hDevHandle)
1297 {
1298         PVRSRV_DEVICE_NODE      *psDeviceNode = hDevHandle;
1299         PVRSRV_RGXDEV_INFO      *psDevInfo = psDeviceNode->pvDevice;
1300         RGXFWIF_KCCB_CMD        sPowCmd;
1301         PVRSRV_ERROR            eError = PVRSRV_OK;
1302
1303         SyncPrimSet(psDevInfo->psPowSyncPrim, 0);
1304
1305         /* Send the IDLE request to the FW */
1306         sPowCmd.eCmdType = RGXFWIF_KCCB_CMD_POW;
1307         sPowCmd.uCmdData.sPowData.ePowType = RGXFWIF_POW_FORCED_IDLE_REQ;
1308         sPowCmd.uCmdData.sPowData.uPoweReqData.bCancelForcedIdle = IMG_TRUE;
1309
1310         PDUMPCOMMENT("RGXForcedIdleRequest: Sending cancel forced idle command");
1311
1312         /* Send cancel forced IDLE command to GP */
1313         eError = RGXSendCommandRaw(psDevInfo,
1314                         RGXFWIF_DM_GP,
1315                         &sPowCmd,
1316                         sizeof(sPowCmd),
1317                         0);
1318
1319         if (eError != PVRSRV_OK)
1320         {
1321                 PDUMPCOMMENT("RGXCancelForcedIdleRequest: Failed to send cancel IDLE request for DM%d", RGXFWIF_DM_GP);
1322                 goto ErrorExit;
1323         }
1324
1325         /* Wait for the firmware to answer. */
1326         eError = PVRSRVPollForValueKM(psDevInfo->psPowSyncPrim->pui32LinAddr, 1, 0xFFFFFFFF);
1327
1328         if (eError != PVRSRV_OK)
1329         {
1330                 PVR_DPF((PVR_DBG_ERROR,"RGXCancelForcedIdleRequest: Timeout waiting for cancel idle request"));
1331                 goto ErrorExit;
1332         }
1333
1334 #if defined(PDUMP)
1335         PDUMPCOMMENT("RGXCancelForcedIdleRequest: Poll for Kernel SyncPrim [0x%p] on DM %d ", psDevInfo->psPowSyncPrim->pui32LinAddr, RGXFWIF_DM_GP);
1336
1337         SyncPrimPDumpPol(psDevInfo->psPowSyncPrim,
1338                                         1,
1339                                         0xffffffff,
1340                                         PDUMP_POLL_OPERATOR_EQUAL,
1341                                         0);
1342 #endif
1343
1344         return eError;
1345
1346 ErrorExit:
1347         PVR_DPF((PVR_DBG_ERROR,"RGXCancelForcedIdleRequest: Firmware potentially left in forced idle state"));
1348         return eError;
1349 }
1350
1351 /******************************************************************************
1352  End of file (rgxpower.c)
1353 ******************************************************************************/